AI media harness for planning, generating, and editing locally
TypeScript
Add scene start frames
Bittif2f3758d250d
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rsindex 5556bba..7ef7707 100644--- a/src-tauri/src/lib.rs+++ b/src-tauri/src/lib.rs@@ -247,6 +247,9 @@ struct GenerationJob {generation_snapshot: Option<String>,#[serde(default)]reference_ids: Vec<String>,+ /// Optional image reference that initializes the scene's first frame.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ start_frame_reference_id: Option<String>,/// The H3 vocabulary tags this shot was built from: group id -> option ids./// zod spells it `.nullish()` (src/lib/project.ts `shotTagSelectionSchema`),/// so `null` IS readable on the frontend — but the key is still skipped@@ -1095,6 +1098,20 @@ fn validate_and_normalize_config(mut config: ProjectConfig) -> Result<ProjectConjob.id, reference_id));}+ if let Some(start_frame_id) = &job.start_frame_reference_id {+ let Some(reference) = config.references.iter().find(|reference| reference.id == *start_frame_id) else {+ return Err(format!(+ "Generation job '{}' uses unknown start-frame reference '{}'.",+ job.id, start_frame_id+ ));+ };+ if reference.kind != "image" {+ return Err(format!(+ "Generation job '{}' start-frame reference '{}' is not an image.",+ job.id, start_frame_id+ ));+ }+ }}Ok(config)}diff --git a/src/components/ProjectWorkspace.test.ts b/src/components/ProjectWorkspace.test.tsindex 0779a8e..8044a52 100644--- a/src/components/ProjectWorkspace.test.ts+++ b/src/components/ProjectWorkspace.test.ts@@ -951,7 +951,8 @@ describe("project workspace timecode", () => {const { container } = render(createElement(GeneratorView, { config, folderPath: "C:\\Ceramic Lamp", onChange: () => undefined, onOpenTimeline: () => undefined, selectedJobId: config.generationJobs[0].id }));expect(container.querySelector(".job-refs")).toBeNull();- expect(screen.queryByText("Lamp photograph")).toBeNull();+ expect(container.querySelector(".scene-settings img, .scene-settings .reference-image-fallback")).toBeNull();+ expect(within(screen.getByRole("combobox", { name: "Start frame for this scene" })).getByRole("option", { name: "Lamp photograph" })).toBeTruthy();});it("keeps scene settings editable while a scene is running", () => {diff --git a/src/components/workspace/GeneratorView.test.tsx b/src/components/workspace/GeneratorView.test.tsxindex 09b64a4..8bc947f 100644--- a/src/components/workspace/GeneratorView.test.tsx+++ b/src/components/workspace/GeneratorView.test.tsx@@ -101,6 +101,30 @@ async function finishFirst(state: ReturnType<typeof setup>) {}describe("Generator scene controls", () => {+ it("offers no inferred Look and sends a selected image as the scene's first frame", async () => {+ const initial = project();+ initial.references = [{+ id: "ref-opening",+ kind: "image",+ name: "Opening still",+ description: "",+ relativePath: "references/opening.jpg",+ intendedUse: [],+ createdAt: initial.createdAt,+ }];+ const state = setup(parseProjectConfig(initial));++ expect(within(screen.getByRole("combobox", { name: "The look of this scene" })).getByRole("option", { name: "None" })).toBeInTheDocument();+ fireEvent.change(screen.getByRole("combobox", { name: "Start frame for this scene" }), { target: { value: "ref-opening" } });+ expect(state.latest().generationJobs[0].startFrameReferenceId).toBe("ref-opening");++ fireEvent.click(within(screen.getByRole("region", { name: "First scene" })).getByRole("button", { name: "Generate" }));+ await waitFor(() => expect(state.latest().generationJobs[0].generationSnapshot).toEqual(expect.any(String)));+ const snapshot = JSON.parse(state.latest().generationJobs[0].generationSnapshot!);+ expect(snapshot.referencePaths).toEqual(["C:\\\\project\\references\\opening.jpg"]);+ expect(snapshot.prompt).toContain("<Picture 1> is the first frame of the video.");+ });+it("shows scene generation controls and sends their values in the render snapshot", async () => {const state = setup();const generation = screen.getByRole("region", { name: "Generation" });diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsxindex fefec43..66c0552 100644--- a/src/components/workspace/GeneratorView.tsx+++ b/src/components/workspace/GeneratorView.tsx@@ -1,6 +1,8 @@import { Plus, Sparkles, Square, WandSparkles } from "lucide-react";+import { invoke } from "@tauri-apps/api/core";import { useEffect, useMemo, useRef, useState } from "react";-import { actionReferenceIds, compileGenerationJobPrompt, compileGenerationJobSegments, createDraftGenerationJob, danglingReferenceTokens, sceneDurationSeconds, sceneGenerationSeed, sceneGenerationSnapshot, sceneGenerationSteps, sceneShots, SCENE_MAX_SECONDS, SCENE_MIN_SECONDS, projectItemPath, usableImageReferences, type GenerationJob, type ProjectConfig, type ProjectReference, type PromptSegment, type SceneShot } from "../../lib/project";+import { actionReferenceIds, compileGenerationJobPrompt, compileGenerationJobSegments, createDraftGenerationJob, danglingReferenceTokens, sceneDurationSeconds, sceneGenerationReferences, sceneGenerationSeed, sceneGenerationSnapshot, sceneGenerationSteps, sceneShots, SCENE_MAX_SECONDS, SCENE_MIN_SECONDS, projectItemPath, usableImageReferences, type GenerationJob, type ProjectConfig, type ProjectReference, type PromptSegment, type SceneShot } from "../../lib/project";+import { isTauri } from "../../lib/persistence";import { cancelVidfabGeneration, enqueueVidfabGeneration, resolveVidfabPlan, type VidfabGenerationRequest, type VidfabStatus } from "../../lib/runtime";import { SceneBoard, type GeneratorSelection } from "./SceneBoard";import { SceneInspector, ShotInspector, STEP_SECONDS, writeShots } from "./SceneEditor";@@ -51,6 +53,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCoconst [, setPlanNotes] = useState<Record<string, string>>({});const [removalTarget, setRemovalTarget] = useState<RemovalTarget | null>(null);const [showDebugPrompt, setShowDebugPrompt] = useState(false);+ const [startFrameError, setStartFrameError] = useState<string | null>(null);useEffect(() => { if (selectedJobId) setSelection({ jobId: selectedJobId, shotId: null }); }, [selectedJobId]);@@ -64,7 +67,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCoconst active = jobs.filter((job) => job.status === "generating" || job.status === "ready");const queued = jobs.filter((job) => job.status === "queued");const cancellable = jobs.filter((job) => job.status === "queued" || job.status === "generating");- const boundRefs = useMemo(() => config.references.filter((ref) => selected?.referenceIds.includes(ref.id)), [config.references, selected]);+ const boundRefs = useMemo(() => selected ? sceneGenerationReferences(selected, config.references) : [], [config.references, selected]);const runtimeReady = runtime?.state === "ready";const updateJob = (id: string, updates: Partial<GenerationJob>) => {@@ -95,6 +98,35 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCoconst updateScene = (job: GenerationJob, updates: Partial<GenerationJob>) =>updateJob(job.id, mergeSceneUpdates(job, updates));+ const addStartFrame = async (job: GenerationJob) => {+ if (!isTauri()) return;+ setStartFrameError(null);+ try {+ const imported = await invoke<{ name: string; relativePath: string } | null>("choose_reference_image", { folderPath });+ if (!imported) return;+ const current = configRef.current;+ const id = `ref-${Date.now()}`;+ const reference: ProjectReference = {+ id,+ kind: "image",+ name: imported.name,+ description: "",+ relativePath: imported.relativePath,+ intendedUse: [],+ createdAt: new Date().toISOString(),+ };+ onChange({+ ...current,+ references: [reference, ...current.references],+ generationJobs: current.generationJobs.map((candidate) => candidate.id === job.id+ ? { ...candidate, startFrameReferenceId: id, updatedAt: new Date().toISOString() }+ : candidate),+ });+ } catch (reason) {+ setStartFrameError(reason instanceof Error ? reason.message : String(reason));+ }+ };+/* --- The shots of a scene ------------------------------------------------Every one of these goes through writeShots, so the mirrored `prompt` and`creativeBrief` can never fall behind the lines they mirror. */@@ -198,7 +230,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo// ordered list drives the <Subject N> / <Picture N> numbering inside the// compiled prompt, because vidfab.rs adds reference_paths sequentially — so// array index 0 must be the asset the prompt calls <Picture 1>.- const bound = configRef.current.references.filter((reference) => job.referenceIds.includes(reference.id));+ const bound = sceneGenerationReferences(job, configRef.current.references);return {jobId: job.id,// Recompiled from current state so edits to a bound reference or a@@ -527,6 +559,10 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCojob={selected}shots={selectedShots}disabled={false}+ references={config.references}+ importAvailable={isTauri()}+ importError={startFrameError}+ onAddStartFrame={() => void addStartFrame(selected)}onChange={(updates) => updateScene(selected, updates)}onShots={(next) => setShots(selected, next)}/>diff --git a/src/components/workspace/ReferencesView.tsx b/src/components/workspace/ReferencesView.tsxindex 6608506..93eb763 100644--- a/src/components/workspace/ReferencesView.tsx+++ b/src/components/workspace/ReferencesView.tsx@@ -79,7 +79,11 @@ export function ReferencesView({ config, folderPath, onChange }: { config: Proje};const remove = () => {if (!selected) return;- onChange({ ...config, references: config.references.filter((ref) => ref.id !== selected.id), generationJobs: config.generationJobs.map((job) => ({ ...job, referenceIds: job.referenceIds.filter((id) => id !== selected.id) })) });+ onChange({ ...config, references: config.references.filter((ref) => ref.id !== selected.id), generationJobs: config.generationJobs.map((job) => ({+ ...job,+ referenceIds: job.referenceIds.filter((id) => id !== selected.id),+ startFrameReferenceId: job.startFrameReferenceId === selected.id ? undefined : job.startFrameReferenceId,+ })) });setSelectedId(config.references.find((ref) => ref.id !== selected.id)?.id);};const openPresetPicker = () => {diff --git a/src/components/workspace/SceneEditor.tsx b/src/components/workspace/SceneEditor.tsxindex c7ef638..53cdb2c 100644--- a/src/components/workspace/SceneEditor.tsx+++ b/src/components/workspace/SceneEditor.tsx@@ -1,4 +1,4 @@-import { Trash2, X } from "lucide-react";+import { ImagePlus, Trash2, X } from "lucide-react";import { useMemo, useRef, useState } from "react";import {actionReferenceIds,@@ -270,10 +270,14 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer/** Scene-wide render controls, the look the description opens with (base guide* §4.1), and the two sound fields defined per prompt (§4.6, §4.7). Length* lives in the scene header where it stays visible. */-export function SceneInspector({ job, shots, disabled, onChange, onShots }: {+export function SceneInspector({ job, shots, references, disabled, importAvailable, importError, onAddStartFrame, onChange, onShots }: {job: GenerationJob;shots: SceneShot[];+ references: ProjectReference[];disabled: boolean;+ importAvailable: boolean;+ importError: string | null;+ onAddStartFrame: () => void;onChange: (updates: Partial<GenerationJob>) => void;onShots: (next: SceneShot[]) => void;}) {@@ -281,6 +285,10 @@ export function SceneInspector({ job, shots, disabled, onChange, onShots }: {// The style is read off the first shot that carries one, so that is where it// is written. One place, never a second field that could disagree with it.const chosen = shots.map((shot) => shot.settings?.[look.id]?.[0]).find(Boolean) ?? "";+ const imageReferences = useMemo(+ () => references.filter((reference) => reference.kind === "image" && isReferenceUsable(reference)),+ [references],+ );return <section className="scene-inspector" aria-label="This scene"><section className="scene-settings" aria-labelledby={`${job.id}-scene-settings`}>@@ -301,10 +309,33 @@ export function SceneInspector({ job, shots, disabled, onChange, onShots }: {}));}}>- <option value="">Guessed from your words</option>+ <option value="">None</option>{look.options.map((option) => <option key={option.id} value={option.id}>{option.label}</option>)}</select></label>+ <div className="scene-settings__field">+ <span>Start frame</span>+ <div className="scene-settings__start-frame">+ <select+ value={job.startFrameReferenceId ?? ""}+ disabled={disabled}+ aria-label="Start frame for this scene"+ onChange={(event) => onChange({ startFrameReferenceId: event.target.value || undefined })}+ >+ <option value="">None</option>+ {imageReferences.map((reference) => <option key={reference.id} value={reference.id}>{reference.name}</option>)}+ </select>+ <button+ type="button"+ className="secondary-button"+ disabled={disabled || !importAvailable}+ title={importAvailable ? "Import an image and use it as this scene's first frame" : "Image import is available in the desktop app"}+ onClick={onAddStartFrame}+ ><ImagePlus size={15} /> Add image</button>+ </div>+ <small>The selected image is sent to VidFab as Picture 1 and anchors the opening frame.</small>+ {importError && <small className="scene-settings__error" role="alert">{importError}</small>}+ </div><label className="scene-settings__field"><span>Sound</span><textareadiff --git a/src/lib/project.test.ts b/src/lib/project.test.tsindex 7070aba..9a64bae 100644--- a/src/lib/project.test.ts+++ b/src/lib/project.test.ts@@ -38,6 +38,7 @@ import {referenceToken,sceneBriefText,sceneDurationSeconds,+ sceneGenerationReferences,sceneShots,splitActionText,DEFAULT_SCENE_SECONDS,@@ -421,6 +422,30 @@ describe("project schema", () => {}});+ it("uses a dedicated image as Picture 1 and the scene's first frame", () => {+ const image = (id: string, name: string, path: string): ProjectReference => projectReferenceSchema.parse({+ id, kind: "image", name, description: "", relativePath: path,+ intendedUse: [], createdAt: new Date().toISOString(),+ });+ const opening = image("opening", "Opening still", "references/opening.jpg");+ const subject = image("subject", "Product", "references/product.jpg");+ const job = createDraftGenerationJob("The product turns towards camera.", {+ id: "scene-with-frame",+ references: [opening, subject],+ referenceIds: ["subject"],+ startFrameReferenceId: "opening",+ });+ const ordered = sceneGenerationReferences(job, [subject, opening]);+ const compiled = compileGenerationJobPrompt(job, ordered);++ expect(ordered.map((reference) => reference.id)).toEqual(["opening", "subject"]);+ expect(compiled).toContain("<Picture 1> is the first frame of the video.");+ expect(compiled).toContain("The video begins with <Picture 1>.");+ expect(compiled).toContain("The opening frame matches <Picture 1>.");+ expect(compiled).toContain("<Subject 1> is the content shown in <Picture 2>.");+ expect(compiled).not.toContain("<Subject 2>");+ });+it("allows a generated scene asset to wait for its render without relaxing imported media", () => {const waiting = structuredClone(createdFixture) as any;waiting.assets = [{diff --git a/src/lib/project.ts b/src/lib/project.tsindex 8a5d3b8..7f114d4 100644--- a/src/lib/project.ts+++ b/src/lib/project.ts@@ -380,6 +380,9 @@ export const generationJobSchema = z.object({* drafts that have never run and on projects saved before change tracking. */generationSnapshot: z.string().min(1).nullish(),referenceIds: z.array(idSchema).default([]),+ /** An image reference used as the scene's opening frame. It is separate from+ * subject citations because a frame is composition, not a named subject. */+ startFrameReferenceId: idSchema.nullish(),// `.nullish()` because Rust holds it as an Option — see the note on// projectAssetSchema.durationMs. Every project written before shot tags// existed has no key here at all, and must keep opening.@@ -673,6 +676,15 @@ export function usableImageReferences(references: ProjectReference[]): ProjectRe.filter(isVisualReference);}+/** References in the exact order both the prompt compiler and VidFab consume.+ * A start frame must be Picture 1; ordinary bound references retain project+ * order after it and a frame that is also bound is still sent only once. */+export function sceneGenerationReferences(job: GenerationJob, references: ProjectReference[]): ProjectReference[] {+ const start = references.find((reference) => reference.id === job.startFrameReferenceId && reference.kind === "image" && isReferenceUsable(reference));+ const bound = references.filter((reference) => job.referenceIds.includes(reference.id) && reference.id !== start?.id);+ return start ? [start, ...bound] : bound;+}+/* Who wrote each piece of the compiled prompt.- "brief" is the user's own prose, character for character.@@ -701,6 +713,7 @@ const addedStop = (text: string): PromptSegment[] => (endSentence(text) === text* to show the same bytes for both. */export interface ScenePrompt {shots: readonly SceneShot[];+ startFrameReferenceId?: string | null;/** Base guide §4.6 / §4.7. Blank or absent falls back to the two default* lines below, which are marked as PolStudio's own writing. */soundscape?: string | null;@@ -805,7 +818,7 @@ export function compileMiniMaxH3PromptSegments(}export function compileGenerationJobSegments(job: GenerationJob, references: ProjectReference[] = []): PromptSegment[] {- return compileScenePromptSegments({ shots: sceneShots(job), soundscape: job.soundscape, music: job.music }, references);+ return compileScenePromptSegments({ shots: sceneShots(job), startFrameReferenceId: job.startFrameReferenceId, soundscape: job.soundscape, music: job.music }, references);}export function compileGenerationJobPrompt(job: GenerationJob, references: ProjectReference[] = []): string {@@ -834,7 +847,8 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje// <Picture N> / reference_paths lockstep note on that function. An// audio-only-tagged reference is dropped here (ref guide §2.1), so a scene// whose only reference is audio-tagged correctly falls back to T2VA.- const usable = references.filter(isReferenceUsable).filter(isVisualReference);+ const startFrame = references.find((reference) => reference.id === scene.startFrameReferenceId && reference.kind === "image" && isReferenceUsable(reference)) ?? null;+ const usable = references.filter(isReferenceUsable).filter(isVisualReference).filter((reference) => reference.id !== startFrame?.id);const subjectNumber = new Map(usable.map((reference, index) => [reference.id, index + 1]));const compiled: CompiledShot[] = shots.map((shot, index) => {@@ -911,7 +925,7 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Projeconst soundSegment = sound ? own(sound) : frame(DEFAULT_SOUNDSCAPE);const musicSegment = music ? own(music) : frame(DEFAULT_MUSIC);- if (usable.length === 0) {+ if (usable.length === 0 && !startFrame) {// T2VA — base guide §2.2 field list and order.return [frame("integrated_multimodal_description: "),@@ -937,9 +951,9 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Projeconst pictureNumber = new Map(images.map((reference, index) => [reference.id, index + 1]));// §2.2: an image that only defines a character, scene, costume or style is- // cited INSIDE its <Subject N> definition — it never becomes a standalone- // <Picture N> entry. This app cannot express a first/last/key frame, so no- // standalone picture entry is ever correct here.+ // cited INSIDE its <Subject N> definition. A scene's explicit first frame is+ // the exception: it describes composition, so it is emitted as a standalone+ // <Picture N> entry instead of pretending to be a subject.// No <Audio N> is emitted either: §2.4 defines it as an actual audio asset,// and every reference in this app is text or image. An "audio" intendedUse// is a note about desired sound, not a signal to copy.@@ -962,6 +976,10 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje...(detail ? [frame(" "), own(detail), ...addedStop(detail)] : []),];});+ const startPicture = startFrame ? pictureNumber.get(startFrame.id) : null;+ const startDefinition = startPicture+ ? [frame(`<Picture ${startPicture}> is the first frame of the video.${definitions.length > 0 ? "\n" : ""}`)]+ : [];const labels = usable.map((reference, index) => referenceLabel(reference, index));const listOf = (values: string[]): string => values.length === 1@@ -979,10 +997,9 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Projeconst anyCited = compiled.some((shot) => shot.citedIds.length > 0);const featured = compiled.map((shot) => anyCited ? shot.citedIds : usable.map((reference) => reference.id));- // §3: task-type prefix. Every reference here provides generation guidance- // without acting as a concrete frame or an edited source video, which is- // exactly `reference generation`. The summary reuses existing labels only,- // and the user's own lines are carried through in order.+ // §3: task-type prefix. Subject references provide generation guidance; an+ // optional opening picture additionally anchors the first frame. The summary+ // reuses existing labels only and carries the user's lines through in order.const summary: PromptSegment[] = [frame("[reference generation] "),...compiled.flatMap((shot) => [@@ -990,7 +1007,8 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje...shot.body,...addedStop(shot.text),]),- frame(` ${labelList} ${labels.length === 1 ? "provides" : "provide"} generation guidance for the ${compiled.length === 1 ? "single shot" : `${compiled.length} shots`} described below.`),+ ...(startPicture ? [frame(` The video begins with <Picture ${startPicture}>.`)] : []),+ ...(labels.length > 0 ? [frame(` ${labelList} ${labels.length === 1 ? "provides" : "provide"} generation guidance for the ${compiled.length === 1 ? "single shot" : `${compiled.length} shots`} described below.`)] : []),];// §4.1: one line per label using the fixed marker vocabulary. Their defined@@ -1009,6 +1027,7 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Projeconst where = appearances.length > 0 ? ` (appears in ${appearances.join(", ")})` : "";return `${referenceLabel(reference, index)}${where}: fully_preserved - the referenced characteristics are retained.`;});+ if (startPicture) retention.unshift(`<Picture ${startPicture}>: first_frame - used as the opening frame.`);// §5.2: in full-reference mode the style opening comes BEFORE [Shot 1], not// after it. §5.3: cite each <Subject N> where it appears in the shot.@@ -1028,11 +1047,13 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje: []),...tail(shot),...dialogue(shot, shot.text.length > 0 || featured[shot.index].length > 0 || shot.tags.clauses.length > 0),+ ...(shot.index === 0 && startPicture ? [frame(` The opening frame matches <Picture ${startPicture}>.`)] : []),]),];return [frame("subject_definitions:\n"),+ ...startDefinition,...definitions,frame("\n\nsummary:\n"),...summary,@@ -1072,6 +1093,7 @@ export function createDraftGenerationJob(id?: string;title?: string;referenceIds?: string[];+ startFrameReferenceId?: string | null;references?: ProjectReference[];shotTags?: ShotTagSelection | null;/** The shots to open the scene with. Omitted means one shot holding@@ -1113,6 +1135,7 @@ export function createDraftGenerationJob(// the scene later holds bound image references). Do not read it as current.compiledPrompt: compileScenePromptSegments(scene, options.references ?? []).map((segment) => segment.value).join(""),referenceIds: options.referenceIds ?? [],+ startFrameReferenceId: options.startFrameReferenceId ?? undefined,shots,durationSeconds,steps: options.steps ?? DEFAULT_GENERATION_STEPS,diff --git a/src/styles/generator.css b/src/styles/generator.cssindex a5ea905..0a0a558 100644--- a/src/styles/generator.css+++ b/src/styles/generator.css@@ -1069,6 +1069,27 @@line-height: var(--leading-snug);}+.scene-settings__start-frame {+ display: grid;+ grid-template-columns: minmax(0, 1fr) auto;+ gap: var(--space-2);+}++.scene-settings__start-frame select {+ width: 100%;+}++.scene-settings__start-frame button {+ min-height: var(--field-height);+ gap: var(--space-2);+ padding-inline: var(--space-3);+ white-space: nowrap;+}++.scene-settings__field .scene-settings__error {+ color: var(--danger);+}+.scene-settings__numbers {display: grid;grid-template-columns: 1fr 1fr;