Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Follow MiniMax H3 prompt guides, send image references, fix desktop save

Bittiebbcac687eb1
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index c4afac0..1dd92da 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -277,14 +277,12 @@ fn validate_and_normalize_config(mut config: ProjectConfig) -> Result<ProjectCon
.map(normalize_project_path)
.transpose()?;
match reference.kind.as_str() {
- "text"
- if !reference.description.trim().is_empty()
- || reference
- .content
- .as_deref()
- .is_some_and(|value| !value.trim().is_empty()) => {}
+ // A text reference may legitimately be blank: the UI creates one the
+ // moment "New definition" is clicked, before the user has typed. It
+ // is marked incomplete there rather than failing the whole save —
+ // refusing here blocked every unrelated edit in the project.
+ "text" => {}
"image" if reference.relative_path.is_some() => {}
- "text" => return Err("Text references require content.".into()),
"image" => return Err("Image references require a relative path.".into()),
_ => return Err(format!("Unsupported reference kind '{}'.", reference.kind)),
}
diff --git a/src/components/ProjectWorkspace.test.ts b/src/components/ProjectWorkspace.test.ts
index f649d9c..8e63ea8 100644
--- a/src/components/ProjectWorkspace.test.ts
+++ b/src/components/ProjectWorkspace.test.ts
@@ -77,7 +77,7 @@ describe("project workspace timecode", () => {
const config = parseProjectConfig({ ...fresh, generationJobs: [{ ...fresh.generationJobs[0], status: "completed", stage: "completed", progress: 1, outputRelativePath: "media/generated/first-scene.mp4" }] });
const onChange = vi.fn();
const onOpenTimeline = vi.fn();
- render(createElement(GeneratorView, { config, onChange, onOpenTimeline, selectedJobId: config.generationJobs[0].id }));
+ render(createElement(GeneratorView, { config, folderPath: "C:\Ceramic Lamp", onChange, onOpenTimeline, selectedJobId: config.generationJobs[0].id }));
fireEvent.click(screen.getByRole("button", { name: "Insert into Story" }));
const next = onChange.mock.calls[0][0];
expect(next.assets[0].relativePath).toBe("media/generated/first-scene.mp4");
@@ -88,7 +88,7 @@ describe("project workspace timecode", () => {
it("labels a cancelled preview as cancelled instead of queued", () => {
const fresh = createProjectConfig({ name: "Ceramic lamp", prompt: "A quiet product film", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 30 });
const config = parseProjectConfig({ ...fresh, generationJobs: [{ ...fresh.generationJobs[0], status: "cancelled", stage: "failed" }] });
- render(createElement(GeneratorView, { config, onChange: () => undefined, onOpenTimeline: () => undefined }));
+ render(createElement(GeneratorView, { config, folderPath: "C:\Ceramic Lamp", onChange: () => undefined, onOpenTimeline: () => undefined }));
expect(screen.getByText("Generation cancelled")).not.toBeNull();
expect(screen.queryByText("Waiting in queue")).toBeNull();
});
diff --git a/src/components/ProjectWorkspace.tsx b/src/components/ProjectWorkspace.tsx
index bcb4950..0e4ce57 100644
--- a/src/components/ProjectWorkspace.tsx
+++ b/src/components/ProjectWorkspace.tsx
@@ -91,7 +91,7 @@ export function ProjectWorkspace({ project, initialView = "timeline", onBack, on
<div className={`project-content project-content--${view}`}>
{view === "timeline" && <TimelineView config={config} onChange={changeConfig} onOpenGenerator={(jobId) => { setSelectedGenerationJobId(jobId); setView("generator"); }} />}
- {view === "generator" && <GeneratorView config={config} runtime={runtime?.vidfab ?? null} onChange={changeConfig} selectedJobId={selectedGenerationJobId} onOpenTimeline={() => setView("timeline")} />}
+ {view === "generator" && <GeneratorView config={config} folderPath={project.folderPath} runtime={runtime?.vidfab ?? null} onChange={changeConfig} selectedJobId={selectedGenerationJobId} onOpenTimeline={() => setView("timeline")} />}
{view === "references" && <ReferencesView config={config} folderPath={project.folderPath} onChange={changeConfig} />}
</div>
<footer className="project-agent-row"><AgentDock
diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsx
index 8fdbdab..f7e2cce 100644
--- a/src/components/workspace/GeneratorView.tsx
+++ b/src/components/workspace/GeneratorView.tsx
@@ -2,11 +2,12 @@ import { listen } from "@tauri-apps/api/event";
import { AlertCircle, Ban, Check, ChevronRight, Clock3, Film, Info, LoaderCircle, Play, RefreshCw, Sparkles, Square, WandSparkles, X } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { isTauri } from "../../lib/persistence";
-import { createDraftGenerationJob, type GenerationJob, type ProjectAsset, type ProjectConfig, type TimelineClip, type TimelineTrack } from "../../lib/project";
+import { compileMiniMaxH3Prompt, createDraftGenerationJob, isReferenceUsable, projectFilePath, usableImageReferences, type GenerationJob, type ProjectAsset, type ProjectConfig, type TimelineClip, type TimelineTrack } from "../../lib/project";
import { cancelVidfabGeneration, enqueueVidfabGeneration, resolveVidfabPlan, type VidfabGenerationRequest, type VidfabStatus } from "../../lib/runtime";
interface GeneratorViewProps {
config: ProjectConfig;
+ folderPath: string;
runtime?: VidfabStatus | null;
onChange: (next: ProjectConfig) => void;
onOpenTimeline: () => void;
@@ -15,7 +16,7 @@ interface GeneratorViewProps {
type JobStatus = GenerationJob["status"];
-export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline, selectedJobId }: GeneratorViewProps) {
+export function GeneratorView({ config, folderPath, runtime = null, onChange, onOpenTimeline, selectedJobId }: GeneratorViewProps) {
const [selectedId, setSelectedId] = useState(selectedJobId ?? config.generationJobs.find((job) => job.status === "generating")?.id ?? config.generationJobs[0]?.id);
const [prompt, setPrompt] = useState("");
const [planNotes, setPlanNotes] = useState<Record<string, string>>({});
@@ -29,6 +30,7 @@ export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline
const drafts = jobs.filter((job) => job.status === "draft");
const completed = jobs.filter((job) => ["ready", "completed", "failed", "cancelled"].includes(job.status));
const boundRefs = useMemo(() => config.references.filter((ref) => selected?.referenceIds.includes(ref.id)), [config.references, selected]);
+ const usableReferences = useMemo(() => config.references.filter(isReferenceUsable), [config.references]);
const runtimeReady = runtime?.state === "ready";
const submitLabel = runtimeReady ? "Generate this shot" : "Save as a draft";
@@ -38,15 +40,26 @@ export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline
const current = configRef.current;
onChange({ ...current, generationJobs: current.generationJobs.map((job) => job.id === id ? { ...job, ...updates } : job) });
};
- const requestFor = (job: GenerationJob): VidfabGenerationRequest => ({
- jobId: job.id,
- prompt: job.compiledPrompt,
- frames: Math.round(6 * config.settings.frameRate),
- steps: 50,
- seed: 482091,
- aspectRatio: config.settings.aspectRatio,
- referencePaths: [],
- });
+ const requestFor = (job: GenerationJob): VidfabGenerationRequest => {
+ // Only the references actually bound to this job, in list order. The same
+ // ordered list drives the <Picture N> numbering inside the compiled prompt,
+ // because vidfab.rs adds reference_paths sequentially — so array index 0
+ // must be the asset the prompt calls <Picture 1>.
+ const bound = configRef.current.references.filter((reference) => job.referenceIds.includes(reference.id));
+ return {
+ jobId: job.id,
+ // Recompiled from current state so edits to a bound reference reach the
+ // engine, rather than sending a prompt frozen at draft-creation time.
+ prompt: compileMiniMaxH3Prompt(job.creativeBrief, bound),
+ frames: Math.round(6 * config.settings.frameRate),
+ steps: 50,
+ seed: 482091,
+ aspectRatio: config.settings.aspectRatio,
+ referencePaths: usableImageReferences(bound)
+ .map((reference) => projectFilePath(folderPath, reference.relativePath ?? ""))
+ .filter((path) => path.length > 0),
+ };
+ };
useEffect(() => {
if (!isTauri()) return;
@@ -83,7 +96,8 @@ export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline
// actually typed, or it gets read back to them as their own request.
const cleanPrompt = prompt.trim();
if (!cleanPrompt) return;
- const draft = createDraftGenerationJob(cleanPrompt, { referenceIds: config.references.slice(0, 2).map((ref) => ref.id) });
+ const bound = config.references.filter(isReferenceUsable).slice(0, 2);
+ const draft = createDraftGenerationJob(cleanPrompt, { referenceIds: bound.map((ref) => ref.id), references: bound });
const job: GenerationJob = runtimeReady ? { ...draft, status: "queued" } : draft;
const next = { ...config, generationJobs: [job, ...jobs] };
onChange(next);
@@ -167,46 +181,10 @@ export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline
<button className="secondary-button" onClick={onOpenTimeline}><Film size={16} /> View timeline</button>
</header>
- <section className="generation-composer">
- <label className="generation-composer__field">
- <span className="generation-composer__label"><Sparkles size={17} /> Describe the shot you want</span>
- <textarea
- ref={composerRef}
- value={prompt}
- onChange={(event) => setPrompt(event.target.value)}
- placeholder="Example: a close shot of hands shaping wet clay on a spinning wheel, warm window light, the camera pushes in slowly, quiet room tone."
- />
- </label>
- <div className="generation-composer__foot">
- <div className="generation-settings">
- <span className="generation-settings__lead">Every shot is made as</span>
- <span className="setting-chip"><em>Length</em><b>6 seconds</b></span>
- <span className="setting-chip"><em>Shape</em><b>{config.settings.aspectRatio}</b></span>
- <span className="setting-chip"><em>Size</em><b>{config.settings.resolution.toUpperCase()}</b></span>
- <span className="setting-chip"><em>Model</em><b>MiniMax H3</b></span>
- </div>
- <button className="primary-button generation-composer__submit" onClick={() => void createJob()} disabled={!prompt.trim()}>
- <WandSparkles size={17} /> {submitLabel}
- </button>
- </div>
- {/* Disclosed at the point of commitment, not after an expensive run. */}
- {runtimeReady && <p className="generation-composer__limit">
- <Info size={16} />
- <span>A finished shot renders frames into memory. Pol Studio can’t save them as a video file yet, so nothing lands in your project folder and there is nothing to add to the timeline.</span>
- </p>}
- <p className="generation-composer__hint">
- {runtimeReady
- ? "Shots render one at a time, so a new shot joins the queue behind anything already running."
- : "Nothing renders on this computer yet, so your shot is saved as a draft you can run later."}
- {/* There is no attach control: shots bind the first two references by
- list order. Say so, rather than implying a choice the user has. */}
- {config.references.length > 0
- ? ` It will automatically use ${config.references.slice(0, 2).map((ref) => ref.name).join(" and ")}${config.references.length > 2 ? `, the first two of your ${config.references.length} references.` : "."}`
- : " No references added yet — add characters, places, or looks to keep shots consistent."}
- </p>
- </section>
-
- {selected ? <section className="job-detail">
+ {/* The draft made from the user's own words leads the screen: they
+ just described their video, so the next step is running that shot,
+ not describing it again in a different unit. */}
+ {selected && <section className="job-detail">
<div className="job-detail__visual">
{/* A labelled placeholder, never invented art — the same rule the scene
thumb, media thumb and reference cards follow. Pol Studio cannot
@@ -266,14 +244,15 @@ export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline
</dl>
<div className="job-refs">
- <span>{boundRefs.length === 0 ? "No references pinned to this shot" : boundRefs.length === 1 ? "1 reference guides this shot" : `${boundRefs.length} references guide this shot`}</span>
+ <span>{boundRefs.length === 0 ? "No references used by this shot" : boundRefs.length === 1 ? "1 reference guides this shot" : `${boundRefs.length} references guide this shot`}</span>
{boundRefs.length === 0
- ? <p className="job-refs__empty">The video engine only follows the words above. Add characters, places, or looks in References to keep shots consistent.</p>
+ ? <p className="job-refs__empty">The video engine only follows the words above.</p>
: boundRefs.map((ref, index) => <div key={ref.id}>
<i className={`ref-mini ref-mini--${index}`} />
<b>{ref.name}</b>
<small>{ref.intendedUse.join(", ")}</small>
</div>)}
+ {boundRefs.length > 0 && <p className="job-refs__empty">Images are sent to the video engine as reference assets; text definitions are written into this shot’s prompt.</p>}
</div>
<div className="job-actions">
@@ -283,7 +262,7 @@ export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline
{selected.status === "draft" && <>
<button className="primary-button" disabled={!runtimeReady} onClick={() => void startDraft(selected)}>
- <WandSparkles size={15} /> {runtimeReady ? "Generate this shot" : "Can’t generate yet"}
+ <WandSparkles size={15} /> {runtimeReady ? (jobs.length === 1 ? "Generate your first shot" : "Generate this shot") : "Can’t generate yet"}
</button>
{!runtimeReady && <p className="job-actions__note">This draft is saved with your project. Pol Studio needs a working video engine before it can render it.</p>}
</>}
@@ -322,7 +301,49 @@ export function GeneratorView({ config, runtime = null, onChange, onOpenTimeline
</>}
</div>
</aside>
- </section> : <div className="job-empty">
+ </section>}
+
+ <section className="generation-composer">
+ <label className="generation-composer__field">
+ <span className="generation-composer__label"><Sparkles size={17} /> {selected ? "Write another shot" : "Describe the shot you want"}</span>
+ <textarea
+ ref={composerRef}
+ value={prompt}
+ onChange={(event) => setPrompt(event.target.value)}
+ placeholder="Example: a close shot of hands shaping wet clay on a spinning wheel, warm window light, the camera pushes in slowly, quiet room tone."
+ />
+ </label>
+ <div className="generation-composer__foot">
+ <div className="generation-settings">
+ <span className="generation-settings__lead">Every shot is made as</span>
+ <span className="setting-chip"><em>Length</em><b>6 seconds</b></span>
+ <span className="setting-chip"><em>Shape</em><b>{config.settings.aspectRatio}</b></span>
+ <span className="setting-chip"><em>Size</em><b>{config.settings.resolution.toUpperCase()}</b></span>
+ <span className="setting-chip"><em>Model</em><b>MiniMax H3</b></span>
+ </div>
+ <button className={`${selected ? "secondary-button" : "primary-button"} generation-composer__submit`} onClick={() => void createJob()} disabled={!prompt.trim()}>
+ <WandSparkles size={17} /> {submitLabel}
+ </button>
+ </div>
+ {/* Disclosed at the point of commitment, not after an expensive run. */}
+ {runtimeReady && <p className="generation-composer__limit">
+ <Info size={16} />
+ <span>A finished shot renders frames into memory. Pol Studio can’t save them as a video file yet, so nothing lands in your project folder and there is nothing to add to the timeline.</span>
+ </p>}
+ <p className="generation-composer__hint">
+ {runtimeReady
+ ? "Shots render one at a time, so a new shot joins the queue behind anything already running."
+ : "Nothing renders on this computer yet, so your shot is saved as a draft you can run later."}
+ {/* Images are sent as real assets (referencePaths) AND cited as
+ <Picture N>; text definitions reach the engine only as prose
+ inside <Subject N>. Say exactly that — no more. */}
+ {usableReferences.length > 0
+ ? ` It will use ${usableReferences.length === 1 ? "your reference" : `the first ${Math.min(2, usableReferences.length)} of your ${usableReferences.length} references`}: images are sent to the video engine, and text definitions are written into the prompt.`
+ : ""}
+ </p>
+ </section>
+
+ {!selected && <div className="job-empty">
<span className="job-empty__icon"><Sparkles size={26} /></span>
<h2>Start with one shot</h2>
<p>Write what should happen on screen in the box above — who or what is in frame, how the camera moves, the light, the sound. Then choose “{submitLabel}”.</p>
diff --git a/src/components/workspace/ReferencesView.tsx b/src/components/workspace/ReferencesView.tsx
index e453564..d6e0758 100644
--- a/src/components/workspace/ReferencesView.tsx
+++ b/src/components/workspace/ReferencesView.tsx
@@ -1,7 +1,7 @@
import { BookOpen, Check, FileText, Image, Link2, Plus, Sparkles, Trash2, Upload, Users } from "lucide-react";
import { invoke } from "@tauri-apps/api/core";
import { useMemo, useState } from "react";
-import type { ProjectConfig, ProjectReference } from "../../lib/project";
+import { isReferenceUsable, type ProjectConfig, type ProjectReference } from "../../lib/project";
import { isTauri } from "../../lib/persistence";
const uses = ["character", "product", "location", "style", "audio"] as const;
@@ -52,7 +52,7 @@ export function ReferencesView({ config, folderPath, onChange }: { config: Proje
<div>
<span className="eyebrow">Consistency library</span>
<h1>References</h1>
- <p>Define the people, places, products, and visual rules Pol should preserve across every generation.</p>
+ <p>Keep the people, places, products, and visual rules for this project in one place. Each new shot uses the first two.</p>
</div>
<div>
<button className="secondary-button" onClick={() => addTextReference()}><FileText size={16} /> New definition</button>
@@ -67,11 +67,11 @@ export function ReferencesView({ config, folderPath, onChange }: { config: Proje
<div className="reference-grid">
{config.references.map((ref) => <button key={ref.id} className={selectedId === ref.id ? "selected" : ""} onClick={() => setSelectedId(ref.id)}>
{ref.kind === "image" ? <span className="reference-art"><Image size={26} /><em>Image file</em></span> : <span className="reference-copy-art"><FileText size={26} /><em>Text definition</em></span>}
- <span className="reference-card__body"><span><b>{ref.name}</b><small>{ref.kind === "text" ? "Text definition" : ref.relativePath}</small></span><p>{ref.description}</p><span className="use-tags">{ref.intendedUse.map((use) => <i key={use}>{use}</i>)}</span></span>
+ <span className="reference-card__body"><span><b>{ref.name}</b><small>{ref.kind === "text" ? "Text definition" : ref.relativePath}</small></span>{isReferenceUsable(ref) ? <p>{ref.description}</p> : <p className="reference-card__incomplete">Not described yet — it won’t be used until you add a definition.</p>}<span className="use-tags">{ref.intendedUse.map((use) => <i key={use}>{use}</i>)}</span></span>
</button>)}
<button className="reference-add-card" onClick={() => void addImage()}><span><Plus size={22} /></span><b>Add a reference</b><small>Import an image or write a definition</small></button>
</div>
- <div className="reference-explainer"><Sparkles size={18} /><div><b>References guide generation, not your timeline</b><p>Every new shot automatically uses the first two references in this list, so characters, products, and art direction stay coherent. Newest additions go to the top. All files are copied into the project’s <code>references/</code> folder.</p></div></div>
+ <div className="reference-explainer"><Sparkles size={18} /><div><b>References guide new shots, not your timeline</b><p>Each new shot uses the first two references in this list. Image files are copied into the project’s <code>references/</code> folder and sent to the video engine; text definitions are written into the shot’s prompt. Newest additions go to the top.</p></div></div>
</section>
<aside className="reference-inspector">
@@ -84,12 +84,12 @@ export function ReferencesView({ config, folderPath, onChange }: { config: Proje
</div>
<section className="intended-use">
<h3>Intended use</h3>
- <p>Tell Pol Studio what must stay consistent. Select any that apply.</p>
+ <p>Note what this reference is for. Select any that apply.</p>
<div>{uses.map((use) => <button key={use} className={selected.intendedUse.includes(use) ? "active" : ""} aria-pressed={selected.intendedUse.includes(use)} onClick={() => update(selected.id, { intendedUse: selected.intendedUse.includes(use) ? selected.intendedUse.filter((item) => item !== use) : [...selected.intendedUse, use] })}>{selected.intendedUse.includes(use) && <Check size={14} />}{use}</button>)}</div>
</section>
<section className="reference-used-by">
<h3>Used by <span>{jobs.length}</span></h3>
- {jobs.length ? jobs.map((job) => <div key={job.id}><span className={`job-link-dot job-link-dot--${job.status}`} /><div><b>{job.title}</b><small>{job.status} · {job.stage}</small></div><Link2 size={16} /></div>) : <p>No shots use this reference yet. Each new shot picks up the first two references in this list, so it will be used once it reaches the top two.</p>}
+ {jobs.length ? jobs.map((job) => <div key={job.id}><span className={`job-link-dot job-link-dot--${job.status}`} /><div><b>{job.title}</b><small>{job.status} · {job.stage}</small></div><Link2 size={16} /></div>) : <p>No shots use this reference yet. Each new shot picks up the first two in this list, so it will be used once it reaches the top two.</p>}
</section>
<section className="portable-path"><BookOpen size={16} /><div><b>Where this lives</b><code>{selected.relativePath ?? "Stored in polstudio.project.json"}</code></div></section>
</> : <div className="reference-empty"><BookOpen size={26} /><b>Select a reference</b><p>Pick one from the library, or add a new one, to edit its definition and see which generations use it.</p></div>}
diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsx
index c66c2c8..0b1e019 100644
--- a/src/components/workspace/TimelineView.tsx
+++ b/src/components/workspace/TimelineView.tsx
@@ -4,7 +4,7 @@ import {
Volume2, VolumeX,
} from "lucide-react";
import { useEffect, useMemo, useState } from "react";
-import { createDraftGenerationJob, type ProjectConfig, type TimelineClip } from "../../lib/project";
+import type { ProjectConfig, TimelineClip } from "../../lib/project";
const MIN_DURATION = 10_000;
const NOT_YET = "Not available yet. This control doesn’t change your project.";
@@ -35,7 +35,7 @@ const timecode = (ms: number) => {
return `00:${String(Math.floor(seconds / 60)).padStart(2, "0")}:${String(seconds % 60).padStart(2, "0")}:${String(frames).padStart(2, "0")}`;
};
-export function TimelineView({ config, onChange, onOpenGenerator }: { config: ProjectConfig; onChange: (next: ProjectConfig) => void; onOpenGenerator: (jobId: string) => void }) {
+export function TimelineView({ config, onChange, onOpenGenerator }: { config: ProjectConfig; onChange: (next: ProjectConfig) => void; onOpenGenerator: (jobId?: string) => void }) {
const firstClip = config.timeline.tracks.flatMap((track) => track.clips)[0];
const [selectedId, setSelectedId] = useState(firstClip?.id ?? "");
const [playhead, setPlayhead] = useState(firstClip?.startMs ?? 0);
@@ -84,19 +84,11 @@ export function TimelineView({ config, onChange, onOpenGenerator }: { config: Pr
};
const addScene = () => {
// A new project already carries an unstarted draft made from the user's own
- // words. Opening that beats stacking a second near-identical shot they have
- // no way to tell apart — and it never re-describes their idea back at them.
- const unstarted = config.generationJobs.find((job) => job.status === "draft");
- if (unstarted) {
- onOpenGenerator(unstarted.id);
- return;
- }
- const job = createDraftGenerationJob(config.brief.prompt, {
- title: "New scene draft",
- referenceIds: config.references.slice(0, 2).map((reference) => reference.id),
- });
- onChange({ ...config, generationJobs: [job, ...config.generationJobs] });
- onOpenGenerator(job.id);
+ // words, so open that rather than stacking a near-identical second shot.
+ // With none left, go to an empty composer: synthesising a shot from
+ // config.brief.prompt would describe the whole VIDEO as if it were a single
+ // shot and hand it back as words the user never wrote.
+ onOpenGenerator(config.generationJobs.find((job) => job.status === "draft")?.id);
};
return (
@@ -158,11 +150,11 @@ export function TimelineView({ config, onChange, onOpenGenerator }: { config: Pr
<div className="monitor-transport">
<strong>{timecode(playhead)}</strong>
<div>
- <button onClick={() => setPlayhead(0)} aria-label="Go to beginning" title="Go to beginning" disabled={clipCount === 0}><SkipBack size={18} /></button>
+ <button onClick={() => setPlayhead(0)} aria-label="Go to beginning" title={clipCount === 0 ? "Nothing to play yet — add or generate a scene first." : "Go to beginning"} disabled={clipCount === 0}><SkipBack size={18} /></button>
{/* Nothing to play means nothing to play: running the clock over
an empty timeline reads as playback of footage that isn't there. */}
<button className="play-button" onClick={() => setPlaying((value) => !value)} aria-label={playing ? "Pause" : "Play"} disabled={clipCount === 0} title={clipCount === 0 ? "Nothing to play yet — add or generate a scene first." : playing ? "Pause" : "Play"}>{playing ? <Pause size={18} fill="currentColor" /> : <Play size={18} fill="currentColor" />}</button>
- <button onClick={() => setPlayhead(Math.min(duration, playhead + 1000))} aria-label="Step forward one second" title="Step forward one second" disabled={clipCount === 0}><SkipForward size={18} /></button>
+ <button onClick={() => setPlayhead(Math.min(duration, playhead + 1000))} aria-label="Step forward one second" title={clipCount === 0 ? "Nothing to play yet — add or generate a scene first." : "Step forward one second"} disabled={clipCount === 0}><SkipForward size={18} /></button>
</div>
<span>{config.settings.resolution.toUpperCase()} · {config.settings.frameRate} fps</span>
</div>
diff --git a/src/lib/project.test.ts b/src/lib/project.test.ts
index ff04a1e..71da5aa 100644
--- a/src/lib/project.test.ts
+++ b/src/lib/project.test.ts
@@ -1,11 +1,18 @@
import { describe, expect, it } from "vitest";
import completeFixture from "../../fixtures/project-v1-complete.json";
import {
+ compileMiniMaxH3Prompt,
createProjectConfig,
+ deriveH3Style,
+ isReferenceUsable,
normalizeProjectPath,
parseProjectConfig,
+ projectFilePath,
projectNameFromPrompt,
+ projectReferenceSchema,
seedProjectWorkspace,
+ usableImageReferences,
+ type ProjectReference,
} from "./project";
describe("project schema", () => {
@@ -45,6 +52,93 @@ describe("project schema", () => {
"non_diegetic_music:",
]);
expect(job.compiledPrompt).toContain(project.brief.prompt);
+ // Base guide §4.1: [Shot 1] opens with the style, and carries no timestamp (§4.2).
+ expect(job.compiledPrompt).toMatch(/integrated_multimodal_description: \[Shot 1\] Live-action, cinematic, /);
+ expect(job.compiledPrompt).not.toMatch(/\[Shot 1\] At \d/);
+ // §4.7: no abstract mood words or statements of emotional function.
+ expect(job.compiledPrompt).toContain("non_diegetic_music: N/A");
+ });
+
+ it("selects the H3 style from the user's own words rather than inventing one", () => {
+ // Base guide §4.1: "for T2VA, select it from the user's text".
+ expect(deriveH3Style("A watercolour short about a heron")).toBe("Watercolor");
+ expect(deriveH3Style("A 2D animated title sequence")).toBe("2D-animated");
+ expect(deriveH3Style("A claymation breakfast scene")).toBe("Claymation");
+ expect(deriveH3Style("A documentary about harbour pilots")).toBe("Live-action, documentary");
+ expect(deriveH3Style("A quiet lighthouse at dawn")).toBe("Live-action, cinematic");
+ });
+
+ it("switches to the six-section full-reference shape when references are bound", () => {
+ const reference = (over: Partial<ProjectReference>): ProjectReference => projectReferenceSchema.parse({
+ id: "ref-1", kind: "text", name: "Mara", description: "Calm architect in charcoal wool.",
+ intendedUse: ["character"], createdAt: new Date().toISOString(), ...over,
+ });
+ const image = reference({ id: "ref-2", kind: "image", name: "Harbor facade", relativePath: "references/harbor.jpg", description: "Pale stone fins." });
+ const compiled = compileMiniMaxH3Prompt("A quiet lighthouse at dawn", [image, reference({})]);
+
+ // Ref guide §1: six sections, this exact order. detailed_description
+ // REPLACES integrated_multimodal_description (§5.2).
+ expect(compiled.match(/^[a-z_]+:/gm)).toEqual([
+ "subject_definitions:",
+ "summary:",
+ "retention_analysis:",
+ "detailed_description:",
+ "overall_soundscape:",
+ "non_diegetic_music:",
+ ]);
+ expect(compiled).not.toContain("integrated_multimodal_description");
+
+ // §2.2: an image defining a character/scene/style is cited INSIDE its
+ // <Subject N> line and never becomes a standalone <Picture N> entry.
+ expect(compiled).toContain("<Subject 1> is Harbor facade, shown in <Picture 1>.");
+ expect(compiled).not.toMatch(/^<Picture \d+> is /m);
+ // A text reference has no asset, so it cites no picture.
+ expect(compiled).toContain("<Subject 2> is Mara.");
+
+ // §3: task-type prefix; no <Audio N> because no audio asset exists (§2.4).
+ expect(compiled).toContain("summary:\n[reference generation] ");
+ expect(compiled).not.toContain("<Audio");
+
+ // §4.1: fixed relationship markers, one line per label. §5.4: no (Sx) here.
+ expect(compiled).toContain("<Subject 1> (appears in [Shot 1]): fully_preserved - ");
+ expect(compiled).toContain("<Subject 2> (appears in [Shot 1]): fully_preserved - ");
+
+ // §5.2: the style opening precedes [Shot 1] in full-reference mode.
+ expect(compiled).toMatch(/detailed_description:\nThe target video is in a live-action, cinematic style\.\n\[Shot 1\] /);
+ });
+
+ it("numbers <Picture N> in the order the engine consumes reference_paths", () => {
+ // vidfab.rs iterates reference_paths sequentially, so index 0 is <Picture 1>.
+ const img = (id: string, name: string, path: string): ProjectReference => projectReferenceSchema.parse({
+ id, kind: "image", name, description: "A reference.", relativePath: path,
+ intendedUse: ["style"], createdAt: new Date().toISOString(),
+ });
+ const first = img("a", "Alpha", "references/a.jpg");
+ const second = img("b", "Beta", "references/b.jpg");
+ const compiled = compileMiniMaxH3Prompt("A shot", [first, second]);
+ expect(compiled).toContain("<Subject 1> is Alpha, shown in <Picture 1>.");
+ expect(compiled).toContain("<Subject 2> is Beta, shown in <Picture 2>.");
+ expect(usableImageReferences([first, second]).map((r) => r.relativePath)).toEqual([
+ "references/a.jpg", "references/b.jpg",
+ ]);
+ });
+
+ it("ignores blank references so they cannot take a binding slot", () => {
+ const blank = projectReferenceSchema.parse({
+ id: "blank", kind: "text", name: "New definition", description: "",
+ intendedUse: ["style"], createdAt: new Date().toISOString(),
+ });
+ expect(isReferenceUsable(blank)).toBe(false);
+ // With only a blank reference, the prompt stays in T2VA shape.
+ expect(compileMiniMaxH3Prompt("A shot", [blank])).toContain("integrated_multimodal_description:");
+ });
+
+ it("resolves project-relative reference paths to absolute for the engine", () => {
+ // enqueue_vidfab_generation never receives the project folder and vidfab.rs
+ // uses the string verbatim, so the frontend must send an absolute path.
+ expect(projectFilePath("C:\\Projects\\Lighthouse", "references/a.jpg")).toBe("C:\\Projects\\Lighthouse\\references\\a.jpg");
+ expect(projectFilePath("/home/nn/lighthouse", "references/a.jpg")).toBe("/home/nn/lighthouse/references/a.jpg");
+ expect(projectFilePath("/home/nn/lighthouse/", "references/a.jpg")).toBe("/home/nn/lighthouse/references/a.jpg");
});
it("parses the complete cross-layer fixture without losing data", () => {
diff --git a/src/lib/project.ts b/src/lib/project.ts
index fd1139e..0c93d90 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -172,18 +172,140 @@ export interface CreateProjectInput {
referenceImages?: PendingReferenceImage[];
}
-export function compileMiniMaxH3Prompt(creativeBrief: string): string {
+/** Absolute on-disk path for a project-relative file.
+ * The generation backend uses `reference_paths` verbatim (vidfab.rs:339 calls
+ * Path::new on each string) and enqueue_vidfab_generation is never told the
+ * project folder, so a relative path would resolve against the process working
+ * directory. Callers must resolve to absolute before sending. */
+export function projectFilePath(folderPath: string, relativePath: string): string {
+ const windowsStyle = folderPath.includes("\\") && !folderPath.includes("/");
+ const separator = windowsStyle ? "\\" : "/";
+ const root = folderPath.replace(/[\\/]+$/, "");
+ return `${root}${separator}${relativePath.split(/[\\/]/).join(separator)}`;
+}
+
+/** A reference the user has actually filled in. Blank ones are a legitimate
+ * in-progress state, but they must not take a binding slot from a real one. */
+export function isReferenceUsable(reference: ProjectReference): boolean {
+ return reference.kind === "image"
+ ? Boolean(reference.relativePath)
+ : Boolean(reference.description.trim() || reference.content?.trim());
+}
+
+/* ---------------------------------------------------------------------------
+ MiniMax H3 prompt compilation.
+
+ Two shapes, both defined by the official guides. Do not "simplify" them into
+ one — the field names and their order are part of the contract:
+
+ - No references -> T2VA, three fields, `integrated_multimodal_description`
+ first and no timestamp on [Shot 1]. (base guide §2.2, §4.2)
+ - Any reference -> full-reference mode, SIX sections in a fixed order, and
+ `detailed_description` REPLACES the T2VA main field. (ref guide §1, §5.2)
+ --------------------------------------------------------------------------- */
+
+/** Base guide §4.1: [Shot 1] opens with the overall style. For T2VA the style is
+ * selected from the user's own text, so match on what they actually wrote and
+ * fall back to the guide's own default rather than inventing a look. */
+const STYLE_PATTERNS: ReadonlyArray<readonly [RegExp, string]> = [
+ [/\bclaymation\b|\bstop[-\s]?motion\b/i, "Claymation"],
+ [/\bwater[-\s]?colou?r\b/i, "Watercolor"],
+ [/\b(2d|hand[-\s]?drawn|anime|cartoon)\b/i, "2D-animated"],
+ [/\b(3d|cgi?|computer[-\s]?animated)\b/i, "3D CG"],
+ [/\b(vintage|super\s?8|8\s?mm|16\s?mm|grainy film)\b/i, "Vintage film"],
+ [/\bdocumentary\b/i, "Live-action, documentary"],
+];
+const DEFAULT_STYLE = "Live-action, cinematic";
+
+export function deriveH3Style(creativeBrief: string): string {
+ return STYLE_PATTERNS.find(([pattern]) => pattern.test(creativeBrief))?.[1] ?? DEFAULT_STYLE;
+}
+
+/* Base guide §4.6: `overall_soundscape` may only be N/A when the user asks for
+ complete silence, so a generic ambience line is emitted instead. §4.7: music
+ must be instrumentation/tempo/dynamics and never abstract mood or emotional
+ function — we emit N/A because the user never asked for a score, and
+ inventing one is a fabrication of its own. */
+const DEFAULT_SOUNDSCAPE = "Ambient room tone continues throughout, along with the incidental sounds of the action described above.";
+const DEFAULT_MUSIC = "N/A";
+
+const referenceLabel = (reference: ProjectReference, index: number) => `<Subject ${index + 1}>`;
+
+/** Image references are the only ones that can be sent to the engine as assets,
+ * so they carry the <Picture N> numbering. That numbering MUST match the order
+ * of `reference_paths` in the generation request: vidfab.rs iterates the array
+ * and calls add_reference sequentially, so index 0 is <Picture 1>. */
+export function usableImageReferences(references: ProjectReference[]): ProjectReference[] {
+ return references.filter((reference) => reference.kind === "image" && isReferenceUsable(reference));
+}
+
+export function compileMiniMaxH3Prompt(creativeBrief: string, references: ProjectReference[] = []): string {
const brief = creativeBrief.trim();
+ const style = deriveH3Style(brief);
+ const usable = references.filter(isReferenceUsable);
+
+ if (usable.length === 0) {
+ // T2VA — base guide §2.2 field list and order.
+ return [
+ `integrated_multimodal_description: [Shot 1] ${style}, ${brief}`,
+ `overall_soundscape: ${DEFAULT_SOUNDSCAPE}`,
+ `non_diegetic_music: ${DEFAULT_MUSIC}`,
+ ].join("\n\n");
+ }
+
+ const images = usableImageReferences(references);
+ const pictureNumber = new Map(images.map((reference, index) => [reference.id, index + 1]));
+
+ // §2.2: an image that only defines a character, scene, costume or style is
+ // cited INSIDE its <Subject N> definition — it never becomes a standalone
+ // <Picture N> entry. This app cannot express a first/last/key frame, so no
+ // standalone picture entry is ever correct here.
+ // No <Audio N> is emitted either: §2.4 defines it as an actual audio asset,
+ // and every reference in this app is text or image. An "audio" intendedUse
+ // is a note about desired sound, not a signal to copy.
+ const definitions = usable.map((reference, index) => {
+ const detail = (reference.description.trim() || reference.content?.trim() || "").replace(/\s+/g, " ");
+ const picture = pictureNumber.get(reference.id);
+ const source = picture ? `, shown in <Picture ${picture}>` : "";
+ return `${referenceLabel(reference, index)} is ${reference.name.trim()}${source}.${detail ? ` ${detail}` : ""}`;
+ });
+
+ const labels = usable.map((reference, index) => referenceLabel(reference, index));
+ const labelList = labels.length === 1
+ ? labels[0]
+ : `${labels.slice(0, -1).join(", ")} and ${labels[labels.length - 1]}`;
+
+ // §3: task-type prefix. Every reference here provides generation guidance
+ // without acting as a concrete frame or an edited source video, which is
+ // exactly `reference generation`. The summary reuses existing labels only.
+ const summary = `[reference generation] ${brief} ${labelList} ${labels.length === 1 ? "provides" : "provide"} generation guidance for the single shot described below.`;
+
+ // §4.1: one line per label using the fixed marker vocabulary. Their defined
+ // role is carried through unchanged, so fully_preserved is the honest marker.
+ // §5.4: never write (Sx) in retention_analysis.
+ const retention = usable.map((reference, index) =>
+ `${referenceLabel(reference, index)} (appears in [Shot 1]): fully_preserved - the defined characteristics of ${reference.name.trim()} are retained.`);
+
+ // §5.2: in full-reference mode the style opening comes BEFORE [Shot 1], not
+ // after it. §5.3: cite each <Subject N> where it appears in the shot.
+ const detailed = [
+ `The target video is in a ${style.toLowerCase()} style.`,
+ `[Shot 1] ${brief} The shot features ${labelList}, matching the definitions above.`,
+ ].join("\n");
+
return [
- `integrated_multimodal_description: [Shot 1] ${brief}`,
- "overall_soundscape: Natural location ambience shaped to the action in the scene.",
- "non_diegetic_music: A restrained cinematic score that supports the story without overpowering it.",
+ `subject_definitions:\n${definitions.join("\n")}`,
+ `summary:\n${summary}`,
+ `retention_analysis:\n${retention.join("\n")}`,
+ `detailed_description:\n${detailed}`,
+ `overall_soundscape:\n${DEFAULT_SOUNDSCAPE}`,
+ `non_diegetic_music:\n${DEFAULT_MUSIC}`,
].join("\n\n");
}
export function createDraftGenerationJob(
creativeBrief: string,
- options: { id?: string; title?: string; referenceIds?: string[]; now?: string } = {},
+ options: { id?: string; title?: string; referenceIds?: string[]; references?: ProjectReference[]; now?: string } = {},
): GenerationJob {
const brief = creativeBrief.trim();
const now = options.now ?? new Date().toISOString();
@@ -196,7 +318,7 @@ export function createDraftGenerationJob(
progress: 0,
providerId: "minimax-h3",
creativeBrief: brief,
- compiledPrompt: compileMiniMaxH3Prompt(brief),
+ compiledPrompt: compileMiniMaxH3Prompt(brief, options.references ?? []),
referenceIds: options.referenceIds ?? [],
createdAt: now,
updatedAt: now,
diff --git a/src/lib/runtime.ts b/src/lib/runtime.ts
index 232dfe4..77b1986 100644
--- a/src/lib/runtime.ts
+++ b/src/lib/runtime.ts
@@ -1,6 +1,6 @@
import { invoke } from "@tauri-apps/api/core";
import { isTauri, saveProject } from "./persistence";
-import { parseProjectConfig, type GenerationJob, type ProjectConfig, type ProjectRecord } from "./project";
+import { compileMiniMaxH3Prompt, parseProjectConfig, type GenerationJob, type ProjectConfig, type ProjectRecord } from "./project";
export type ProviderId = "claude" | "codex";
export interface ProviderStatus {
@@ -133,11 +133,13 @@ function demoTurn(config: ProjectConfig, prompt: string): AgentTurnResult {
const job: GenerationJob = {
id: `job-demo-${Date.now()}`, title: clean.split(/\s+/).slice(0, 5).join(" "), prompt: clean,
status: "draft", stage: "queued", progress: 0, providerId: "minimax-h3", creativeBrief: clean,
- compiledPrompt: `integrated_multimodal_description: [Shot 1] ${clean}\n\noverall_soundscape: Quiet environmental ambience.\n\nnon_diegetic_music: N/A`,
+ // Share the one compiler so the demo agent cannot drift from the guides.
+ // Bound to no references, so it compiles to the T2VA three-field shape.
+ compiledPrompt: compileMiniMaxH3Prompt(clean, []),
referenceIds: [], createdAt: now, updatedAt: now,
};
const project = parseProjectConfig({ ...config, generationJobs: [job, ...config.generationJobs] });
- return { kind: "mutation", summary: "Added a schema-valid draft shot with an official three-field H3 prompt. No media was generated.", project };
+ return { kind: "mutation", summary: "Added a schema-valid draft shot using the official MiniMax H3 prompt format. No media was generated.", project };
}
return { kind: "answer", content: `I reviewed the project context. A safe next step is to refine “${clean.slice(0, 72)}” into a shot brief before generation.` };
}
diff --git a/src/styles/references.css b/src/styles/references.css
index 2287d6f..82f6bd9 100644
--- a/src/styles/references.css
+++ b/src/styles/references.css
@@ -715,3 +715,9 @@
grid-template-columns: minmax(0, 1fr);
}
}
+
+/* A blank reference is a legitimate in-progress state, but it must be visibly
+ inert: binding skips it, so the card says so rather than looking finished. */
+.reference-card__incomplete {
+ color: var(--warning);
+}