Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add Pose scenes

Bitti55b20e035ac1
diff --git a/docs/scene-types.md b/docs/scene-types.md
index 6df8280..12fc1f9 100644
--- a/docs/scene-types.md
+++ b/docs/scene-types.md
@@ -4,6 +4,7 @@ The Scene panel saves a type on each scene:
- **First & Last Frame** keeps the existing shot descriptions, speech, Look, sound, music and optional frame anchors. Older scenes without a type use this type.
- **Animate** uses an Animate generator, one driving video and one repainted scene frame. It does not send a text prompt.
+- **Pose** uses a selected video for body pose, movement and timing. The Scene panel places the video selector above the shot prompts, where optional references can define the target subject and setting. It uses the video's saved clip range without its soundtrack, attached stills or refmods. Only the selected pose video and references cited in the prompts are sent; frame anchors and previous-scene continuation are inactive. Pose requires a nonempty prompt and reference-capable MiniMax weights (References or Singularity), with the existing limits of three videos, nine images and 15 seconds of reference video.
- **Character Replace** gives each shot a source video selector, a new character reference selector and an optional description identifying the character to replace. The character reference must contain an image. Blank target descriptions mean the main character.
- **Extend** takes a source video in the Scene panel and uses the shot descriptions as the continuation prompt.
- **Bridge** takes start and end video references in the Scene panel and uses the shot descriptions as the prompt for the connecting segment.
diff --git a/src-tauri/src/project/scenes.rs b/src-tauri/src/project/scenes.rs
index 4f71d26..8a9765e 100644
--- a/src-tauri/src/project/scenes.rs
+++ b/src-tauri/src/project/scenes.rs
@@ -44,6 +44,8 @@ pub(crate) struct GenerationJob {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) scene_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) pose_video_reference_id: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) start_video_reference_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) end_video_reference_id: Option<String>,
@@ -141,6 +143,7 @@ impl GenerationJob {
Self {
id: id.to_string(),
scene_type: None,
+ pose_video_reference_id: None,
start_video_reference_id: None,
end_video_reference_id: None,
title: title.to_string(),
diff --git a/src-tauri/src/project/validation/scenes.rs b/src-tauri/src/project/validation/scenes.rs
index d30e53c..6cbda7a 100644
--- a/src-tauri/src/project/validation/scenes.rs
+++ b/src-tauri/src/project/validation/scenes.rs
@@ -7,14 +7,18 @@ pub(super) fn validate(config: &mut ProjectConfig) -> Result<(), String> {
if job.scene_type.as_deref().is_some_and(|value| {
!matches!(
value,
- "first-last-frame" | "animate" | "character-replace" | "extend" | "bridge"
+ "first-last-frame" | "animate" | "pose" | "character-replace" | "extend" | "bridge"
)
}) {
return Err(format!("Scene '{}' has an unsupported scene type.", job.id));
}
- if [&job.start_video_reference_id, &job.end_video_reference_id]
- .iter()
- .any(|id| id.as_deref() == Some(""))
+ if [
+ &job.start_video_reference_id,
+ &job.end_video_reference_id,
+ &job.pose_video_reference_id,
+ ]
+ .iter()
+ .any(|id| id.as_deref() == Some(""))
{
return Err(format!(
"Scene '{}' has an empty video reference id.",
diff --git a/src-tauri/src/tests/scenes.rs b/src-tauri/src/tests/scenes.rs
index 678062a..99d2554 100644
--- a/src-tauri/src/tests/scenes.rs
+++ b/src-tauri/src/tests/scenes.rs
@@ -268,6 +268,7 @@ fn scene_types_and_character_replacement_inputs_survive_save_and_reopen() {
for scene_type in [
"first-last-frame",
"animate",
+ "pose",
"character-replace",
"extend",
"bridge",
@@ -275,6 +276,7 @@ fn scene_types_and_character_replacement_inputs_survive_save_and_reopen() {
let mut config = scene_fixture();
let job = &mut config.generation_jobs[0];
job.scene_type = Some(scene_type.into());
+ job.pose_video_reference_id = Some("pose-video".into());
job.start_video_reference_id = Some("start-video".into());
job.end_video_reference_id = Some("end-video".into());
let shot = &mut job.shots.as_mut().unwrap()[0];
@@ -284,6 +286,12 @@ fn scene_types_and_character_replacement_inputs_survive_save_and_reopen() {
let normalized = validate_and_normalize_config(config).unwrap();
let saved = serde_json::to_string(&normalized).unwrap();
let restored: ProjectConfig = serde_json::from_str(&saved).unwrap();
+ assert_eq!(
+ restored.generation_jobs[0]
+ .pose_video_reference_id
+ .as_deref(),
+ Some("pose-video")
+ );
assert_eq!(
restored.generation_jobs[0]
.start_video_reference_id
@@ -310,6 +318,9 @@ fn scene_types_and_character_replacement_inputs_survive_save_and_reopen() {
let mut config = scene_fixture();
config.generation_jobs[0].scene_type = Some("unknown".into());
assert!(validate_and_normalize_config(config).is_err());
+ let mut config = scene_fixture();
+ config.generation_jobs[0].pose_video_reference_id = Some(String::new());
+ assert!(validate_and_normalize_config(config).is_err());
}
#[test]
diff --git a/src/components/workspace/GeneratorView.test.tsx b/src/components/workspace/GeneratorView.test.tsx
index f06ffd5..beb0f88 100644
--- a/src/components/workspace/GeneratorView.test.tsx
+++ b/src/components/workspace/GeneratorView.test.tsx
@@ -134,6 +134,51 @@ it("saves scene types and submits Character Replace with the exact selected refe
expect(generate).toBeDisabled();
});
+it("submits Pose from the scene panel with its video and optional prompt references", () => {
+ const initial = project();
+ initial.generationJobs = [createDraftGenerationJob("", { id: "pose", title: "Pose scene" })];
+ initial.references = [
+ { id: "motion", kind: "video", name: "Motion", description: "", intendedUse: [], sourcePath: "C:/motion.mp4",
+ video: { startSeconds: 1, durationSeconds: 3, includeAudio: true }, createdAt: "2026-09-22T00:00:00.000Z" },
+ { id: "hero", kind: "image", name: "Hero", description: "", intendedUse: [], relativePath: "references/hero.png", createdAt: "2026-09-22T00:00:00.000Z" },
+ ];
+ let latest = initial;
+ const submitted = vi.fn();
+ function Harness() {
+ const [config, setConfig] = useState(initial);
+ latest = config;
+ return <GeneratorView config={config} folderPath="C:/project" runtime={readyRuntime}
+ onChange={setConfig} onGenerate={submitted} onOpenTimeline={() => undefined} />;
+ }
+ render(<Harness />);
+ fireEvent.change(screen.getByLabelText("Scene type"), { target: { value: "pose" } });
+ expect(screen.queryByLabelText("Start frame for this scene")).toBeNull();
+ const generate = within(screen.getByRole("region", { name: "Pose scene" })).getByRole("button", { name: "Generate" });
+ expect(generate).toBeDisabled();
+ fireEvent.change(screen.getByLabelText("Pose video reference for this scene"), { target: { value: "motion" } });
+ expect(generate).toBeDisabled();
+ fireEvent.change(screen.getByLabelText("Describe shot 1"), { target: { value: "A dancer on a rainy street." } });
+ expect(generate).toBeEnabled();
+ fireEvent.click(generate);
+ expect(submitted.mock.calls[0][0][0].request).toMatchObject({
+ prompt: expect.stringContaining("Apply the body pose, movement and timing from <Video 1>"),
+ referencePaths: [],
+ referenceVideos: [{ name: "Motion", sourcePath: "C:/motion.mp4", startSeconds: 1, durationSeconds: 3, includeAudio: false }],
+ });
+ fireEvent.click(screen.getByRole("button", { name: /Not in this scene yet Hero/ }));
+ expect(latest.generationJobs[0].shots![0].action).toContain("@[ref:hero]");
+ fireEvent.click(generate);
+ expect(submitted.mock.calls.at(-1)![0][0].request.referencePaths).toEqual(["C:/project/references/hero.png"]);
+ expect(parseProjectConfig(JSON.parse(JSON.stringify(latest))).generationJobs[0]).toMatchObject({ sceneType: "pose", poseVideoReferenceId: "motion" });
+ fireEvent.change(screen.getByLabelText("Scene type"), { target: { value: "first-last-frame" } });
+ fireEvent.change(screen.getByLabelText("Scene type"), { target: { value: "pose" } });
+ expect(screen.getByLabelText("Pose video reference for this scene")).toHaveValue("motion");
+ fireEvent.change(screen.getByLabelText("Pose video reference for this scene"), { target: { value: "" } });
+ expect(generate).toBeDisabled();
+ expect(templateSceneBlocker("pose", minimaxOriginalTemplate())).toContain("References or Singularity");
+ expect(templateSceneBlocker("pose", viggleAnimateTemplate())).toContain("MiniMax prompt");
+});
+
it("keeps an explicitly selected scene type independent of the global generator", () => {
const initial = project();
initial.generationJobs[0].sceneType = "animate";
diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsx
index ae1b2b1..fe70b64 100644
--- a/src/components/workspace/GeneratorView.tsx
+++ b/src/components/workspace/GeneratorView.tsx
@@ -1,7 +1,7 @@
import type { GenerationSubmission } from "../../lib/workQueue";
import { loadLoras, subscribeLoras } from "../../lib/loras";
import { refreshDownloadedLoras } from "../../lib/weightDownloads";
-import { characterReplaceBlocker, isVideoTransition, videoTransitionBlocker, usableVideoReferences, type SceneType } from "../../lib/project";
+import { characterReplaceBlocker, poseBlocker, isVideoTransition, videoTransitionBlocker, usableVideoReferences, type SceneType } from "../../lib/project";
import { referenceRefmodInputs } from "../../lib/project";
import { ChevronDown, Plus, Square, Trash2, WandSparkles } from "lucide-react";
import { invoke } from "@tauri-apps/api/core";
@@ -167,7 +167,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
if (updates.sceneType && templateSceneBlocker(updates.sceneType, selectedTemplate)) {
const candidates = templateSettings.templates.filter((template) => !templateNeedsDownload(template)
&& !templateSceneBlocker(updates.sceneType!, template));
- const compatible = (updates.sceneType === "character-replace" || isVideoTransition(updates)
+ const compatible = (updates.sceneType === "character-replace" || updates.sceneType === "pose" || isVideoTransition(updates)
? candidates.find((template) => /ref2v/i.test(template.paths.transformer)) : undefined) ?? candidates[0];
if (compatible) chooseGeneratorTemplate(compatible.id);
}
@@ -399,10 +399,10 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
const batchScenesReady: GenerationJob[] = [];
for (const [index, job] of jobs.entries()) {
if (job.status === "queued" || job.status === "generating" || job.status === "ready") continue;
- if (!isVideoTransition(job) && job.sceneType !== "character-replace" && job.usePreviousSceneLastFrame && index === 0) continue;
+ if (!isVideoTransition(job) && job.sceneType !== "pose" && job.sceneType !== "character-replace" && job.usePreviousSceneLastFrame && index === 0) continue;
if (templateSceneBlocker(sceneTypeFor(job), selectedTemplate) || sendBlocker(job, config.references)) continue;
const previous = jobs[index - 1];
- const previousWillRender = !isVideoTransition(job) && job.usePreviousSceneLastFrame && previous &&
+ const previousWillRender = !isVideoTransition(job) && job.sceneType !== "pose" && job.usePreviousSceneLastFrame && previous &&
(batchScenesReady.includes(previous) || ["queued", "generating", "ready"].includes(previous.status));
if (previousWillRender || sceneGenerationSeed(job) === RANDOM_GENERATION_SEED || job.status !== "completed"
|| !job.outputRelativePath
@@ -416,7 +416,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
const generationBlocker = (job: GenerationJob): string | null => {
if (!runtimeReady) return runtimeError ?? generatorRuntime?.detail ?? "The video generator is not ready.";
if (job.status === "ready") return "This scene is being saved now.";
- if (!isVideoTransition(job) && job.sceneType !== "character-replace" && job.usePreviousSceneLastFrame && jobs[0]?.id === job.id) return "This scene needs a previous scene to continue.";
+ if (!isVideoTransition(job) && job.sceneType !== "pose" && job.sceneType !== "character-replace" && job.usePreviousSceneLastFrame && jobs[0]?.id === job.id) return "This scene needs a previous scene to continue.";
return templateSceneBlocker(sceneTypeFor(job), selectedTemplate) ?? sendBlocker(job, config.references);
};
@@ -632,6 +632,10 @@ function sendBlocker(job: GenerationJob, references: ProjectReference[]): string
const shots = sceneShots(job);
const animate = job.sceneType === "animate";
const characterReplace = job.sceneType === "character-replace";
+ if (job.sceneType === "pose") {
+ const blocker = poseBlocker(job, references);
+ if (blocker) return blocker;
+ }
if (isVideoTransition(job)) {
const blocker = videoTransitionBlocker(job, references);
if (blocker) return blocker;
@@ -664,7 +668,7 @@ function sendBlocker(job: GenerationJob, references: ProjectReference[]): string
export function templateSceneBlocker(type: SceneType, template: GeneratorTemplate): string | null {
if (type === "animate" && template.mode !== "animate") return "Select an Animate generator for this scene.";
if (type !== "animate" && template.mode === "animate") return "Select a MiniMax prompt generator for this scene.";
- if ((type === "character-replace" || type === "extend" || type === "bridge") && /fl2v/i.test(template.paths.transformer)) return `Select a References or Singularity generator for ${type === "character-replace" ? "Character Replace" : type === "extend" ? "Extend" : "Bridge"}.`;
+ if ((type === "pose" || type === "character-replace" || type === "extend" || type === "bridge") && /fl2v/i.test(template.paths.transformer)) return `Select a References or Singularity generator for ${type === "pose" ? "Pose" : type === "character-replace" ? "Character Replace" : type === "extend" ? "Extend" : "Bridge"}.`;
return null;
}
diff --git a/src/components/workspace/SceneEditor.tsx b/src/components/workspace/SceneEditor.tsx
index a1c3967..60a8df7 100644
--- a/src/components/workspace/SceneEditor.tsx
+++ b/src/components/workspace/SceneEditor.tsx
@@ -14,6 +14,7 @@ import {
sceneDurationSeconds,
sceneGenerationSeed,
sceneGenerationSteps,
+ sceneGenerationReferences,
splitActionText,
DEFAULT_SPEECH_LANGUAGE,
MAX_GENERATION_STEPS,
@@ -62,7 +63,8 @@ export const citableReferences = (references: readonly ProjectReference[]): Proj
* The board and the panel both read this, because "Reference 2" on a card and
* "Reference 2" in the field have to be the same reference. */
export function referenceOrder(job: GenerationJob, shots: readonly SceneShot[], references: readonly ProjectReference[]): string[] {
- const order = citableReferences(references).filter((reference) => job.referenceIds.includes(reference.id)).map((reference) => reference.id);
+ const boundIds = job.sceneType === "pose" ? sceneGenerationReferences(job, [...references]).map((reference) => reference.id) : job.referenceIds;
+ const order = citableReferences(references).filter((reference) => boundIds.includes(reference.id)).map((reference) => reference.id);
for (const shot of shots) {
for (const id of actionReferenceIds(shot.action)) if (!order.includes(id)) order.push(id);
}
@@ -96,7 +98,8 @@ export function writeShots(job: GenerationJob, next: readonly SceneShot[], extra
/** Everything about ONE shot: when it starts, the line the user wrote for it,
* the references that line cites, and the settings hung off it. This is what
* clicking a card on the board opens. */
-export function ShotInspector({ job, shots, shot, index, endsAt, duration, references, disabled, onChange }: {
+export function ShotInspector({ job, shots, shot, index, endsAt, duration, references, disabled, onChange, promptOnly = false }: {
+ promptOnly?: boolean;
job: GenerationJob;
/** Every shot of the scene, so the panel can number the references the same
* way the compiler does and warn about the ones no line can still cite. */
@@ -116,8 +119,8 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
}) {
const fieldRef = useRef<HTMLTextAreaElement | null>(null);
const citable = useMemo(() => citableReferences(references), [references]);
- const numbered = useMemo(() => citable.filter((reference) => job.referenceIds.includes(reference.id)), [citable, job.referenceIds]);
const tokenOrder = useMemo(() => referenceOrder(job, shots, references), [job, shots, references]);
+ const numbered = useMemo(() => citable.filter((reference) => tokenOrder.includes(reference.id)), [citable, tokenOrder]);
const referenceById = useMemo(() => new Map(references.map((reference) => [reference.id, reference])), [references]);
const dangling = useMemo(() => danglingReferenceTokens(shots, references), [shots, references]);
const settings = shot.settings ?? {};
@@ -164,7 +167,7 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
};
return <section className="shot-inspector" aria-label={`Shot ${shotNumber}`}>
- <div className="shot-inspector__timing">
+ {!promptOnly && <div className="shot-inspector__timing">
<label className="shot-card__start">
<span>Starts at</span>
<input
@@ -180,11 +183,11 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
/>
<em>to {seconds(endsAt)}</em>
</label>
- </div>
+ </div>}
{job.sceneType === "character-replace" ? <CharacterReplaceInputs shot={shot} references={references} disabled={disabled} onChange={onChange} /> : <>
<div className="shot-card__action">
- <label className="shot-card__sublabel" htmlFor={`shot-action-${shot.id}`}>Describe the shot</label>
+ <label className="shot-card__sublabel" htmlFor={`shot-action-${shot.id}`}>{job.sceneType === "pose" ? "Describe where the pose should be used" : "Describe the shot"}</label>
<div className="shot-action-editor">
<textarea
id={`shot-action-${shot.id}`}
@@ -192,7 +195,7 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
value={display}
disabled={disabled}
aria-label={`Describe shot ${shotNumber}`}
- placeholder="Example: she walks towards the camera and stops under the awning."
+ placeholder={job.sceneType === "pose" ? "Example: a dancer in a red coat performs this movement on a rainy city street." : "Example: she walks towards the camera and stops under the awning."}
onChange={(event) => onChange({ action: store(event.target.value) })}
onDragOver={(event) => { if (event.dataTransfer.types.includes(REFERENCE_DRAG_TYPE)) event.preventDefault(); }}
onDrop={(event) => {
@@ -221,7 +224,7 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
</div>
</div>
- <div className="shot-speech">
+ {!promptOnly && <div className="shot-speech">
<div className="shot-speech__heading">
<label className="shot-card__sublabel" htmlFor={`shot-speech-${shot.id}`}>Speech</label>
<label className="shot-speech__language">
@@ -245,7 +248,7 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
onChange={(event) => onChange({ speech: event.target.value || null })}
/>
<p className="shot-speech__hint">Added to the final prompt as MiniMax dialogue.</p>
- </div>
+ </div>}
<ReferencePalette
citable={citable}
@@ -256,12 +259,12 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
onInsert={insertAtCaret}
/>
- <ShotSettings
+ {!promptOnly && <ShotSettings
settings={settings}
disabled={disabled}
shotNumber={shotNumber}
onChange={(next) => onChange({ settings: normalizeShotTagSelection(next) })}
- />
+ />}
</>}
</section>;
}
@@ -325,6 +328,7 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
onShots: (next: SceneShot[]) => void;
}) {
const animate = sceneType === "animate";
+ const pose = sceneType === "pose";
const characterReplace = sceneType === "character-replace";
const transition = isVideoTransition({ sceneType });
const look = SHOT_TAG_GROUPS.find((group) => group.id === "visualStyle")!;
@@ -346,11 +350,28 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
...(event.target.value === "animate" ? { endFrameReferenceId: undefined } : {}) })}>
<option value="first-last-frame">First &amp; Last Frame</option>
<option value="animate">Animate</option>
+ <option value="pose">Pose</option>
<option value="character-replace">Character Replace</option>
<option value="extend">Extend</option>
<option value="bridge">Bridge</option>
</select>
</label>
+ {pose && <>
+ <label className="scene-settings__field">
+ <span>Pose video reference</span>
+ <select aria-label="Pose video reference for this scene" value={job.poseVideoReferenceId ?? ""} disabled={disabled}
+ onChange={(event) => onChange({ poseVideoReferenceId: event.target.value || null })}>
+ <option value="">Select a video</option>
+ {job.poseVideoReferenceId && !usableVideoReferences(references).some((reference) => reference.id === job.poseVideoReferenceId) && <option value={job.poseVideoReferenceId}>Unavailable video reference</option>}
+ {usableVideoReferences(references).map((reference) => <option key={reference.id} value={reference.id}>{reference.name}</option>)}
+ </select>
+ <small>Uses the saved clip range for pose and motion. Describe the subject and setting below, with optional references.</small>
+ </label>
+ {shots.map((shot, index) => <ShotInspector key={shot.id} job={job} shots={shots} shot={shot} index={index}
+ endsAt={shots[index + 1]?.startSeconds ?? sceneDurationSeconds(job)} duration={sceneDurationSeconds(job)}
+ references={references} disabled={disabled} promptOnly
+ onChange={(updates) => onShots(shots.map((item) => item.id === shot.id ? { ...item, ...updates } : item))} />)}
+ </>}
{characterReplace && <p>Open a shot to select its source video and new character reference.</p>}
{transition && <>
<label className="scene-settings__field">
@@ -408,7 +429,7 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
</select>
{!chosen && defaultLook && <small>Using project Look: {look.options.find((option) => option.id === defaultLook)?.label ?? defaultLook}.</small>}
</label>}
- {!transition && <div className="scene-settings__field">
+ {!transition && !pose && <div className="scene-settings__field">
<span>{animate ? "Repainted scene frame" : "Start frame"}</span>
<div className="scene-settings__start-frame">
<select
@@ -442,7 +463,7 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
? previousScene ? `Continues “${previousScene.title}” using its saved latents with 22 overlapping frames. Generate that scene first, or use Generate All.` : "Move this scene after another scene to continue it."
: "The selected image anchors the opening frame."}</small>
</div>}
- {!animate && !transition && <div className="scene-settings__field">
+ {!animate && !transition && !pose && <div className="scene-settings__field">
<span>Last frame</span>
<div className="scene-settings__start-frame">
<select value={job.endFrameReferenceId ?? ""} disabled={disabled} aria-label="Last frame for this scene"
diff --git a/src/lib/project.pose.test.ts b/src/lib/project.pose.test.ts
new file mode 100644
index 0000000..00c9034
--- /dev/null
+++ b/src/lib/project.pose.test.ts
@@ -0,0 +1,73 @@
+import { expect, it } from "vitest";
+import { compileGenerationJobPrompt, createDraftGenerationJob, createProjectConfig, generationJobSchema,
+ poseBlocker, referenceToken, sceneFrameInputs, sceneGenerationReferences, type ProjectReference } from "./project";
+
+const reference = (id: string, kind: "image" | "video"): ProjectReference => ({
+ id, kind, name: id, description: "", intendedUse: [], createdAt: "2026-09-22T00:00:00.000Z",
+ sourcePath: `C:/${id}.${kind === "video" ? "mp4" : "png"}`,
+ ...(kind === "video" ? { video: { startSeconds: 1, durationSeconds: 3, includeAudio: true } } : {}),
+});
+const scene = () => ({ ...createDraftGenerationJob("A dancer on a rainy street.", { sceneType: "pose" }), poseVideoReferenceId: "motion" });
+
+it("saves Pose and its video choice while keeping older scenes unchanged", () => {
+ const job = scene();
+ expect(generationJobSchema.parse(JSON.parse(JSON.stringify(job)))).toEqual(job);
+ expect(createDraftGenerationJob("Legacy")).not.toHaveProperty("poseVideoReferenceId");
+ expect(generationJobSchema.safeParse({ ...scene(), poseVideoReferenceId: "" }).success).toBe(false);
+});
+
+it("uses the pose video without requiring optional references or copying its appearance", () => {
+ const job = scene();
+ const refs = [reference("motion", "video")];
+ expect(poseBlocker(job, refs)).toBeNull();
+ const prompt = compileGenerationJobPrompt(job, refs);
+ expect(prompt).toContain("pose and motion guidance from <Video 1>");
+ expect(prompt).toContain("A dancer on a rainy street.");
+ expect(prompt).toContain("partially_preserved - retain body pose, movement and timing only");
+ expect(prompt).not.toContain("fully_preserved");
+ expect(prompt).not.toContain("The shot features <Subject 1>");
+});
+
+it("sends only the pose and cited references with matching payload numbering", () => {
+ const job = { ...scene(), referenceIds: ["stale"], startFrameReferenceId: "stale", usePreviousSceneLastFrame: true };
+ job.shots![0].action = `${referenceToken("hero")} dances in ${referenceToken("setting")}.`;
+ const motion = reference("motion", "video");
+ motion.images = [{ id: "still", name: "Still", sourcePath: "C:/still.png" }];
+ const refs = [reference("setting", "video"), reference("stale", "image"), reference("hero", "image"), motion];
+ const config = createProjectConfig({ name: "Test", prompt: "", aspectRatio: "16:9", resolution: "416p", targetDurationSeconds: 6 });
+ config.references = refs;
+ config.generationJobs = [job];
+ const inputs = sceneFrameInputs(job, config);
+ expect(inputs).not.toHaveProperty("previousSceneId");
+ expect(inputs.references.map(({ id }) => id)).toEqual(["setting", "hero", "motion"]);
+ expect(inputs.references[2]).toMatchObject({ images: [], refmods: [], video: { startSeconds: 1, durationSeconds: 3, includeAudio: false } });
+ expect(motion.video!.includeAudio).toBe(true);
+ const prompt = compileGenerationJobPrompt(job, inputs.references);
+ expect(prompt).toContain("<Subject 2> is the content shown in <Picture 1>");
+ expect(prompt).toContain("<Subject 3> is pose and motion guidance from <Video 2>");
+ expect(prompt).toContain("<Subject 2> dances in <Subject 1>");
+ expect(prompt).not.toMatch(/first frame|last frame|<Picture 2>/);
+ job.shots![0].action = "A dancer.";
+ expect(sceneGenerationReferences(job, refs).map(({ id }) => id)).toEqual(["motion"]);
+});
+
+it("requires an available video and prompt and enforces payload limits", () => {
+ const job = scene();
+ expect(poseBlocker(job, [])).toContain("pose video");
+ expect(poseBlocker(job, [reference("motion", "image")])).toContain("pose video");
+ expect(poseBlocker(job, [{ ...reference("motion", "video"), sourcePath: undefined }])).toContain("pose video");
+ const motion = reference("motion", "video");
+ job.shots![0].action = "";
+ expect(poseBlocker(job, [motion])).toContain("Describe where");
+ job.shots![0].action = "A dancer.";
+ motion.video!.durationSeconds = 16;
+ expect(poseBlocker(job, [motion])).toContain("15 seconds");
+ motion.video!.durationSeconds = 3;
+ const extra = ["a", "b", "c"].map((id) => reference(id, "video"));
+ job.shots![0].action = extra.map(({ id }) => referenceToken(id)).join(" ");
+ expect(poseBlocker(job, [motion, ...extra])).toContain("three video");
+ const hero = reference("hero", "image");
+ hero.images = Array.from({ length: 9 }, (_, i) => ({ id: `image-${i}`, name: "Angle", sourcePath: `C:/angle-${i}.png` }));
+ job.shots![0].action = referenceToken("hero");
+ expect(poseBlocker(job, [motion, hero])).toContain("nine reference images");
+});
diff --git a/src/lib/project.ts b/src/lib/project.ts
index 9765567..3fab18b 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -356,7 +356,7 @@ export const SCENE_MIN_SECONDS = 0;
* length is read as — the length every shot has always been generated at. */
export const DEFAULT_SCENE_SECONDS = 6;
-export const sceneTypeSchema = z.enum(["first-last-frame", "animate", "character-replace", "extend", "bridge"]);
+export const sceneTypeSchema = z.enum(["first-last-frame", "animate", "pose", "character-replace", "extend", "bridge"]);
export type SceneType = z.infer<typeof sceneTypeSchema>;
export const isVideoTransition = (job: Pick<GenerationJob, "sceneType">): boolean => job.sceneType === "extend" || job.sceneType === "bridge";
@@ -435,6 +435,7 @@ export function actionReferenceIds(action: string): string[] {
export const generationJobSchema = z.object({
id: idSchema,
sceneType: sceneTypeSchema.nullish(),
+ poseVideoReferenceId: idSchema.nullish(),
startVideoReferenceId: idSchema.nullish(),
endVideoReferenceId: idSchema.nullish(),
title: z.string().min(1),
@@ -808,6 +809,15 @@ export function usableVideoReferences(references: ProjectReference[]): ProjectRe
* order. Frame anchors use their first image, even on a multi-image reference;
* the same anchor selected for both ends is sent only once. */
export function sceneGenerationReferences(job: GenerationJob, references: ProjectReference[]): ProjectReference[] {
+ if (job.sceneType === "pose") {
+ // Optional guidance is explicit in the prompt. Old bindings and frame
+ // anchors belong to other modes and must not redefine the target scene.
+ const cited = sceneShots(job).flatMap((shot) => actionReferenceIds(shot.action));
+ return references.filter((reference) => reference.id === job.poseVideoReferenceId || cited.includes(reference.id))
+ .map((reference) => reference.id === job.poseVideoReferenceId
+ ? { ...reference, images: [], refmods: [], video: { startSeconds: 0, durationSeconds: 2, ...reference.video, includeAudio: false } }
+ : reference);
+ }
if (isVideoTransition(job)) {
const ids = [job.startVideoReferenceId, ...(job.sceneType === "bridge" ? [job.endVideoReferenceId] : [])];
// Role order, not library order: native video 1 is the tail; video 2 is the head.
@@ -830,7 +840,7 @@ export function sceneGenerationReferences(job: GenerationJob, references: Projec
/** Resolve the adjacent scene when preparing a request; queued work captures
* its id so subsequent board edits cannot change the source of that run. */
export function sceneFrameInputs(job: GenerationJob, config: ProjectConfig) {
- if (isVideoTransition(job) || job.sceneType === "character-replace" || !job.usePreviousSceneLastFrame) return { job, references: sceneGenerationReferences(job, config.references) };
+ if (isVideoTransition(job) || job.sceneType === "pose" || job.sceneType === "character-replace" || !job.usePreviousSceneLastFrame) return { job, references: sceneGenerationReferences(job, config.references) };
const previous = config.generationJobs[config.generationJobs.findIndex((candidate) => candidate.id === job.id) - 1];
const resolved = { ...job, startFrameReferenceId: undefined };
return { job: resolved, references: sceneGenerationReferences(resolved, config.references),
@@ -874,6 +884,7 @@ const addedStop = (text: string): PromptSegment[] => (endSentence(text) === text
* to show the same bytes for both. */
export interface ScenePrompt {
shots: readonly SceneShot[];
+ poseVideoReferenceId?: string | null;
videoTransition?: "extend" | "bridge";
startFrameReferenceId?: string | null;
endFrameReferenceId?: string | null;
@@ -994,6 +1005,9 @@ export function compileMiniMaxH3PromptSegments(
}
export function compileGenerationJobSegments(job: GenerationJob, references: ProjectReference[] = [], defaultLook?: string | null): PromptSegment[] {
+ if (job.sceneType === "pose") return compileScenePromptSegments({
+ shots: sceneShots(job), poseVideoReferenceId: job.poseVideoReferenceId, soundscape: job.soundscape, music: job.music,
+ }, sceneGenerationReferences(job, references), defaultLook);
if (job.sceneType === "character-replace") return compileCharacterReplaceSegments(job, references);
if (job.sceneType === "extend" || job.sceneType === "bridge") return compileScenePromptSegments({
shots: sceneShots(job), videoTransition: job.sceneType, soundscape: job.soundscape, music: job.music,
@@ -1001,6 +1015,16 @@ export function compileGenerationJobSegments(job: GenerationJob, references: Pro
return compileScenePromptSegments({ shots: sceneShots(job), startFrameReferenceId: job.startFrameReferenceId, endFrameReferenceId: job.endFrameReferenceId, soundscape: job.soundscape, music: job.music }, references, defaultLook);
}
+export function poseBlocker(job: GenerationJob, references: ProjectReference[]): string | null {
+ if (!usableVideoReferences(references).some((reference) => reference.id === job.poseVideoReferenceId)) return "Select a pose video reference for this scene.";
+ if (sceneShots(job).every((shot) => !shot.action.trim())) return "Describe where the pose should be used before generating this scene.";
+ const bound = sceneGenerationReferences(job, references);
+ if (usableVideoReferences(bound).length > 3) return "Pose supports up to three video references per scene.";
+ if (usableReferenceImages(bound).length > 9) return "Pose supports up to nine reference images per scene.";
+ if (usableVideoReferences(bound).reduce((total, reference) => total + (reference.video?.durationSeconds ?? 2), 0) > 15) return "The selected video clips must total no more than 15 seconds.";
+ return null;
+}
+
export function videoTransitionBlocker(job: GenerationJob, references: ProjectReference[]): string | null {
const videos = usableVideoReferences(references);
const start = videos.find((reference) => reference.id === job.startVideoReferenceId);
@@ -1241,6 +1265,9 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
const video = videoNumbers.get(reference.id);
const label = referenceLabel(reference, index);
const lead = index === 0 ? "" : "\n";
+ if (reference.id === scene.poseVideoReferenceId && video) {
+ return [frame(`${lead}${label} is pose and motion guidance from <Video ${video}>. Use its body positions, movement and timing; the prompt and other references define the target subject and setting.`)];
+ }
if (!detail && pictures.length === 0 && !video && activeReferenceRefmods(reference).length) {
return [frame(`${lead}${label} uses the attached pre-encoded reference.`)];
}
@@ -1270,7 +1297,10 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
scene written before references could be dropped into the text means, and
what keeps a single untokened shot compiling to the bytes it always did. */
const anyCited = compiled.some((shot) => shot.citedIds.length > 0);
- const featured = compiled.map((shot) => anyCited ? shot.citedIds : usable.map((reference) => reference.id));
+ const featured = compiled.map((shot) => (anyCited ? shot.citedIds : usable.map((reference) => reference.id))
+ .filter((id) => id !== scene.poseVideoReferenceId));
+ const poseVideo = scene.poseVideoReferenceId ? videoNumbers.get(scene.poseVideoReferenceId) : undefined;
+ const poseDirection = poseVideo ? `Apply the body pose, movement and timing from <Video ${poseVideo}> to the subject described below. Use the prompt and optional references for identity, clothing, environment, lighting and camera framing. Do not copy the pose video's appearance, background or soundtrack. ` : "";
// §3: task-type prefix. Subject references provide generation guidance; an
// optional opening picture additionally anchors the first frame. The summary
@@ -1278,6 +1308,7 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
const summary: PromptSegment[] = [
frame(scene.videoTransition === "extend" ? "[video continuation] Continue from the final frame of <Video 1>. "
: scene.videoTransition === "bridge" ? "[video continuation + reference generation] Generate the missing segment after <Video 1> and before <Video 2>. " : "[reference generation] "),
+ ...(poseDirection ? [frame(poseDirection)] : []),
...compiled.flatMap((shot) => [
...(shot.index === 0 ? [] : [frame(" ")]),
...shot.body,
@@ -1299,6 +1330,7 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
// bound to the scene but written into none of its lines gets no "appears in"
// claim at all, because there is no shot it can honestly be placed in.
const retention = usable.map((reference, index) => {
+ if (reference.id === scene.poseVideoReferenceId && poseVideo) return `${referenceLabel(reference, index)} / <Video ${poseVideo}>: partially_preserved - retain body pose, movement and timing only; generate the subject and setting from the prompt and optional references.`;
const appearances = compiled.filter((shot) => featured[shot.index].includes(reference.id))
.map((shot) => `[Shot ${shot.index + 1}]`);
const where = appearances.length > 0 ? ` (appears in ${appearances.join(", ")})` : "";
@@ -1313,6 +1345,7 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
frame("The target video is in a "),
styleFromTag ? tagged(midSentenceStyle(style)) : frame(midSentenceStyle(style)),
frame(" style.\n"),
+ ...(poseDirection ? [frame(`${poseDirection.trim()}\n`)] : []),
...(scene.videoTransition ? [frame(scene.videoTransition === "extend"
? "Begin immediately after the final frame of <Video 1>, continuing its subject identity, pose, motion, lighting and camera trajectory. Follow the text direction below. Output only the new continuation; do not replay the source clip.\n"
: "Begin immediately after the final frame of <Video 1>. Follow the text direction below, developing continuous action that reaches the subject positions, motion, lighting and camera framing immediately before the opening frame of <Video 2>. Output only the new connecting segment; do not replay either source clip.\n")] : []),