Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add voice reference chips to shot speech

Bitti07eb5e616166
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 1e4731a..9871ed6 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -25,6 +25,7 @@ This alpha update adds DMAD LoRA support, resumable model downloads, and improve
## Fixes and improvements
+- Add voice reference chips directly in a shot's Speech field. Voice guidance stays separate from the words spoken.
- Removing a reference from a scene's last prompt citation now removes it from generation inputs and “Used by.” References selected as frame or motion inputs remain active.
- Reference chips now highlight when included in a prompt's text selection.
- The Video scene list scrollbar no longer overlaps the Generate and Cancel buttons.
diff --git a/src/components/workspace/GeneratorView.test.tsx b/src/components/workspace/GeneratorView.test.tsx
index ea2fa4a..584fcd7 100644
--- a/src/components/workspace/GeneratorView.test.tsx
+++ b/src/components/workspace/GeneratorView.test.tsx
@@ -552,11 +552,11 @@ describe("Generator scene controls", () => {
const speech = screen.getByRole("textbox", { name: "Speech for shot 1" });
const language = screen.getByRole("combobox", { name: "Speech language for shot 1" });
- expect(speech).toHaveValue("");
+ expect(speech).toHaveTextContent("");
expect(comboValue(language)).toBe("English");
chooseOption(language, "Korean");
- fireEvent.change(speech, { target: { value: "문을 열어 주세요." } });
+ typePrompt(speech, "문을 열어 주세요.");
expect(sceneShots(state.latest().generationJobs[0])[0]).toEqual(expect.objectContaining({
speech: "문을 열어 주세요.",
speechLanguage: "Korean",
@@ -579,6 +579,46 @@ describe("Generator scene controls", () => {
expect(optionNames(language)).toContain("Klingon");
});
+ it("inserts a voice chip into Speech and removes its generation input when the chip is removed", () => {
+ const initial = project();
+ initial.references = [
+ { id: "voice", kind: "audio", name: "Narrator voice", description: "", intendedUse: [], sourcePath: "C:/voice.wav",
+ audio: { startSeconds: 2, durationSeconds: 4 }, createdAt: initial.createdAt },
+ { id: "hero", kind: "image", name: "Hero", description: "", intendedUse: [], sourcePath: "C:/hero.png", createdAt: initial.createdAt },
+ ];
+ initial.generationJobs[0].startFrameReferenceId = "hero";
+ let latest = initial;
+ const submitted = vi.fn();
+ function Harness() {
+ const [config, setConfig] = useState(initial);
+ latest = config;
+ return <GeneratorView config={config} folderPath="C:/project" runtime={readyRuntime}
+ onChange={setConfig} onGenerate={submitted} onOpenTimeline={() => undefined} />;
+ }
+ render(<Harness />);
+ fireEvent.click(screen.getByRole("button", { name: "Shot 1 of First scene" }));
+ const speech = screen.getByRole("textbox", { name: "Speech for shot 1" });
+ typePrompt(speech, "Welcome home.");
+ placePromptCaret(speech);
+ fireEvent.click(screen.getByRole("button", { name: "Voice reference" }));
+ expect(within(screen.getByRole("dialog")).queryByRole("button", { name: /^Hero/ })).toBeNull();
+ fireEvent.click(screen.getByRole("button", { name: /^Narrator voice/ }));
+ expect(speech.querySelector(".prompt-chip")).toHaveTextContent("Narrator voice");
+ expect(latest.generationJobs[0].referenceIds).toEqual(["voice"]);
+ expect(parseProjectConfig(JSON.parse(JSON.stringify(latest))).generationJobs[0].shots![0].speech).toContain("@[ref:voice]");
+ const generate = within(screen.getByRole("region", { name: "First scene" })).getByRole("button", { name: "Generate" });
+ fireEvent.click(generate);
+ expect(submitted.mock.calls.at(-1)![0][0].request).toMatchObject({
+ prompt: expect.stringContaining("<d>[English] Welcome home.</d>"),
+ referenceAudios: [{ name: "Narrator voice", sourcePath: "C:/voice.wav", startSeconds: 2, durationSeconds: 4 }],
+ });
+ changePromptChip(speech, 0, null);
+ expect(latest.generationJobs[0].referenceIds).toEqual([]);
+ fireEvent.click(generate);
+ expect(submitted.mock.calls.at(-1)![0][0].request.referenceAudios).toEqual([]);
+ expect(submitted.mock.calls.at(-1)![0][0].request.prompt).not.toContain("<Audio");
+ });
+
it("reorders shots inside a scene while keeping its cut slots", () => {
const state = setup();
const shotTransfer = transfer();
diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsx
index 89ba485..a3f3ba2 100644
--- a/src/components/workspace/GeneratorView.tsx
+++ b/src/components/workspace/GeneratorView.tsx
@@ -10,7 +10,7 @@ import { useShortcut } from "../../lib/commands";
import { askNative } from "../../lib/nativeShell";
import { ProjectStatus } from "./ProjectStatus";
import { CommandBar, CommandBarButton, CommandBarSeparator, ComboBox, EmptyState, ItemHeader, Splitter, tooltipProps, usePaneSize } from "../ui";
-import { actionReferenceIds, compileGenerationJobPrompt, compileGenerationJobSegments, createDraftGenerationJob, danglingReferenceTokens, GENERATION_FRAME_RATE, RANDOM_GENERATION_SEED, sceneDurationSeconds, sceneFrameInputs, sceneGenerationReferences, sceneGenerationSeed, sceneGenerationSnapshot, sceneGenerationSteps, sceneShots, SCENE_MAX_SECONDS, SCENE_MIN_SECONDS, projectItemPath, usableReferenceImages, type GenerationJob, type ProjectConfig, type ProjectReference, type SceneShot } from "../../lib/project";
+import { shotReferenceIds, compileGenerationJobPrompt, compileGenerationJobSegments, createDraftGenerationJob, danglingReferenceTokens, GENERATION_FRAME_RATE, RANDOM_GENERATION_SEED, sceneDurationSeconds, sceneFrameInputs, sceneGenerationReferences, sceneGenerationSeed, sceneGenerationSnapshot, sceneGenerationSteps, sceneShots, SCENE_MAX_SECONDS, SCENE_MIN_SECONDS, projectItemPath, usableReferenceImages, type GenerationJob, type ProjectConfig, type ProjectReference, type SceneShot } from "../../lib/project";
import { generationDimensions } from "../../lib/export";
import { isTauri } from "../../lib/persistence";
import { getEngineStatus, type SlopfabGenerationRequest, type SlopfabStatus } from "../../lib/runtime";
@@ -174,7 +174,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
if (updates.shots && (updates.sceneType ?? job.sceneType) !== "animate") {
const referenceIds: string[] = [];
for (const shot of updates.shots) {
- for (const id of actionReferenceIds(shot.action)) {
+ for (const id of shotReferenceIds(shot)) {
if (!referenceIds.includes(id) && config.references.some((reference) => reference.id === id)) referenceIds.push(id);
}
}
diff --git a/src/components/workspace/PromptTextField.tsx b/src/components/workspace/PromptTextField.tsx
index 85a54a8..2d8217d 100644
--- a/src/components/workspace/PromptTextField.tsx
+++ b/src/components/workspace/PromptTextField.tsx
@@ -69,11 +69,12 @@ interface Chips {
missingLabel: (key: string) => string;
}
-export function PromptTextField({ value, onChange, references, format = REFERENCE_TOKENS, missingLabel = () => "Deleted reference", missingTooltip = "Left out of the prompt — click to swap it for another reference or remove it", onInsertReference, disabled = false, placeholder, rows = 4, id, className, "aria-label": ariaLabel, "aria-labelledby": ariaLabelledBy }: {
+export function PromptTextField({ value, onChange, references, referenceButtonLabel = "Reference", format = REFERENCE_TOKENS, missingLabel = () => "Deleted reference", missingTooltip = "Left out of the prompt — click to swap it for another reference or remove it", onInsertReference, disabled = false, placeholder, rows = 4, id, className, "aria-label": ariaLabel, "aria-labelledby": ariaLabelledBy }: {
value: string;
onChange: (value: string) => void;
/** The references the picker offers; a chip for any other key is missing. */
references: readonly PromptReference[];
+ referenceButtonLabel?: string;
format?: PromptTokenFormat;
missingLabel?: (key: string) => string;
missingTooltip?: string;
@@ -211,7 +212,7 @@ export function PromptTextField({ value, onChange, references, format = REFERENC
const at = selection.current ?? { start: value.length, end: value.length };
openPicker(addButton.current!, Math.min(at.start, value.length), Math.min(at.end, value.length), null);
}}
- ><Add14 aria-hidden="true" />Reference</button>
+ ><Add14 aria-hidden="true" />{referenceButtonLabel}</button>
</div>
<div
ref={field}
diff --git a/src/components/workspace/ReferencesView.test.tsx b/src/components/workspace/ReferencesView.test.tsx
index f46c228..8568c94 100644
--- a/src/components/workspace/ReferencesView.test.tsx
+++ b/src/components/workspace/ReferencesView.test.tsx
@@ -447,6 +447,16 @@ describe("Reference type presets", () => {
});
describe("Reference library views and selection", () => {
+ it("counts voice references cited only in Speech as usage", () => {
+ const initial = project();
+ initial.references[0] = { ...initial.references[0], kind: "audio", sourcePath: "C:/voice.wav", audio: { startSeconds: 0, durationSeconds: 4 } };
+ initial.generationJobs[0].shots![0].speech = "@[ref:ref-hero] Hello";
+ setup(initial);
+ expect(screen.queryByText("No scenes yet.")).toBeNull();
+ const usage = document.querySelector(".reference-used-by")! as HTMLElement;
+ expect(within(usage).getByRole("button", { name: initial.generationJobs[0].title })).toBeInTheDocument();
+ });
+
it("does not count stale scene bindings as usage", () => {
const initial = project();
initial.generationJobs[0].referenceIds = ["ref-hero"];
diff --git a/src/components/workspace/SceneEditor.tsx b/src/components/workspace/SceneEditor.tsx
index 00d8031..d18c0cd 100644
--- a/src/components/workspace/SceneEditor.tsx
+++ b/src/components/workspace/SceneEditor.tsx
@@ -2,7 +2,7 @@ import { Audio16, Dismiss12, Image16, ImageAdd16, Text16, Video16 } from "../ui/
import { useId, useMemo, useState, type ReactNode } from "react";
import { ComboBox, InfoBar, PropRow, PropSection } from "../ui";
import {
- actionReferenceIds,
+ shotReferenceIds,
danglingReferenceTokens,
isAudioReference,
isReferenceUsable,
@@ -67,7 +67,7 @@ export function referenceOrder(job: GenerationJob, shots: readonly SceneShot[],
const boundIds = sceneGenerationReferences({ ...job, shots: [...shots] }, [...references]).map((reference) => reference.id);
const order = citableReferences(references).filter((reference) => boundIds.includes(reference.id)).map((reference) => reference.id);
for (const shot of shots) {
- for (const id of actionReferenceIds(shot.action)) if (!order.includes(id)) order.push(id);
+ for (const id of shotReferenceIds(shot)) if (!order.includes(id)) order.push(id);
}
return order;
}
@@ -188,15 +188,17 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
{!promptOnly && <div className="shot-speech">
<label className="shot-card__sublabel" htmlFor={`shot-speech-${shot.id}`}>Speech</label>
- <textarea
+ <PromptTextField
id={`shot-speech-${shot.id}`}
- className="text-field"
value={shot.speech ?? ""}
+ references={pickable.filter((reference) => referenceById.get(reference.id)?.kind === "audio")}
+ referenceButtonLabel="Voice reference"
+ missingLabel={(referenceId) => referenceById.get(referenceId)?.name ?? "Deleted reference"}
+ missingTooltip="Speech references must be sound clips — click to replace or remove"
disabled={disabled}
aria-label={`Speech for shot ${shotNumber}`}
placeholder="Words spoken in this shot"
- data-tooltip="Added to the prompt as MiniMax dialogue"
- onChange={(event) => onChange({ speech: event.target.value || null })}
+ onChange={(speech) => onChange({ speech: speech || null })}
/>
<PropRow label="Language" htmlFor={`shot-language-${shot.id}`}>
<ComboBox
diff --git a/src/lib/project.audio.test.ts b/src/lib/project.audio.test.ts
index ff75ff8..e1184ac 100644
--- a/src/lib/project.audio.test.ts
+++ b/src/lib/project.audio.test.ts
@@ -1,7 +1,7 @@
import { expect, it } from "vitest";
import {
audioReferenceBlocker, compileMiniMaxH3Prompt, compileScenePromptSegments, createDraftGenerationJob, danglingReferenceTokens, isReferenceUsable,
- projectReferenceSchema, referenceAudioNumbers, referenceToken, sceneGenerationSnapshot, usableAudioReferences, usableReferenceImages, type ProjectReference,
+ projectReferenceSchema, referenceAudioNumbers, referenceToken, sceneGenerationSnapshot, sceneGenerationReferences, compileGenerationJobPrompt, usableAudioReferences, usableReferenceImages, type ProjectReference,
} from "./project";
import { referenceAudioPcm, referenceAudioRange } from "./referenceAudio";
@@ -39,6 +39,43 @@ it("defines an undescribed sound without inventing its role and keeps it out of
expect(usableReferenceImages(references)).toHaveLength(1);
});
+it("takes voice guidance from Speech chips while keeping only words inside dialogue tags", () => {
+ const refs = [reference("voice", "audio"), reference("video", "video")];
+ const job = createDraftGenerationJob("", { shots: [
+ { id: "a", startSeconds: 0, action: "@[ref:video]", speech: "@[ref:voice] Hello, world!", speechLanguage: "French" },
+ { id: "b", startSeconds: 3, action: "The camera pans.", speech: "Goodbye." },
+ ] });
+ const bound = sceneGenerationReferences(job, refs);
+ expect(bound).toEqual(refs);
+ const prompt = compileGenerationJobPrompt(job, bound);
+ expect(prompt).toContain("Use the voice characteristics of <Audio 2> for the scene's speaker (S1).");
+ expect(prompt).toContain("<d>[French] Hello, world!</d>");
+ expect(prompt).toContain("<d>[English] Goodbye.</d>");
+ expect(prompt.match(/Use the voice characteristics/g)).toHaveLength(1);
+ expect(prompt).not.toContain("@[ref:");
+ expect(prompt).not.toMatch(/<d>[^<]*<Audio/);
+ job.shots![0].speech = "Hello, world!";
+ expect(sceneGenerationReferences(job, refs).map(({ id }) => id)).toEqual(["video"]);
+});
+
+it("rejects missing or non-audio Speech references and deduplicates citations", () => {
+ const refs = [reference("voice", "audio"), reference("hero", "image")];
+ const shots = [{ id: "a", startSeconds: 0, action: "@[ref:hero]", speech: "@[ref:voice] @[ref:voice] Hello" }];
+ expect(danglingReferenceTokens(shots, refs)).toEqual([]);
+ const job = createDraftGenerationJob("", { shots });
+ expect(sceneGenerationReferences(job, refs)).toHaveLength(2);
+ expect(compileGenerationJobPrompt(job, refs).match(/Use the voice characteristics of <Audio 1>/g)).toHaveLength(1);
+ shots[0].speech = "@[ref:hero] @[ref:missing] Hello";
+ expect(danglingReferenceTokens(shots, refs)).toEqual(["hero", "missing"]);
+});
+
+it("keeps a voice-only chip as guidance without inventing dialogue", () => {
+ const job = createDraftGenerationJob("A speaker", { shots: [{ id: "a", startSeconds: 0, action: "A speaker", speech: "@[ref:voice]" }] });
+ const prompt = compileGenerationJobPrompt(job, [reference("voice", "audio"), reference("hero", "image")]);
+ expect(prompt).toContain("Use the voice characteristics of <Audio 1>");
+ expect(prompt).not.toContain("<d>");
+});
+
it("lets shots cite sounds and blocks sounds without visual media", () => {
const job = { ...createDraftGenerationJob("Scene"), referenceIds: ["voice"] };
const voice = reference("voice", "audio");
diff --git a/src/lib/project.ts b/src/lib/project.ts
index 0369099..6ad2c78 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -429,6 +429,7 @@ export const sceneShotSchema = z.object({
action: z.string(),
/** Dialogue is separate from the visual action so the compiler can guarantee
* that every spoken character lands inside MiniMax H3's `<d>...</d>` syntax.
+ * Reference tokens select voice guidance outside the spoken dialogue.
* Both fields are optional to preserve projects written before speech UI. */
speech: z.string().nullish(),
speechLanguage: speechLanguageSchema.nullish(),
@@ -482,6 +483,11 @@ export function actionReferenceIds(action: string): string[] {
return seen;
}
+/** References in either editable prompt of a shot. */
+export function shotReferenceIds(shot: Pick<SceneShot, "action" | "speech">): string[] {
+ return [...new Set([...actionReferenceIds(shot.action), ...actionReferenceIds(shot.speech ?? "")])];
+}
+
export const generationJobSchema = z.object({
id: idSchema,
sceneType: sceneTypeSchema.nullish(),
@@ -911,7 +917,7 @@ export function sceneGenerationReferences(job: GenerationJob, references: Projec
if (job.sceneType === "pose") {
// Optional guidance is explicit in the prompt. Old bindings and frame
// anchors belong to other modes and must not redefine the target scene.
- const cited = sceneShots(job).flatMap((shot) => actionReferenceIds(shot.action));
+ const cited = sceneShots(job).flatMap(shotReferenceIds);
return references.filter((reference) => reference.id === job.poseVideoReferenceId || cited.includes(reference.id))
.map((reference) => reference.id === job.poseVideoReferenceId
? { ...reference, images: [], refmods: [], video: { startSeconds: 0, durationSeconds: 2, ...reference.video, includeAudio: false } }
@@ -935,7 +941,7 @@ export function sceneGenerationReferences(job: GenerationJob, references: Projec
.map((reference) => ({ ...reference, kind: "text" as const, video: undefined, relativePath: null, sourcePath: null, intendedUse: [], images: referenceImages(reference).slice(0, 1) }));
// Animate stores its dedicated motion/frame selections in referenceIds.
// Other scenes take guidance from their current prompts, never stale bindings.
- const ids = job.sceneType === "animate" ? job.referenceIds : sceneShots(job).flatMap((shot) => actionReferenceIds(shot.action));
+ const ids = job.sceneType === "animate" ? job.referenceIds : sceneShots(job).flatMap(shotReferenceIds);
return [...anchors, ...references.filter((reference) => ids.includes(reference.id) && !anchors.some((anchor) => anchor.id === reference.id))];
}
@@ -1229,6 +1235,7 @@ interface CompiledShot {
citedIds: string[];
tags: ShotTagClauses;
speech: string | null;
+ voiceNumbers: number[];
speechLanguage: SpeechLanguage;
}
@@ -1270,6 +1277,15 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
body.push(frame(`<Subject ${number}>`));
text += `<Subject ${number}>`;
}
+ // Chips in Speech select voice guidance; they are not spoken words.
+ const speechParts = splitActionText(shot.speech ?? "");
+ const speechText = speechParts.some((part) => part.kind === "reference")
+ ? speechParts.filter((part) => part.kind === "text").map((part) => part.value).join("").trim()
+ : shot.speech ?? "";
+ const voiceNumbers = actionReferenceIds(shot.speech ?? "").flatMap((id) => {
+ const number = audioNumber.get(id);
+ return number ? [number] : [];
+ });
return {
index,
startSeconds: shot.startSeconds,
@@ -1277,7 +1293,8 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
text,
citedIds,
tags: shotTagClauses(shot.settings ?? null),
- speech: shot.speech?.trim().length ? shot.speech : null,
+ speech: speechText.trim().length ? speechText : null,
+ voiceNumbers,
speechLanguage: shot.speechLanguage ?? DEFAULT_SPEECH_LANGUAGE,
};
});
@@ -1308,18 +1325,23 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
* outside `<d>`, while only the language marker and the user's exact text are
* inside. One scene-level speech track means the same (S1) voice is retained
* across its shots. The pieces stay split so prompt provenance remains honest. */
- const dialogue = (shot: CompiledShot, separate: boolean): PromptSegment[] => shot.speech === null ? [] : [
- frame(`${separate ? " " : ""}The scene's speaker (S1) says, <d>[`),
- tagged(shot.speechLanguage),
- frame("] "),
- own(shot.speech),
- frame("</d>"),
- ];
+ const dialogue = (shot: CompiledShot, separate: boolean): PromptSegment[] => {
+ const guidance = shot.voiceNumbers.length
+ ? `Use the voice characteristics of ${shot.voiceNumbers.map((number) => `<Audio ${number}>`).join(" and ")} for the scene's speaker (S1).`
+ : "";
+ return [
+ ...(guidance ? [frame(`${separate ? " " : ""}${guidance}`)] : []),
+ ...(shot.speech === null ? [] : [
+ frame(`${separate || guidance ? " " : ""}The scene's speaker (S1) says, <d>[`),
+ tagged(shot.speechLanguage), frame("] "), own(shot.speech), frame("</d>"),
+ ]),
+ ];
+ };
/* A stop is added when something follows the line — another clause, or
another shot. A lone untagged shot keeps whatever terminal punctuation the
user gave it, so every prompt this app has ever produced is unchanged. */
const stop = (shot: CompiledShot): PromptSegment[] =>
- shot.tags.clauses.length > 0 || shot.speech !== null || shot.index < compiled.length - 1 ? addedStop(shot.text) : [];
+ shot.tags.clauses.length > 0 || shot.speech !== null || shot.voiceNumbers.length > 0 || shot.index < compiled.length - 1 ? addedStop(shot.text) : [];
const sound = (scene.soundscape ?? "").trim();
const music = (scene.music ?? "").trim();
@@ -1520,11 +1542,15 @@ export function compileScenePromptSegments(scene: ScenePrompt, references: Proje
* why, and sending is blocked until they are re-pointed or removed. */
export function danglingReferenceTokens(shots: readonly SceneShot[], references: ProjectReference[]): string[] {
const usable = new Set(references.filter(isReferenceUsable).filter((reference) => isVisualReference(reference) || isAudioReference(reference)).map((reference) => reference.id));
+ const sounds = new Set(usableAudioReferences(references).map((reference) => reference.id));
const missing: string[] = [];
for (const shot of shots) {
for (const id of actionReferenceIds(shot.action)) {
if (!usable.has(id) && !missing.includes(id)) missing.push(id);
}
+ for (const id of actionReferenceIds(shot.speech ?? "")) {
+ if (!sounds.has(id) && !missing.includes(id)) missing.push(id);
+ }
}
return missing;
}
diff --git a/src/styles/generator.css b/src/styles/generator.css
index 40e7bfd..7725547 100644
--- a/src/styles/generator.css
+++ b/src/styles/generator.css
@@ -780,8 +780,7 @@
margin-top: var(--space-2);
}
-.scene-settings__field textarea,
-.shot-speech > textarea {
+.scene-settings__field textarea {
width: 100%;
height: auto;
min-height: 80px;