Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Create music drafts when editing generated songs

Bittic45f91b273b2
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 9d51ad3..409c74f 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -20,6 +20,7 @@ Changes since 0.3.2.
## Fixes and improvements
+- Editing generated music creates a separate draft; cards distinguish drafts from playable songs.
- Single Shot continuation conditions both generation passes on the previous clip's final latents to preserve continuity.
- Updated First/Last Frame Fast to the eight-step FastH3 Omni model and corrected Omni and First/Last Frame weight fallbacks.
- Image edits preserve source images, prompts and settings; queued results get separate entries, and whole-image edits disable feathering.
diff --git a/src/components/workspace/MusicView.test.tsx b/src/components/workspace/MusicView.test.tsx
index a2205d7..3b0a41c 100644
--- a/src/components/workspace/MusicView.test.tsx
+++ b/src/components/workspace/MusicView.test.tsx
@@ -231,6 +231,84 @@ it("selects the new song returned by generation while keeping the original visib
expect(screen.getByLabelText("Song title")).toHaveValue("Evening (2)");
});
+it.each([
+ ["Music style", "Acoustic folk"], ["Music lyrics", "[Verse]\nA new day"], ["Music seed", "12"],
+ ["Music steps", "16"], ["Song duration", "45"], ["Song title", "Morning"],
+])("creates one selected draft when editing %s on generated music", (label, value) => {
+ const folder = newMusicFolder("Jazz", null);
+ const original = { ...newSong("yue2", 32, folder.id), title: "Evening", style: "Jazz", status: "completed" as const, assetId: "original" };
+ original.generation = { generatorId: original.generatorId, style: original.style, settings: { ...original.settings }, steps: original.steps, seed: 42 };
+ const asset = { id: "original", kind: "audio" as const, name: "Evening", relativePath: "media/evening.wav", durationMs: 30000, mimeType: "audio/wav", createdAt: original.createdAt };
+ render(<Harness initial={{ ...createConfig(), musicFolders: [folder], musicSongs: [original], assets: [asset] }} />);
+ fireEvent.click(screen.getByRole("button", { name: "Open Jazz" }));
+ // Focusing and committing an unchanged number must not create a copy.
+ fireEvent.blur(screen.getByLabelText("Music steps"));
+ expect(latest.musicSongs).toEqual([original]);
+ fireEvent.change(screen.getByLabelText(label), { target: { value } });
+ expect(latest.musicSongs).toHaveLength(2);
+ expect(latest.musicSongs![0]).toEqual(original);
+ const draft = latest.musicSongs![1];
+ expect(draft.id).not.toBe(original.id);
+ expect(draft).toMatchObject({ folderId: folder.id, status: "draft", assetId: null, generation: null, error: null });
+ expect(screen.getByRole("button", { name: `Select ${draft.title}` })).toHaveAttribute("aria-pressed", "true");
+ expect(screen.getByLabelText(label)).toHaveValue(["Music seed", "Music steps", "Song duration"].includes(label) ? Number(value) : value);
+ fireEvent.change(screen.getByLabelText("Music style"), { target: { value: "Acoustic folk with strings" } });
+ fireEvent.change(screen.getByLabelText("Music lyrics"), { target: { value: "Keep writing" } });
+ expect(latest.musicSongs).toHaveLength(2);
+ expect(latest.musicSongs![0]).toEqual(original);
+ expect(latest.musicSongs![1]).toMatchObject({ id: draft.id, style: "Acoustic folk with strings", settings: { lyrics: "Keep writing" } });
+ expect(latest.assets).toEqual([asset]);
+ fireEvent.click(screen.getByRole("button", { name: "Generate" }));
+ expect(generate).toHaveBeenCalledWith(draft.id, expect.objectContaining({ id: "yue2" }));
+});
+
+it("copies generated music to a draft when changing its generator", () => {
+ const settings = loadGeneratorTemplateSettings(), source = settings.templates.find((template) => template.id === "yue2")!;
+ saveGeneratorTemplateSettings({ ...settings, templates: [...settings.templates, { ...source, id: "other-music", name: "Other music generator", defaultSteps: 16 }] });
+ const original = { ...newSong(), title: "Evening", style: "Jazz", status: "completed" as const, assetId: "original" };
+ render(<Harness initial={{ ...createConfig(), musicSongs: [original] }} />);
+ fireEvent.click(screen.getByRole("combobox", { name: "Generator" }));
+ fireEvent.click(screen.getByRole("option", { name: /Other music generator/ }));
+ expect(latest.musicSongs).toHaveLength(2);
+ expect(latest.musicSongs![0]).toEqual(original);
+ expect(latest.musicSongs![1]).toMatchObject({ generatorId: "other-music", steps: 16, status: "draft", assetId: null });
+});
+
+it.each(["Grid", "List"])("distinguishes playable songs from drafts and missing audio in %s view", (view) => {
+ const original = { ...newSong(), title: "Evening", status: "completed" as const, assetId: "original" };
+ const draft = newSongVariation(original, [original]);
+ const missing = { ...newSong(), title: "Missing", status: "completed" as const, assetId: "missing" };
+ const asset = { id: "original", kind: "audio" as const, name: "Evening", relativePath: "media/evening.wav", durationMs: 30000, mimeType: "audio/wav", createdAt: original.createdAt };
+ render(<Harness initial={{ ...createConfig(), musicSongs: [original, draft, missing], assets: [asset] }} />);
+ fireEvent.click(screen.getByRole("button", { name: `${view} view` }));
+ const readyCard = screen.getByRole("button", { name: "Select Evening" }).closest('[role="listitem"]')!;
+ const draftCard = screen.getByRole("button", { name: "Select Evening (2)" }).closest('[role="listitem"]')!;
+ expect(readyCard).toHaveClass("music-song--playable");
+ expect(within(readyCard as HTMLElement).getByText("Ready · 0:30")).toBeInTheDocument();
+ expect(within(readyCard as HTMLElement).getByRole("button", { name: "Play Evening" })).toBeEnabled();
+ expect(draftCard).toHaveClass("music-song--draft");
+ expect(within(draftCard as HTMLElement).getByText("Draft · Not generated")).toBeInTheDocument();
+ expect(screen.queryByRole("button", { name: "Play Evening (2)" })).not.toBeInTheDocument();
+ expect(screen.getByText("Audio unavailable")).toBeInTheDocument();
+ expect(screen.queryByRole("button", { name: "Play Missing" })).not.toBeInTheDocument();
+});
+
+it("keeps the original audio playing when editing creates a draft", async () => {
+ const original = { ...newSong(), title: "Evening", style: "Jazz", status: "completed" as const, assetId: "original" };
+ const asset = { id: "original", kind: "audio" as const, name: "Evening", relativePath: "media/evening.wav", durationMs: 30000, mimeType: "audio/wav", createdAt: original.createdAt };
+ const { container } = render(<Harness initial={{ ...createConfig(), musicSongs: [original], assets: [asset] }} />);
+ fireEvent.click(screen.getByRole("button", { name: "Play Evening" }));
+ await waitFor(() => expect(container.querySelector("audio")).toHaveAttribute("src", "blob:song"));
+ fireEvent.canPlay(container.querySelector("audio")!);
+ fireEvent.play(container.querySelector("audio")!);
+ const pauses = vi.mocked(HTMLMediaElement.prototype.pause).mock.calls.length;
+ fireEvent.change(screen.getByLabelText("Music lyrics"), { target: { value: "New lyrics" } });
+ expect(HTMLMediaElement.prototype.pause).toHaveBeenCalledTimes(pauses);
+ expect(readMediaFileUrl).toHaveBeenCalledOnce();
+ expect(screen.getByRole("button", { name: "Pause music" })).toBeEnabled();
+ expect(latest.musicSongs![0]).toEqual(original);
+});
+
it("plays generated audio and creates a reference for the context-clicked song", async () => {
const first = { ...newSong(), title: "First", status: "completed" as const, assetId: "song-a" };
const second = { ...newSong(), title: "Second", status: "completed" as const, assetId: "song-b" };
diff --git a/src/components/workspace/MusicView.tsx b/src/components/workspace/MusicView.tsx
index 8cc81b6..e16fc29 100644
--- a/src/components/workspace/MusicView.tsx
+++ b/src/components/workspace/MusicView.tsx
@@ -2,10 +2,10 @@ import { useEffect, useRef, useState } from "react";
import type { ProjectAsset, ProjectConfig } from "../../lib/project";
import { isTauri, readMediaFileUrl } from "../../lib/persistence";
import { loadGeneratorTemplateSettings, subscribeGeneratorTemplates, templateSupportsMusic, templateUsable, generatorPathFields, type GeneratorTemplate } from "../../lib/settings";
-import { musicSettingsForModel, moveSong, newMusicFolder, newSong, removeMusicFolder, songReference, updateSong, type MusicSettings, type MusicSong } from "../../lib/music";
+import { musicSettingsForModel, moveSong, newMusicFolder, newSong, newSongVariation, removeMusicFolder, songReference, updateSong, type MusicSettings, type MusicSong } from "../../lib/music";
import { isWorkActive, type WorkItem } from "../../lib/workQueue";
import { ComboBox, CommandBarButton, ProgressBar, PropRow, PropSection, Slider, Splitter, useContextMenu, usePaneSize, type MenuEntry } from "../ui";
-import { Add16, Audio16, Delete16, FolderAdd16, FolderOpen16, GoToEnd16, GoToStart16, GridView16, ListView16, Pause16, Play16, References16, Speaker14, Sparkle16, Stop14 } from "../ui/icons";
+import { Add16, Audio16, Delete16, Edit16, FolderAdd16, FolderOpen16, GoToEnd16, GoToStart16, GridView16, ListView16, Pause16, Play16, References16, Speaker14, Sparkle16, Stop14 } from "../ui/icons";
import { generatorOptions } from "../generatorOptions";
import { SliderNumberInput, GENERATION_STEPS_RANGE } from "./SliderNumberInput";
import type { ConfigUpdate } from "./TimelineView";
@@ -63,7 +63,14 @@ export function MusicView({ config, folderPath, onChange, onGenerate, onCancel,
: !ready ? { state: "modelsMissing", label: "Model weights missing", detail: "Download or select music model weights in Settings → Generators." }
: { state: "ready", label: "Music generator ready", detail: template.name };
const edit = (patch: Partial<MusicSong>) => {
- if (selected && !busy) onChange((current) => updateSong(current, selected.id, patch), `music:${selected.id}`);
+ if (!selected || busy || Object.entries(patch).every(([key, value]) => JSON.stringify(value) === JSON.stringify(selected[key as keyof MusicSong]))) return;
+ if (selected.assetId || selected.generation || selected.status === "completed") {
+ const draft = { ...newSongVariation(selected, config.musicSongs ?? []), ...patch };
+ onChange((current) => ({ ...current, musicSongs: current.musicSongs?.flatMap((song) => song.id === selected.id ? [song, draft] : [song]) }), `music:${draft.id}`);
+ setSelectedId(draft.id);
+ } else {
+ onChange((current) => updateSong(current, selected.id, patch), `music:${selected.id}`);
+ }
};
const setting = (patch: Partial<MusicSettings>) => { if (selected) edit({ settings: { ...selected.settings, ...patch } }); };
const add = () => {
@@ -265,19 +272,21 @@ export function MusicView({ config, folderPath, onChange, onGenerate, onCancel,
</button>
</div>)}
{visibleSongs.map((song, index) => {
- const asset = config.assets.find((item) => item.id === song.assetId);
+ const asset = config.assets.find((item) => item.id === song.assetId && item.kind === "audio" && item.relativePath);
const work = workItems.find((item) => item.sceneId === song.id && isWorkActive(item));
- return <div className={`music-song${selected?.id === song.id ? " music-song--selected" : ""}`} role="listitem" key={`song:${song.id}`}
+ const generated = Boolean(song.assetId || song.generation || song.status === "completed");
+ const state = asset ? "playable" : generated ? "unavailable" : "draft";
+ return <div className={`music-song music-song--${state}${selected?.id === song.id ? " music-song--selected" : ""}`} role="listitem" key={`song:${song.id}`}
draggable onDragStart={(event) => { event.dataTransfer.effectAllowed = "move"; event.dataTransfer.setData(SONG_DRAG_TYPE, JSON.stringify({ projectId: config.id, songId: song.id })); }} onDragEnd={() => setDropFolder(undefined)}
onContextMenu={(event) => context(event, song)}>
- <button type="button" className="music-song__select" aria-label={`Select ${song.title}`} aria-pressed={selected?.id === song.id}
+ <button type="button" className="music-song__select" aria-label={`Select ${song.title}`} aria-pressed={selected?.id === song.id} aria-describedby={`music-status-${song.id}`}
onClick={() => setSelectedId(song.id)} onDoubleClick={() => { if (asset) startSong(asset); }}
onKeyDown={(event) => { if (event.key === "ContextMenu" || event.shiftKey && event.key === "F10") context(event, song); }}>
- <span className="music-song__art" aria-hidden="true"><Audio16 /><span>{String(index + 1).padStart(2, "0")}</span></span>
+ <span className="music-song__art" aria-hidden="true">{generated ? <Audio16 /> : <Edit16 />}<span>{String(index + 1).padStart(2, "0")}</span></span>
<span className="music-song__info"><strong>{song.title}</strong><span>{song.style || "Add a style to get started"}</span></span>
- <span className="music-song__status">{work ? `${work.detail} · ${Math.round(work.progress * 100)}%` : song.status === "completed" && asset ? time((asset.durationMs ?? 0) / 1000) : song.status}</span>
+ <span className="music-song__status" id={`music-status-${song.id}`}>{work ? `${work.detail} · ${Math.round(work.progress * 100)}%` : asset ? `Ready · ${time((asset.durationMs ?? 0) / 1000)}` : generated ? "Audio unavailable" : song.status === "draft" ? "Draft · Not generated" : song.status}</span>
</button>
- <button type="button" className="icon-button music-song__play" aria-label={`Play ${song.title}`} disabled={!asset?.relativePath} onClick={() => asset && startSong(asset)}><Play16 /></button>
+ {asset && <button type="button" className="icon-button music-song__play" aria-label={`Play ${song.title}`} onClick={() => startSong(asset)}><Play16 /></button>}
</div>;
})}
</div>}
diff --git a/src/styles/music.css b/src/styles/music.css
index 01bdd88..36000bb 100644
--- a/src/styles/music.css
+++ b/src/styles/music.css
@@ -24,6 +24,9 @@
.music-songs--grid { grid-template-columns: repeat(auto-fill, minmax(170px, 1fr)); }
.music-song { position: relative; min-width: 0; border: 1px solid var(--line); border-radius: var(--radius-lg); background: var(--surface-pane); }
.music-song--selected { border-color: var(--accent); }
+.music-song--draft { border-style: dashed; }
+.music-song--draft .music-song__art { background: transparent; color: var(--text-secondary); }
+.music-song--playable .music-song__status { color: var(--accent); }
.music-song__select { display: flex; flex-direction: column; gap: var(--space-2); width: 100%; height: 100%; padding: var(--space-3); border: 0; border-radius: inherit; text-align: left; color: var(--text-primary); background: transparent; }
.music-song__select:hover { background: var(--fill-subtle-hover); }
.music-song__art { display: flex; align-items: center; justify-content: center; gap: var(--space-3); width: 100%; aspect-ratio: 1; border-radius: var(--radius-sm); background: var(--fill-subtle-hover); color: var(--accent); }