Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat: titles speak - generate voice from text clips

Jareerd017e1290015
diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx
index 32e5da32..8fd5a23c 100644
--- a/desktop/src/App.tsx
+++ b/desktop/src/App.tsx
@@ -278,7 +278,9 @@ function Editor({
);
const [settingsOpen, setSettingsOpen] = useState(false);
- const [speechOpen, setSpeechOpen] = useState(false);
+ /** The speech sheet: open with optional prefilled text and a fixed landing
+ * time (a text clip's start), or null when closed. */
+ const [speech, setSpeech] = useState<{ text?: string; at?: number } | null>(null);
// The save-as-template sheet: null closed, otherwise whether the host is
// packing the bundle right now.
const [templateDialog, setTemplateDialog] = useState<null | { busy: boolean }>(null);
@@ -359,9 +361,10 @@ function Editor({
);
// A finished narration WAV: imported like any dropped file, then placed at
- // the playhead. Throwing lets the speech sheet show the failure in place.
+ // `at` - a text clip's start when spoken from one, the playhead otherwise.
+ // Throwing lets the speech sheet show the failure in place.
const insertNarration = useCallback(
- async (path: string) => {
+ async (path: string, at?: number) => {
const summary = await probeMedia(path);
const mediaId = await dispatch({
op: "addMedia",
@@ -379,7 +382,11 @@ function Editor({
},
session.path,
);
- await dispatch({ op: "addClipAtFirstFree", mediaId, start: latest.current.playhead });
+ await dispatch({
+ op: "addClipAtFirstFree",
+ mediaId,
+ start: at ?? latest.current.playhead,
+ });
pushToast(t("toast.narrationAdded"), false);
},
[dispatch, session.path, pushToast, t],
@@ -540,11 +547,27 @@ function Editor({
onToast: pushToast,
});
- /** The audio/video tools dropdown. Hidden entirely for soundless clips. */
+ /** The clip tools dropdown. Hidden for clips with nothing to offer. */
const clipTools = useMemo<MenuOption[][]>(() => {
const clip = selectedClipIds.length === 1 ? findClip(project, selectedClipIds[0]) : null;
- const media = clip ? findMedia(project, clip.mediaId) : null;
- if (!clip || (clip.kind !== "video" && clip.kind !== "audio")) return [];
+ if (!clip) return [];
+
+ // A title's one tool: speak its text. The sheet opens prefilled, and the
+ // narration lands where the title starts.
+ if (clip.kind === "text") {
+ return [
+ [
+ {
+ label: t("menu.clip.generateVoice"),
+ icon: "volume" as const,
+ onSelect: () => setSpeech({ text: clip.text?.content ?? "", at: clip.start }),
+ },
+ ],
+ ];
+ }
+
+ const media = findMedia(project, clip.mediaId);
+ if (clip.kind !== "video" && clip.kind !== "audio") return [];
const hasSound = Boolean(media?.hasAudio);
const canDetach = Boolean(
@@ -843,7 +866,7 @@ function Editor({
const openModifyProject = useCallback(() => setModifyingProject({ busy: false }), []);
const openSettings = useCallback(() => setSettingsOpen(true), []);
- const openSpeech = useCallback(() => setSpeechOpen(true), []);
+ const openSpeech = useCallback(() => setSpeech({}), []);
const openExport = useCallback(() => setExporting(true), []);
// ── keyboard ─────────────────────────────────────────────────────────────
@@ -1419,98 +1442,125 @@ function Editor({
setContext({
x,
y,
- items: [
- {
- label: tp("contextMenu.splitClips", target.length),
- icon: "razor",
- hint: "S",
- onSelect: splitAtPlayhead,
- },
- ...(whyNotMerge(project, target) === null
- ? [
- {
- label: tp("contextMenu.mergeClips", target.length),
- icon: "merge" as const,
- hint: "M",
- onSelect: mergeSelected,
- },
- ]
- : []),
- ...(() => {
- const media = clip ? findMedia(project, clip.mediaId) : null;
- return !many &&
- clip &&
- (clip.kind === "video" || clip.kind === "audio") &&
- media?.hasAudio &&
- !transcribing
+ // Related actions share a group; the menu draws a divider
+ // between groups and drops any that come up empty.
+ groups: [
+ // Cutting.
+ [
+ {
+ label: tp("contextMenu.splitClips", target.length),
+ icon: "razor",
+ hint: "S",
+ onSelect: splitAtPlayhead,
+ },
+ ...(whyNotMerge(project, target) === null
? [
{
- label: t("menu.clip.autoCaptions"),
- icon: "type" as const,
- onSelect: () => void autoCaption(clip, media),
+ label: tp("contextMenu.mergeClips", target.length),
+ icon: "merge" as const,
+ hint: "M",
+ onSelect: mergeSelected,
},
]
- : [];
- })(),
- ...(!many &&
- clip?.kind === "video" &&
- !clip.muted &&
- findMedia(project, clip.mediaId)?.hasAudio &&
- detachedAudioOf(project, clip.id).length === 0
- ? [
- {
- label: t("menu.clip.detachAudio"),
- icon: "waveform" as const,
- onSelect: () => void dispatch({ op: "detachAudio", clipId }),
- },
- ]
- : []),
- ...(!many &&
- clip &&
- ((clip.kind === "video" && detachedAudioOf(project, clip.id).length > 0) ||
- (clip.kind === "audio" &&
- clip.detachedFrom &&
- findClip(project, clip.detachedFrom)))
- ? [
- {
- label: t("menu.clip.reattachAudio"),
- icon: "merge" as const,
- onSelect: () => {
- void dispatch({ op: "reattachAudio", clipId });
- setSelectedClipIds([]);
- },
- },
- ]
- : []),
- {
- label: t("contextMenu.moveToPlayhead"),
- icon: "select",
- onSelect: () => {
- const moving = findClip(latest.current.project, clipId);
- if (moving) {
- void dispatch({
- op: "moveClips",
- moves: [
+ : []),
+ ],
+ // Generation: captions out of speech, speech out of titles.
+ [
+ ...(() => {
+ const media = clip ? findMedia(project, clip.mediaId) : null;
+ return !many &&
+ clip &&
+ (clip.kind === "video" || clip.kind === "audio") &&
+ media?.hasAudio &&
+ !transcribing
+ ? [
{
- clipId,
- start: latest.current.playhead,
- trackId: moving.trackId,
+ label: t("menu.clip.autoCaptions"),
+ icon: "type" as const,
+ onSelect: () => void autoCaption(clip, media),
+ },
+ ]
+ : [];
+ })(),
+ ...(!many && clip?.kind === "text"
+ ? [
+ {
+ label: t("menu.clip.generateVoice"),
+ icon: "volume" as const,
+ onSelect: () =>
+ setSpeech({ text: clip.text?.content ?? "", at: clip.start }),
+ },
+ ]
+ : []),
+ ],
+ // The audio attached to a video clip.
+ [
+ ...(!many &&
+ clip?.kind === "video" &&
+ !clip.muted &&
+ findMedia(project, clip.mediaId)?.hasAudio &&
+ detachedAudioOf(project, clip.id).length === 0
+ ? [
+ {
+ label: t("menu.clip.detachAudio"),
+ icon: "waveform" as const,
+ onSelect: () => void dispatch({ op: "detachAudio", clipId }),
+ },
+ ]
+ : []),
+ ...(!many &&
+ clip &&
+ ((clip.kind === "video" && detachedAudioOf(project, clip.id).length > 0) ||
+ (clip.kind === "audio" &&
+ clip.detachedFrom &&
+ findClip(project, clip.detachedFrom)))
+ ? [
+ {
+ label: t("menu.clip.reattachAudio"),
+ icon: "merge" as const,
+ onSelect: () => {
+ void dispatch({ op: "reattachAudio", clipId });
+ setSelectedClipIds([]);
},
- ],
- });
- }
+ },
+ ]
+ : []),
+ ],
+ // Arranging.
+ [
+ {
+ label: t("contextMenu.moveToPlayhead"),
+ icon: "select",
+ onSelect: () => {
+ const moving = findClip(latest.current.project, clipId);
+ if (moving) {
+ void dispatch({
+ op: "moveClips",
+ moves: [
+ {
+ clipId,
+ start: latest.current.playhead,
+ trackId: moving.trackId,
+ },
+ ],
+ });
+ }
+ },
},
- },
- {
- label: tp("contextMenu.deleteClips", target.length),
- icon: "trash",
- hint: "Del",
- danger: true,
- onSelect: () => {
- void dispatch({ op: "removeClips", clipIds: target });
- setSelectedClipIds([]);
+ ],
+ // Destruction, kept in its own section at the bottom.
+ [
+ {
+ label: tp("contextMenu.deleteClips", target.length),
+ icon: "trash",
+ hint: "Del",
+ danger: true,
+ onSelect: () => {
+ void dispatch({ op: "removeClips", clipIds: target });
+ setSelectedClipIds([]);
+ },
},
- },
+ ],
],
});
}}
@@ -1520,11 +1570,12 @@ function Editor({
{context && <ContextMenu target={context} onClose={() => setContext(null)} />}
{settingsOpen && <SettingsDialog onClose={() => setSettingsOpen(false)} />}
- {speechOpen && (
+ {speech && (
<TtsDialog
projectPath={session.path}
- onGenerated={insertNarration}
- onClose={() => setSpeechOpen(false)}
+ initialText={speech.text}
+ onGenerated={(path) => insertNarration(path, speech.at)}
+ onClose={() => setSpeech(null)}
/>
)}
diff --git a/desktop/src/components/ContextMenu.tsx b/desktop/src/components/ContextMenu.tsx
index ed101520..f90c3925 100644
--- a/desktop/src/components/ContextMenu.tsx
+++ b/desktop/src/components/ContextMenu.tsx
@@ -14,7 +14,8 @@ export interface ContextItem {
export interface ContextTarget {
x: number;
y: number;
- items: ContextItem[];
+ /** Related actions grouped together; a hairline divides the groups. */
+ groups: ContextItem[][];
}
/**
@@ -69,30 +70,40 @@ export function ContextMenu({ target, onClose }: { target: ContextTarget; onClos
onContextMenu={(event) => event.preventDefault()}
className="surface fixed z-50 min-w-48 rounded-xl px-1.5 py-1.5 text-primary"
>
- {target.items.map((item) => (
- <button
- key={item.label}
- role="menuitem"
- type="button"
- onClick={() => {
- item.onSelect();
- onClose();
- }}
- className={`flex w-full cursor-pointer items-center gap-2.5 rounded-lg px-2.5 py-1.5
- text-left text-xs transition-colors hover:bg-hover
- ${item.danger ? "text-danger" : ""}`}
- >
- {item.icon ? (
- <Icon name={item.icon} size={14} className="shrink-0 opacity-70" />
- ) : (
- <span className="w-3.5 shrink-0" />
- )}
- <span className="min-w-0 flex-1 truncate">{item.label}</span>
- {item.hint && (
- <span className="shrink-0 font-technical text-[10px] text-tertiary">{item.hint}</span>
- )}
- </button>
- ))}
+ {target.groups
+ .filter((group) => group.length > 0)
+ .map((group, index) => (
+ // Index as key is fine here: the menu is rebuilt from scratch on
+ // every open and never reorders while visible.
+ <div key={index} className={index > 0 ? "mt-1 border-t border-hairline pt-1" : ""}>
+ {group.map((item) => (
+ <button
+ key={item.label}
+ role="menuitem"
+ type="button"
+ onClick={() => {
+ item.onSelect();
+ onClose();
+ }}
+ className={`flex w-full cursor-pointer items-center gap-2.5 rounded-lg px-2.5 py-1.5
+ text-left text-xs transition-colors hover:bg-hover
+ ${item.danger ? "text-danger" : ""}`}
+ >
+ {item.icon ? (
+ <Icon name={item.icon} size={14} className="shrink-0 opacity-70" />
+ ) : (
+ <span className="w-3.5 shrink-0" />
+ )}
+ <span className="min-w-0 flex-1 truncate">{item.label}</span>
+ {item.hint && (
+ <span className="shrink-0 font-technical text-[10px] text-tertiary">
+ {item.hint}
+ </span>
+ )}
+ </button>
+ ))}
+ </div>
+ ))}
</div>,
document.body,
);
diff --git a/desktop/src/components/TimelinePanel.tsx b/desktop/src/components/TimelinePanel.tsx
index 58027400..f147488f 100644
--- a/desktop/src/components/TimelinePanel.tsx
+++ b/desktop/src/components/TimelinePanel.tsx
@@ -984,8 +984,9 @@ export function TimelinePanel({
active={snap}
onClick={() => onSnapChange(!snap)}
/>
- {/* Only while a sound-capable clip is selected: a title has nothing
- to detach or transcribe, and a menu of grey rows teaches nothing. */}
+ {/* Only while the selected clip has tools to offer - sound to detach
+ or transcribe, or a title's text to speak. A menu of grey rows
+ teaches nothing. */}
{clipTools.length > 0 && (
<>
<Divider />
diff --git a/desktop/src/components/TtsDialog.tsx b/desktop/src/components/TtsDialog.tsx
index 9d26c218..56bdf1dd 100644
--- a/desktop/src/components/TtsDialog.tsx
+++ b/desktop/src/components/TtsDialog.tsx
@@ -65,18 +65,21 @@ function voiceLabel(name: string): string {
*/
export function TtsDialog({
projectPath,
+ initialText,
onGenerated,
onClose,
}: {
/** The open project's folder - where the host writes the WAV. */
projectPath: string;
+ /** Prefills the text box - a title clip's content when spoken from one. */
+ initialText?: string;
/** Places the finished WAV on the timeline; throwing lands in the sheet's
* error notice. The sheet closes itself afterwards. */
onGenerated: (path: string, duration: number) => Promise<void>;
onClose: () => void;
}) {
const { t } = useLocale();
- const [text, setText] = useState("");
+ const [text, setText] = useState(initialText ?? "");
const [voice, setVoice] = useState(getTtsVoice());
const [speed, setSpeed] = useState(1.0);
const [phase, setPhase] = useState<Phase>({ kind: "idle" });
diff --git a/desktop/src/locales/en.json b/desktop/src/locales/en.json
index 73e8e421..84288f3d 100644
--- a/desktop/src/locales/en.json
+++ b/desktop/src/locales/en.json
@@ -257,6 +257,7 @@
"mediaBin.videoEffects": "Video Effects",
"menu.clip.autoCaptions": "Auto captions",
"menu.clip.detachAudio": "Detach audio",
+ "menu.clip.generateVoice": "Generate voice...",
"menu.clip.reattachAudio": "Reattach audio",
"menu.clip.transcribing": "Transcribing...",
"menu.edit.deleteClips.one": "Delete clip",
diff --git a/desktop/src/locales/zh-CN.json b/desktop/src/locales/zh-CN.json
index bd33d269..04fd7bf1 100644
--- a/desktop/src/locales/zh-CN.json
+++ b/desktop/src/locales/zh-CN.json
@@ -257,6 +257,7 @@
"mediaBin.videoEffects": "视频特效",
"menu.clip.autoCaptions": "自动字幕",
"menu.clip.detachAudio": "分离音频",
+ "menu.clip.generateVoice": "生成语音...",
"menu.clip.reattachAudio": "重新附加音频",
"menu.clip.transcribing": "正在转录...",
"menu.edit.deleteClips.one": "删除片段",