Skip to content

Open source CapCut alternative.

TypeScript

feat(site): true speech-cut edges into real pauses mechanically [rebuild]

David Le7d735c9c1661
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index e4dd1e0d..8cb23fc9 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -30,6 +30,7 @@ import {
makeStillSheetClientSide,
renderAudioSpanWav,
} from "./media";
+import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";
import { requestSidePanel, SIDE_PANEL_TABS } from "./panelRequest";
import { blobToInlineAudio, refToInlineAudio, visualRefs, type InlineImage } from "./refMedia";
import { characterPrompt, stockTitle } from "./stock";
@@ -250,46 +251,18 @@ export async function runAiTool(
case "detect_silence": {
const { projectId, asset, clip, speed, from, to } = resolveWatchRange(s, input);
if (asset.type === "image") throw new ToolError(`"${asset.name}" is an image — it has no audio.`);
- interface SilenceBody {
- silences: { start: number; end: number; duration: number }[];
- error?: string;
- }
- let body: SilenceBody;
- if (getBackend().kind === "cloud") {
- // Same defaults and clamps as the engine's silence handler.
- body = {
- silences: await detectSilenceClientSide(asset.url, {
- from,
- ...(to !== undefined ? { to } : {}),
- thresholdDb: clamp(isNum(input.threshold_db) ? input.threshold_db : -30, -90, 0),
- minSilence: clamp(isNum(input.min_silence) ? input.min_silence : 0.35, 0.05, 10),
- }).catch((e) => {
- throw new ToolError(
- e instanceof Error ? e.message : "Could not scan for silence."
- );
- }),
- };
- } else {
- const res = await apiFetch(`/api/cut/projects/${projectId}/silence`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({
- file: asset.fileName,
- from,
- ...(to !== undefined ? { to } : {}),
- ...(isNum(input.threshold_db) ? { threshold_db: input.threshold_db } : {}),
- ...(isNum(input.min_silence) ? { min_silence: input.min_silence } : {}),
- }),
- });
- body = await apiJson<SilenceBody>(res);
- if (!res.ok) throw new ToolError(body.error ?? "Could not scan for silence.");
- }
+ const silences = await fetchSilences(projectId, asset, {
+ from,
+ ...(to !== undefined ? { to } : {}),
+ ...(isNum(input.threshold_db) ? { thresholdDb: input.threshold_db } : {}),
+ ...(isNum(input.min_silence) ? { minSilence: input.min_silence } : {}),
+ });
// Pre-map each silence's overlap with the clip's trimmed range onto the
// timeline so the model cuts on ready numbers.
const toTimeline = (t: number) =>
round2(clip!.start + (clamp(t, clip!.in, clip!.out) - clip!.in) / speed);
return {
- silences: body.silences.map((x) => ({
+ silences: silences.map((x) => ({
start: round2(x.start),
end: round2(x.end),
duration: round2(x.duration),
@@ -309,7 +282,7 @@ export async function runAiTool(
},
}
: {}),
- ...(body.silences.length === 0
+ ...(silences.length === 0
? { note: "No silence at these settings — a higher threshold_db or shorter min_silence hears more." }
: {}),
};
@@ -522,6 +495,139 @@ export async function runAiTool(
return { in: nextIn, out: nextOut, len: Math.round((nextOut - nextIn) * 100) / 100 };
}
+ case "refine_speech_cuts": {
+ const projectId = s.projectId;
+ if (!projectId) throw new ToolError("No project open.");
+ const ids = Array.isArray(input.clip_ids) ? input.clip_ids.map(String) : [];
+ if (ids.length === 0)
+ throw new ToolError("clip_ids is required — the recut clips whose edges are speech cuts.");
+ const chosen = ids.map((id) => requireItem(s.clips, id, "video clip"));
+
+ // Every trim is planned before anything is written, so the re-lay below
+ // can preserve the spacing the cut already has.
+ const plans = new Map<string, { in: number; out: number }>();
+ const flags = new Map<string, string[]>();
+ const flag = (id: string, note: string) =>
+ flags.set(id, [...(flags.get(id) ?? []), note]);
+ await Promise.all(
+ chosen.map(async (clip) => {
+ const asset = s.assets.find((a) => a.id === clip.assetId);
+ if (!asset || asset.type === "image" || !(asset.duration > 0)) {
+ flag(clip.id, "no audio-bearing source — skipped");
+ return;
+ }
+ // One scan spanning both edges when the clip is short, else one
+ // window around each. Fine pauses matter here, so the scan listens
+ // for stretches down to a single breath.
+ const win = REACH + 0.6;
+ const ranges: [number, number][] =
+ clip.out - clip.in <= 2 * win
+ ? [[clip.in - win, clip.out + win]]
+ : [
+ [clip.in - win, clip.in + win],
+ [clip.out - win, clip.out + win],
+ ];
+ const spans: SilenceSpan[] = (
+ await Promise.all(
+ ranges.map(([a, b]) =>
+ fetchSilences(projectId, asset, {
+ from: clamp(a, 0, asset.duration),
+ to: clamp(b, 0, asset.duration),
+ thresholdDb: -30,
+ minSilence: BREATH,
+ })
+ )
+ )
+ ).flat();
+ const inEdge = refineEdge(spans, clip.in, "in");
+ const outEdge = refineEdge(spans, clip.out, "out");
+ if (!inEdge.pause)
+ flag(clip.id, "no pause within reach of the in edge — speech runs continuously; left alone");
+ if (!outEdge.pause)
+ flag(clip.id, "no pause within reach of the out edge — speech runs continuously; left alone");
+ const nextIn = clamp(inEdge.t, 0, asset.duration);
+ const nextOut = clamp(outEdge.t, 0, asset.duration);
+ if (nextOut - nextIn >= 0.15) plans.set(clip.id, { in: nextIn, out: nextOut });
+ else flag(clip.id, "refining would collapse the clip — left as cut");
+ })
+ );
+
+ // Two sides of one removed stretch never cross back into each other's
+ // material, so no word plays twice across a joint.
+ const spd = (c: VideoClip) => (c.speed && c.speed > 0 ? c.speed : 1);
+ const rows = new Map<number, VideoClip[]>();
+ for (const c of s.clips) rows.set(c.track, [...(rows.get(c.track) ?? []), c]);
+ for (const row of rows.values()) {
+ row.sort((a, b) => a.start - b.start);
+ for (let i = 0; i + 1 < row.length; i++) {
+ const a = row[i];
+ const b = row[i + 1];
+ if (a.assetId !== b.assetId || a.out > b.in + 1e-3) continue;
+ const pa = plans.get(a.id);
+ const pb = plans.get(b.id);
+ if ((pa?.out ?? a.out) <= (pb?.in ?? b.in) + 1e-6) continue;
+ if (pa) pa.out = a.out;
+ if (pb) pb.in = b.in;
+ for (const c of [a, b])
+ if (plans.has(c.id)) flag(c.id, "its joint's edges would cross the removed stretch — that edge left as cut");
+ }
+ }
+
+ // Drop no-op plans so the report says which edges already sat right.
+ for (const [id, plan] of [...plans]) {
+ const c = chosen.find((x) => x.id === id)!;
+ if (Math.abs(plan.in - c.in) < 0.005 && Math.abs(plan.out - c.out) < 0.005)
+ plans.delete(id);
+ }
+
+ // One write for the whole call: each planned clip takes its new trim in
+ // place, and every clip after it on the track shifts by the length
+ // change — butted joints stay butted, deliberate beats keep their width,
+ // and a tightened clip closes up instead of opening a gap.
+ if (plans.size > 0) {
+ s.pushHistory();
+ const patches: { id: string; patch: Partial<VideoClip> }[] = [];
+ for (const row of rows.values()) {
+ let delta = 0;
+ for (const c of row) {
+ const plan = plans.get(c.id);
+ const patch: Partial<VideoClip> = {};
+ if (Math.abs(delta) > 1e-6) patch.start = Math.max(0, c.start + delta);
+ if (plan) {
+ patch.in = plan.in;
+ patch.out = plan.out;
+ delta += (plan.out - plan.in - (c.out - c.in)) / spd(c);
+ }
+ if (plan || patch.start !== undefined) patches.push({ id: c.id, patch });
+ }
+ }
+ useEditor.getState().updateClipsTransient(patches);
+ useEditor.getState().sortClips();
+ }
+
+ return {
+ refined: chosen.map((clip) => {
+ const plan = plans.get(clip.id);
+ const notes = flags.get(clip.id);
+ return {
+ clipId: clip.id,
+ ...(plan
+ ? {
+ in: { from: round2(clip.in), to: round2(plan.in) },
+ out: { from: round2(clip.out), to: round2(plan.out) },
+ }
+ : {}),
+ ...(notes
+ ? { notes }
+ : plan
+ ? {}
+ : { notes: ["edges already sit in their pauses"] }),
+ };
+ }),
+ note: `Each moved edge sits ~${BREATH}s inside a real pause; clip spacing is preserved. Listen at any flagged edge.`,
+ };
+ }
+
case "set_clip_muted": {
const clip = requireItem(s.clips, input.clipId, "video clip");
s.updateClip(clip.id, { muted: Boolean(input.muted) });
@@ -1986,6 +2092,47 @@ function resolveWatchRange(
return { projectId, asset, clip, speed, from, to };
}
+/** Silent stretches of a source range, in source seconds — the engine's
+ * ffmpeg scan locally, the browser's RMS fold in cloud mode. Shared by
+ * detect_silence (the model's ears) and refine_speech_cuts (the mechanical
+ * re-trim). */
+async function fetchSilences(
+ projectId: string,
+ asset: MediaAsset,
+ opts: { from: number; to?: number; thresholdDb?: number; minSilence?: number }
+): Promise<{ start: number; end: number; duration: number }[]> {
+ // Same defaults and clamps as the engine's silence handler.
+ const thresholdDb = clamp(opts.thresholdDb ?? -30, -90, 0);
+ const minSilence = clamp(opts.minSilence ?? 0.35, 0.05, 10);
+ if (getBackend().kind === "cloud") {
+ return detectSilenceClientSide(asset.url, {
+ from: opts.from,
+ ...(opts.to !== undefined ? { to: opts.to } : {}),
+ thresholdDb,
+ minSilence,
+ }).catch((e) => {
+ throw new ToolError(e instanceof Error ? e.message : "Could not scan for silence.");
+ });
+ }
+ const res = await apiFetch(`/api/cut/projects/${projectId}/silence`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ file: asset.fileName,
+ from: opts.from,
+ ...(opts.to !== undefined ? { to: opts.to } : {}),
+ threshold_db: thresholdDb,
+ min_silence: minSilence,
+ }),
+ });
+ const body = await apiJson<{
+ silences: { start: number; end: number; duration: number }[];
+ error?: string;
+ }>(res);
+ if (!res.ok) throw new ToolError(body.error ?? "Could not scan for silence.");
+ return body.silences;
+}
+
/** Pull a source's audio track off (video and audio alike) and inline it for
* the model; null when the range clears the inline cap. Local: the engine
* extracts mono AAC. Cloud: the browser renders the span to 16 kHz mono WAV. */
diff --git a/site/src/cut/lib/cutRefine.test.ts b/site/src/cut/lib/cutRefine.test.ts
new file mode 100644
index 00000000..3cb7df2c
--- /dev/null
+++ b/site/src/cut/lib/cutRefine.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, test } from "bun:test";
+import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";
+
+/**
+ * Edges refine against detected silence: speech runs everywhere a span
+ * doesn't. The spans here draw a take with pauses at known places, and each
+ * case asks where a model-made edge should really sit.
+ */
+
+const spans: SilenceSpan[] = [
+ { start: 0, end: 1 }, // lead-in silence
+ { start: 4, end: 5 }, // pause between sentences
+ { start: 9, end: 9.3 }, // a short breath
+ { start: 12, end: 20 }, // long trailing silence
+];
+
+describe("out edges (a clip's end)", () => {
+ test("an edge inside a pause settles a breath past the speech", () => {
+ expect(refineEdge(spans, 4.8, "out")).toEqual({ t: 4 + BREATH, pause: true });
+ });
+
+ test("an edge clipping a word moves outward to the next pause", () => {
+ expect(refineEdge(spans, 3.6, "out")).toEqual({ t: 4 + BREATH, pause: true });
+ });
+
+ test("a pause shorter than a breath still holds the cut at its far side", () => {
+ expect(refineEdge(spans, 9.1, "out")).toEqual({ t: 9.25, pause: true });
+ });
+
+ test("continuous speech leaves the edge alone, flagged", () => {
+ expect(refineEdge([{ start: 0, end: 1 }], 6, "out")).toEqual({ t: 6, pause: false });
+ });
+
+ test("a long pause tightens only within reach", () => {
+ expect(refineEdge(spans, 19, "out")).toEqual({ t: 19 - REACH, pause: true });
+ });
+});
+
+describe("in edges (a clip's start)", () => {
+ test("an edge inside a pause settles a breath ahead of the speech", () => {
+ expect(refineEdge(spans, 4.2, "in")).toEqual({ t: 5 - BREATH, pause: true });
+ });
+
+ test("an edge clipping a lead word moves back to the pause before it", () => {
+ expect(refineEdge(spans, 5.4, "in")).toEqual({ t: 5 - BREATH, pause: true });
+ });
+
+ test("an edge at the source head trims the lead-in to a breath", () => {
+ expect(refineEdge(spans, 0, "in")).toEqual({ t: 1 - BREATH, pause: true });
+ });
+
+ test("continuous speech leaves the edge alone, flagged", () => {
+ expect(refineEdge(spans, 7.5, "in")).toEqual({ t: 7.5, pause: false });
+ });
+
+ test("a pause further back than reach does not pull the edge", () => {
+ expect(refineEdge([{ start: 0, end: 1 }], 3.5, "in")).toEqual({ t: 3.5, pause: false });
+ });
+});
diff --git a/site/src/cut/lib/cutRefine.ts b/site/src/cut/lib/cutRefine.ts
new file mode 100644
index 00000000..5a5665b1
--- /dev/null
+++ b/site/src/cut/lib/cutRefine.ts
@@ -0,0 +1,75 @@
+// True up speech-cut edges against the silence around them.
+//
+// The assistant plans cuts from cue timings and silence spans, but the numbers
+// it emits are hand-derived floats: an edge lands a hair inside a word, or
+// leaves stray dead air. The split of labor is the usual one — the model says
+// WHICH edges are speech cuts, and the arithmetic here decides exactly where
+// each one sits: a breath inside the real pause.
+//
+// The rules err toward keeping words. An edge already inside a pause slides to
+// a breath past the speech it closes (an out point) or ahead of the speech it
+// opens (an in point). An edge sitting in speech — a clipped word — only moves
+// outward, to the first pause in the direction that makes the word whole
+// again. With no pause within reach the speech runs continuously there, so the
+// edge stays where the model put it and comes back flagged for a listen.
+
+/** A silent stretch of the source, in source seconds. */
+export interface SilenceSpan {
+ start: number;
+ end: number;
+}
+
+/** The pause kept on the speech side of a refined edge, seconds. */
+export const BREATH = 0.25;
+
+/** How far an edge may travel to find or settle into a pause. Beyond this a
+ * long pause reads as a held beat — a choice, not slack — and a cut in speech
+ * reads as deliberate. */
+export const REACH = 2;
+
+/** Which side of the clip an edge trims: "in" opens the clip, "out" closes it. */
+export type EdgeKind = "in" | "out";
+
+export interface RefinedEdge {
+ /** The refined source time — the input `t` when no pause was within reach. */
+ t: number;
+ /** False when speech runs continuously at this edge and it was left alone. */
+ pause: boolean;
+}
+
+const EPS = 1e-3;
+const round = (n: number) => Math.round(n * 1000) / 1000;
+const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));
+
+/**
+ * Place one trim edge a breath inside the pause it belongs to. `spans` are the
+ * silences detected around `t`, sorted by start and disjoint.
+ */
+export function refineEdge(
+ spans: readonly SilenceSpan[],
+ t: number,
+ kind: EdgeKind
+): RefinedEdge {
+ const within = spans.find((s) => t >= s.start - EPS && t <= s.end + EPS);
+ if (within) {
+ // A pause shorter than two breaths still holds a cut — the edge settles at
+ // the pause's far boundary and the whole pause is the breath.
+ const ideal =
+ kind === "out"
+ ? Math.min(within.end, within.start + BREATH)
+ : Math.max(within.start, within.end - BREATH);
+ return { t: round(clamp(ideal, t - REACH, t + REACH)), pause: true };
+ }
+ if (kind === "out") {
+ const next = spans.find((s) => s.start > t && s.start - t <= REACH);
+ if (next) return { t: round(Math.min(next.end, next.start + BREATH)), pause: true };
+ } else {
+ let prev: SilenceSpan | undefined;
+ for (const s of spans) {
+ if (s.end < t && t - s.end <= REACH) prev = s;
+ if (s.end >= t) break;
+ }
+ if (prev) return { t: round(Math.max(prev.start, prev.end - BREATH)), pause: true };
+ }
+ return { t, pause: false };
+}
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 60d7634a..a58978b9 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -145,6 +145,21 @@ export const AI_TOOLS: AiToolDef[] = [
"Set a video clip's trim points inside its source media (seconds). Changing `in` hides the leading part; `out` the trailing part.",
inputSchema: obj({ clipId: str("Video clip id"), in: num("New in point (optional)"), out: num("New out point (optional)") }, ["clipId"]),
},
+ {
+ name: "refine_speech_cuts",
+ description:
+ "True up speech-cut edges mechanically: re-scans the audio around each clip's in/out and re-trims so every edge sits a ~0.25s breath inside a real pause — an edge that clips a word moves outward to the next pause, stray dead air trims off. Clip spacing survives: a tightened clip closes up, butted joints stay butted, and deliberate gaps keep their width. An edge with no pause within ~2s stays put and comes back flagged. Run it on every clip you recut for speech, after cutting and before listening; leave out clips whose edge you placed mid-speech on purpose.",
+ inputSchema: obj(
+ {
+ clip_ids: {
+ type: "array",
+ items: { type: "string" },
+ description: "Video clip ids whose in/out edges are speech cuts",
+ },
+ },
+ ["clip_ids"]
+ ),
+ },
{
name: "set_clip_muted",
description: "Mute or unmute a video clip's own audio.",
@@ -804,16 +819,17 @@ The flow for "edit this for me":
1. get_state — every clip, its trim (in/out), speed, gaps, and the other tracks.
2. Speech? subtitles_generate first — cue timings are timeline seconds and say what is said when.
3. watch_video each distinct source (or the ranges in question); note scene changes and what happens where.
-4. Plan in thoughts: group the cues into sentences and thoughts, decide which thoughts stay whole, then detect_silence to find where the speaker actually pauses. Cue timings approximate the audio — a trailing word often spills past its cue — so place every speech cut inside a detected silence span, padded ~0.2s in from both of its edges; a cue boundary with no silence span around it means the speech runs continuously there, so leave that joint alone.
+4. Plan in thoughts: group the cues into sentences and thoughts, decide which thoughts stay whole, then detect_silence to find where the speaker actually pauses. Cue timings approximate the audio — a trailing word often spills past its cue — so aim every speech cut into a detected silence span, and aim loose: up to a second of spare air on each side beats a tight cut, because the refine step trues the edges after. A cue boundary with no silence span around it means the speech runs continuously there, so leave that joint alone.
5. Execute: split_at (timeline s) to divide, delete_item to drop a segment, trim_clip (source s) to tighten edges, place_clip/move_clip to close gaps or reorder, set_speed to compress slow stretches.
-6. Verify by ear: listen_audio each recut clip (its trim is the default range; scope a long one with from/to to ~2s around each new edge) and check the first and last words arrive whole and the hand-off into the next thought sounds natural. A clipped word → widen that trim deeper into the silence and listen again. After a heavy edit, listen once end-to-end. seek + capture_frame when the change is visual.
+6. Refine: refine_speech_cuts with every recut clip id. It re-scans the audio at each edge and re-trims it to sit a ~0.25s breath inside the real pause, keeping the clips' spacing. It reads sound, not intent — leave out a clip whose edge you placed mid-speech on purpose.
+7. Verify by ear: listen_audio at each edge the refine flagged, plus a spot-check or two (scope from/to to ~2s around the joint), and check the first and last words arrive whole and the hand-off into the next thought sounds natural. A clipped word → widen that trim deeper into the silence and listen again. After a heavy edit, listen once end-to-end. seek + capture_frame when the change is visual.
Time math (get this right):
- watch_video, detect_silence, and trim_clip speak SOURCE seconds; split_at and place_clip speak TIMELINE seconds.
- timeline_t = clip.start + (source_t − clip.in) / clip.speed, valid while source_t is inside [in, out]. Each watch result's clip block carries this formula with the real numbers filled in.
- The same source can appear in several clips — map per clip.
-Cutting speech well: pace is part of the message. Keep a beat of the speaker's own pause between sentences (~0.3s) and a slightly longer one between thoughts (~0.5s) — shorten a long pause to that beat instead of deleting it, and speech butted directly together reads as rushed. Leave ~0.2s of breath on each side of every cut so edges land in true silence. Prefer split_at + delete_item, then place_clip to close the gap (a beat of black may be wanted — ask the cut, not the tool); trim_clip only tightens a clip's edges.`,
+Cutting speech well: pace is part of the message. Keep a beat of the speaker's own pause between sentences (~0.3s) and a slightly longer one between thoughts (~0.5s) — shorten a long pause to that beat instead of deleting it, and speech butted directly together reads as rushed. Cut loose and let refine_speech_cuts set the breath at each edge. Prefer split_at + delete_item, then place_clip to close the gap (a beat of black may be wanted — ask the cut, not the tool); trim_clip only tightens a clip's edges.`,
"transitions-and-fades": `# Transitions, animations, looks & fades
Route the ask to the right feature: