Open source CapCut alternative.
TypeScript
feat(site): true speech-cut edges into real pauses mechanically [rebuild]
David Le7d735c9c1661
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex e4dd1e0d..8cb23fc9 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -30,6 +30,7 @@ import {makeStillSheetClientSide,renderAudioSpanWav,} from "./media";+import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";import { requestSidePanel, SIDE_PANEL_TABS } from "./panelRequest";import { blobToInlineAudio, refToInlineAudio, visualRefs, type InlineImage } from "./refMedia";import { characterPrompt, stockTitle } from "./stock";@@ -250,46 +251,18 @@ export async function runAiTool(case "detect_silence": {const { projectId, asset, clip, speed, from, to } = resolveWatchRange(s, input);if (asset.type === "image") throw new ToolError(`"${asset.name}" is an image — it has no audio.`);- interface SilenceBody {- silences: { start: number; end: number; duration: number }[];- error?: string;- }- let body: SilenceBody;- if (getBackend().kind === "cloud") {- // Same defaults and clamps as the engine's silence handler.- body = {- silences: await detectSilenceClientSide(asset.url, {- from,- ...(to !== undefined ? { to } : {}),- thresholdDb: clamp(isNum(input.threshold_db) ? input.threshold_db : -30, -90, 0),- minSilence: clamp(isNum(input.min_silence) ? input.min_silence : 0.35, 0.05, 10),- }).catch((e) => {- throw new ToolError(- e instanceof Error ? e.message : "Could not scan for silence."- );- }),- };- } else {- const res = await apiFetch(`/api/cut/projects/${projectId}/silence`, {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- file: asset.fileName,- from,- ...(to !== undefined ? { to } : {}),- ...(isNum(input.threshold_db) ? { threshold_db: input.threshold_db } : {}),- ...(isNum(input.min_silence) ? { min_silence: input.min_silence } : {}),- }),- });- body = await apiJson<SilenceBody>(res);- if (!res.ok) throw new ToolError(body.error ?? "Could not scan for silence.");- }+ const silences = await fetchSilences(projectId, asset, {+ from,+ ...(to !== undefined ? { to } : {}),+ ...(isNum(input.threshold_db) ? { thresholdDb: input.threshold_db } : {}),+ ...(isNum(input.min_silence) ? { minSilence: input.min_silence } : {}),+ });// Pre-map each silence's overlap with the clip's trimmed range onto the// timeline so the model cuts on ready numbers.const toTimeline = (t: number) =>round2(clip!.start + (clamp(t, clip!.in, clip!.out) - clip!.in) / speed);return {- silences: body.silences.map((x) => ({+ silences: silences.map((x) => ({start: round2(x.start),end: round2(x.end),duration: round2(x.duration),@@ -309,7 +282,7 @@ export async function runAiTool(},}: {}),- ...(body.silences.length === 0+ ...(silences.length === 0? { note: "No silence at these settings — a higher threshold_db or shorter min_silence hears more." }: {}),};@@ -522,6 +495,139 @@ export async function runAiTool(return { in: nextIn, out: nextOut, len: Math.round((nextOut - nextIn) * 100) / 100 };}+ case "refine_speech_cuts": {+ const projectId = s.projectId;+ if (!projectId) throw new ToolError("No project open.");+ const ids = Array.isArray(input.clip_ids) ? input.clip_ids.map(String) : [];+ if (ids.length === 0)+ throw new ToolError("clip_ids is required — the recut clips whose edges are speech cuts.");+ const chosen = ids.map((id) => requireItem(s.clips, id, "video clip"));++ // Every trim is planned before anything is written, so the re-lay below+ // can preserve the spacing the cut already has.+ const plans = new Map<string, { in: number; out: number }>();+ const flags = new Map<string, string[]>();+ const flag = (id: string, note: string) =>+ flags.set(id, [...(flags.get(id) ?? []), note]);+ await Promise.all(+ chosen.map(async (clip) => {+ const asset = s.assets.find((a) => a.id === clip.assetId);+ if (!asset || asset.type === "image" || !(asset.duration > 0)) {+ flag(clip.id, "no audio-bearing source — skipped");+ return;+ }+ // One scan spanning both edges when the clip is short, else one+ // window around each. Fine pauses matter here, so the scan listens+ // for stretches down to a single breath.+ const win = REACH + 0.6;+ const ranges: [number, number][] =+ clip.out - clip.in <= 2 * win+ ? [[clip.in - win, clip.out + win]]+ : [+ [clip.in - win, clip.in + win],+ [clip.out - win, clip.out + win],+ ];+ const spans: SilenceSpan[] = (+ await Promise.all(+ ranges.map(([a, b]) =>+ fetchSilences(projectId, asset, {+ from: clamp(a, 0, asset.duration),+ to: clamp(b, 0, asset.duration),+ thresholdDb: -30,+ minSilence: BREATH,+ })+ )+ )+ ).flat();+ const inEdge = refineEdge(spans, clip.in, "in");+ const outEdge = refineEdge(spans, clip.out, "out");+ if (!inEdge.pause)+ flag(clip.id, "no pause within reach of the in edge — speech runs continuously; left alone");+ if (!outEdge.pause)+ flag(clip.id, "no pause within reach of the out edge — speech runs continuously; left alone");+ const nextIn = clamp(inEdge.t, 0, asset.duration);+ const nextOut = clamp(outEdge.t, 0, asset.duration);+ if (nextOut - nextIn >= 0.15) plans.set(clip.id, { in: nextIn, out: nextOut });+ else flag(clip.id, "refining would collapse the clip — left as cut");+ })+ );++ // Two sides of one removed stretch never cross back into each other's+ // material, so no word plays twice across a joint.+ const spd = (c: VideoClip) => (c.speed && c.speed > 0 ? c.speed : 1);+ const rows = new Map<number, VideoClip[]>();+ for (const c of s.clips) rows.set(c.track, [...(rows.get(c.track) ?? []), c]);+ for (const row of rows.values()) {+ row.sort((a, b) => a.start - b.start);+ for (let i = 0; i + 1 < row.length; i++) {+ const a = row[i];+ const b = row[i + 1];+ if (a.assetId !== b.assetId || a.out > b.in + 1e-3) continue;+ const pa = plans.get(a.id);+ const pb = plans.get(b.id);+ if ((pa?.out ?? a.out) <= (pb?.in ?? b.in) + 1e-6) continue;+ if (pa) pa.out = a.out;+ if (pb) pb.in = b.in;+ for (const c of [a, b])+ if (plans.has(c.id)) flag(c.id, "its joint's edges would cross the removed stretch — that edge left as cut");+ }+ }++ // Drop no-op plans so the report says which edges already sat right.+ for (const [id, plan] of [...plans]) {+ const c = chosen.find((x) => x.id === id)!;+ if (Math.abs(plan.in - c.in) < 0.005 && Math.abs(plan.out - c.out) < 0.005)+ plans.delete(id);+ }++ // One write for the whole call: each planned clip takes its new trim in+ // place, and every clip after it on the track shifts by the length+ // change — butted joints stay butted, deliberate beats keep their width,+ // and a tightened clip closes up instead of opening a gap.+ if (plans.size > 0) {+ s.pushHistory();+ const patches: { id: string; patch: Partial<VideoClip> }[] = [];+ for (const row of rows.values()) {+ let delta = 0;+ for (const c of row) {+ const plan = plans.get(c.id);+ const patch: Partial<VideoClip> = {};+ if (Math.abs(delta) > 1e-6) patch.start = Math.max(0, c.start + delta);+ if (plan) {+ patch.in = plan.in;+ patch.out = plan.out;+ delta += (plan.out - plan.in - (c.out - c.in)) / spd(c);+ }+ if (plan || patch.start !== undefined) patches.push({ id: c.id, patch });+ }+ }+ useEditor.getState().updateClipsTransient(patches);+ useEditor.getState().sortClips();+ }++ return {+ refined: chosen.map((clip) => {+ const plan = plans.get(clip.id);+ const notes = flags.get(clip.id);+ return {+ clipId: clip.id,+ ...(plan+ ? {+ in: { from: round2(clip.in), to: round2(plan.in) },+ out: { from: round2(clip.out), to: round2(plan.out) },+ }+ : {}),+ ...(notes+ ? { notes }+ : plan+ ? {}+ : { notes: ["edges already sit in their pauses"] }),+ };+ }),+ note: `Each moved edge sits ~${BREATH}s inside a real pause; clip spacing is preserved. Listen at any flagged edge.`,+ };+ }+case "set_clip_muted": {const clip = requireItem(s.clips, input.clipId, "video clip");s.updateClip(clip.id, { muted: Boolean(input.muted) });@@ -1986,6 +2092,47 @@ function resolveWatchRange(return { projectId, asset, clip, speed, from, to };}+/** Silent stretches of a source range, in source seconds — the engine's+ * ffmpeg scan locally, the browser's RMS fold in cloud mode. Shared by+ * detect_silence (the model's ears) and refine_speech_cuts (the mechanical+ * re-trim). */+async function fetchSilences(+ projectId: string,+ asset: MediaAsset,+ opts: { from: number; to?: number; thresholdDb?: number; minSilence?: number }+): Promise<{ start: number; end: number; duration: number }[]> {+ // Same defaults and clamps as the engine's silence handler.+ const thresholdDb = clamp(opts.thresholdDb ?? -30, -90, 0);+ const minSilence = clamp(opts.minSilence ?? 0.35, 0.05, 10);+ if (getBackend().kind === "cloud") {+ return detectSilenceClientSide(asset.url, {+ from: opts.from,+ ...(opts.to !== undefined ? { to: opts.to } : {}),+ thresholdDb,+ minSilence,+ }).catch((e) => {+ throw new ToolError(e instanceof Error ? e.message : "Could not scan for silence.");+ });+ }+ const res = await apiFetch(`/api/cut/projects/${projectId}/silence`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ file: asset.fileName,+ from: opts.from,+ ...(opts.to !== undefined ? { to: opts.to } : {}),+ threshold_db: thresholdDb,+ min_silence: minSilence,+ }),+ });+ const body = await apiJson<{+ silences: { start: number; end: number; duration: number }[];+ error?: string;+ }>(res);+ if (!res.ok) throw new ToolError(body.error ?? "Could not scan for silence.");+ return body.silences;+}+/** Pull a source's audio track off (video and audio alike) and inline it for* the model; null when the range clears the inline cap. Local: the engine* extracts mono AAC. Cloud: the browser renders the span to 16 kHz mono WAV. */diff --git a/site/src/cut/lib/cutRefine.test.ts b/site/src/cut/lib/cutRefine.test.tsnew file mode 100644index 00000000..3cb7df2c--- /dev/null+++ b/site/src/cut/lib/cutRefine.test.ts@@ -0,0 +1,59 @@+import { describe, expect, test } from "bun:test";+import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";++/**+ * Edges refine against detected silence: speech runs everywhere a span+ * doesn't. The spans here draw a take with pauses at known places, and each+ * case asks where a model-made edge should really sit.+ */++const spans: SilenceSpan[] = [+ { start: 0, end: 1 }, // lead-in silence+ { start: 4, end: 5 }, // pause between sentences+ { start: 9, end: 9.3 }, // a short breath+ { start: 12, end: 20 }, // long trailing silence+];++describe("out edges (a clip's end)", () => {+ test("an edge inside a pause settles a breath past the speech", () => {+ expect(refineEdge(spans, 4.8, "out")).toEqual({ t: 4 + BREATH, pause: true });+ });++ test("an edge clipping a word moves outward to the next pause", () => {+ expect(refineEdge(spans, 3.6, "out")).toEqual({ t: 4 + BREATH, pause: true });+ });++ test("a pause shorter than a breath still holds the cut at its far side", () => {+ expect(refineEdge(spans, 9.1, "out")).toEqual({ t: 9.25, pause: true });+ });++ test("continuous speech leaves the edge alone, flagged", () => {+ expect(refineEdge([{ start: 0, end: 1 }], 6, "out")).toEqual({ t: 6, pause: false });+ });++ test("a long pause tightens only within reach", () => {+ expect(refineEdge(spans, 19, "out")).toEqual({ t: 19 - REACH, pause: true });+ });+});++describe("in edges (a clip's start)", () => {+ test("an edge inside a pause settles a breath ahead of the speech", () => {+ expect(refineEdge(spans, 4.2, "in")).toEqual({ t: 5 - BREATH, pause: true });+ });++ test("an edge clipping a lead word moves back to the pause before it", () => {+ expect(refineEdge(spans, 5.4, "in")).toEqual({ t: 5 - BREATH, pause: true });+ });++ test("an edge at the source head trims the lead-in to a breath", () => {+ expect(refineEdge(spans, 0, "in")).toEqual({ t: 1 - BREATH, pause: true });+ });++ test("continuous speech leaves the edge alone, flagged", () => {+ expect(refineEdge(spans, 7.5, "in")).toEqual({ t: 7.5, pause: false });+ });++ test("a pause further back than reach does not pull the edge", () => {+ expect(refineEdge([{ start: 0, end: 1 }], 3.5, "in")).toEqual({ t: 3.5, pause: false });+ });+});diff --git a/site/src/cut/lib/cutRefine.ts b/site/src/cut/lib/cutRefine.tsnew file mode 100644index 00000000..5a5665b1--- /dev/null+++ b/site/src/cut/lib/cutRefine.ts@@ -0,0 +1,75 @@+// True up speech-cut edges against the silence around them.+//+// The assistant plans cuts from cue timings and silence spans, but the numbers+// it emits are hand-derived floats: an edge lands a hair inside a word, or+// leaves stray dead air. The split of labor is the usual one — the model says+// WHICH edges are speech cuts, and the arithmetic here decides exactly where+// each one sits: a breath inside the real pause.+//+// The rules err toward keeping words. An edge already inside a pause slides to+// a breath past the speech it closes (an out point) or ahead of the speech it+// opens (an in point). An edge sitting in speech — a clipped word — only moves+// outward, to the first pause in the direction that makes the word whole+// again. With no pause within reach the speech runs continuously there, so the+// edge stays where the model put it and comes back flagged for a listen.++/** A silent stretch of the source, in source seconds. */+export interface SilenceSpan {+ start: number;+ end: number;+}++/** The pause kept on the speech side of a refined edge, seconds. */+export const BREATH = 0.25;++/** How far an edge may travel to find or settle into a pause. Beyond this a+ * long pause reads as a held beat — a choice, not slack — and a cut in speech+ * reads as deliberate. */+export const REACH = 2;++/** Which side of the clip an edge trims: "in" opens the clip, "out" closes it. */+export type EdgeKind = "in" | "out";++export interface RefinedEdge {+ /** The refined source time — the input `t` when no pause was within reach. */+ t: number;+ /** False when speech runs continuously at this edge and it was left alone. */+ pause: boolean;+}++const EPS = 1e-3;+const round = (n: number) => Math.round(n * 1000) / 1000;+const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));++/**+ * Place one trim edge a breath inside the pause it belongs to. `spans` are the+ * silences detected around `t`, sorted by start and disjoint.+ */+export function refineEdge(+ spans: readonly SilenceSpan[],+ t: number,+ kind: EdgeKind+): RefinedEdge {+ const within = spans.find((s) => t >= s.start - EPS && t <= s.end + EPS);+ if (within) {+ // A pause shorter than two breaths still holds a cut — the edge settles at+ // the pause's far boundary and the whole pause is the breath.+ const ideal =+ kind === "out"+ ? Math.min(within.end, within.start + BREATH)+ : Math.max(within.start, within.end - BREATH);+ return { t: round(clamp(ideal, t - REACH, t + REACH)), pause: true };+ }+ if (kind === "out") {+ const next = spans.find((s) => s.start > t && s.start - t <= REACH);+ if (next) return { t: round(Math.min(next.end, next.start + BREATH)), pause: true };+ } else {+ let prev: SilenceSpan | undefined;+ for (const s of spans) {+ if (s.end < t && t - s.end <= REACH) prev = s;+ if (s.end >= t) break;+ }+ if (prev) return { t: round(Math.max(prev.start, prev.end - BREATH)), pause: true };+ }+ return { t, pause: false };+}diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 60d7634a..a58978b9 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -145,6 +145,21 @@ export const AI_TOOLS: AiToolDef[] = ["Set a video clip's trim points inside its source media (seconds). Changing `in` hides the leading part; `out` the trailing part.",inputSchema: obj({ clipId: str("Video clip id"), in: num("New in point (optional)"), out: num("New out point (optional)") }, ["clipId"]),},+ {+ name: "refine_speech_cuts",+ description:+ "True up speech-cut edges mechanically: re-scans the audio around each clip's in/out and re-trims so every edge sits a ~0.25s breath inside a real pause — an edge that clips a word moves outward to the next pause, stray dead air trims off. Clip spacing survives: a tightened clip closes up, butted joints stay butted, and deliberate gaps keep their width. An edge with no pause within ~2s stays put and comes back flagged. Run it on every clip you recut for speech, after cutting and before listening; leave out clips whose edge you placed mid-speech on purpose.",+ inputSchema: obj(+ {+ clip_ids: {+ type: "array",+ items: { type: "string" },+ description: "Video clip ids whose in/out edges are speech cuts",+ },+ },+ ["clip_ids"]+ ),+ },{name: "set_clip_muted",description: "Mute or unmute a video clip's own audio.",@@ -804,16 +819,17 @@ The flow for "edit this for me":1. get_state — every clip, its trim (in/out), speed, gaps, and the other tracks.2. Speech? subtitles_generate first — cue timings are timeline seconds and say what is said when.3. watch_video each distinct source (or the ranges in question); note scene changes and what happens where.-4. Plan in thoughts: group the cues into sentences and thoughts, decide which thoughts stay whole, then detect_silence to find where the speaker actually pauses. Cue timings approximate the audio — a trailing word often spills past its cue — so place every speech cut inside a detected silence span, padded ~0.2s in from both of its edges; a cue boundary with no silence span around it means the speech runs continuously there, so leave that joint alone.+4. Plan in thoughts: group the cues into sentences and thoughts, decide which thoughts stay whole, then detect_silence to find where the speaker actually pauses. Cue timings approximate the audio — a trailing word often spills past its cue — so aim every speech cut into a detected silence span, and aim loose: up to a second of spare air on each side beats a tight cut, because the refine step trues the edges after. A cue boundary with no silence span around it means the speech runs continuously there, so leave that joint alone.5. Execute: split_at (timeline s) to divide, delete_item to drop a segment, trim_clip (source s) to tighten edges, place_clip/move_clip to close gaps or reorder, set_speed to compress slow stretches.-6. Verify by ear: listen_audio each recut clip (its trim is the default range; scope a long one with from/to to ~2s around each new edge) and check the first and last words arrive whole and the hand-off into the next thought sounds natural. A clipped word → widen that trim deeper into the silence and listen again. After a heavy edit, listen once end-to-end. seek + capture_frame when the change is visual.+6. Refine: refine_speech_cuts with every recut clip id. It re-scans the audio at each edge and re-trims it to sit a ~0.25s breath inside the real pause, keeping the clips' spacing. It reads sound, not intent — leave out a clip whose edge you placed mid-speech on purpose.+7. Verify by ear: listen_audio at each edge the refine flagged, plus a spot-check or two (scope from/to to ~2s around the joint), and check the first and last words arrive whole and the hand-off into the next thought sounds natural. A clipped word → widen that trim deeper into the silence and listen again. After a heavy edit, listen once end-to-end. seek + capture_frame when the change is visual.Time math (get this right):- watch_video, detect_silence, and trim_clip speak SOURCE seconds; split_at and place_clip speak TIMELINE seconds.- timeline_t = clip.start + (source_t − clip.in) / clip.speed, valid while source_t is inside [in, out]. Each watch result's clip block carries this formula with the real numbers filled in.- The same source can appear in several clips — map per clip.-Cutting speech well: pace is part of the message. Keep a beat of the speaker's own pause between sentences (~0.3s) and a slightly longer one between thoughts (~0.5s) — shorten a long pause to that beat instead of deleting it, and speech butted directly together reads as rushed. Leave ~0.2s of breath on each side of every cut so edges land in true silence. Prefer split_at + delete_item, then place_clip to close the gap (a beat of black may be wanted — ask the cut, not the tool); trim_clip only tightens a clip's edges.`,+Cutting speech well: pace is part of the message. Keep a beat of the speaker's own pause between sentences (~0.3s) and a slightly longer one between thoughts (~0.5s) — shorten a long pause to that beat instead of deleting it, and speech butted directly together reads as rushed. Cut loose and let refine_speech_cuts set the breath at each edge. Prefer split_at + delete_item, then place_clip to close the gap (a beat of black may be wanted — ask the cut, not the tool); trim_clip only tightens a clip's edges.`,"transitions-and-fades": `# Transitions, animations, looks & fadesRoute the ask to the right feature: