Open source CapCut alternative.
TypeScript
feat(site): speech cuts are placed on the words themselves [rebuild]
David Le1f7cdd2a9d4c
diff --git a/site/src/cut/components/Timeline.tools.ts b/site/src/cut/components/Timeline.tools.tsindex dd163145..8bd7589d 100644--- a/site/src/cut/components/Timeline.tools.ts+++ b/site/src/cut/components/Timeline.tools.ts@@ -63,7 +63,7 @@ export const TIMELINE_TOOLS = [{name: "refine_speech_cuts",description:- "True up speech-cut edges mechanically: re-scans the audio around each clip's in/out and re-trims so every edge sits a ~0.25s breath inside a real pause — an edge that clips a word moves outward to the next pause, stray dead air trims off. Clip spacing survives: a tightened clip closes up, butted joints stay butted, and deliberate gaps keep their width. An edge with no pause within ~2s stays put and comes back flagged. Run it on every clip you recut for speech, after cutting and before listening; leave out clips whose edge you placed mid-speech on purpose.",+ "True up speech-cut edges against the words themselves: re-reads the audio around each clip's in/out, measures that recording's own room tone and voice level, finds where the words actually start and finish — tails and soft attacks included — and re-trims so every edge sits a measured beat clear of one. An edge inside a word moves outward until the word is whole; stray dead air trims off. THIS IS WHERE PACE IS SET: `pace` fast|natural|relaxed chooses how much air is left (~0.14s/0.26s/0.45s after the last word, ~0.09s/0.16s/0.28s before the next), and even fast never cuts closer than 0.08s to a word. Cutting closer than the pace is not faster, it is clipped. Clip spacing survives: a tightened clip closes up, butted joints stay butted, deliberate gaps keep their width, and where a pause is too short for both sides' air the two edges meet inside it. An edge with no word within ~2s stays put and comes back flagged. Run it on every clip you recut for speech, after cutting and before listening; leave out clips whose edge you placed mid-speech on purpose.",inputSchema: obj({clip_ids: {@@ -71,6 +71,12 @@ export const TIMELINE_TOOLS = [items: { type: "string" },description: "Video clip ids whose in/out edges are speech cuts",},+ pace: {+ type: "string",+ enum: ["fast", "natural", "relaxed"],+ description:+ "How much air each edge keeps around the words — fast for a punchy recut, relaxed to let it breathe. Default natural.",+ },},["clip_ids"]),diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 9eb50fef..76711e45 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -79,10 +79,11 @@ import {makeStillFrame,renderAudioSpanWav,sampleWatchFrames,+ scanSourceSpeech,} from "./media";import { convertAssetToMp4 } from "./mediaConvert";import { isLottieAsset } from "./lottieAssets";-import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";+import { AIR, MIN_AIR, placeEdge, REACH, settleJoint, type Pace, type PlacedEdge } from "./cutRefine";import { requestSidePanel } from "./panelRequest";import { blobToInlineAudio, refToInlineAudio, visualRefs, type InlineImage } from "./refMedia";import { characterPrompt, stockAspectDims, stockTitle } from "./stock";@@ -1218,58 +1219,76 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {if (ids.length === 0)throw new ToolError("clip_ids is required — the recut clips whose edges are speech cuts.");const chosen = ids.map((id) => requireItem(s.clips, id, "video clip"));+ const pace: Pace =+ input.pace === "fast" || input.pace === "relaxed" ? input.pace : "natural";// Every trim is planned before anything is written, so the re-lay below- // can preserve the spacing the cut already has.- const plans = new Map<string, { in: number; out: number }>();+ // can preserve the spacing the cut already has. A plan carries the word+ // boundaries its edges were placed against: repairing a joint has to+ // know where the words are too, or the repair clips one.+ interface Plan { in: number; out: number; inBound: number; outBound: number }+ const plans = new Map<string, Plan>();const flags = new Map<string, string[]>();const flag = (id: string, note: string) =>flags.set(id, [...(flags.get(id) ?? []), note]);- await Promise.all(- chosen.map(async (clip) => {+ await eachScanned(chosen, async (clip) => {const asset = s.assets.find((a) => a.id === clip.assetId);if (!asset || asset.type === "image" || !(asset.duration > 0)) {flag(clip.id, "no audio-bearing source — skipped");return;}- // One scan spanning both edges when the clip is short, else one- // window around each. Fine pauses matter here, so the scan listens- // for stretches down to a single breath.- const win = REACH + 0.6;- const ranges: [number, number][] =- clip.out - clip.in <= 2 * win- ? [[clip.in - win, clip.out + win]]- : [- [clip.in - win, clip.in + win],- [clip.out - win, clip.out + win],- ];- const spans: SilenceSpan[] = (- await Promise.all(- ranges.map(([a, b]) =>- fetchSilences(projectId, asset, {- from: clamp(a, 0, asset.duration),- to: clamp(b, 0, asset.duration),- thresholdDb: -30,- minSilence: BREATH,- })- )+ // The scan needs the reach an edge may travel plus enough room+ // around it to measure the recording's own levels. One window+ // spanning both edges when the clip is short, else one per edge.+ const win = REACH + 1;+ const short = clip.out - clip.in <= 2 * win;+ const ranges: [number, number][] = short+ ? [[clip.in - win, clip.out + win]]+ : [+ [clip.in - win, clip.in + win],+ [clip.out - win, clip.out + win],+ ];+ const scans = await Promise.all(+ ranges.map(([a, b]) =>+ fetchSpeech(asset, clamp(a, 0, asset.duration), clamp(b, 0, asset.duration)))- ).flat();- const inEdge = refineEdge(spans, clip.in, "in");- const outEdge = refineEdge(spans, clip.out, "out");- if (!inEdge.pause)- flag(clip.id, "no pause within reach of the in edge — speech runs continuously; left alone");- if (!outEdge.pause)- flag(clip.id, "no pause within reach of the out edge — speech runs continuously; left alone");+ ).catch((e: unknown) => {+ flag(clip.id, e instanceof Error ? `${e.message} — left as cut` : "could not read the audio — left as cut");+ return null;+ });+ if (!scans) return;+ const inScan = scans[0];+ const outScan = short ? scans[0] : scans[1];+ if (inScan.segments.length === 0 && outScan.segments.length === 0) {+ flag(clip.id, "no speech around either edge — left as cut");+ return;+ }+ if (!inScan.separated || !outScan.separated)+ flag(clip.id, "room tone sits close to the voice here — placed on the best read of it");+ const inEdge = placeEdge(inScan.segments, clip.in, "in", pace, inScan);+ const outEdge = placeEdge(outScan.segments, clip.out, "out", pace, outScan);+ if (!inEdge.placed)+ flag(clip.id, "no word within reach of the in edge — left alone");+ if (!outEdge.placed)+ flag(clip.id, "no word within reach of the out edge — left alone");const nextIn = clamp(inEdge.t, 0, asset.duration);const nextOut = clamp(outEdge.t, 0, asset.duration);- if (nextOut - nextIn >= 0.15) plans.set(clip.id, { in: nextIn, out: nextOut });+ if (nextOut - nextIn >= 0.15)+ plans.set(clip.id, {+ in: nextIn,+ out: nextOut,+ inBound: clamp(inEdge.bound, 0, asset.duration),+ outBound: clamp(outEdge.bound, 0, asset.duration),+ });else flag(clip.id, "refining would collapse the clip — left as cut");- })- );+ });// Two sides of one removed stretch never cross back into each other's- // material, so no word plays twice across a joint.+ // material, so no word plays twice across a joint. Where the pace would+ // push them past each other the pause between the words was shorter than+ // the air both sides wanted, so both give ground and the joint sits+ // inside that pause — reverting to the cut's own numbers is what put an+ // edge back on top of a word.const spd = (c: VideoClip) => (c.speed && c.speed > 0 ? c.speed : 1);const rows = new Map<number, VideoClip[]>();for (const c of s.clips) rows.set(c.track, [...(rows.get(c.track) ?? []), c]);@@ -1281,11 +1300,21 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {if (a.assetId !== b.assetId || a.out > b.in + 1e-3) continue;const pa = plans.get(a.id);const pb = plans.get(b.id);- if ((pa?.out ?? a.out) <= (pb?.in ?? b.in) + 1e-6) continue;- if (pa) pa.out = a.out;- if (pb) pb.in = b.in;+ const left: PlacedEdge = { t: pa?.out ?? a.out, placed: true, bound: pa?.outBound ?? a.out };+ const right: PlacedEdge = { t: pb?.in ?? b.in, placed: true, bound: pb?.inBound ?? b.in };+ if (left.t <= right.t + 1e-6) continue;+ const joint = settleJoint(left, right);+ if (joint === null) {+ if (pa) pa.out = a.out;+ if (pb) pb.in = b.in;+ for (const c of [a, b])+ if (plans.has(c.id)) flag(c.id, "speech runs across this joint — that edge left as cut");+ continue;+ }+ if (pa) pa.out = joint;+ if (pb) pb.in = joint;for (const c of [a, b])- if (plans.has(c.id)) flag(c.id, "its joint's edges would cross the removed stretch — that edge left as cut");+ if (plans.has(c.id)) flag(c.id, "the pause at this joint was short — the two edges meet inside it");}}@@ -1326,6 +1355,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {}return {+ pace,refined: chosen.map((clip) => {const plan = plans.get(clip.id);const notes = flags.get(clip.id);@@ -1341,10 +1371,10 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {? { notes }: plan? {}- : { notes: ["edges already sit in their pauses"] }),+ : { notes: ["edges already sit where the words leave them"] }),};}),- note: `Each moved edge sits ~${BREATH}s inside a real pause; clip spacing is preserved. Listen at any flagged edge.`,+ note: `Every moved edge is measured from the word itself: ~${AIR[pace].out}s after the last one finishes, ~${AIR[pace].in}s before the next one starts, never under ${MIN_AIR.out}s/${MIN_AIR.in}s. Clip spacing is preserved. Listen at any flagged edge.`,...tracksAfter(),};},@@ -3601,6 +3631,30 @@ function resolveWatchRange(return { projectId, asset, clip, speed, from, to };}+/** Scans, a few clips at a time. Each one opens a decoder over its own span of+ * the source, and a recut hands this tool every clip it made. */+async function eachScanned<T>(items: T[], run: (item: T) => Promise<void>) {+ const width = 3;+ let next = 0;+ await Promise.all(+ Array.from({ length: Math.min(width, items.length) }, async () => {+ while (next < items.length) await run(items[next++]);+ })+ );+}++/** Where the words are in a source range, in source seconds. One reader for+ * every residency: the page decodes its own media whether the engine or the+ * cloud is serving it, and the headless runner installs the same decoders.+ * This is what refine_speech_cuts places its edges against — a fixed dB+ * threshold cannot answer it, because a quiet room and a loud one disagree+ * about what silence is and a cut placed on the wrong answer lands in a word. */+async function fetchSpeech(asset: MediaAsset, from: number, to: number) {+ return scanSourceSpeech(asset.url, { from, to }).catch((e) => {+ throw new ToolError(e instanceof Error ? e.message : "Could not read the audio.");+ });+}+/** Silent stretches of a source range, in source seconds — the engine's* ffmpeg scan locally, the browser's RMS fold in cloud mode. Shared by* detect_silence (the model's ears) and refine_speech_cuts (the mechanicaldiff --git a/site/src/cut/lib/audioScan.test.ts b/site/src/cut/lib/audioScan.test.tsindex 70bd2e85..d9186627 100644--- a/site/src/cut/lib/audioScan.test.ts+++ b/site/src/cut/lib/audioScan.test.ts@@ -1,5 +1,5 @@import { describe, expect, test } from "bun:test";-import { scanSilence, type PcmChunk } from "./audioScan";+import { scanSilence, scanSpeech, type PcmChunk } from "./audioScan";const RATE = 8000;@@ -82,3 +82,135 @@ describe("scanSilence", () => {expect(await scanSilence((async function* (): AsyncGenerator<PcmChunk> {})(), opts)).toEqual([]);});});++// ── Speech ──────────────────────────────────────────────────────────────────++/** Deterministic noise: a scan that passes four times in five is no test. */+const rng = (seed: number) => () => {+ seed = (seed * 1103515245 + 12345) % 2147483648;+ return seed / 2147483648 - 0.5;+};++interface Spoken {+ start: number;+ end: number;+ /** Peak amplitude, so a whispered aside can sit under a shouted line. */+ level?: number;+}++/**+ * A take: bursts shaped like words — attack, hold, decay, a wobble inside —+ * over a room. `floorDb` is the room, `bedDb` lays a steady tone under+ * everything, the case where nothing is ever truly quiet.+ */+async function* take(+ words: Spoken[],+ opts: { floorDb: number; bedDb?: number; duration: number; seed?: number }+): AsyncGenerator<PcmChunk> {+ const rate = 16000;+ const rnd = rng(opts.seed ?? 7);+ const out = new Float32Array(Math.ceil(opts.duration * rate));+ const floor = Math.pow(10, opts.floorDb / 20);+ for (let i = 0; i < out.length; i++) out[i] = rnd() * floor * 2;+ if (opts.bedDb !== undefined) {+ const bed = Math.pow(10, opts.bedDb / 20);+ for (let i = 0; i < out.length; i++) out[i] += Math.sin((2 * Math.PI * 110 * i) / rate) * bed;+ }+ for (const w of words) {+ const from = Math.round(w.start * rate);+ const to = Math.min(out.length, Math.round(w.end * rate));+ const len = Math.max(1, to - from);+ for (let i = from; i < to; i++) {+ const p = (i - from) / len;+ const env = Math.min(1, p / 0.1) * Math.min(1, (1 - p) / 0.15);+ const wobble = 0.7 + 0.3 * Math.sin(p * Math.PI * 4);+ out[i] += rnd() * (w.level ?? 0.6) * env * wobble;+ }+ }+ // Handed over in uneven spans, the way a decoder hands them over.+ for (let start = 0; start < out.length; start += 4321) {+ const end = Math.min(out.length, start + 4321);+ yield { channels: [out.slice(start, end)], timestamp: start / rate, sampleRate: rate };+ }+}++const SPOKEN: Spoken[] = [+ { start: 0.6, end: 1.4 },+ { start: 1.6, end: 2.5 },+ { start: 3.4, end: 4.2 },+];++/** Every word accounted for, each within `slack` of where it was spoken. */+function expectWords(+ segments: { start: number; end: number }[],+ words: Spoken[],+ slack: { early: number; late: number }+) {+ expect(segments).toHaveLength(words.length);+ words.forEach((w, i) => {+ // A word may be opened early and closed late — never the other way round,+ // which is the cut landing inside it.+ expect(segments[i].start).toBeLessThanOrEqual(w.start + 0.02);+ expect(segments[i].start).toBeGreaterThanOrEqual(w.start - slack.early);+ expect(segments[i].end).toBeGreaterThanOrEqual(w.end - 0.02);+ expect(segments[i].end).toBeLessThanOrEqual(w.end + slack.late);+ });+}++describe("scanSpeech", () => {+ test("finds the words in a quiet room", async () => {+ const scan = await scanSpeech(take(SPOKEN, { floorDb: -60, duration: 5 }), { from: 0 });+ expect(scan.separated).toBe(true);+ expectWords(scan.segments, SPOKEN, { early: 0.12, late: 0.2 });+ });++ test("finds the same words in a room 22dB louder", async () => {+ const scan = await scanSpeech(take(SPOKEN, { floorDb: -38, duration: 5 }), { from: 0 });+ expect(scan.separated).toBe(true);+ expectWords(scan.segments, SPOKEN, { early: 0.12, late: 0.2 });+ });++ test("finds them in a quiet recording nobody normalized", async () => {+ const quiet = SPOKEN.map((w) => ({ ...w, level: 0.05 }));+ const scan = await scanSpeech(take(quiet, { floorDb: -78, duration: 5 }), { from: 0 });+ expectWords(scan.segments, quiet, { early: 0.12, late: 0.2 });+ });++ test("reads the gaps under a steady music bed", async () => {+ const scan = await scanSpeech(take(SPOKEN, { floorDb: -60, bedDb: -34, duration: 5 }), { from: 0 });+ expect(scan.segments.length).toBe(SPOKEN.length);+ });++ test("keeps a word whole across the closure inside it", async () => {+ // "sta-p": 40ms of nothing in the middle of one word is not a pause.+ const scan = await scanSpeech(+ take([{ start: 1, end: 1.4 }, { start: 1.44, end: 1.7 }], { floorDb: -60, duration: 3 }),+ { from: 0 }+ );+ expect(scan.segments).toHaveLength(1);+ expect(scan.segments[0].end).toBeGreaterThan(1.65);+ });++ test("a click is not a word", async () => {+ const scan = await scanSpeech(+ take([{ start: 1, end: 1.02, level: 0.9 }, { start: 2, end: 2.6 }], { floorDb: -60, duration: 3.5 }),+ { from: 0 }+ );+ expect(scan.segments).toHaveLength(1);+ expect(scan.segments[0].start).toBeGreaterThan(1.5);+ });++ test("a room with nothing said in it holds no words", async () => {+ const scan = await scanSpeech(take([], { floorDb: -60, duration: 3 }), { from: 0 });+ expect(scan.segments).toEqual([]);+ });++ test("hears the same words however the range is windowed", async () => {+ const whole = await scanSpeech(take(SPOKEN, { floorDb: -50, duration: 5 }), { from: 0 });+ const part = await scanSpeech(take(SPOKEN, { floorDb: -50, duration: 5 }), { from: 3, to: 5 });+ const last = whole.segments[whole.segments.length - 1];+ expect(part.segments).toHaveLength(1);+ expect(part.segments[0].start).toBeCloseTo(last.start, 1);+ expect(part.segments[0].end).toBeCloseTo(last.end, 1);+ });+});diff --git a/site/src/cut/lib/audioScan.ts b/site/src/cut/lib/audioScan.tsindex 68f67a66..d83856b8 100644--- a/site/src/cut/lib/audioScan.ts+++ b/site/src/cut/lib/audioScan.ts@@ -2,9 +2,13 @@* Folding decoded audio into an answer, a chunk at a time.** Audio arrives from the reader in whatever spans the decoder hands back, so- * anything measured over a window — silence, in practice — has to carry its+ * anything measured over a window — level, in practice — has to carry its* running state across chunk seams. Keeping that fold here, over plain sample* arrays, is what lets it be tested without a decoder.+ *+ * Everything else in this file reads the one thing the fold produces: an+ * envelope, a level per frame. Silence is a fixed threshold over it; speech is+ * a measurement of the recording's own two levels, which is what a cut needs.*//** Decoded PCM for one span: one array per channel, all the same length. */@@ -34,57 +38,57 @@ export interface SilenceOptions {/** The measurement window, in seconds — what ffmpeg's silencedetect uses. */const WINDOW = 0.02;+/** The speech scan's frame. Half the silence window, because a word boundary+ * placed to the nearest 20ms is audibly late and one placed to 10ms is not. */+const SPEECH_HOP = 0.01;++/** Digital silence reads as this many dBFS, so every level is a real number. */+const QUIET = -100;+const round2 = (n: number) => Math.round(n * 100) / 100;+const round3 = (n: number) => Math.round(n * 1000) / 1000;++/** Level per frame across a span of source audio. */+export interface Envelope {+ /** Frame level in dBFS; NaN where the decoder handed back no samples. */+ db: Float32Array;+ /** Source seconds at the left edge of frame 0. */+ from: number;+ /** Frame width, seconds. */+ hop: number;+ /** Source seconds where the audio ran out. */+ end: number;+}/**- * Silent spans in `chunks`, by RMS over 20ms windows.+ * Fold `chunks` into one level per frame.*- * A window is silent when its RMS across every channel falls under the- * threshold; consecutive silent windows merge, and a run shorter than- * `minSilence` is dropped. Windows are measured against the timeline the- * chunks carry, so a decoder that hands back spans of uneven length — or one- * that starts a span mid-window — reports the same silences as one that- * doesn't.+ * Frames sit on a fixed grid anchored at `from` rather than starting wherever+ * the previous chunk ended. That is what makes the answer independent of how+ * the decoder happened to split the audio: a boundary is a property of the+ * timeline, so a chunk that starts mid-frame is measured into the frame it+ * belongs to instead of starting a new one. A frame no sample landed in reads+ * NaN — the decoder said nothing about it, which is different from silence.*/-export async function scanSilence(+export async function scanEnvelope(chunks: AsyncIterable<PcmChunk>,- opts: SilenceOptions-): Promise<SilenceSpan[]> {- const { from, to, minSilence } = opts;- const threshold = Math.pow(10, opts.thresholdDb / 20);- const silences: SilenceSpan[] = [];-- // Windows sit on a fixed grid anchored at `from` rather than starting- // wherever the previous one ended. That is what makes the answer independent- // of how the decoder happened to split the audio: a boundary is a property of- // the timeline, so a chunk that starts mid-window is measured into the window- // it belongs to instead of starting a new one.- const windowAt = (index: number) => from + index * WINDOW;+ opts: { from: number; to?: number; hop?: number }+): Promise<Envelope> {+ const { from, to } = opts;+ const hop = opts.hop ?? WINDOW;+ const db: number[] = [];- let open: number | null = null;let index = -1;let end = from;let sum = 0;let count = 0;- /** End an open silent run at `endT`, keeping it if it ran long enough. */- const close = (endT: number) => {- if (open !== null && endT - open >= minSilence) {- const start = round2(Math.max(from, open));- const stop = round2(endT);- if (stop > start) silences.push({ start, end: stop, duration: round2(stop - start) });- }- open = null;- };-- /** Judge the window that just filled and clear the accumulator. */+ /** Write the frame that just filled and clear the accumulator. */const settle = () => {- if (index >= 0 && count > 0) {- if (Math.sqrt(sum / count) < threshold) {- open ??= windowAt(index);- } else {- close(windowAt(index));- }+ if (index >= 0) {+ while (db.length < index) db.push(NaN);+ const rms = count > 0 ? Math.sqrt(sum / count) : 0;+ db[index] = count > 0 ? Math.max(QUIET, 20 * Math.log10(rms || 1e-12)) : NaN;}sum = 0;count = 0;@@ -94,24 +98,232 @@ export async function scanSilence(if (channels.length === 0) continue;const length = channels[0].length;// Sample times rebuild from an integer sample index, so a chunk split- // cannot move a boundary sample across a window by a rounding error.+ // cannot move a boundary sample across a frame by a rounding error.const base = Math.round(timestamp * sampleRate);for (let i = 0; i < length; i++) {const at = (base + i) / sampleRate;if (at < from) continue;if (to !== undefined && at >= to) break;- const window = Math.floor((at - from) / WINDOW);- if (window !== index) {+ const frame = Math.floor((at - from) / hop);+ if (frame !== index) {settle();- index = window;+ index = frame;}for (const ch of channels) sum += ch[i] * ch[i];count += channels.length;end = at + 1 / sampleRate;}}-settle();- close(to !== undefined ? Math.min(to, end) : end);+ return { db: Float32Array.from(db), from, hop, end };+}++/**+ * Silent spans of an envelope: runs of frames under `thresholdDb` lasting at+ * least `minSilence`. Frames the decoder never filled neither open nor close a+ * run, so a gap in the decode is not heard as a pause.+ */+export function silencesFrom(+ env: Envelope,+ opts: { thresholdDb: number; minSilence: number; to?: number }+): SilenceSpan[] {+ const { from, hop, db } = env;+ const silences: SilenceSpan[] = [];+ const at = (index: number) => from + index * hop;+ let open: number | null = null;++ const close = (endT: number) => {+ if (open !== null && endT - open >= opts.minSilence) {+ const start = round2(Math.max(from, open));+ const stop = round2(endT);+ if (stop > start) silences.push({ start, end: stop, duration: round2(stop - start) });+ }+ open = null;+ };++ for (let i = 0; i < db.length; i++) {+ if (!Number.isFinite(db[i])) continue;+ if (db[i] < opts.thresholdDb) open ??= at(i);+ else close(at(i));+ }+ close(opts.to !== undefined ? Math.min(opts.to, env.end) : env.end);return silences;}++/** Silent spans in `chunks`, by RMS over 20ms windows. */+export async function scanSilence(+ chunks: AsyncIterable<PcmChunk>,+ opts: SilenceOptions+): Promise<SilenceSpan[]> {+ const env = await scanEnvelope(chunks, { from: opts.from, to: opts.to });+ return silencesFrom(env, opts);+}++/** A run of speech, tail and soft attack included. */+export interface SpeechSegment {+ start: number;+ end: number;+ /** The loudest frame in the run, dBFS — how much of the voice this is. */+ peakDb: number;+}++export interface SpeechScan {+ /** Speech runs in source seconds, in order, never touching. */+ segments: SpeechSegment[];+ from: number;+ to: number;+ /** Room tone as measured, dBFS. */+ floorDb: number;+ /** The level the voice sits at, dBFS. */+ speechDb: number;+ /** What a frame has to clear to open a word, dBFS. */+ onsetDb: number;+ /** True when quiet and loud separate cleanly. False under a music bed or in+ * a range that is speech end to end — the segments are still the best read+ * of it, they just carry less authority. */+ separated: boolean;+}++// How the two levels are told apart. Every number here is about human speech:+// a talker's quiet decile is room tone, their loud decile is a vowel, and the+// distance between the two is the recording's headroom.+const FLOOR_Q = 0.1;+const SPEECH_Q = 0.95;+/** Headroom that makes a room readable — quiet and loud clearly two things. */+const SEPARATION = 20;+/** In a readable room the word opens this far over the floor, at least. */+const LIFT_MIN = 6;+/** …or this share of the headroom, whichever asks more. */+const LIFT_SHARE = 0.35;+/** The threshold never comes within this of the voice itself. */+const HEADROOM = 10;+/** Where the two levels sit on top of each other, read down from the voice. */+const CROWDED = 12;+/** A word holds while it decays this far under the level that opened it. */+const RELEASE = 8;+/** …and is over once it has stayed down for this long. */+const HANG = 0.05;+/** Energy this brief is a click, a lip smack, a chair. */+const MIN_ONSET = 0.03;+/** A gap this short is inside a word — the closure in a "t", a glottal stop. */+const BRIDGE = 0.09;+/** A run shorter than this is not a word. */+const MIN_SEGMENT = 0.06;+/** Nothing in a range this quiet is speech, however the levels fall out. */+const SILENT_RANGE = -55;+/** No threshold reaches below this, whatever the floor measures. */+const DEEPEST = -75;++const quantile = (sorted: number[], q: number) =>+ sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(q * sorted.length)))];++/**+ * Where the words are.+ *+ * A fixed dB threshold answers this badly: it calls a noisy room continuous+ * speech and a quiet one continuous silence, and either way the cut lands in+ * the wrong place. So the levels come from the recording — its floor and its+ * voice — and the threshold is set between them. A word opens when the level+ * clears that threshold and stays open while it decays, so the tail of a word+ * belongs to the word; the soft attack ahead of it does too. Gaps too short to+ * be a pause are bridged, and energy too brief to be a word is dropped.+ */+export function speechFrom(env: Envelope): SpeechScan {+ const { db, from, hop } = env;+ const to = Math.min(env.end, from + db.length * hop);+ const at = (index: number) => from + index * hop;++ const heard: number[] = [];+ for (let i = 0; i < db.length; i++) if (Number.isFinite(db[i])) heard.push(db[i]);+ const empty = (floorDb: number, speechDb: number, onsetDb: number, separated: boolean) => ({+ segments: [],+ from: round3(from),+ to: round3(to),+ floorDb: round2(floorDb),+ speechDb: round2(speechDb),+ onsetDb: round2(onsetDb),+ separated,+ });+ if (heard.length === 0) return empty(QUIET, QUIET, QUIET, false);++ heard.sort((a, b) => a - b);+ const floorDb = quantile(heard, FLOOR_Q);+ const speechDb = quantile(heard, SPEECH_Q);+ if (speechDb < SILENT_RANGE) return empty(floorDb, speechDb, SILENT_RANGE, true);++ const spread = speechDb - floorDb;+ const separated = spread >= SEPARATION;+ const onsetDb = Math.max(+ DEEPEST,+ separated+ ? Math.min(floorDb + Math.max(LIFT_MIN, spread * LIFT_SHARE), speechDb - HEADROOM)+ : speechDb - CROWDED+ );+ const releaseDb = Math.min(onsetDb, Math.max(onsetDb - RELEASE, floorDb + 3));++ const onsetFrames = Math.max(1, Math.round(MIN_ONSET / hop));+ const raw: { start: number; end: number }[] = [];+ let open = -1;+ let last = -1;+ let run = 0;+ for (let i = 0; i < db.length; i++) {+ const level = db[i];+ if (!Number.isFinite(level)) continue;+ if (open < 0) {+ if (level < onsetDb) {+ run = 0;+ continue;+ }+ if (++run < onsetFrames) continue;+ // Back over the soft attack: a fricative or an intake of breath ahead of+ // the vowel sits under the onset level and is still part of the word.+ let start = i - run + 1;+ while (start > 0 && Number.isFinite(db[start - 1]) && db[start - 1] >= releaseDb) start--;+ open = start;+ last = i;+ run = 0;+ } else if (level >= releaseDb) {+ last = i;+ } else if (at(i) - at(last) >= HANG) {+ raw.push({ start: at(open), end: at(last) + hop });+ open = -1;+ }+ }+ if (open >= 0) raw.push({ start: at(open), end: Math.min(at(last) + hop, to) });++ const merged: { start: number; end: number }[] = [];+ for (const seg of raw) {+ const prev = merged[merged.length - 1];+ if (prev && seg.start - prev.end < BRIDGE) prev.end = seg.end;+ else merged.push({ ...seg });+ }++ const segments: SpeechSegment[] = [];+ for (const seg of merged) {+ if (seg.end - seg.start < MIN_SEGMENT) continue;+ let peak = QUIET;+ const first = Math.max(0, Math.floor((seg.start - from) / hop));+ const stop = Math.min(db.length, Math.ceil((seg.end - from) / hop));+ for (let i = first; i < stop; i++) if (Number.isFinite(db[i]) && db[i] > peak) peak = db[i];+ segments.push({ start: round3(seg.start), end: round3(Math.min(seg.end, to)), peakDb: round2(peak) });+ }++ return {+ segments,+ from: round3(from),+ to: round3(to),+ floorDb: round2(floorDb),+ speechDb: round2(speechDb),+ onsetDb: round2(onsetDb),+ separated,+ };+}++/** Where the words are in `chunks`. */+export async function scanSpeech(+ chunks: AsyncIterable<PcmChunk>,+ opts: { from: number; to?: number }+): Promise<SpeechScan> {+ const env = await scanEnvelope(chunks, { ...opts, hop: SPEECH_HOP });+ return speechFrom(env);+}diff --git a/site/src/cut/lib/cutRefine.test.ts b/site/src/cut/lib/cutRefine.test.tsindex 3cb7df2c..93437ee2 100644--- a/site/src/cut/lib/cutRefine.test.ts+++ b/site/src/cut/lib/cutRefine.test.ts@@ -1,59 +1,116 @@import { describe, expect, test } from "bun:test";-import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";+import { AIR, MIN_AIR, placeEdge, REACH, settleJoint, type SpeechSegment } from "./cutRefine";/**- * Edges refine against detected silence: speech runs everywhere a span- * doesn't. The spans here draw a take with pauses at known places, and each- * case asks where a model-made edge should really sit.+ * Edges are placed against the words, so the take below is written as words:+ * a sentence, a long pause, a sentence, a breath, a sentence, and open air+ * either end. Each case asks where a model-made edge should really sit.*/-const spans: SilenceSpan[] = [- { start: 0, end: 1 }, // lead-in silence- { start: 4, end: 5 }, // pause between sentences- { start: 9, end: 9.3 }, // a short breath- { start: 12, end: 20 }, // long trailing silence+const word = (start: number, end: number): SpeechSegment => ({ start, end, peakDb: -18 });++const take: SpeechSegment[] = [+ word(1, 4), // first sentence+ word(5, 9), // second, a second of pause before it+ word(9.3, 12), // third, only a breath before it];+const bounds = { from: 0, to: 20 };++const near = (n: number) => Math.round(n * 1000) / 1000;describe("out edges (a clip's end)", () => {- test("an edge inside a pause settles a breath past the speech", () => {- expect(refineEdge(spans, 4.8, "out")).toEqual({ t: 4 + BREATH, pause: true });+ test("an edge in the pause sits the pace's air past the last word", () => {+ expect(placeEdge(take, 4.8, "out", "natural", bounds)).toEqual({+ t: near(4 + AIR.natural.out),+ placed: true,+ bound: 4,+ });});- test("an edge clipping a word moves outward to the next pause", () => {- expect(refineEdge(spans, 3.6, "out")).toEqual({ t: 4 + BREATH, pause: true });+ test("an edge inside a word runs on until the word is whole", () => {+ expect(placeEdge(take, 3.6, "out", "natural", bounds).t).toBe(near(4 + AIR.natural.out));});- test("a pause shorter than a breath still holds the cut at its far side", () => {- expect(refineEdge(spans, 9.1, "out")).toEqual({ t: 9.25, pause: true });+ test("pace changes the air and nothing else", () => {+ expect(placeEdge(take, 4.8, "out", "fast", bounds).t).toBe(near(4 + AIR.fast.out));+ expect(placeEdge(take, 4.8, "out", "relaxed", bounds).t).toBe(near(4 + AIR.relaxed.out));});- test("continuous speech leaves the edge alone, flagged", () => {- expect(refineEdge([{ start: 0, end: 1 }], 6, "out")).toEqual({ t: 6, pause: false });+ test("a pause too short for the air stops clear of the next word", () => {+ // 0.3s of breath cannot hold a relaxed 0.45s tail, so the edge stops just+ // before the next word rather than eating its first syllable.+ const edge = placeEdge(take, 9.1, "out", "relaxed", bounds);+ expect(edge.t).toBeLessThan(9.3);+ expect(edge.t).toBeGreaterThanOrEqual(9 + MIN_AIR.out);});- test("a long pause tightens only within reach", () => {- expect(refineEdge(spans, 19, "out")).toEqual({ t: 19 - REACH, pause: true });+ test("the fastest pace still lets the word finish", () => {+ const edge = placeEdge(take, 3.98, "out", "fast", bounds);+ expect(edge.t - 4).toBeGreaterThanOrEqual(MIN_AIR.out);+ });++ test("an edge out in open air pulls back to the last word", () => {+ expect(placeEdge(take, 13.5, "out", "natural", bounds).t).toBe(near(12 + AIR.natural.out));+ });++ test("no word within reach leaves the edge alone, flagged", () => {+ expect(placeEdge([word(1, 4)], 4 + REACH + 0.5, "out", "natural", bounds).placed).toBe(false);+ });++ test("with nothing after it the edge takes its full air", () => {+ expect(placeEdge([word(1, 4)], 4.1, "out", "relaxed", bounds).t).toBe(near(4 + AIR.relaxed.out));});});describe("in edges (a clip's start)", () => {- test("an edge inside a pause settles a breath ahead of the speech", () => {- expect(refineEdge(spans, 4.2, "in")).toEqual({ t: 5 - BREATH, pause: true });+ test("an edge in the pause leads the next word by the pace's air", () => {+ expect(placeEdge(take, 4.2, "in", "natural", bounds)).toEqual({+ t: near(5 - AIR.natural.in),+ placed: true,+ bound: 5,+ });+ });++ test("an edge inside a word backs up so the word starts whole", () => {+ expect(placeEdge(take, 5.4, "in", "natural", bounds).t).toBe(near(5 - AIR.natural.in));+ });++ test("an edge at the source head trims the lead-in to the pace's air", () => {+ expect(placeEdge(take, 0, "in", "natural", bounds).t).toBe(near(1 - AIR.natural.in));+ });++ test("a breath too short for the air stops clear of the word before it", () => {+ const edge = placeEdge(take, 9.25, "in", "relaxed", bounds);+ expect(edge.t).toBeGreaterThan(9);+ expect(edge.t).toBeLessThanOrEqual(9.3 - MIN_AIR.in);});- test("an edge clipping a lead word moves back to the pause before it", () => {- expect(refineEdge(spans, 5.4, "in")).toEqual({ t: 5 - BREATH, pause: true });+ test("no word within reach leaves the edge alone, flagged", () => {+ expect(placeEdge([word(1, 4)], 1 - REACH - 0.5, "in", "natural", bounds).placed).toBe(false);});- test("an edge at the source head trims the lead-in to a breath", () => {- expect(refineEdge(spans, 0, "in")).toEqual({ t: 1 - BREATH, pause: true });+ test("an edge past the last word has nothing to open on", () => {+ expect(placeEdge(take, 13, "in", "natural", bounds).placed).toBe(false);+ });+});++describe("a joint whose pause cannot hold both sides", () => {+ const out = { t: 4.26, placed: true, bound: 4 };+ const into = { t: 4.14, placed: true, bound: 4.3 };++ test("the two edges meet inside the pause", () => {+ const joint = settleJoint(out, into)!;+ expect(joint).toBeGreaterThanOrEqual(4 + Math.min(MIN_AIR.out, 0.3 * 0.6));+ expect(joint).toBeLessThanOrEqual(4.3 - Math.min(MIN_AIR.in, 0.3 * 0.4));});- test("continuous speech leaves the edge alone, flagged", () => {- expect(refineEdge(spans, 7.5, "in")).toEqual({ t: 7.5, pause: false });+ test("a pause of nothing at all still leaves each word whole", () => {+ const joint = settleJoint({ t: 4.1, placed: true, bound: 4 }, { t: 4.02, placed: true, bound: 4.06 })!;+ expect(joint).toBeGreaterThanOrEqual(4);+ expect(joint).toBeLessThanOrEqual(4.06);});- test("a pause further back than reach does not pull the edge", () => {- expect(refineEdge([{ start: 0, end: 1 }], 3.5, "in")).toEqual({ t: 3.5, pause: false });+ test("words that overlap are not a pause, so there is no joint to settle", () => {+ expect(settleJoint({ t: 4.3, placed: true, bound: 4.3 }, { t: 4.1, placed: true, bound: 4.1 })).toBeNull();});});diff --git a/site/src/cut/lib/cutRefine.ts b/site/src/cut/lib/cutRefine.tsindex 5a5665b1..4f7af1ba 100644--- a/site/src/cut/lib/cutRefine.ts+++ b/site/src/cut/lib/cutRefine.ts@@ -1,40 +1,59 @@-// True up speech-cut edges against the silence around them.+// Place a speech cut against the words around it.//-// The assistant plans cuts from cue timings and silence spans, but the numbers+// The assistant plans cuts from cue timings and what it heard, but the numbers// it emits are hand-derived floats: an edge lands a hair inside a word, or// leaves stray dead air. The split of labor is the usual one — the model says// WHICH edges are speech cuts, and the arithmetic here decides exactly where-// each one sits: a breath inside the real pause.+// each one sits.//-// The rules err toward keeping words. An edge already inside a pause slides to-// a breath past the speech it closes (an out point) or ahead of the speech it-// opens (an in point). An edge sitting in speech — a clipped word — only moves-// outward, to the first pause in the direction that makes the word whole-// again. With no pause within reach the speech runs continuously there, so the-// edge stays where the model put it and comes back flagged for a listen.+// What makes a joint pleasant is air, and the amount is small and specific: a+// beat after the last word finishes decaying, a shorter one before the next+// one starts. Both are measured from the word, never from the edge the model+// guessed, so a cut cannot land inside a word and cannot butt two words+// together. Pace changes how much air, never whether there is any — the+// fastest cut in this file still lets every word finish.-/** A silent stretch of the source, in source seconds. */-export interface SilenceSpan {- start: number;- end: number;-}+import type { SpeechSegment } from "./audioScan";++export type { SpeechSegment };++/** How much room the cut leaves around the words. */+export type Pace = "fast" | "natural" | "relaxed";++/** Air kept after the last word ("out") and before the next one ("in"),+ * seconds. Out runs longer than in: the ear wants a word to land before the+ * picture moves on, and it takes the next word's own attack as the lead. */+export const AIR: Record<Pace, { out: number; in: number }> = {+ fast: { out: 0.14, in: 0.09 },+ natural: { out: 0.26, in: 0.16 },+ relaxed: { out: 0.45, in: 0.28 },+};-/** The pause kept on the speech side of a refined edge, seconds. */-export const BREATH = 0.25;+/** The air no pace goes under. Under this the cut is heard as clipping the+ * word even when the waveform says it did not. */+export const MIN_AIR = { out: 0.08, in: 0.06 };-/** How far an edge may travel to find or settle into a pause. Beyond this a- * long pause reads as a held beat — a choice, not slack — and a cut in speech- * reads as deliberate. */+/** Kept clear of the neighbouring word when the pause cannot hold the air. */+const GUARD = 0.02;++/** How far an edge may travel to reach a word boundary. Beyond this a long+ * pause reads as a held beat — a choice, not slack — and a cut inside a long+ * run of speech reads as deliberate. */export const REACH = 2;/** Which side of the clip an edge trims: "in" opens the clip, "out" closes it. */export type EdgeKind = "in" | "out";-export interface RefinedEdge {- /** The refined source time — the input `t` when no pause was within reach. */+export interface PlacedEdge {+ /** The placed source time — the input `t` when no word was within reach. */t: number;- /** False when speech runs continuously at this edge and it was left alone. */- pause: boolean;+ /** False when the scan found no word to place against and the edge was left+ * where the model put it. */+ placed: boolean;+ /** The word boundary this edge protects: the latest an out edge may sit+ * before it, the earliest an in edge may sit after it. Repairing a joint+ * works from these, so the repair cannot clip a word either. */+ bound: number;}const EPS = 1e-3;@@ -42,34 +61,79 @@ const round = (n: number) => Math.round(n * 1000) / 1000;const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));/**- * Place one trim edge a breath inside the pause it belongs to. `spans` are the- * silences detected around `t`, sorted by start and disjoint.+ * Place one trim edge against the speech around it. `segments` are the words+ * the scan found, in order and disjoint; `bounds` is the span it looked at, so+ * an edge with open air on one side does not wander past what was measured.+ *+ * An out edge follows the word it keeps: past that word's tail by the pace's+ * air, and never into the word after it. An in edge leads the word it opens by+ * the same reasoning. An edge sitting inside a word moves outward, which is+ * what makes the word whole again.*/-export function refineEdge(- spans: readonly SilenceSpan[],+export function placeEdge(+ segments: readonly SpeechSegment[],t: number,- kind: EdgeKind-): RefinedEdge {- const within = spans.find((s) => t >= s.start - EPS && t <= s.end + EPS);- if (within) {- // A pause shorter than two breaths still holds a cut — the edge settles at- // the pause's far boundary and the whole pause is the breath.- const ideal =- kind === "out"- ? Math.min(within.end, within.start + BREATH)- : Math.max(within.start, within.end - BREATH);- return { t: round(clamp(ideal, t - REACH, t + REACH)), pause: true };- }+ kind: EdgeKind,+ pace: Pace = "natural",+ bounds?: { from: number; to: number }+): PlacedEdge {+ const air = AIR[pace];+ const unplaced: PlacedEdge = { t, placed: false, bound: t };+if (kind === "out") {- const next = spans.find((s) => s.start > t && s.start - t <= REACH);- if (next) return { t: round(Math.min(next.end, next.start + BREATH)), pause: true };- } else {- let prev: SilenceSpan | undefined;- for (const s of spans) {- if (s.end < t && t - s.end <= REACH) prev = s;- if (s.end >= t) break;+ // The word this clip keeps: the last one that has started by `t`. An edge+ // inside it extends to its end; an edge past it pulls back to the same+ // place, so both cases are the one rule.+ let keep: SpeechSegment | undefined;+ for (const s of segments) {+ if (s.start - EPS > t) break;+ keep = s;}- if (prev) return { t: round(Math.max(prev.start, prev.end - BREATH)), pause: true };+ if (!keep || Math.abs(keep.end - t) > REACH) return unplaced;+ const next = segments.find((s) => s.start > keep!.end + EPS);+ const ceiling = Math.min(next ? next.start - GUARD : Infinity, bounds?.to ?? Infinity);+ return {+ t: round(clamp(keep.end + air.out, keep.end, Math.max(keep.end, ceiling))),+ placed: true,+ bound: round(keep.end),+ };}- return { t, pause: false };++ // The word this clip opens on: the first one still sounding at `t`.+ const word = segments.find((s) => s.end - EPS > t);+ if (!word || Math.abs(word.start - t) > REACH) return unplaced;+ let prev: SpeechSegment | undefined;+ for (const s of segments) {+ if (s.end - EPS > word.start) break;+ prev = s;+ }+ const floorT = Math.max(prev ? prev.end + GUARD : -Infinity, bounds?.from ?? -Infinity);+ return {+ t: round(clamp(word.start - air.in, Math.min(word.start, floorT), word.start)),+ placed: true,+ bound: round(word.start),+ };+}++/**+ * Where two edges of one removed stretch land when the pace would push them+ * past each other — the pause left between the words is shorter than the air+ * both sides want.+ *+ * Both sides give ground rather than either reverting to the model's guess:+ * the joint sits at one point inside the pause, as close to the middle as the+ * two words allow. Returns null when the words themselves overlap, which means+ * the stretch being removed is not a pause at all.+ */+export function settleJoint(out: PlacedEdge, into: PlacedEdge): number | null {+ const lo = out.bound;+ const hi = into.bound;+ if (hi - lo < EPS) return null;+ const wanted = (out.t + into.t) / 2;+ const room = hi - lo;+ // Split what there is: the out side takes the larger share, the same way it+ // takes the longer air when there is room for both.+ const lowest = lo + Math.min(MIN_AIR.out, room * 0.6);+ const highest = hi - Math.min(MIN_AIR.in, room * 0.4);+ return round(clamp(wanted, Math.min(lowest, highest), Math.max(lowest, highest)));}diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 14e3e65a..2757a0c7 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1,6 +1,6 @@"use client";-import { scanSilence, type PcmChunk } from "./audioScan";+import { scanSilence, scanSpeech, type PcmChunk, type SpeechScan } from "./audioScan";import { apiFetch, apiJson, getBackend, type CutBackend } from "./backend";import { cloudBackend } from "./backend/cloud";import { pollCloudJob } from "./cloudJob";@@ -1054,6 +1054,31 @@ export async function captureFreezeFrame(const round2 = (n: number) => Math.round(n * 100) / 100;+/** Decoded audio for a span of a source, chunk by chunk, so scanning a long+ * file costs one decoded chunk at a time rather than a decoded copy of the+ * whole thing. Throws when the file carries no audio at all — whether it does+ * is asked of the file, not of the span, so a range past the end of a+ * perfectly good recording is an empty answer instead of an error. */+async function* sourcePcm(+ sourceUrl: string,+ from: number,+ to: number | undefined+): AsyncGenerator<PcmChunk> {+ const input = openMedia(sourceUrl);+ const track = await audioTrackOf(input).finally(() => input.dispose());+ if (!track) throw new Error("This file has no audio track.");+ if (to !== undefined && !(to > from)) return;+ for await (const { buffer, timestamp } of audioChunks(sourceUrl, from, to)) {+ yield {+ channels: Array.from({ length: buffer.numberOfChannels }, (_, c) =>+ buffer.getChannelData(c)+ ),+ timestamp,+ sampleRate: buffer.sampleRate,+ };+ }+}+/** Cloud twin of the engine's silence route (ffmpeg silencedetect): read the* source's audio and scan 20ms RMS windows against the same dB threshold and* minimum-duration rules. Times are absolute source seconds. */@@ -1062,30 +1087,18 @@ export async function detectSilenceClientSide(opts: { from: number; to?: number; thresholdDb: number; minSilence: number }): Promise<{ start: number; end: number; duration: number }[]> {const from = Math.max(0, opts.from);- const to = opts.to;- // Whether the file has sound is asked of the file, not of the span: a range- // past the end of a perfectly good recording is an empty answer, not a file- // with no audio in it.- const input = openMedia(sourceUrl);- const track = await audioTrackOf(input).finally(() => input.dispose());- if (!track) throw new Error("This file has no audio track.");- if (to !== undefined && !(to > from)) return [];-- // The audio is folded into windows as it arrives, so scanning a long file- // costs one decoded chunk at a time rather than a decoded copy of the whole- // thing.- async function* pcm(): AsyncGenerator<PcmChunk> {- for await (const { buffer, timestamp } of audioChunks(sourceUrl, from, to)) {- yield {- channels: Array.from({ length: buffer.numberOfChannels }, (_, c) =>- buffer.getChannelData(c)- ),- timestamp,- sampleRate: buffer.sampleRate,- };- }- }- return scanSilence(pcm(), { ...opts, from });+ return scanSilence(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });+}++/** Where the words are in a span of a source. One implementation for every+ * residency: the page reads its own media this way in local and cloud mode+ * alike, and the headless runner installs the same decoders. */+export async function scanSourceSpeech(+ sourceUrl: string,+ opts: { from: number; to?: number }+): Promise<SpeechScan> {+ const from = Math.max(0, opts.from);+ return scanSpeech(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });}/** Cloud twin of the engine's audio-extract route: render a span of thediff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex fa2543bc..fbbf1ea5 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -138,7 +138,7 @@ The flow for "edit this for me":3. watch_video each distinct source (or the ranges in question); note scene changes and what happens where.4. Plan in thoughts: group the cues into sentences and thoughts, decide which thoughts stay whole, then detect_silence to find where the speaker actually pauses. Cue timings approximate the audio — a trailing word often spills past its cue — so aim every speech cut into a detected silence span, and aim loose: up to a second of spare air on each side beats a tight cut, because the refine step trues the edges after. A cue boundary with no silence span around it means the speech runs continuously there, so leave that joint alone.5. Execute: split_at (timeline s) to divide, delete_item to drop a segment, trim_clip (source s) to tighten edges, place_clip/move_clip to close gaps or reorder, set_speed to compress slow stretches. Batch each wave in one round: splits don't move times, so issue every planned split_at together, then every delete together. Every timeline mutation returns the updated track-0 and soundtrack rows (ids, starts, lengths) — read the new ids from the result and keep cutting; get_state between steps is a wasted round.-6. Refine: refine_speech_cuts with every recut clip id. It re-scans the audio at each edge and re-trims it to sit a ~0.25s breath inside the real pause, keeping the clips' spacing. It reads sound, not intent — leave out a clip whose edge you placed mid-speech on purpose.+6. Refine: refine_speech_cuts with every recut clip id, and with the \`pace\` the request asked for. It measures the recording's own room tone and voice level, finds where each word truly starts and finishes, and re-trims every edge to sit a beat clear of one, keeping the clips' spacing. It reads sound, not intent — leave out a clip whose edge you placed mid-speech on purpose.7. Verify by ear: listen_audio at each edge the refine flagged, plus a spot-check or two (scope from/to to ~2s around the joint), and check the first and last words arrive whole and the hand-off into the next thought sounds natural. A clipped word → widen that trim deeper into the silence and listen again. After a heavy edit, listen once end-to-end. seek + capture_frame when the change is visual.Time math (get this right):@@ -146,7 +146,7 @@ Time math (get this right):- timeline_t = clip.start + (source_t − clip.in) / clip.speed, valid while source_t is inside [in, out]. Each watch result's clip block carries this formula with the real numbers filled in.- The same source can appear in several clips — map per clip.-Cutting speech well: pace is part of the message. Keep a beat of the speaker's own pause between sentences (~0.3s) and a slightly longer one between thoughts (~0.5s) — shorten a long pause to that beat instead of deleting it, and speech butted directly together reads as rushed. Cut loose and let refine_speech_cuts set the breath at each edge. Prefer split_at + delete_item, then place_clip to close the gap (a beat of black may be wanted — ask the cut, not the tool); trim_clip only tightens a clip's edges.`,+Cutting speech well: pace is part of the message, and it is set in one place — refine_speech_cuts' \`pace\`. "Fast paced", "punchy", "tight" means pace: fast there; a calm piece means relaxed. Placing your own edges closer to the words is not a faster cut, it is a clipped one: a word's tail runs past its cue, the ear hears the truncation, and no amount of tightening reads as energy. So cut loose — up to a second of spare air on each side — and let the refine step take it back to the beat the pace asks for. Shorten a long pause to that beat rather than deleting it; speech butted directly together reads as rushed however fast the piece is meant to be. Prefer split_at + delete_item, then place_clip to close the gap (a beat of black may be wanted — ask the cut, not the tool); trim_clip only tightens a clip's edges.`,"transitions-and-fades": `# Transitions, animations, looks & fadesRoute the ask to the right feature: