Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(ai-edition): silence cuts keep breath next to kept speech
EtienneLescote484b9cab9a4
diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.tsindex c128e7976..4179d54d9 100644--- a/electron/ai-edition/agent-tools.test.ts+++ b/electron/ai-edition/agent-tools.test.ts@@ -2430,3 +2430,59 @@ describe("setWordText", () => {expect(result.document).toBeUndefined();});});++describe("addTrim shapes the cut like the transcript pane", () => {+ // A 1.0-1.5, B 2.0-2.4, C 3.0-3.5: the production shape, one word per segment.+ function withWords(): AxcutDocument {+ const doc = fixtureDocument();+ const words = [+ { id: "w_a", segmentId: "seg_1", startSec: 1, endSec: 1.5, text: "A" },+ { id: "w_b", segmentId: "seg_1", startSec: 2, endSec: 2.4, text: "B" },+ { id: "w_c", segmentId: "seg_1", startSec: 3, endSec: 3.5, text: "C" },+ ];+ return { ...doc, transcripts: [{ ...doc.transcripts[0], words }] };+ }+ const lastTrim = (doc: AxcutDocument | undefined) => doc?.timeline.trimRanges.at(-1);++ it("keeps breath next to the kept words when a word goes with both its silences", () => {+ const result = executeAgentTool(+ withWords(),+ "addTrim",+ JSON.stringify({ startSec: 1.5, endSec: 3 }),+ );+ expect(result.ok).toBe(true);+ expect(lastTrim(result.document)?.startSec).toBeCloseTo(1.57);+ expect(lastTrim(result.document)?.endSec).toBeCloseTo(2.93);+ });++ it("does not breathe next to a word an earlier cut already took (addTrims)", () => {+ const doc = withWords();+ doc.timeline.trimRanges.push({+ id: "trim_a",+ assetId: "asset_1",+ clipId: "clip_1",+ startSec: 1,+ endSec: 1.5,+ reason: "",+ origin: "agent",+ });+ const result = executeAgentTool(+ doc,+ "addTrims",+ JSON.stringify({ ranges: [{ startSec: 1.5, endSec: 2 }] }),+ );+ expect(lastTrim(result.document)?.startSec).toBe(1.5);+ expect(lastTrim(result.document)?.endSec).toBeCloseTo(1.93);+ });++ it("closes a gap under two frames against a cut already on the clip", () => {+ // trim_1 covers 10-12 on asset_1; 50 ms of picture would flash between the two.+ const result = executeAgentTool(+ fixtureDocument(),+ "addTrim",+ JSON.stringify({ startSec: 12.05, endSec: 15 }),+ );+ expect(lastTrim(result.document)?.startSec).toBe(12);+ expect(lastTrim(result.document)?.endSec).toBe(15);+ });+});diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.tsindex 4ca0bae0e..e90fb9feb 100644--- a/electron/ai-edition/agent-tools.ts+++ b/electron/ai-edition/agent-tools.ts@@ -40,6 +40,7 @@ import {buildCursorTrack,type CursorTrackSample,} from "../../src/lib/ai-edition/timeline/cursor-track";+import { breatheCut, mergeCloseCuts } from "../../src/lib/ai-edition/timeline/cut-breath";import {anchorRegionsWithDerivedMs,coalesceRegionsForRuler,@@ -1423,8 +1424,8 @@ export function executeAgentTool(if (!document.assets.some((a) => a.id === assetId)) {return failure(`Unknown asset: ${assetId}`);}- const startSec = Math.min(parsed.data.startSec, parsed.data.endSec);- const endSec = Math.max(parsed.data.startSec, parsed.data.endSec);+ let startSec = Math.min(parsed.data.startSec, parsed.data.endSec);+ let endSec = Math.max(parsed.data.startSec, parsed.data.endSec);// Which clip the cut sits on. Named explicitly when the model says so; otherwise// inferred from the source range — but only when the answer is unique. Two clips@@ -1453,6 +1454,30 @@ export function executeAgentTool();}+ // Breath next to the speech that stays, and no one-frame flash against a cut+ // already on the same clip — the same shaping the transcript pane applies.+ const clip = document.timeline.clips.find((c) => c.id === clipId);+ const siblings = document.timeline.trimRanges.filter((t) =>+ clip ? trimAppliesToClip(t, clip) : t.assetId === assetId,+ );+ const transcript =+ document.transcripts.find((t) => t.assetId === assetId) ??+ (document.transcript?.assetId === assetId ? document.transcript : null);+ const words = (transcript?.words ?? [])+ .filter((w) => w.source !== "synth")+ .map((w) => {+ const mid = (w.startSec + w.endSec) / 2;+ const inClip =+ !clip || (mid >= clip.sourceStartSec && mid < (clip.sourceEndSec ?? Infinity));+ const cut = siblings.some((t) => mid >= t.startSec && mid < t.endSec);+ return { startSec: w.startSec, endSec: w.endSec, kept: inClip && !cut };+ });+ ({ startSec, endSec } = mergeCloseCuts(+ breatheCut({ startSec, endSec }, words),+ siblings,+ document.assets.find((a) => a.id === assetId)?.video?.fps,+ ));+const trim = {id: createId("trim"),assetId,diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsxindex 85145645d..b164bf65a 100644--- a/src/components/ai-edition/NewEditorShell.tsx+++ b/src/components/ai-edition/NewEditorShell.tsx@@ -44,9 +44,11 @@ import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus";import { useSequentialTimelineOps } from "@/lib/ai-edition/store/useSequentialTimelineOps";import { useTimeline } from "@/lib/ai-edition/store/useTimeline";import { isGeneratedAssetId } from "@/lib/ai-edition/timeline/clip-parts";+import { mergeCloseCuts } from "@/lib/ai-edition/timeline/cut-breath";import { newRegionDurationSec } from "@/lib/ai-edition/timeline/newRegionDuration";import {dropTrimPillsByIds,+ trimAppliesToClip,ventilateTimelineSpanToTrims,} from "@/lib/ai-edition/timeline/trim-mapping";import { firstTimelineBusyView } from "@/lib/ai-edition/transcription/status";@@ -729,15 +731,24 @@ export function NewEditorShell() {toast.error(te("errors.trimNoFilm"));return;}- const rows = ranges.map((range) => ({- id: createId("trim"),- assetId: range.assetId,- clipId: range.clipId,- startSec: range.sourceStartSec,- endSec: range.sourceEndSec,- reason,- origin: "user" as const,- }));+ const rows = ranges.map((range) => {+ // No one-frame flash between this cut and one already on the same clip.+ const merged = mergeCloseCuts(+ { startSec: range.sourceStartSec, endSec: range.sourceEndSec },+ doc.timeline.trimRanges.filter((t) =>+ trimAppliesToClip(t, { id: range.clipId, assetId: range.assetId }),+ ),+ doc.assets.find((a) => a.id === range.assetId)?.video?.fps,+ );+ return {+ id: createId("trim"),+ assetId: range.assetId,+ clipId: range.clipId,+ ...merged,+ reason,+ origin: "user" as const,+ };+ });await saveDocument({...doc,diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsxindex d38872432..c04736717 100644--- a/src/components/ai-edition/RightPanes.tsx+++ b/src/components/ai-edition/RightPanes.tsx@@ -90,6 +90,7 @@ import {voiceoverPlacements,} from "@/lib/ai-edition/timeline/aggregated-transcript";import { assetCameraSource, hasAnyClipWithCamera } from "@/lib/ai-edition/timeline/camera";+import { breatheCut } from "@/lib/ai-edition/timeline/cut-breath";import { formatMs } from "@/lib/ai-edition/timeline/format";import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time";import type {@@ -1174,12 +1175,19 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({pendingCaretWordIdRef.current = keptRange[0].id;const startSec = Math.min(...keptRange.map((w) => w.word.startSec));const endSec = Math.max(...keptRange.map((w) => w.word.endSec));+ // Breath next to the speech that stays, so a cut silence does not glue two words.+ const breathed = breatheCut(+ { startSec, endSec },+ words+ .filter((cw) => !isSilenceWord(cw.word) && !isInsertedWord(cw.word))+ .map((cw) => ({ startSec: cw.word.startSec, endSec: cw.word.endSec, kept: cw.kept })),+ );onTrimTimelineSpan(- ...toRawSpan(startSec, endSec),+ ...toRawSpan(breathed.startSec, breathed.endSec),`Skip ${formatMs(startSec * 1000)}-${formatMs(endSec * 1000)} from ${clip.assetId}.`,);},- [busy, clip.assetId, toRawSpan, onTrimTimelineSpan, onRemoveWords],+ [busy, clip.assetId, words, toRawSpan, onTrimTimelineSpan, onRemoveWords],);const removeTrimRun = useCallback(diff --git a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsxindex a19330610..859f7aede 100644--- a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx+++ b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx@@ -72,13 +72,14 @@ function renderPane(trimRanges: AxcutTrimRange[],onTrimTimelineSpan = vi.fn(),busyAssetIds: string[] = [],+ transcript: AxcutTranscript = TRANSCRIPT,) {const view = render(<I18nProvider><TranscriptPaneclips={[CLIP]}audioTracks={[]}- transcripts={[TRANSCRIPT]}+ transcripts={[transcript]}assets={[ASSET]}trimRanges={trimRanges}busyAssetIds={busyAssetIds}@@ -266,3 +267,37 @@ describe("keyboard cut with the caret between words", () => {expect(struck).toEqual(["clip_1:w1", "clip_1:w2", "clip_1:w3", "clip_1:w4"]);});});++// Cutting a silence must leave air next to the speech that stays, or the last syllable is+// glued to the next word. With 500 ms gaps, each side keeps 70 ms.+describe("a cut silence breathes", () => {+ const GAPPED: AxcutTranscript = {+ ...TRANSCRIPT,+ words: [+ { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "un" },+ { id: "w2", segmentId: "s", startSec: 1.5, endSec: 2.5, text: "deux" },+ { id: "w3", segmentId: "s", startSec: 3, endSec: 4, text: "trois" },+ ],+ };+ // un, [silence], deux, [silence], trois+ const BEFORE_TROIS = 4;++ it("keeps breath on both sides of a silence between two kept words", () => {+ const { editor, onTrimTimelineSpan } = renderPane([], vi.fn(), [], GAPPED);+ caretBeforeWordAt(editor, BEFORE_TROIS);+ fireEvent.keyDown(editor, { key: "Backspace" });+ const cut = cutRange(onTrimTimelineSpan);+ expect(cut?.[0]).toBeCloseTo(2.57);+ expect(cut?.[1]).toBeCloseTo(2.93);+ });++ it("does not breathe next to a word that is already cut", () => {+ const deuxCut = { ...W2_TRIMMED, startSec: 1.5, endSec: 2.5 };+ const { editor, onTrimTimelineSpan } = renderPane([deuxCut], vi.fn(), [], GAPPED);+ caretBeforeWordAt(editor, BEFORE_TROIS);+ fireEvent.keyDown(editor, { key: "Backspace" });+ const cut = cutRange(onTrimTimelineSpan);+ expect(cut?.[0]).toBe(2.5);+ expect(cut?.[1]).toBeCloseTo(2.93);+ });+});diff --git a/src/lib/ai-edition/timeline/cut-breath.test.ts b/src/lib/ai-edition/timeline/cut-breath.test.tsnew file mode 100644index 000000000..f6e33b020--- /dev/null+++ b/src/lib/ai-edition/timeline/cut-breath.test.ts@@ -0,0 +1,108 @@+import { describe, expect, it } from "vitest";+import { type BreathWord, breatheCut, mergeCloseCuts } from "./cut-breath";++const w = (startSec: number, endSec: number, kept = true): BreathWord => ({+ startSec,+ endSec,+ kept,+});++// A 1.0-1.5, B 2.0-2.4, C 3.0-3.5: two 0.5-0.6 s silences around B.+const A = w(1, 1.5);+const B = w(2, 2.4);+const C = w(3, 3.5);++describe("breatheCut", () => {+ it("leaves 70 ms of air on each side of a cut silence", () => {+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [A, B, C]);+ expect(cut.startSec).toBeCloseTo(1.57);+ expect(cut.endSec).toBeCloseTo(1.93);+ });++ it("word plus both silences: breath only at the kept edges, no sliver around the word", () => {+ const cut = breatheCut({ startSec: 1.5, endSec: 3 }, [A, B, C]);+ expect(cut.startSec).toBeCloseTo(1.57);+ expect(cut.endSec).toBeCloseTo(2.93);+ // B stays wholly inside the cut.+ expect(cut.startSec).toBeLessThan(B.startSec);+ expect(cut.endSec).toBeGreaterThan(B.endSec);+ });++ it("never goes past the middle of a short gap", () => {+ const cut = breatheCut({ startSec: 1.5, endSec: 1.6 }, [A, w(1.6, 2)]);+ expect(cut).toEqual({ startSec: 1.5, endSec: 1.6 });+ const wider = breatheCut({ startSec: 1.5, endSec: 1.62 }, [A, w(1.62, 2)]);+ expect(wider).toEqual({ startSec: 1.5, endSec: 1.62 });+ });++ it("breathes up to the middle when the gap has room for less than two breaths", () => {+ // Gaps of 100 ms on each side of the word removed with them (1.6-1.9).+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [A, w(1.6, 1.9), w(2, 2.5)]);+ expect(cut.startSec).toBeCloseTo(1.55);+ expect(cut.endSec).toBeCloseTo(1.95);+ });++ it("does not breathe next to a word that is already cut", () => {+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [w(1, 1.5, false), B, C]);+ expect(cut.startSec).toBe(1.5);+ expect(cut.endSec).toBeCloseTo(1.93);+ });++ it("leaves an edge inside a word alone", () => {+ expect(breatheCut({ startSec: 1.2, endSec: 2.2 }, [A, B, C])).toEqual({+ startSec: 1.2,+ endSec: 2.2,+ });+ });++ it("leaves a cut on exact word bounds alone", () => {+ expect(breatheCut({ startSec: 2, endSec: 2.4 }, [A, B, C])).toEqual({+ startSec: 2,+ endSec: 2.4,+ });+ });++ it("ignores words with no duration", () => {+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [A, w(1.5, 1.5), w(2, 2), B]);+ expect(cut.startSec).toBeCloseTo(1.57);+ expect(cut.endSec).toBeCloseTo(1.93);+ });++ it("leaves a side alone when the cut is too short to give it back", () => {+ // A 40 ms nibble just after A: breathing would push the start past the end.+ expect(breatheCut({ startSec: 1.5, endSec: 1.54 }, [A, C])).toEqual({+ startSec: 1.5,+ endSec: 1.54,+ });+ });+});++describe("mergeCloseCuts", () => {+ const fps = 30;++ it("snaps onto a neighbour less than two frames away", () => {+ expect(+ mergeCloseCuts({ startSec: 2.05, endSec: 3 }, [{ startSec: 1, endSec: 2 }], fps),+ ).toEqual({ startSec: 2, endSec: 3 });+ expect(+ mergeCloseCuts({ startSec: 1, endSec: 2 }, [{ startSec: 2.05, endSec: 3 }], fps),+ ).toEqual({ startSec: 1, endSec: 2.05 });+ });++ it("keeps a gap of two frames or more", () => {+ const cut = { startSec: 2.07, endSec: 3 };+ expect(mergeCloseCuts(cut, [{ startSec: 1, endSec: 2 }], fps)).toEqual(cut);+ });++ it("uses the asset's frame rate, 30 fps when it has none", () => {+ const cut = { startSec: 2.05, endSec: 3 };+ const before = [{ startSec: 1, endSec: 2 }];+ expect(mergeCloseCuts(cut, before, 60)).toEqual(cut);+ expect(mergeCloseCuts(cut, before, 0)).toEqual({ startSec: 2, endSec: 3 });+ });++ it("ignores overlapping neighbours", () => {+ const cut = { startSec: 1.5, endSec: 3 };+ expect(mergeCloseCuts(cut, [{ startSec: 1, endSec: 2 }], fps)).toEqual(cut);+ });+});diff --git a/src/lib/ai-edition/timeline/cut-breath.ts b/src/lib/ai-edition/timeline/cut-breath.tsnew file mode 100644index 000000000..8af827a10--- /dev/null+++ b/src/lib/ai-edition/timeline/cut-breath.ts@@ -0,0 +1,88 @@+// Silence cuts that breathe.+//+// A cut that swallows the whole gap between two words glues the last syllable to the+// next word. Speech needs a little air on each side, so the edge of a cut that sits in a+// gap next to KEPT speech is pulled back by `CUT_BREATH_SEC`, never past the middle of+// that gap. An edge that touches removed material, or sits inside a word, is left where+// it is: breathing there would only leave a sliver of kept media between two cuts.+//+// Only cuts authored from speech go through this (the transcript pane, the agent). A cut+// drawn on the timeline by hand is the user's and is never reshaped.++import type { Interval } from "./intervals";++/** Air kept between a cut and the speech it touches, per side (~140 ms across a cut). */+export const CUT_BREATH_SEC = 0.07;++/** Frame rate {@link mergeCloseCuts} assumes when the asset reports none (audio, unprobed). */+const FALLBACK_FPS = 30;++export interface BreathWord extends Interval {+ /** Whether this word is still in the film, BEFORE the cut being shaped is applied. */+ kept: boolean;+}++const EPS = 1e-6;++/**+ * `cut` with each edge that sits in a gap next to a kept word pulled back to leave+ * `breathSec` of that gap, capped at the gap's middle. All times on one source clock.+ * Words with no duration carry no audio and are ignored. A side the cut is too short to+ * give back is left alone.+ */+export function breatheCut(+ cut: Interval,+ words: readonly BreathWord[],+ breathSec = CUT_BREATH_SEC,+): Interval {+ const spoken = words.filter((w) => w.endSec > w.startSec);+ let startSec = cut.startSec;+ let endSec = cut.endSec;++ // Start edge: the word that ends last at or before it, and the gap that follows it.+ const before = spoken+ .filter((w) => w.endSec <= cut.startSec + EPS)+ .reduce<BreathWord | null>((a, w) => (!a || w.endSec > a.endSec ? w : a), null);+ if (before?.kept) {+ const nextStarts = spoken.filter((w) => w.startSec >= before.endSec - EPS && w !== before);+ const gapEnd = Math.min(...nextStarts.map((w) => w.startSec));+ // gapEnd below the edge means the edge is inside a word, not in the gap.+ if (gapEnd >= cut.startSec - EPS) {+ const target = Math.min(before.endSec + breathSec, (before.endSec + gapEnd) / 2);+ if (target > startSec && target < cut.endSec) startSec = target;+ }+ }++ // End edge, mirrored: the word that starts first at or after it, and the gap before it.+ const after = spoken+ .filter((w) => w.startSec >= cut.endSec - EPS)+ .reduce<BreathWord | null>((a, w) => (!a || w.startSec < a.startSec ? w : a), null);+ if (after?.kept) {+ const prevEnds = spoken.filter((w) => w.endSec <= after.startSec + EPS && w !== after);+ const gapStart = Math.max(...prevEnds.map((w) => w.endSec));+ if (gapStart <= cut.endSec + EPS) {+ const target = Math.max(after.startSec - breathSec, (gapStart + after.startSec) / 2);+ if (target < endSec && target > cut.startSec) endSec = target;+ }+ }++ return endSec > startSec ? { startSec, endSec } : cut;+}++/**+ * `cut` with an edge snapped onto a neighbouring cut of the SAME clip when the media+ * left between them would be shorter than two frames — a one-frame flash of picture+ * between two cuts reads as a glitch, never as content. `neighbours` are that clip's+ * existing cuts, on the same clock as `cut`; `fps` is the asset's, 0 when unknown.+ */+export function mergeCloseCuts(cut: Interval, neighbours: readonly Interval[], fps = 0): Interval {+ const maxGap = 2 / (fps > 0 ? fps : FALLBACK_FPS);+ let { startSec, endSec } = cut;+ for (const n of neighbours) {+ const gapBefore = cut.startSec - n.endSec;+ if (gapBefore > 0 && gapBefore < maxGap) startSec = Math.min(startSec, n.endSec);+ const gapAfter = n.startSec - cut.endSec;+ if (gapAfter > 0 && gapAfter < maxGap) endSec = Math.max(endSec, n.startSec);+ }+ return { startSec, endSec };+}