Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(ai-edition): silence cuts keep breath next to kept speech

EtienneLescote484b9cab9a4
diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts
index c128e7976..4179d54d9 100644
--- a/electron/ai-edition/agent-tools.test.ts
+++ b/electron/ai-edition/agent-tools.test.ts
@@ -2430,3 +2430,59 @@ describe("setWordText", () => {
expect(result.document).toBeUndefined();
});
});
+
+describe("addTrim shapes the cut like the transcript pane", () => {
+ // A 1.0-1.5, B 2.0-2.4, C 3.0-3.5: the production shape, one word per segment.
+ function withWords(): AxcutDocument {
+ const doc = fixtureDocument();
+ const words = [
+ { id: "w_a", segmentId: "seg_1", startSec: 1, endSec: 1.5, text: "A" },
+ { id: "w_b", segmentId: "seg_1", startSec: 2, endSec: 2.4, text: "B" },
+ { id: "w_c", segmentId: "seg_1", startSec: 3, endSec: 3.5, text: "C" },
+ ];
+ return { ...doc, transcripts: [{ ...doc.transcripts[0], words }] };
+ }
+ const lastTrim = (doc: AxcutDocument | undefined) => doc?.timeline.trimRanges.at(-1);
+
+ it("keeps breath next to the kept words when a word goes with both its silences", () => {
+ const result = executeAgentTool(
+ withWords(),
+ "addTrim",
+ JSON.stringify({ startSec: 1.5, endSec: 3 }),
+ );
+ expect(result.ok).toBe(true);
+ expect(lastTrim(result.document)?.startSec).toBeCloseTo(1.57);
+ expect(lastTrim(result.document)?.endSec).toBeCloseTo(2.93);
+ });
+
+ it("does not breathe next to a word an earlier cut already took (addTrims)", () => {
+ const doc = withWords();
+ doc.timeline.trimRanges.push({
+ id: "trim_a",
+ assetId: "asset_1",
+ clipId: "clip_1",
+ startSec: 1,
+ endSec: 1.5,
+ reason: "",
+ origin: "agent",
+ });
+ const result = executeAgentTool(
+ doc,
+ "addTrims",
+ JSON.stringify({ ranges: [{ startSec: 1.5, endSec: 2 }] }),
+ );
+ expect(lastTrim(result.document)?.startSec).toBe(1.5);
+ expect(lastTrim(result.document)?.endSec).toBeCloseTo(1.93);
+ });
+
+ it("closes a gap under two frames against a cut already on the clip", () => {
+ // trim_1 covers 10-12 on asset_1; 50 ms of picture would flash between the two.
+ const result = executeAgentTool(
+ fixtureDocument(),
+ "addTrim",
+ JSON.stringify({ startSec: 12.05, endSec: 15 }),
+ );
+ expect(lastTrim(result.document)?.startSec).toBe(12);
+ expect(lastTrim(result.document)?.endSec).toBe(15);
+ });
+});
diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts
index 4ca0bae0e..e90fb9feb 100644
--- a/electron/ai-edition/agent-tools.ts
+++ b/electron/ai-edition/agent-tools.ts
@@ -40,6 +40,7 @@ import {
buildCursorTrack,
type CursorTrackSample,
} from "../../src/lib/ai-edition/timeline/cursor-track";
+import { breatheCut, mergeCloseCuts } from "../../src/lib/ai-edition/timeline/cut-breath";
import {
anchorRegionsWithDerivedMs,
coalesceRegionsForRuler,
@@ -1423,8 +1424,8 @@ export function executeAgentTool(
if (!document.assets.some((a) => a.id === assetId)) {
return failure(`Unknown asset: ${assetId}`);
}
- const startSec = Math.min(parsed.data.startSec, parsed.data.endSec);
- const endSec = Math.max(parsed.data.startSec, parsed.data.endSec);
+ let startSec = Math.min(parsed.data.startSec, parsed.data.endSec);
+ let endSec = Math.max(parsed.data.startSec, parsed.data.endSec);
// Which clip the cut sits on. Named explicitly when the model says so; otherwise
// inferred from the source range — but only when the answer is unique. Two clips
@@ -1453,6 +1454,30 @@ export function executeAgentTool(
);
}
+ // Breath next to the speech that stays, and no one-frame flash against a cut
+ // already on the same clip — the same shaping the transcript pane applies.
+ const clip = document.timeline.clips.find((c) => c.id === clipId);
+ const siblings = document.timeline.trimRanges.filter((t) =>
+ clip ? trimAppliesToClip(t, clip) : t.assetId === assetId,
+ );
+ const transcript =
+ document.transcripts.find((t) => t.assetId === assetId) ??
+ (document.transcript?.assetId === assetId ? document.transcript : null);
+ const words = (transcript?.words ?? [])
+ .filter((w) => w.source !== "synth")
+ .map((w) => {
+ const mid = (w.startSec + w.endSec) / 2;
+ const inClip =
+ !clip || (mid >= clip.sourceStartSec && mid < (clip.sourceEndSec ?? Infinity));
+ const cut = siblings.some((t) => mid >= t.startSec && mid < t.endSec);
+ return { startSec: w.startSec, endSec: w.endSec, kept: inClip && !cut };
+ });
+ ({ startSec, endSec } = mergeCloseCuts(
+ breatheCut({ startSec, endSec }, words),
+ siblings,
+ document.assets.find((a) => a.id === assetId)?.video?.fps,
+ ));
+
const trim = {
id: createId("trim"),
assetId,
diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx
index 85145645d..b164bf65a 100644
--- a/src/components/ai-edition/NewEditorShell.tsx
+++ b/src/components/ai-edition/NewEditorShell.tsx
@@ -44,9 +44,11 @@ import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus";
import { useSequentialTimelineOps } from "@/lib/ai-edition/store/useSequentialTimelineOps";
import { useTimeline } from "@/lib/ai-edition/store/useTimeline";
import { isGeneratedAssetId } from "@/lib/ai-edition/timeline/clip-parts";
+import { mergeCloseCuts } from "@/lib/ai-edition/timeline/cut-breath";
import { newRegionDurationSec } from "@/lib/ai-edition/timeline/newRegionDuration";
import {
dropTrimPillsByIds,
+ trimAppliesToClip,
ventilateTimelineSpanToTrims,
} from "@/lib/ai-edition/timeline/trim-mapping";
import { firstTimelineBusyView } from "@/lib/ai-edition/transcription/status";
@@ -729,15 +731,24 @@ export function NewEditorShell() {
toast.error(te("errors.trimNoFilm"));
return;
}
- const rows = ranges.map((range) => ({
- id: createId("trim"),
- assetId: range.assetId,
- clipId: range.clipId,
- startSec: range.sourceStartSec,
- endSec: range.sourceEndSec,
- reason,
- origin: "user" as const,
- }));
+ const rows = ranges.map((range) => {
+ // No one-frame flash between this cut and one already on the same clip.
+ const merged = mergeCloseCuts(
+ { startSec: range.sourceStartSec, endSec: range.sourceEndSec },
+ doc.timeline.trimRanges.filter((t) =>
+ trimAppliesToClip(t, { id: range.clipId, assetId: range.assetId }),
+ ),
+ doc.assets.find((a) => a.id === range.assetId)?.video?.fps,
+ );
+ return {
+ id: createId("trim"),
+ assetId: range.assetId,
+ clipId: range.clipId,
+ ...merged,
+ reason,
+ origin: "user" as const,
+ };
+ });
await saveDocument(
{
...doc,
diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx
index d38872432..c04736717 100644
--- a/src/components/ai-edition/RightPanes.tsx
+++ b/src/components/ai-edition/RightPanes.tsx
@@ -90,6 +90,7 @@ import {
voiceoverPlacements,
} from "@/lib/ai-edition/timeline/aggregated-transcript";
import { assetCameraSource, hasAnyClipWithCamera } from "@/lib/ai-edition/timeline/camera";
+import { breatheCut } from "@/lib/ai-edition/timeline/cut-breath";
import { formatMs } from "@/lib/ai-edition/timeline/format";
import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time";
import type {
@@ -1174,12 +1175,19 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({
pendingCaretWordIdRef.current = keptRange[0].id;
const startSec = Math.min(...keptRange.map((w) => w.word.startSec));
const endSec = Math.max(...keptRange.map((w) => w.word.endSec));
+ // Breath next to the speech that stays, so a cut silence does not glue two words.
+ const breathed = breatheCut(
+ { startSec, endSec },
+ words
+ .filter((cw) => !isSilenceWord(cw.word) && !isInsertedWord(cw.word))
+ .map((cw) => ({ startSec: cw.word.startSec, endSec: cw.word.endSec, kept: cw.kept })),
+ );
onTrimTimelineSpan(
- ...toRawSpan(startSec, endSec),
+ ...toRawSpan(breathed.startSec, breathed.endSec),
`Skip ${formatMs(startSec * 1000)}-${formatMs(endSec * 1000)} from ${clip.assetId}.`,
);
},
- [busy, clip.assetId, toRawSpan, onTrimTimelineSpan, onRemoveWords],
+ [busy, clip.assetId, words, toRawSpan, onTrimTimelineSpan, onRemoveWords],
);
const removeTrimRun = useCallback(
diff --git a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx
index a19330610..859f7aede 100644
--- a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx
+++ b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx
@@ -72,13 +72,14 @@ function renderPane(
trimRanges: AxcutTrimRange[],
onTrimTimelineSpan = vi.fn(),
busyAssetIds: string[] = [],
+ transcript: AxcutTranscript = TRANSCRIPT,
) {
const view = render(
<I18nProvider>
<TranscriptPane
clips={[CLIP]}
audioTracks={[]}
- transcripts={[TRANSCRIPT]}
+ transcripts={[transcript]}
assets={[ASSET]}
trimRanges={trimRanges}
busyAssetIds={busyAssetIds}
@@ -266,3 +267,37 @@ describe("keyboard cut with the caret between words", () => {
expect(struck).toEqual(["clip_1:w1", "clip_1:w2", "clip_1:w3", "clip_1:w4"]);
});
});
+
+// Cutting a silence must leave air next to the speech that stays, or the last syllable is
+// glued to the next word. With 500 ms gaps, each side keeps 70 ms.
+describe("a cut silence breathes", () => {
+ const GAPPED: AxcutTranscript = {
+ ...TRANSCRIPT,
+ words: [
+ { id: "w1", segmentId: "s", startSec: 0, endSec: 1, text: "un" },
+ { id: "w2", segmentId: "s", startSec: 1.5, endSec: 2.5, text: "deux" },
+ { id: "w3", segmentId: "s", startSec: 3, endSec: 4, text: "trois" },
+ ],
+ };
+ // un, [silence], deux, [silence], trois
+ const BEFORE_TROIS = 4;
+
+ it("keeps breath on both sides of a silence between two kept words", () => {
+ const { editor, onTrimTimelineSpan } = renderPane([], vi.fn(), [], GAPPED);
+ caretBeforeWordAt(editor, BEFORE_TROIS);
+ fireEvent.keyDown(editor, { key: "Backspace" });
+ const cut = cutRange(onTrimTimelineSpan);
+ expect(cut?.[0]).toBeCloseTo(2.57);
+ expect(cut?.[1]).toBeCloseTo(2.93);
+ });
+
+ it("does not breathe next to a word that is already cut", () => {
+ const deuxCut = { ...W2_TRIMMED, startSec: 1.5, endSec: 2.5 };
+ const { editor, onTrimTimelineSpan } = renderPane([deuxCut], vi.fn(), [], GAPPED);
+ caretBeforeWordAt(editor, BEFORE_TROIS);
+ fireEvent.keyDown(editor, { key: "Backspace" });
+ const cut = cutRange(onTrimTimelineSpan);
+ expect(cut?.[0]).toBe(2.5);
+ expect(cut?.[1]).toBeCloseTo(2.93);
+ });
+});
diff --git a/src/lib/ai-edition/timeline/cut-breath.test.ts b/src/lib/ai-edition/timeline/cut-breath.test.ts
new file mode 100644
index 000000000..f6e33b020
--- /dev/null
+++ b/src/lib/ai-edition/timeline/cut-breath.test.ts
@@ -0,0 +1,108 @@
+import { describe, expect, it } from "vitest";
+import { type BreathWord, breatheCut, mergeCloseCuts } from "./cut-breath";
+
+const w = (startSec: number, endSec: number, kept = true): BreathWord => ({
+ startSec,
+ endSec,
+ kept,
+});
+
+// A 1.0-1.5, B 2.0-2.4, C 3.0-3.5: two 0.5-0.6 s silences around B.
+const A = w(1, 1.5);
+const B = w(2, 2.4);
+const C = w(3, 3.5);
+
+describe("breatheCut", () => {
+ it("leaves 70 ms of air on each side of a cut silence", () => {
+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [A, B, C]);
+ expect(cut.startSec).toBeCloseTo(1.57);
+ expect(cut.endSec).toBeCloseTo(1.93);
+ });
+
+ it("word plus both silences: breath only at the kept edges, no sliver around the word", () => {
+ const cut = breatheCut({ startSec: 1.5, endSec: 3 }, [A, B, C]);
+ expect(cut.startSec).toBeCloseTo(1.57);
+ expect(cut.endSec).toBeCloseTo(2.93);
+ // B stays wholly inside the cut.
+ expect(cut.startSec).toBeLessThan(B.startSec);
+ expect(cut.endSec).toBeGreaterThan(B.endSec);
+ });
+
+ it("never goes past the middle of a short gap", () => {
+ const cut = breatheCut({ startSec: 1.5, endSec: 1.6 }, [A, w(1.6, 2)]);
+ expect(cut).toEqual({ startSec: 1.5, endSec: 1.6 });
+ const wider = breatheCut({ startSec: 1.5, endSec: 1.62 }, [A, w(1.62, 2)]);
+ expect(wider).toEqual({ startSec: 1.5, endSec: 1.62 });
+ });
+
+ it("breathes up to the middle when the gap has room for less than two breaths", () => {
+ // Gaps of 100 ms on each side of the word removed with them (1.6-1.9).
+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [A, w(1.6, 1.9), w(2, 2.5)]);
+ expect(cut.startSec).toBeCloseTo(1.55);
+ expect(cut.endSec).toBeCloseTo(1.95);
+ });
+
+ it("does not breathe next to a word that is already cut", () => {
+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [w(1, 1.5, false), B, C]);
+ expect(cut.startSec).toBe(1.5);
+ expect(cut.endSec).toBeCloseTo(1.93);
+ });
+
+ it("leaves an edge inside a word alone", () => {
+ expect(breatheCut({ startSec: 1.2, endSec: 2.2 }, [A, B, C])).toEqual({
+ startSec: 1.2,
+ endSec: 2.2,
+ });
+ });
+
+ it("leaves a cut on exact word bounds alone", () => {
+ expect(breatheCut({ startSec: 2, endSec: 2.4 }, [A, B, C])).toEqual({
+ startSec: 2,
+ endSec: 2.4,
+ });
+ });
+
+ it("ignores words with no duration", () => {
+ const cut = breatheCut({ startSec: 1.5, endSec: 2 }, [A, w(1.5, 1.5), w(2, 2), B]);
+ expect(cut.startSec).toBeCloseTo(1.57);
+ expect(cut.endSec).toBeCloseTo(1.93);
+ });
+
+ it("leaves a side alone when the cut is too short to give it back", () => {
+ // A 40 ms nibble just after A: breathing would push the start past the end.
+ expect(breatheCut({ startSec: 1.5, endSec: 1.54 }, [A, C])).toEqual({
+ startSec: 1.5,
+ endSec: 1.54,
+ });
+ });
+});
+
+describe("mergeCloseCuts", () => {
+ const fps = 30;
+
+ it("snaps onto a neighbour less than two frames away", () => {
+ expect(
+ mergeCloseCuts({ startSec: 2.05, endSec: 3 }, [{ startSec: 1, endSec: 2 }], fps),
+ ).toEqual({ startSec: 2, endSec: 3 });
+ expect(
+ mergeCloseCuts({ startSec: 1, endSec: 2 }, [{ startSec: 2.05, endSec: 3 }], fps),
+ ).toEqual({ startSec: 1, endSec: 2.05 });
+ });
+
+ it("keeps a gap of two frames or more", () => {
+ const cut = { startSec: 2.07, endSec: 3 };
+ expect(mergeCloseCuts(cut, [{ startSec: 1, endSec: 2 }], fps)).toEqual(cut);
+ });
+
+ it("uses the asset's frame rate, 30 fps when it has none", () => {
+ const cut = { startSec: 2.05, endSec: 3 };
+ const before = [{ startSec: 1, endSec: 2 }];
+ expect(mergeCloseCuts(cut, before, 60)).toEqual(cut);
+ expect(mergeCloseCuts(cut, before, 0)).toEqual({ startSec: 2, endSec: 3 });
+ });
+
+ it("ignores overlapping neighbours", () => {
+ const cut = { startSec: 1.5, endSec: 3 };
+ expect(mergeCloseCuts(cut, [{ startSec: 1, endSec: 2 }], fps)).toEqual(cut);
+ });
+});
diff --git a/src/lib/ai-edition/timeline/cut-breath.ts b/src/lib/ai-edition/timeline/cut-breath.ts
new file mode 100644
index 000000000..8af827a10
--- /dev/null
+++ b/src/lib/ai-edition/timeline/cut-breath.ts
@@ -0,0 +1,88 @@
+// Silence cuts that breathe.
+//
+// A cut that swallows the whole gap between two words glues the last syllable to the
+// next word. Speech needs a little air on each side, so the edge of a cut that sits in a
+// gap next to KEPT speech is pulled back by `CUT_BREATH_SEC`, never past the middle of
+// that gap. An edge that touches removed material, or sits inside a word, is left where
+// it is: breathing there would only leave a sliver of kept media between two cuts.
+//
+// Only cuts authored from speech go through this (the transcript pane, the agent). A cut
+// drawn on the timeline by hand is the user's and is never reshaped.
+
+import type { Interval } from "./intervals";
+
+/** Air kept between a cut and the speech it touches, per side (~140 ms across a cut). */
+export const CUT_BREATH_SEC = 0.07;
+
+/** Frame rate {@link mergeCloseCuts} assumes when the asset reports none (audio, unprobed). */
+const FALLBACK_FPS = 30;
+
+export interface BreathWord extends Interval {
+ /** Whether this word is still in the film, BEFORE the cut being shaped is applied. */
+ kept: boolean;
+}
+
+const EPS = 1e-6;
+
+/**
+ * `cut` with each edge that sits in a gap next to a kept word pulled back to leave
+ * `breathSec` of that gap, capped at the gap's middle. All times on one source clock.
+ * Words with no duration carry no audio and are ignored. A side the cut is too short to
+ * give back is left alone.
+ */
+export function breatheCut(
+ cut: Interval,
+ words: readonly BreathWord[],
+ breathSec = CUT_BREATH_SEC,
+): Interval {
+ const spoken = words.filter((w) => w.endSec > w.startSec);
+ let startSec = cut.startSec;
+ let endSec = cut.endSec;
+
+ // Start edge: the word that ends last at or before it, and the gap that follows it.
+ const before = spoken
+ .filter((w) => w.endSec <= cut.startSec + EPS)
+ .reduce<BreathWord | null>((a, w) => (!a || w.endSec > a.endSec ? w : a), null);
+ if (before?.kept) {
+ const nextStarts = spoken.filter((w) => w.startSec >= before.endSec - EPS && w !== before);
+ const gapEnd = Math.min(...nextStarts.map((w) => w.startSec));
+ // gapEnd below the edge means the edge is inside a word, not in the gap.
+ if (gapEnd >= cut.startSec - EPS) {
+ const target = Math.min(before.endSec + breathSec, (before.endSec + gapEnd) / 2);
+ if (target > startSec && target < cut.endSec) startSec = target;
+ }
+ }
+
+ // End edge, mirrored: the word that starts first at or after it, and the gap before it.
+ const after = spoken
+ .filter((w) => w.startSec >= cut.endSec - EPS)
+ .reduce<BreathWord | null>((a, w) => (!a || w.startSec < a.startSec ? w : a), null);
+ if (after?.kept) {
+ const prevEnds = spoken.filter((w) => w.endSec <= after.startSec + EPS && w !== after);
+ const gapStart = Math.max(...prevEnds.map((w) => w.endSec));
+ if (gapStart <= cut.endSec + EPS) {
+ const target = Math.max(after.startSec - breathSec, (gapStart + after.startSec) / 2);
+ if (target < endSec && target > cut.startSec) endSec = target;
+ }
+ }
+
+ return endSec > startSec ? { startSec, endSec } : cut;
+}
+
+/**
+ * `cut` with an edge snapped onto a neighbouring cut of the SAME clip when the media
+ * left between them would be shorter than two frames — a one-frame flash of picture
+ * between two cuts reads as a glitch, never as content. `neighbours` are that clip's
+ * existing cuts, on the same clock as `cut`; `fps` is the asset's, 0 when unknown.
+ */
+export function mergeCloseCuts(cut: Interval, neighbours: readonly Interval[], fps = 0): Interval {
+ const maxGap = 2 / (fps > 0 ? fps : FALLBACK_FPS);
+ let { startSec, endSec } = cut;
+ for (const n of neighbours) {
+ const gapBefore = cut.startSec - n.endSec;
+ if (gapBefore > 0 && gapBefore < maxGap) startSec = Math.min(startSec, n.endSec);
+ const gapAfter = n.startSec - cut.endSec;
+ if (gapAfter > 0 && gapAfter < maxGap) endSec = Math.max(endSec, n.startSec);
+ }
+ return { startSec, endSec };
+}