Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(audio): keep each take on its own clips through a clip edit (#1011)
EtienneLescotf22cc8ae6e7a
diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.tsindex b0055cb3a..5af2592ef 100644--- a/src/lib/ai-edition/document/audioTracks.ts+++ b/src/lib/ai-edition/document/audioTracks.ts@@ -19,7 +19,11 @@import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "../schema";import { isGeneratedAssetId } from "../timeline/clip-parts";-import { anchorRegionsWithDerivedMs, clampSpanAgainstNeighbours } from "../timeline/timelineMap";+import {+ anchorRegionsWithDerivedMs,+ clampSpanAgainstNeighbours,+ hasCompleteClipAnchor,+} from "../timeline/timelineMap";/** Every fragment of one user-visible track shares this key. */export function trackGroupId(track: AxcutAudioTrack): string {@@ -75,14 +79,79 @@ export function reanchorAudioTracks(clips: AxcutClip[],makeId: () => string,): AxcutAudioTrack[] {- // Coalesce back to one raw span per track FIRST: re-anchoring the stored- // fragments individually would re-ventilate each one and multiply them.+ // Coalesce back to one raw span per take FIRST: re-anchoring the stored+ // fragments individually would re-ventilate each one and multiply them. Per+ // take, not per `trackId`: the edit may have pulled one take apart.//// Joined at the PILL level, before ventilation, for the same reason: ventilation// deliberately produces fragments that meet and whose offsets continue, so joining// after it would undo the split it just made.- return joinContiguousTakes(collapseTracksToPills(tracks)).flatMap((track) =>- anchorAudioTrackFragments(track, clips, makeId),+ return joinContiguousTakes(collapseTracksToPills(splitDetachedTakes(tracks, clips))).flatMap(+ (track) => anchorAudioTrackFragments(track, clips, makeId),+ );+}++/**+ * Give each stretch of a take that no longer hangs together a `trackId` of its own (#1011).+ *+ * A structural clip edit carries every fragment with its own clip, so after a reorder the+ * fragments of one take may no longer meet, or meet with the file jumping. Folded into ONE span+ * per `trackId`, as every reader folds them, the take swallowed whatever clip now sat between its+ * pieces: re-anchoring that span gave the clip audio it never had and ran `offsetMs` past the end+ * of the file. Cut where it came apart, each piece keeps its clip, its length and its `offsetMs`,+ * and is a take of its own, keyed by its first fragment.+ */+function splitDetachedTakes(tracks: AxcutAudioTrack[], clips: AxcutClip[]): AxcutAudioTrack[] {+ const assetOf = new Map(clips.map((clip) => [clip.id, clip.assetId]));+ const takes = new Map<string, AxcutAudioTrack[]>();+ for (const track of tracks) {+ const key = trackGroupId(track);+ const bucket = takes.get(key);+ if (bucket) bucket.push(track);+ else takes.set(key, [track]);+ }+ return [...takes.values()].flatMap((fragments) => {+ const ordered = [...fragments].sort((a, b) => a.startMs - b.startMs);+ let head = ordered[0];+ return ordered.map((fragment, index) => {+ const previous = ordered[index - 1];+ if (previous && !hangsTogether(previous, fragment, assetOf)) head = fragment;+ return { ...fragment, trackId: head.id };+ });+ });+}++/**+ * The file runs on from `left` into `right`, and nothing plays between them: they meet on the+ * ruler, or sit either side of an inserted word.+ *+ * Running on includes keeping the offset: a loop never advances it, and re-ventilating an orphan+ * or an unanchored take copies it verbatim into every piece, which re-anchoring then repairs.+ * Within a millisecond, because a re-laid boundary rounds to whole ms.+ */+function hangsTogether(+ left: AxcutAudioTrack,+ right: AxcutAudioTrack,+ assetOf: Map<string, string>,+): boolean {+ const runsOn =+ Math.abs(left.offsetMs - right.offsetMs) <= 1 ||+ (!right.loop && Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1);+ return runsOn && (Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf));+}++/** One stretch of one media, cut in two: what an inserted word leaves either side of it. */+function mediaRunsOn(+ left: AxcutAudioTrack,+ right: AxcutAudioTrack,+ assetOf: Map<string, string>,+): boolean {+ if (!hasCompleteClipAnchor(left) || !hasCompleteClipAnchor(right)) return false;+ const assetId = assetOf.get(left.clipId);+ return (+ assetId !== undefined &&+ assetId === assetOf.get(right.clipId) &&+ Math.abs(left.sourceEndSec - right.sourceStartSec) < 0.001);}@@ -109,21 +178,35 @@ function joinContiguousTakes(pills: AxcutAudioTrack[]): AxcutAudioTrack[] {}/** Same file, meeting on the ruler, and the file's own timecode continuing across the join —- * plus every payload the two would otherwise have to disagree about. */+ * plus every payload the two would otherwise have to disagree about. A loop's offset never+ * advances, so its file says nothing about the join: two loops at one offset are one take when+ * neither fades across the seam, which is the seam splitting one leaves (#1011). */function takesJoin(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean {const spanMs = left.endMs - left.startMs;return (left.assetId === right.assetId &&left.kind === right.kind &&- !left.loop &&- !right.loop &&+ left.loop === right.loop &&left.gainDb === right.gainDb &&left.muted === right.muted &&- Math.abs(left.endMs - right.startMs) < 1 &&- Math.abs(left.offsetMs + spanMs - right.offsetMs) < 1+ Math.abs(left.endMs - right.startMs) <= 1 &&+ (left.loop+ ? left.offsetMs === right.offsetMs && left.fadeOutMs === 0 && right.fadeInMs === 0+ : Math.abs(left.offsetMs + spanMs - right.offsetMs) <= 1));}+/**+ * A clip edit that cuts a fragment's head off cuts the same stretch off its take's file, so what+ * is left goes on playing the audio over its own footage (#1011). A loop restarts on its own and+ * keeps its offset; a region with no file passes through.+ */+export function cutTakeHead<T extends object>(region: T, cutSec: number): T {+ const take = region as T & { offsetMs?: unknown; loop?: unknown };+ if (!(cutSec > 0) || typeof take.offsetMs !== "number" || take.loop === true) return region;+ return { ...region, offsetMs: take.offsetMs + Math.round(cutSec * 1000) };+}+/*** The user-visible tracks: fragments folded back into one span per `trackId`,* carrying the FIRST fragment's payload (its `offsetMs` is the track's realdiff --git a/src/lib/ai-edition/document/insertion.test.ts b/src/lib/ai-edition/document/insertion.test.tsindex 23efb0dae..8e87531a2 100644--- a/src/lib/ai-edition/document/insertion.test.ts+++ b/src/lib/ai-edition/document/insertion.test.ts@@ -179,6 +179,55 @@ describe("removeGeneratedClips", () => {});});+describe("a music bed over the cut (#1011)", () => {+ // 2–8 s of c1, so the word typed at 4 s cuts it in two.+ const withBed = () =>+ doc({+ audioTracks: [+ {+ id: "bed",+ trackId: "bed",+ assetId: "music",+ kind: "music",+ clipId: "c1",+ sourceStartSec: 2,+ sourceEndSec: 8,+ startMs: 2000,+ endMs: 8000,+ durationSec: 30,+ offsetMs: 0,+ gainDb: -18,+ loop: false,+ fadeInMs: 1000,+ fadeOutMs: 1000,+ muted: false,+ label: "bed",+ origin: "user",+ },+ ],+ } as Partial<AxcutDocument>);+ /** Each piece of the bed in ruler order: c1, the word, or the right half (a minted id). */+ const spans = (d: AxcutDocument) =>+ [...d.audioTracks]+ .sort((a, b) => a.startMs - b.startMs)+ .map((t) => [+ t.clipId?.startsWith("clip_") ? "right" : t.clipId,+ t.startMs,+ t.endMs,+ t.offsetMs,+ ]);++ it("plays on through the word, as one take", () => {+ const next = withInsertion(withBed());+ expect(spans(next)).toEqual([+ ["c1", 2000, 4000, 0],+ ["ext:synth_1", 4000, 4150, 2000],+ ["right", 4150, 8150, 2150],+ ]);+ expect(new Set(next.audioTracks.map((t) => t.trackId)).size).toBe(1);+ });+});+describe("deleting the amber clip from the TIMELINE, not from the transcript", () => {// The trash icon on the clip calls `removeClip`, and so does the agent's tool. Neither// goes through `removeGeneratedClips`, so if the media were only dropped there, thediff --git a/src/lib/ai-edition/document/timeline.test.ts b/src/lib/ai-edition/document/timeline.test.tsindex a4eb137f3..a33f1c6e7 100644--- a/src/lib/ai-edition/document/timeline.test.ts+++ b/src/lib/ai-edition/document/timeline.test.ts@@ -5,6 +5,7 @@ import {type AxcutTrimRange,axcutSchemaVersion,} from "../schema";+import { audioLanePills } from "./audioTracks";import {buildTimelineFromIntervals,duplicateClip,@@ -25,6 +26,7 @@ import {setClipSourceRange,subtractInterval,timelineIntervals,+ withClipsChanged,} from "./timeline";function makeDoc(overrides: Partial<AxcutDocument> = {}): AxcutDocument {@@ -1081,7 +1083,7 @@ describe("duplicateClip / moveClip", () => {});// Left out on purpose, see `duplicateClip`: an imported take is not a modifier of the- // clip, and the audio fold (#1011) has no safe place for a copied fragment.+ // clip.it("duplicateClip leaves an imported audio take on the original only", () => {const take: AxcutDocument["audioTracks"][number] = {id: "vo",@@ -1264,6 +1266,369 @@ describe("duplicateClip / moveClip", () => {});});+// #1011. A take is stored as one fragment per clip it covers, and a structural edit carries+// each fragment with its own clip. Folding every fragment of a `trackId` back into one span+// afterwards, first to last, swallowed whatever clip had come to sit between them.+describe("imported audio stays on the clips it was laid over (#1011)", () => {+ type AudioTrack = AxcutDocument["audioTracks"][number];+ const fragment = (+ over: Pick<+ AudioTrack,+ "id" | "trackId" | "clipId" | "sourceStartSec" | "sourceEndSec" | "startMs" | "endMs"+ > &+ Partial<AudioTrack>,+ ): AudioTrack => ({+ assetId: "bed",+ kind: "music",+ durationSec: 30,+ offsetMs: 0,+ gainDb: -18,+ loop: false,+ fadeInMs: 0,+ fadeOutMs: 0,+ muted: false,+ label: "",+ origin: "user",+ ...over,+ });+ /** Every fragment in ruler order: which file, on which clip, where, and from where in the file. */+ const spans = (doc: AxcutDocument) =>+ [...doc.audioTracks]+ .sort((a, b) => a.startMs - b.startMs)+ .map((t) => [t.assetId, t.clipId, t.startMs, t.endMs, t.offsetMs]);+ /** What each clip carries: the file, the stretch of the clip, the place in the file. */+ const carried = (doc: AxcutDocument) =>+ doc.audioTracks+ .map((t) =>+ [+ t.clipId,+ t.assetId,+ Math.round((t.sourceStartSec ?? Number.NaN) * 1000),+ Math.round((t.sourceEndSec ?? Number.NaN) * 1000),+ t.offsetMs,+ ].join(" "),+ )+ .sort();+ const musicLane = (doc: AxcutDocument) =>+ audioLanePills(doc.audioTracks, "music").map((p) => [p.startMs, p.endMs]);++ describe("the report's project", () => {+ // Three clips; a 30 s bed across the first junction, a looping 4 s track across the+ // second, both music.+ const reported = () =>+ makeDoc({+ timeline: {+ ...makeDoc().timeline,+ clips: [+ makeClip({+ id: "b6cdac",+ assetId: "rec_a",+ sourceEndSec: 102.1,+ timelineEndSec: 102.1,+ }),+ makeClip({+ id: "63fedd",+ assetId: "rec_b",+ sourceEndSec: 61.87,+ timelineStartSec: 102.1,+ timelineEndSec: 163.97,+ }),+ makeClip({+ id: "d51b65",+ assetId: "rec_a",+ sourceEndSec: 102.1,+ timelineStartSec: 163.97,+ timelineEndSec: 266.07,+ }),+ ],+ },+ audioTracks: [+ fragment({+ id: "bed",+ trackId: "bed",+ clipId: "b6cdac",+ sourceStartSec: 89.869,+ sourceEndSec: 102.1,+ startMs: 89_869,+ endMs: 102_100,+ fadeInMs: 1000,+ }),+ fragment({+ id: "bed_2",+ trackId: "bed",+ clipId: "63fedd",+ sourceStartSec: 0,+ sourceEndSec: 17.769,+ startMs: 102_100,+ endMs: 119_869,+ offsetMs: 12_231,+ fadeOutMs: 1000,+ }),+ fragment({+ id: "loop",+ trackId: "loop",+ assetId: "loop",+ durationSec: 4,+ loop: true,+ clipId: "63fedd",+ sourceStartSec: 17.769,+ sourceEndSec: 61.87,+ startMs: 119_869,+ endMs: 163_970,+ fadeInMs: 1000,+ }),+ fragment({+ id: "loop_2",+ trackId: "loop",+ assetId: "loop",+ durationSec: 4,+ loop: true,+ clipId: "d51b65",+ sourceStartSec: 0,+ sourceEndSec: 36.007,+ startMs: 163_970,+ endMs: 199_977,+ fadeOutMs: 1000,+ }),+ ],+ });++ it("moves each fragment with its own clip when the middle clip goes to the end", () => {+ const before = reported();+ const moved = moveClip(before, "63fedd", 2);+ // Was: the bed stretched over d51b65 at offset 12231, and the loop pushed to 221.969–+ // 385.939 s, past the 266.07 s end.+ expect(spans(moved)).toEqual([+ ["bed", "b6cdac", 89_869, 102_100, 0],+ ["loop", "d51b65", 102_100, 138_107, 0],+ ["bed", "63fedd", 204_200, 221_969, 12_231],+ ["loop", "63fedd", 221_969, 266_070, 0],+ ]);+ expect(carried(moved)).toEqual(carried(before));+ // Four takes now, side by side on the one music row: nothing for the lane to push.+ expect(musicLane(moved)).toEqual([+ [89_869, 102_100],+ [102_100, 138_107],+ [204_200, 221_969],+ [221_969, 266_070],+ ]);+ });++ it("puts both takes back together when the clip moves back", () => {+ const before = reported();+ const back = moveClip(moveClip(before, "63fedd", 2), "63fedd", 1);+ expect(spans(back)).toEqual(spans(before));+ expect(carried(back)).toEqual(carried(before));+ expect(musicLane(back)).toEqual([+ [89_869, 119_869],+ [119_869, 199_977],+ ]);+ expect(back.audioTracks.map((t) => [t.trackId, t.fadeInMs, t.fadeOutMs])).toEqual([+ ["bed", 1000, 0],+ ["bed", 0, 1000],+ ["loop", 1000, 0],+ ["loop", 0, 1000],+ ]);+ });+ });++ it("keeps each half's place in the file when the two clips under a take swap", () => {+ // The halves still meet on the ruler; only the file says they are no longer one take.+ const doc = makeDoc({+ timeline: {+ ...makeDoc().timeline,+ clips: [+ makeClip({ id: "clip_a", assetId: "rec_a", sourceEndSec: 10, timelineEndSec: 10 }),+ makeClip({+ id: "clip_b",+ assetId: "rec_b",+ sourceEndSec: 10,+ timelineStartSec: 10,+ timelineEndSec: 20,+ }),+ ],+ },+ audioTracks: [+ fragment({+ id: "bed",+ trackId: "bed",+ clipId: "clip_a",+ sourceStartSec: 0,+ sourceEndSec: 10,+ startMs: 0,+ endMs: 10_000,+ }),+ fragment({+ id: "bed_2",+ trackId: "bed",+ clipId: "clip_b",+ sourceStartSec: 0,+ sourceEndSec: 10,+ startMs: 10_000,+ endMs: 20_000,+ offsetMs: 10_000,+ }),+ ],+ });+ expect(spans(moveClip(doc, "clip_a", 1))).toEqual([+ ["bed", "clip_b", 0, 10_000, 10_000],+ ["bed", "clip_a", 10_000, 20_000, 0],+ ]);+ });++ describe("a bed across three clips", () => {+ // A: rec_a 0–10 s, B: rec_b 0–10 s, C: rec_a 20–30 s; the bed covers 5–25 s on the ruler.+ const threeClips = () =>+ makeDoc({+ timeline: {+ ...makeDoc().timeline,+ clips: [+ makeClip({ id: "clip_a", assetId: "rec_a", sourceEndSec: 10, timelineEndSec: 10 }),+ makeClip({+ id: "clip_b",+ assetId: "rec_b",+ sourceEndSec: 10,+ timelineStartSec: 10,+ timelineEndSec: 20,+ }),+ makeClip({+ id: "clip_c",+ assetId: "rec_a",+ sourceStartSec: 20,+ sourceEndSec: 30,+ timelineStartSec: 20,+ timelineEndSec: 30,+ }),+ ],+ },+ audioTracks: [+ fragment({+ id: "bed",+ trackId: "bed",+ clipId: "clip_a",+ sourceStartSec: 5,+ sourceEndSec: 10,+ startMs: 5000,+ endMs: 10_000,+ }),+ fragment({+ id: "bed_2",+ trackId: "bed",+ clipId: "clip_b",+ sourceStartSec: 0,+ sourceEndSec: 10,+ startMs: 10_000,+ endMs: 20_000,+ offsetMs: 5000,+ }),+ fragment({+ id: "bed_3",+ trackId: "bed",+ clipId: "clip_c",+ sourceStartSec: 20,+ sourceEndSec: 25,+ startMs: 20_000,+ endMs: 25_000,+ offsetMs: 15_000,+ }),+ ],+ });++ it("gives a clip inserted inside it none of it", () => {+ const before = threeClips();+ const added = makeClip({+ id: "clip_new",+ assetId: "rec_c",+ sourceEndSec: 4,+ timelineEndSec: 4,+ });+ const [a, ...rest] = before.timeline.clips;+ const next = withClipsChanged(before, [a, added, ...rest]);+ expect(carried(next)).toEqual(carried(before));+ expect(spans(next)).toEqual([+ ["bed", "clip_a", 5000, 10_000, 0],+ ["bed", "clip_b", 14_000, 24_000, 5000],+ ["bed", "clip_c", 24_000, 29_000, 15_000],+ ]);+ });++ it("gives a duplicated clip none of it", () => {+ const before = threeClips();+ const next = duplicateClip(before, "clip_a");+ expect(carried(next)).toEqual(carried(before));+ });++ it("keeps the rest of it on its own footage when a clip under it is deleted", () => {+ const next = removeClip(threeClips(), "clip_b");+ expect(spans(next)).toEqual([+ ["bed", "clip_a", 5000, 10_000, 0],+ ["bed", "clip_c", 10_000, 15_000, 15_000],+ ]);+ });++ it("cuts the file with the footage when a clip under it is narrowed", () => {+ // Two seconds off clip_b's head take the two seconds of bed that played over them:+ // the jump in the file is where the footage was cut, and clip_c keeps its music.+ const next = setClipSourceRange(threeClips(), "clip_b", 2, 10);+ expect(spans(next)).toEqual([+ ["bed", "clip_a", 5000, 10_000, 0],+ ["bed", "clip_b", 10_000, 18_000, 7000],+ ["bed", "clip_c", 18_000, 23_000, 15_000],+ ]);+ expect(musicLane(next)).toEqual([+ [5000, 10_000],+ [10_000, 23_000],+ ]);+ });+ });++ it("still advances the offset of a fragment a rebuild re-cuts across two clips", () => {+ // The repair the fold was added for: re-ventilating an orphan copies `offsetMs`+ // verbatim into every piece, and the second piece must pick up where the first stops.+ const doc = makeDoc({+ timeline: {+ ...makeDoc().timeline,+ clips: [+ makeClip({ id: "clip_a", sourceEndSec: 10, timelineEndSec: 10 }),+ makeClip({+ id: "clip_b",+ sourceStartSec: 20,+ sourceEndSec: 30,+ timelineStartSec: 10,+ timelineEndSec: 20,+ }),+ ],+ },+ audioTracks: [+ fragment({+ id: "bed",+ trackId: "bed",+ clipId: "clip_b",+ sourceStartSec: 22,+ sourceEndSec: 28,+ startMs: 12_000,+ endMs: 18_000,+ offsetMs: 1000,+ }),+ ],+ });+ const rebuilt = replaceTimeline(+ doc,+ [+ { startSec: 0, endSec: 10 },+ { startSec: 20, endSec: 24 },+ { startSec: 26, endSec: 30 },+ ],+ "split clip_b",+ );+ expect(spans(rebuilt).map(([, , start, end, offset]) => [start, end, offset])).toEqual([+ [12_000, 14_000, 1000],+ [14_000, 18_000, 3000],+ ]);+ expect(musicLane(rebuilt)).toEqual([[12_000, 18_000]]);+ });+});+function makeZoom(overrides: Partial<AxcutDocument["zoomRanges"][number]> = {}) {return {id: "z1",diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.tsindex 53b7c1120..06e7964ed 100644--- a/src/lib/ai-edition/document/timeline.ts+++ b/src/lib/ai-edition/document/timeline.ts@@ -25,6 +25,7 @@ import {} from "../timeline/timelineMap";import { dropTrimPillsByIds, trimAppliesToClip } from "../timeline/trim-mapping";import {+ cutTakeHead,dropUnusedGeneratedMedia,reanchorAudioTracks,removeAudioTrack,@@ -475,7 +476,7 @@ function rederiveAnchoredRegion<if (!span) return [];return [{- ...region,+ ...cutTakeHead(region, sourceStartSec - region.sourceStartSec),sourceStartSec,sourceEndSec,startMs: Math.round(span.startSec * 1000),@@ -1007,8 +1008,7 @@ export function moveClip(// The modifiers anchored to the clip are copied the same way, across every collection// `mapAllRegionCollections` walks; `withClipsChanged` then places the copies on the new// clip. Imported audio is left out on purpose: a take is media on its own lane, not an-// effect on the clip, and a copied fragment cannot be placed safely while every structural-// edit folds each `trackId` back into one span (`reanchorAudioTracks`, #1011).+// effect on the clip.export function duplicateClip(document: AxcutDocument,clipId: string,