Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(audio): keep each take on its own clips through a clip edit (#1011)

EtienneLescotf22cc8ae6e7a
diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.ts
index b0055cb3a..5af2592ef 100644
--- a/src/lib/ai-edition/document/audioTracks.ts
+++ b/src/lib/ai-edition/document/audioTracks.ts
@@ -19,7 +19,11 @@
import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "../schema";
import { isGeneratedAssetId } from "../timeline/clip-parts";
-import { anchorRegionsWithDerivedMs, clampSpanAgainstNeighbours } from "../timeline/timelineMap";
+import {
+ anchorRegionsWithDerivedMs,
+ clampSpanAgainstNeighbours,
+ hasCompleteClipAnchor,
+} from "../timeline/timelineMap";
/** Every fragment of one user-visible track shares this key. */
export function trackGroupId(track: AxcutAudioTrack): string {
@@ -75,14 +79,79 @@ export function reanchorAudioTracks(
clips: AxcutClip[],
makeId: () => string,
): AxcutAudioTrack[] {
- // Coalesce back to one raw span per track FIRST: re-anchoring the stored
- // fragments individually would re-ventilate each one and multiply them.
+ // Coalesce back to one raw span per take FIRST: re-anchoring the stored
+ // fragments individually would re-ventilate each one and multiply them. Per
+ // take, not per `trackId`: the edit may have pulled one take apart.
//
// Joined at the PILL level, before ventilation, for the same reason: ventilation
// deliberately produces fragments that meet and whose offsets continue, so joining
// after it would undo the split it just made.
- return joinContiguousTakes(collapseTracksToPills(tracks)).flatMap((track) =>
- anchorAudioTrackFragments(track, clips, makeId),
+ return joinContiguousTakes(collapseTracksToPills(splitDetachedTakes(tracks, clips))).flatMap(
+ (track) => anchorAudioTrackFragments(track, clips, makeId),
+ );
+}
+
+/**
+ * Give each stretch of a take that no longer hangs together a `trackId` of its own (#1011).
+ *
+ * A structural clip edit carries every fragment with its own clip, so after a reorder the
+ * fragments of one take may no longer meet, or meet with the file jumping. Folded into ONE span
+ * per `trackId`, as every reader folds them, the take swallowed whatever clip now sat between its
+ * pieces: re-anchoring that span gave the clip audio it never had and ran `offsetMs` past the end
+ * of the file. Cut where it came apart, each piece keeps its clip, its length and its `offsetMs`,
+ * and is a take of its own, keyed by its first fragment.
+ */
+function splitDetachedTakes(tracks: AxcutAudioTrack[], clips: AxcutClip[]): AxcutAudioTrack[] {
+ const assetOf = new Map(clips.map((clip) => [clip.id, clip.assetId]));
+ const takes = new Map<string, AxcutAudioTrack[]>();
+ for (const track of tracks) {
+ const key = trackGroupId(track);
+ const bucket = takes.get(key);
+ if (bucket) bucket.push(track);
+ else takes.set(key, [track]);
+ }
+ return [...takes.values()].flatMap((fragments) => {
+ const ordered = [...fragments].sort((a, b) => a.startMs - b.startMs);
+ let head = ordered[0];
+ return ordered.map((fragment, index) => {
+ const previous = ordered[index - 1];
+ if (previous && !hangsTogether(previous, fragment, assetOf)) head = fragment;
+ return { ...fragment, trackId: head.id };
+ });
+ });
+}
+
+/**
+ * The file runs on from `left` into `right`, and nothing plays between them: they meet on the
+ * ruler, or sit either side of an inserted word.
+ *
+ * Running on includes keeping the offset: a loop never advances it, and re-ventilating an orphan
+ * or an unanchored take copies it verbatim into every piece, which re-anchoring then repairs.
+ * Within a millisecond, because a re-laid boundary rounds to whole ms.
+ */
+function hangsTogether(
+ left: AxcutAudioTrack,
+ right: AxcutAudioTrack,
+ assetOf: Map<string, string>,
+): boolean {
+ const runsOn =
+ Math.abs(left.offsetMs - right.offsetMs) <= 1 ||
+ (!right.loop && Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1);
+ return runsOn && (Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf));
+}
+
+/** One stretch of one media, cut in two: what an inserted word leaves either side of it. */
+function mediaRunsOn(
+ left: AxcutAudioTrack,
+ right: AxcutAudioTrack,
+ assetOf: Map<string, string>,
+): boolean {
+ if (!hasCompleteClipAnchor(left) || !hasCompleteClipAnchor(right)) return false;
+ const assetId = assetOf.get(left.clipId);
+ return (
+ assetId !== undefined &&
+ assetId === assetOf.get(right.clipId) &&
+ Math.abs(left.sourceEndSec - right.sourceStartSec) < 0.001
);
}
@@ -109,21 +178,35 @@ function joinContiguousTakes(pills: AxcutAudioTrack[]): AxcutAudioTrack[] {
}
/** Same file, meeting on the ruler, and the file's own timecode continuing across the join —
- * plus every payload the two would otherwise have to disagree about. */
+ * plus every payload the two would otherwise have to disagree about. A loop's offset never
+ * advances, so its file says nothing about the join: two loops at one offset are one take when
+ * neither fades across the seam, which is the seam splitting one leaves (#1011). */
function takesJoin(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean {
const spanMs = left.endMs - left.startMs;
return (
left.assetId === right.assetId &&
left.kind === right.kind &&
- !left.loop &&
- !right.loop &&
+ left.loop === right.loop &&
left.gainDb === right.gainDb &&
left.muted === right.muted &&
- Math.abs(left.endMs - right.startMs) < 1 &&
- Math.abs(left.offsetMs + spanMs - right.offsetMs) < 1
+ Math.abs(left.endMs - right.startMs) <= 1 &&
+ (left.loop
+ ? left.offsetMs === right.offsetMs && left.fadeOutMs === 0 && right.fadeInMs === 0
+ : Math.abs(left.offsetMs + spanMs - right.offsetMs) <= 1)
);
}
+/**
+ * A clip edit that cuts a fragment's head off cuts the same stretch off its take's file, so what
+ * is left goes on playing the audio over its own footage (#1011). A loop restarts on its own and
+ * keeps its offset; a region with no file passes through.
+ */
+export function cutTakeHead<T extends object>(region: T, cutSec: number): T {
+ const take = region as T & { offsetMs?: unknown; loop?: unknown };
+ if (!(cutSec > 0) || typeof take.offsetMs !== "number" || take.loop === true) return region;
+ return { ...region, offsetMs: take.offsetMs + Math.round(cutSec * 1000) };
+}
+
/**
* The user-visible tracks: fragments folded back into one span per `trackId`,
* carrying the FIRST fragment's payload (its `offsetMs` is the track's real
diff --git a/src/lib/ai-edition/document/insertion.test.ts b/src/lib/ai-edition/document/insertion.test.ts
index 23efb0dae..8e87531a2 100644
--- a/src/lib/ai-edition/document/insertion.test.ts
+++ b/src/lib/ai-edition/document/insertion.test.ts
@@ -179,6 +179,55 @@ describe("removeGeneratedClips", () => {
});
});
+describe("a music bed over the cut (#1011)", () => {
+ // 2–8 s of c1, so the word typed at 4 s cuts it in two.
+ const withBed = () =>
+ doc({
+ audioTracks: [
+ {
+ id: "bed",
+ trackId: "bed",
+ assetId: "music",
+ kind: "music",
+ clipId: "c1",
+ sourceStartSec: 2,
+ sourceEndSec: 8,
+ startMs: 2000,
+ endMs: 8000,
+ durationSec: 30,
+ offsetMs: 0,
+ gainDb: -18,
+ loop: false,
+ fadeInMs: 1000,
+ fadeOutMs: 1000,
+ muted: false,
+ label: "bed",
+ origin: "user",
+ },
+ ],
+ } as Partial<AxcutDocument>);
+ /** Each piece of the bed in ruler order: c1, the word, or the right half (a minted id). */
+ const spans = (d: AxcutDocument) =>
+ [...d.audioTracks]
+ .sort((a, b) => a.startMs - b.startMs)
+ .map((t) => [
+ t.clipId?.startsWith("clip_") ? "right" : t.clipId,
+ t.startMs,
+ t.endMs,
+ t.offsetMs,
+ ]);
+
+ it("plays on through the word, as one take", () => {
+ const next = withInsertion(withBed());
+ expect(spans(next)).toEqual([
+ ["c1", 2000, 4000, 0],
+ ["ext:synth_1", 4000, 4150, 2000],
+ ["right", 4150, 8150, 2150],
+ ]);
+ expect(new Set(next.audioTracks.map((t) => t.trackId)).size).toBe(1);
+ });
+});
+
describe("deleting the amber clip from the TIMELINE, not from the transcript", () => {
// The trash icon on the clip calls `removeClip`, and so does the agent's tool. Neither
// goes through `removeGeneratedClips`, so if the media were only dropped there, the
diff --git a/src/lib/ai-edition/document/timeline.test.ts b/src/lib/ai-edition/document/timeline.test.ts
index a4eb137f3..a33f1c6e7 100644
--- a/src/lib/ai-edition/document/timeline.test.ts
+++ b/src/lib/ai-edition/document/timeline.test.ts
@@ -5,6 +5,7 @@ import {
type AxcutTrimRange,
axcutSchemaVersion,
} from "../schema";
+import { audioLanePills } from "./audioTracks";
import {
buildTimelineFromIntervals,
duplicateClip,
@@ -25,6 +26,7 @@ import {
setClipSourceRange,
subtractInterval,
timelineIntervals,
+ withClipsChanged,
} from "./timeline";
function makeDoc(overrides: Partial<AxcutDocument> = {}): AxcutDocument {
@@ -1081,7 +1083,7 @@ describe("duplicateClip / moveClip", () => {
});
// Left out on purpose, see `duplicateClip`: an imported take is not a modifier of the
- // clip, and the audio fold (#1011) has no safe place for a copied fragment.
+ // clip.
it("duplicateClip leaves an imported audio take on the original only", () => {
const take: AxcutDocument["audioTracks"][number] = {
id: "vo",
@@ -1264,6 +1266,369 @@ describe("duplicateClip / moveClip", () => {
});
});
+// #1011. A take is stored as one fragment per clip it covers, and a structural edit carries
+// each fragment with its own clip. Folding every fragment of a `trackId` back into one span
+// afterwards, first to last, swallowed whatever clip had come to sit between them.
+describe("imported audio stays on the clips it was laid over (#1011)", () => {
+ type AudioTrack = AxcutDocument["audioTracks"][number];
+ const fragment = (
+ over: Pick<
+ AudioTrack,
+ "id" | "trackId" | "clipId" | "sourceStartSec" | "sourceEndSec" | "startMs" | "endMs"
+ > &
+ Partial<AudioTrack>,
+ ): AudioTrack => ({
+ assetId: "bed",
+ kind: "music",
+ durationSec: 30,
+ offsetMs: 0,
+ gainDb: -18,
+ loop: false,
+ fadeInMs: 0,
+ fadeOutMs: 0,
+ muted: false,
+ label: "",
+ origin: "user",
+ ...over,
+ });
+ /** Every fragment in ruler order: which file, on which clip, where, and from where in the file. */
+ const spans = (doc: AxcutDocument) =>
+ [...doc.audioTracks]
+ .sort((a, b) => a.startMs - b.startMs)
+ .map((t) => [t.assetId, t.clipId, t.startMs, t.endMs, t.offsetMs]);
+ /** What each clip carries: the file, the stretch of the clip, the place in the file. */
+ const carried = (doc: AxcutDocument) =>
+ doc.audioTracks
+ .map((t) =>
+ [
+ t.clipId,
+ t.assetId,
+ Math.round((t.sourceStartSec ?? Number.NaN) * 1000),
+ Math.round((t.sourceEndSec ?? Number.NaN) * 1000),
+ t.offsetMs,
+ ].join(" "),
+ )
+ .sort();
+ const musicLane = (doc: AxcutDocument) =>
+ audioLanePills(doc.audioTracks, "music").map((p) => [p.startMs, p.endMs]);
+
+ describe("the report's project", () => {
+ // Three clips; a 30 s bed across the first junction, a looping 4 s track across the
+ // second, both music.
+ const reported = () =>
+ makeDoc({
+ timeline: {
+ ...makeDoc().timeline,
+ clips: [
+ makeClip({
+ id: "b6cdac",
+ assetId: "rec_a",
+ sourceEndSec: 102.1,
+ timelineEndSec: 102.1,
+ }),
+ makeClip({
+ id: "63fedd",
+ assetId: "rec_b",
+ sourceEndSec: 61.87,
+ timelineStartSec: 102.1,
+ timelineEndSec: 163.97,
+ }),
+ makeClip({
+ id: "d51b65",
+ assetId: "rec_a",
+ sourceEndSec: 102.1,
+ timelineStartSec: 163.97,
+ timelineEndSec: 266.07,
+ }),
+ ],
+ },
+ audioTracks: [
+ fragment({
+ id: "bed",
+ trackId: "bed",
+ clipId: "b6cdac",
+ sourceStartSec: 89.869,
+ sourceEndSec: 102.1,
+ startMs: 89_869,
+ endMs: 102_100,
+ fadeInMs: 1000,
+ }),
+ fragment({
+ id: "bed_2",
+ trackId: "bed",
+ clipId: "63fedd",
+ sourceStartSec: 0,
+ sourceEndSec: 17.769,
+ startMs: 102_100,
+ endMs: 119_869,
+ offsetMs: 12_231,
+ fadeOutMs: 1000,
+ }),
+ fragment({
+ id: "loop",
+ trackId: "loop",
+ assetId: "loop",
+ durationSec: 4,
+ loop: true,
+ clipId: "63fedd",
+ sourceStartSec: 17.769,
+ sourceEndSec: 61.87,
+ startMs: 119_869,
+ endMs: 163_970,
+ fadeInMs: 1000,
+ }),
+ fragment({
+ id: "loop_2",
+ trackId: "loop",
+ assetId: "loop",
+ durationSec: 4,
+ loop: true,
+ clipId: "d51b65",
+ sourceStartSec: 0,
+ sourceEndSec: 36.007,
+ startMs: 163_970,
+ endMs: 199_977,
+ fadeOutMs: 1000,
+ }),
+ ],
+ });
+
+ it("moves each fragment with its own clip when the middle clip goes to the end", () => {
+ const before = reported();
+ const moved = moveClip(before, "63fedd", 2);
+ // Was: the bed stretched over d51b65 at offset 12231, and the loop pushed to 221.969–
+ // 385.939 s, past the 266.07 s end.
+ expect(spans(moved)).toEqual([
+ ["bed", "b6cdac", 89_869, 102_100, 0],
+ ["loop", "d51b65", 102_100, 138_107, 0],
+ ["bed", "63fedd", 204_200, 221_969, 12_231],
+ ["loop", "63fedd", 221_969, 266_070, 0],
+ ]);
+ expect(carried(moved)).toEqual(carried(before));
+ // Four takes now, side by side on the one music row: nothing for the lane to push.
+ expect(musicLane(moved)).toEqual([
+ [89_869, 102_100],
+ [102_100, 138_107],
+ [204_200, 221_969],
+ [221_969, 266_070],
+ ]);
+ });
+
+ it("puts both takes back together when the clip moves back", () => {
+ const before = reported();
+ const back = moveClip(moveClip(before, "63fedd", 2), "63fedd", 1);
+ expect(spans(back)).toEqual(spans(before));
+ expect(carried(back)).toEqual(carried(before));
+ expect(musicLane(back)).toEqual([
+ [89_869, 119_869],
+ [119_869, 199_977],
+ ]);
+ expect(back.audioTracks.map((t) => [t.trackId, t.fadeInMs, t.fadeOutMs])).toEqual([
+ ["bed", 1000, 0],
+ ["bed", 0, 1000],
+ ["loop", 1000, 0],
+ ["loop", 0, 1000],
+ ]);
+ });
+ });
+
+ it("keeps each half's place in the file when the two clips under a take swap", () => {
+ // The halves still meet on the ruler; only the file says they are no longer one take.
+ const doc = makeDoc({
+ timeline: {
+ ...makeDoc().timeline,
+ clips: [
+ makeClip({ id: "clip_a", assetId: "rec_a", sourceEndSec: 10, timelineEndSec: 10 }),
+ makeClip({
+ id: "clip_b",
+ assetId: "rec_b",
+ sourceEndSec: 10,
+ timelineStartSec: 10,
+ timelineEndSec: 20,
+ }),
+ ],
+ },
+ audioTracks: [
+ fragment({
+ id: "bed",
+ trackId: "bed",
+ clipId: "clip_a",
+ sourceStartSec: 0,
+ sourceEndSec: 10,
+ startMs: 0,
+ endMs: 10_000,
+ }),
+ fragment({
+ id: "bed_2",
+ trackId: "bed",
+ clipId: "clip_b",
+ sourceStartSec: 0,
+ sourceEndSec: 10,
+ startMs: 10_000,
+ endMs: 20_000,
+ offsetMs: 10_000,
+ }),
+ ],
+ });
+ expect(spans(moveClip(doc, "clip_a", 1))).toEqual([
+ ["bed", "clip_b", 0, 10_000, 10_000],
+ ["bed", "clip_a", 10_000, 20_000, 0],
+ ]);
+ });
+
+ describe("a bed across three clips", () => {
+ // A: rec_a 0–10 s, B: rec_b 0–10 s, C: rec_a 20–30 s; the bed covers 5–25 s on the ruler.
+ const threeClips = () =>
+ makeDoc({
+ timeline: {
+ ...makeDoc().timeline,
+ clips: [
+ makeClip({ id: "clip_a", assetId: "rec_a", sourceEndSec: 10, timelineEndSec: 10 }),
+ makeClip({
+ id: "clip_b",
+ assetId: "rec_b",
+ sourceEndSec: 10,
+ timelineStartSec: 10,
+ timelineEndSec: 20,
+ }),
+ makeClip({
+ id: "clip_c",
+ assetId: "rec_a",
+ sourceStartSec: 20,
+ sourceEndSec: 30,
+ timelineStartSec: 20,
+ timelineEndSec: 30,
+ }),
+ ],
+ },
+ audioTracks: [
+ fragment({
+ id: "bed",
+ trackId: "bed",
+ clipId: "clip_a",
+ sourceStartSec: 5,
+ sourceEndSec: 10,
+ startMs: 5000,
+ endMs: 10_000,
+ }),
+ fragment({
+ id: "bed_2",
+ trackId: "bed",
+ clipId: "clip_b",
+ sourceStartSec: 0,
+ sourceEndSec: 10,
+ startMs: 10_000,
+ endMs: 20_000,
+ offsetMs: 5000,
+ }),
+ fragment({
+ id: "bed_3",
+ trackId: "bed",
+ clipId: "clip_c",
+ sourceStartSec: 20,
+ sourceEndSec: 25,
+ startMs: 20_000,
+ endMs: 25_000,
+ offsetMs: 15_000,
+ }),
+ ],
+ });
+
+ it("gives a clip inserted inside it none of it", () => {
+ const before = threeClips();
+ const added = makeClip({
+ id: "clip_new",
+ assetId: "rec_c",
+ sourceEndSec: 4,
+ timelineEndSec: 4,
+ });
+ const [a, ...rest] = before.timeline.clips;
+ const next = withClipsChanged(before, [a, added, ...rest]);
+ expect(carried(next)).toEqual(carried(before));
+ expect(spans(next)).toEqual([
+ ["bed", "clip_a", 5000, 10_000, 0],
+ ["bed", "clip_b", 14_000, 24_000, 5000],
+ ["bed", "clip_c", 24_000, 29_000, 15_000],
+ ]);
+ });
+
+ it("gives a duplicated clip none of it", () => {
+ const before = threeClips();
+ const next = duplicateClip(before, "clip_a");
+ expect(carried(next)).toEqual(carried(before));
+ });
+
+ it("keeps the rest of it on its own footage when a clip under it is deleted", () => {
+ const next = removeClip(threeClips(), "clip_b");
+ expect(spans(next)).toEqual([
+ ["bed", "clip_a", 5000, 10_000, 0],
+ ["bed", "clip_c", 10_000, 15_000, 15_000],
+ ]);
+ });
+
+ it("cuts the file with the footage when a clip under it is narrowed", () => {
+ // Two seconds off clip_b's head take the two seconds of bed that played over them:
+ // the jump in the file is where the footage was cut, and clip_c keeps its music.
+ const next = setClipSourceRange(threeClips(), "clip_b", 2, 10);
+ expect(spans(next)).toEqual([
+ ["bed", "clip_a", 5000, 10_000, 0],
+ ["bed", "clip_b", 10_000, 18_000, 7000],
+ ["bed", "clip_c", 18_000, 23_000, 15_000],
+ ]);
+ expect(musicLane(next)).toEqual([
+ [5000, 10_000],
+ [10_000, 23_000],
+ ]);
+ });
+ });
+
+ it("still advances the offset of a fragment a rebuild re-cuts across two clips", () => {
+ // The repair the fold was added for: re-ventilating an orphan copies `offsetMs`
+ // verbatim into every piece, and the second piece must pick up where the first stops.
+ const doc = makeDoc({
+ timeline: {
+ ...makeDoc().timeline,
+ clips: [
+ makeClip({ id: "clip_a", sourceEndSec: 10, timelineEndSec: 10 }),
+ makeClip({
+ id: "clip_b",
+ sourceStartSec: 20,
+ sourceEndSec: 30,
+ timelineStartSec: 10,
+ timelineEndSec: 20,
+ }),
+ ],
+ },
+ audioTracks: [
+ fragment({
+ id: "bed",
+ trackId: "bed",
+ clipId: "clip_b",
+ sourceStartSec: 22,
+ sourceEndSec: 28,
+ startMs: 12_000,
+ endMs: 18_000,
+ offsetMs: 1000,
+ }),
+ ],
+ });
+ const rebuilt = replaceTimeline(
+ doc,
+ [
+ { startSec: 0, endSec: 10 },
+ { startSec: 20, endSec: 24 },
+ { startSec: 26, endSec: 30 },
+ ],
+ "split clip_b",
+ );
+ expect(spans(rebuilt).map(([, , start, end, offset]) => [start, end, offset])).toEqual([
+ [12_000, 14_000, 1000],
+ [14_000, 18_000, 3000],
+ ]);
+ expect(musicLane(rebuilt)).toEqual([[12_000, 18_000]]);
+ });
+});
+
function makeZoom(overrides: Partial<AxcutDocument["zoomRanges"][number]> = {}) {
return {
id: "z1",
diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts
index 53b7c1120..06e7964ed 100644
--- a/src/lib/ai-edition/document/timeline.ts
+++ b/src/lib/ai-edition/document/timeline.ts
@@ -25,6 +25,7 @@ import {
} from "../timeline/timelineMap";
import { dropTrimPillsByIds, trimAppliesToClip } from "../timeline/trim-mapping";
import {
+ cutTakeHead,
dropUnusedGeneratedMedia,
reanchorAudioTracks,
removeAudioTrack,
@@ -475,7 +476,7 @@ function rederiveAnchoredRegion<
if (!span) return [];
return [
{
- ...region,
+ ...cutTakeHead(region, sourceStartSec - region.sourceStartSec),
sourceStartSec,
sourceEndSec,
startMs: Math.round(span.startSec * 1000),
@@ -1007,8 +1008,7 @@ export function moveClip(
// The modifiers anchored to the clip are copied the same way, across every collection
// `mapAllRegionCollections` walks; `withClipsChanged` then places the copies on the new
// clip. Imported audio is left out on purpose: a take is media on its own lane, not an
-// effect on the clip, and a copied fragment cannot be placed safely while every structural
-// edit folds each `trackId` back into one span (`reanchorAudioTracks`, #1011).
+// effect on the clip.
export function duplicateClip(
document: AxcutDocument,
clipId: string,