Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(audio): play a track once across its pill, so speed regions keep it on the clock

EtienneLescotf34971a8a474
diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx
index 4e4ec9e55..19c0fb117 100644
--- a/src/components/ai-edition/VirtualPreview.tsx
+++ b/src/components/ai-edition/VirtualPreview.tsx
@@ -11,8 +11,8 @@ import {
takeCrossedClickHits,
} from "@/lib/ai-edition/clickSound";
import {
- collapseTracksToPills,
resolveFadeSecs,
+ takesByHeadFragment,
trackGroupId,
} from "@/lib/ai-edition/document/audioTracks";
import {
@@ -875,23 +875,22 @@ export function VirtualPreview({
// One walk per take, recomputed only when the cuts or the insertions move. The rAF asks
// it every frame per track, and walking on each would be wasteful.
const takePiecesRef = useRef<Map<string, TakePiece[]>>(new Map());
- const takeHeadsRef = useRef<Map<string, string>>(new Map());
+ // Each take keyed by the fragment whose element plays it, as the export mixes it.
+ const takesRef = useRef<Map<string, AxcutAudioTrack>>(new Map());
const removedRef = useRef(removedRawSpans(clips, trimRanges));
removedRef.current = useMemo(() => removedRawSpans(clips, trimRanges), [clips, trimRanges]);
const takeWalks = useMemo(() => {
const pieces = new Map<string, TakePiece[]>();
- const heads = new Map<string, string>();
const removed = removedRawSpans(clips, trimRanges);
- for (const pill of collapseTracksToPills(audioTracks)) {
+ const takes = takesByHeadFragment(audioTracks);
+ for (const pill of takes.values()) {
if (pill.kind !== "voiceover" || pill.loop) continue;
- const groupId = trackGroupId(pill);
- heads.set(groupId, pill.id);
- pieces.set(groupId, takeProgramme(pill, removed));
+ pieces.set(trackGroupId(pill), takeProgramme(pill, removed));
}
- return { pieces, heads };
+ return { pieces, takes };
}, [audioTracks, clips, trimRanges]);
takePiecesRef.current = takeWalks.pieces;
- takeHeadsRef.current = takeWalks.heads;
+ takesRef.current = takeWalks.takes;
// Trim-narrowed (`resolvePlaybackSegments`) — used ONLY to detect "has the <video>'s own
// currentTime drifted into a trim" and where to jump it back out to. Everything ELSE in
// this component (`clips`/`clipsRef` above, virtualTimeSec, zoom/speed region lookups,
@@ -1033,9 +1032,18 @@ export function VirtualPreview({
}
}
duck.db = nextMusicDuckDb(duck.db, (now - duck.voiceAt) / 1000, frameSec);
- for (const track of audioTracksRef.current) {
- const el = audioTrackElsRef.current.get(track.id);
+ for (const fragment of audioTracksRef.current) {
+ const el = audioTrackElsRef.current.get(fragment.id);
if (!el) continue;
+ // A take plays once, from its head fragment's element, as its whole pill: the
+ // document keeps one fragment per clip the take covers, and letting each play its
+ // own slice put a bed back on the ruler under a speed region (see
+ // `takesByHeadFragment`), and a voiceover on top of itself.
+ const track = takesRef.current.get(fragment.id);
+ if (!track) {
+ if (!el.paused) el.pause();
+ continue;
+ }
const outputStartSec = projectRawTimelineSecToPlayback(
clipsRef.current,
trimRangesRef.current,
@@ -1062,22 +1070,15 @@ export function VirtualPreview({
),
);
// A voiceover follows the cuts AND its own insertions, through one walk over
- // its PILL. Every fragment but the head is silenced: the document keeps one
- // per clip the take covers, and letting each play its own slice would put the
- // take on top of itself, exactly as it would in the export.
+ // its pill.
//
// A bed plays through a cut and ends early, on purpose. A looping voiceover
// keeps the bed's treatment — step 6 of #560 refuses that combination, and
// inventing semantics for it would be the worse answer.
- const takeGroupId = trackGroupId(track);
const takePieces =
track.kind === "voiceover" && !track.loop
- ? takePiecesRef.current.get(takeGroupId)
+ ? takePiecesRef.current.get(trackGroupId(track))
: undefined;
- if (takePieces && takeHeadsRef.current.get(takeGroupId) !== track.id) {
- if (!el.paused) el.pause();
- continue;
- }
const trackTarget = takePieces
? (takePlaybackAt(takePieces, virtualTimeSecRef.current) ?? {
targetTimeSec: track.offsetMs / 1000,
@@ -1109,7 +1110,7 @@ export function VirtualPreview({
} else {
trackGainDb += duck.db;
}
- const trackGainNode = audioTrackGainNodesRef.current.get(track.id);
+ const trackGainNode = audioTrackGainNodesRef.current.get(fragment.id);
if (trackGainNode) {
trackGainNode.gain.value = audioGainScalar(trackGainDb) * fade;
if (el.volume !== 1) el.volume = 1;
diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.ts
index 83dd74a1e..a65d59d41 100644
--- a/src/lib/ai-edition/document/audioTracks.ts
+++ b/src/lib/ai-edition/document/audioTracks.ts
@@ -239,6 +239,22 @@ export function collapseTracksToPills(tracks: AxcutAudioTrack[]): AxcutAudioTrac
});
}
+/**
+ * Every take once, keyed by the id of the fragment that plays it: its first. The export and the
+ * preview both play a take from there, at 1x, across the whole pill. Played fragment by fragment,
+ * each one its own slice, a bed was back on the ruler under a speed region: silent at the end of a
+ * slowed clip, then behind; on top of itself through a sped-up one.
+ */
+export function takesByHeadFragment(tracks: AxcutAudioTrack[]): Map<string, AxcutAudioTrack> {
+ const heads = new Map<string, string>();
+ for (const track of [...tracks].sort((a, b) => a.startMs - b.startMs)) {
+ if (!heads.has(trackGroupId(track))) heads.set(trackGroupId(track), track.id);
+ }
+ return new Map(
+ collapseTracksToPills(tracks).map((pill) => [heads.get(trackGroupId(pill)) ?? pill.id, pill]),
+ );
+}
+
/** Drop every fragment of a track, and its asset when nothing else needs it.
*
* An imported audio asset is only ever reachable through its track — audio is
diff --git a/src/native/sceneDescription.test.ts b/src/native/sceneDescription.test.ts
index 63a682072..c873190f6 100644
--- a/src/native/sceneDescription.test.ts
+++ b/src/native/sceneDescription.test.ts
@@ -2953,6 +2953,52 @@ describe("buildSceneDescription.audioTracks", () => {
expect(entry.startSec).toBeCloseTo(3, 6);
});
+ it("plays a bed spanning a slowed clip as one take, on the clock, not on the ruler", () => {
+ // The document stores a track as one fragment per clip it covers, each fragment's offset
+ // advanced by the ruler time before it. Mixed fragment by fragment, the bed under a 0.5x
+ // clip played its 4 ruler seconds of file over 8 output seconds, went silent for the other
+ // 4, then came back 4 seconds behind the music it should have been playing.
+ const screen = makeAsset({ id: "scr", originalPath: "/screen.mp4", durationSec: 20 });
+ const fragment = { ...track, trackId: "trk1", startMs: 0, offsetMs: 0, endMs: 4000 };
+ const doc = makeDoc({
+ assets: [screen, audioAsset],
+ clips: [
+ makeClip({
+ id: "c1",
+ assetId: "scr",
+ sourceStartSec: 0,
+ sourceEndSec: 4,
+ timelineStartSec: 0,
+ timelineEndSec: 4,
+ }),
+ makeClip({
+ id: "c2",
+ assetId: "scr",
+ sourceStartSec: 4,
+ sourceEndSec: 8,
+ timelineStartSec: 4,
+ timelineEndSec: 8,
+ }),
+ ],
+ legacyEditor: { speedRegions: [{ id: "s1", startMs: 0, endMs: 4000, speed: 0.5 }] },
+ audioTracks: [
+ { ...fragment, clipId: "c1", sourceStartSec: 0, sourceEndSec: 4 },
+ {
+ ...fragment,
+ id: "trk1-b",
+ clipId: "c2",
+ sourceStartSec: 4,
+ sourceEndSec: 8,
+ startMs: 4000,
+ endMs: 8000,
+ offsetMs: 4000,
+ },
+ ],
+ });
+ const entries = buildSceneDescription(doc).audioTracks;
+ expect(entries.map((e) => [e.startSec, e.trimStartSec, e.trimEndSec])).toEqual([[0, 0, 8]]);
+ });
+
it("drops a track whose asset has no resolvable path", () => {
const doc = makeDoc({ assets: [], audioTracks: [track] });
expect(buildSceneDescription(doc).audioTracks).toEqual([]);
diff --git a/src/native/sceneDescription.ts b/src/native/sceneDescription.ts
index 20caf7a20..791196a99 100644
--- a/src/native/sceneDescription.ts
+++ b/src/native/sceneDescription.ts
@@ -31,7 +31,7 @@ import {
getCaptionTranslations,
} from "@/lib/ai-edition/captions";
import { clickSoundForDocument } from "@/lib/ai-edition/clickSound";
-import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks";
+import { takesByHeadFragment } from "@/lib/ai-edition/document/audioTracks";
import { createId } from "@/lib/ai-edition/document/ids";
import { isFormatFillActive, pickOutputDims } from "@/lib/ai-edition/document/outputFormat";
import {
@@ -884,15 +884,12 @@ export function buildSceneDescription(
// question, and it does not depend on the track.
// Placed once: the projection below counts them, so a track after a pause lands where
const removed = removedRawSpans(projectedClips, document.timeline.trimRanges);
- // The take's pills, keyed by group. A voiceover is walked ONCE per pill and never per
- // stored fragment: the document keeps one fragment per clip a take covers, so walking
- // them separately would emit overlapping entries and `overlay_track_pcm` sums with `+=`
- // at an absolute offset — the export would contain the take playing on top of itself.
- const voiceoverPills = new Map(
- collapseTracksToPills(document.audioTracks)
- .filter((pill) => pill.kind === "voiceover" && !pill.loop)
- .map((pill) => [trackGroupId(pill), pill]),
- );
+ // Every take is mixed ONCE, from its head fragment, as its whole pill — never per stored
+ // fragment. The document keeps one fragment per clip a take covers: a voiceover walked per
+ // fragment would play on top of itself (`overlay_track_pcm` sums with `+=`), and a bed laid
+ // per fragment, each at its projected head with its ruler-advanced offset, went silent at the
+ // end of a slowed clip and came back late (see `takesByHeadFragment`).
+ const takes = takesByHeadFragment(document.audioTracks);
// Click hits for the "mouse clicks" toggle, placed on the programme this export assembles.
// Synchronous: the cues were read from the take's sidecar when the sound was switched on.
// Its row lives under "Show cursor", like the click impact: a hidden cursor makes no sound.
@@ -900,8 +897,9 @@ export function buildSceneDescription(
clickSound: settings.cursorShow && settings.cursor.clickSound,
clickSoundGainDb: settings.cursor.clickSoundGainDb,
});
- const audioTracks = document.audioTracks.flatMap((track) => {
- if (track.muted) return [];
+ const audioTracks = document.audioTracks.flatMap((fragment) => {
+ const track = takes.get(fragment.id);
+ if (!track || track.muted) return [];
const asset = assetById.get(track.assetId);
if (!asset?.originalPath) return [];
const sourceDurationSec = asset.durationSec ?? track.durationSec;
@@ -957,17 +955,12 @@ export function buildSceneDescription(
// #560 refuses that combination outright, and inventing semantics for something
// about to be banned would be the worse answer.
if (track.kind === "voiceover" && !track.loop) {
- const groupId = trackGroupId(track);
- const pill = voiceoverPills.get(groupId);
- // Emitted from the group's HEAD fragment only — every other fragment of the same
- // take is already covered by the pill's own walk.
- if (!pill || pill.id !== track.id) return [];
- const rawSpanSec = Math.max(0, pill.endMs / 1000 - pill.startMs / 1000);
+ const rawSpanSec = Math.max(0, track.endMs / 1000 - track.startMs / 1000);
// Unprobed assets have no real duration to cap with; the RAW span is how much
// file the take covers, which is the honest fallback once the cuts are taken out.
const voWindowSec =
sourceDurationSec > 0 ? Math.max(0, sourceDurationSec - offsetSec) : rawSpanSec;
- const kept = takeProgramme(pill, removed)
+ const kept = takeProgramme(track, removed)
.filter((piece) => piece.kind === "play")
.map((piece) => ({
...base,