Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(audio): play a track once across its pill, so speed regions keep it on the clock
EtienneLescotf34971a8a474
diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsxindex 4e4ec9e55..19c0fb117 100644--- a/src/components/ai-edition/VirtualPreview.tsx+++ b/src/components/ai-edition/VirtualPreview.tsx@@ -11,8 +11,8 @@ import {takeCrossedClickHits,} from "@/lib/ai-edition/clickSound";import {- collapseTracksToPills,resolveFadeSecs,+ takesByHeadFragment,trackGroupId,} from "@/lib/ai-edition/document/audioTracks";import {@@ -875,23 +875,22 @@ export function VirtualPreview({// One walk per take, recomputed only when the cuts or the insertions move. The rAF asks// it every frame per track, and walking on each would be wasteful.const takePiecesRef = useRef<Map<string, TakePiece[]>>(new Map());- const takeHeadsRef = useRef<Map<string, string>>(new Map());+ // Each take keyed by the fragment whose element plays it, as the export mixes it.+ const takesRef = useRef<Map<string, AxcutAudioTrack>>(new Map());const removedRef = useRef(removedRawSpans(clips, trimRanges));removedRef.current = useMemo(() => removedRawSpans(clips, trimRanges), [clips, trimRanges]);const takeWalks = useMemo(() => {const pieces = new Map<string, TakePiece[]>();- const heads = new Map<string, string>();const removed = removedRawSpans(clips, trimRanges);- for (const pill of collapseTracksToPills(audioTracks)) {+ const takes = takesByHeadFragment(audioTracks);+ for (const pill of takes.values()) {if (pill.kind !== "voiceover" || pill.loop) continue;- const groupId = trackGroupId(pill);- heads.set(groupId, pill.id);- pieces.set(groupId, takeProgramme(pill, removed));+ pieces.set(trackGroupId(pill), takeProgramme(pill, removed));}- return { pieces, heads };+ return { pieces, takes };}, [audioTracks, clips, trimRanges]);takePiecesRef.current = takeWalks.pieces;- takeHeadsRef.current = takeWalks.heads;+ takesRef.current = takeWalks.takes;// Trim-narrowed (`resolvePlaybackSegments`) — used ONLY to detect "has the <video>'s own// currentTime drifted into a trim" and where to jump it back out to. Everything ELSE in// this component (`clips`/`clipsRef` above, virtualTimeSec, zoom/speed region lookups,@@ -1033,9 +1032,18 @@ export function VirtualPreview({}}duck.db = nextMusicDuckDb(duck.db, (now - duck.voiceAt) / 1000, frameSec);- for (const track of audioTracksRef.current) {- const el = audioTrackElsRef.current.get(track.id);+ for (const fragment of audioTracksRef.current) {+ const el = audioTrackElsRef.current.get(fragment.id);if (!el) continue;+ // A take plays once, from its head fragment's element, as its whole pill: the+ // document keeps one fragment per clip the take covers, and letting each play its+ // own slice put a bed back on the ruler under a speed region (see+ // `takesByHeadFragment`), and a voiceover on top of itself.+ const track = takesRef.current.get(fragment.id);+ if (!track) {+ if (!el.paused) el.pause();+ continue;+ }const outputStartSec = projectRawTimelineSecToPlayback(clipsRef.current,trimRangesRef.current,@@ -1062,22 +1070,15 @@ export function VirtualPreview({),);// A voiceover follows the cuts AND its own insertions, through one walk over- // its PILL. Every fragment but the head is silenced: the document keeps one- // per clip the take covers, and letting each play its own slice would put the- // take on top of itself, exactly as it would in the export.+ // its pill.//// A bed plays through a cut and ends early, on purpose. A looping voiceover// keeps the bed's treatment — step 6 of #560 refuses that combination, and// inventing semantics for it would be the worse answer.- const takeGroupId = trackGroupId(track);const takePieces =track.kind === "voiceover" && !track.loop- ? takePiecesRef.current.get(takeGroupId)+ ? takePiecesRef.current.get(trackGroupId(track)): undefined;- if (takePieces && takeHeadsRef.current.get(takeGroupId) !== track.id) {- if (!el.paused) el.pause();- continue;- }const trackTarget = takePieces? (takePlaybackAt(takePieces, virtualTimeSecRef.current) ?? {targetTimeSec: track.offsetMs / 1000,@@ -1109,7 +1110,7 @@ export function VirtualPreview({} else {trackGainDb += duck.db;}- const trackGainNode = audioTrackGainNodesRef.current.get(track.id);+ const trackGainNode = audioTrackGainNodesRef.current.get(fragment.id);if (trackGainNode) {trackGainNode.gain.value = audioGainScalar(trackGainDb) * fade;if (el.volume !== 1) el.volume = 1;diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.tsindex 83dd74a1e..a65d59d41 100644--- a/src/lib/ai-edition/document/audioTracks.ts+++ b/src/lib/ai-edition/document/audioTracks.ts@@ -239,6 +239,22 @@ export function collapseTracksToPills(tracks: AxcutAudioTrack[]): AxcutAudioTrac});}+/**+ * Every take once, keyed by the id of the fragment that plays it: its first. The export and the+ * preview both play a take from there, at 1x, across the whole pill. Played fragment by fragment,+ * each one its own slice, a bed was back on the ruler under a speed region: silent at the end of a+ * slowed clip, then behind; on top of itself through a sped-up one.+ */+export function takesByHeadFragment(tracks: AxcutAudioTrack[]): Map<string, AxcutAudioTrack> {+ const heads = new Map<string, string>();+ for (const track of [...tracks].sort((a, b) => a.startMs - b.startMs)) {+ if (!heads.has(trackGroupId(track))) heads.set(trackGroupId(track), track.id);+ }+ return new Map(+ collapseTracksToPills(tracks).map((pill) => [heads.get(trackGroupId(pill)) ?? pill.id, pill]),+ );+}+/** Drop every fragment of a track, and its asset when nothing else needs it.** An imported audio asset is only ever reachable through its track — audio isdiff --git a/src/native/sceneDescription.test.ts b/src/native/sceneDescription.test.tsindex 63a682072..c873190f6 100644--- a/src/native/sceneDescription.test.ts+++ b/src/native/sceneDescription.test.ts@@ -2953,6 +2953,52 @@ describe("buildSceneDescription.audioTracks", () => {expect(entry.startSec).toBeCloseTo(3, 6);});+ it("plays a bed spanning a slowed clip as one take, on the clock, not on the ruler", () => {+ // The document stores a track as one fragment per clip it covers, each fragment's offset+ // advanced by the ruler time before it. Mixed fragment by fragment, the bed under a 0.5x+ // clip played its 4 ruler seconds of file over 8 output seconds, went silent for the other+ // 4, then came back 4 seconds behind the music it should have been playing.+ const screen = makeAsset({ id: "scr", originalPath: "/screen.mp4", durationSec: 20 });+ const fragment = { ...track, trackId: "trk1", startMs: 0, offsetMs: 0, endMs: 4000 };+ const doc = makeDoc({+ assets: [screen, audioAsset],+ clips: [+ makeClip({+ id: "c1",+ assetId: "scr",+ sourceStartSec: 0,+ sourceEndSec: 4,+ timelineStartSec: 0,+ timelineEndSec: 4,+ }),+ makeClip({+ id: "c2",+ assetId: "scr",+ sourceStartSec: 4,+ sourceEndSec: 8,+ timelineStartSec: 4,+ timelineEndSec: 8,+ }),+ ],+ legacyEditor: { speedRegions: [{ id: "s1", startMs: 0, endMs: 4000, speed: 0.5 }] },+ audioTracks: [+ { ...fragment, clipId: "c1", sourceStartSec: 0, sourceEndSec: 4 },+ {+ ...fragment,+ id: "trk1-b",+ clipId: "c2",+ sourceStartSec: 4,+ sourceEndSec: 8,+ startMs: 4000,+ endMs: 8000,+ offsetMs: 4000,+ },+ ],+ });+ const entries = buildSceneDescription(doc).audioTracks;+ expect(entries.map((e) => [e.startSec, e.trimStartSec, e.trimEndSec])).toEqual([[0, 0, 8]]);+ });+it("drops a track whose asset has no resolvable path", () => {const doc = makeDoc({ assets: [], audioTracks: [track] });expect(buildSceneDescription(doc).audioTracks).toEqual([]);diff --git a/src/native/sceneDescription.ts b/src/native/sceneDescription.tsindex 20caf7a20..791196a99 100644--- a/src/native/sceneDescription.ts+++ b/src/native/sceneDescription.ts@@ -31,7 +31,7 @@ import {getCaptionTranslations,} from "@/lib/ai-edition/captions";import { clickSoundForDocument } from "@/lib/ai-edition/clickSound";-import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks";+import { takesByHeadFragment } from "@/lib/ai-edition/document/audioTracks";import { createId } from "@/lib/ai-edition/document/ids";import { isFormatFillActive, pickOutputDims } from "@/lib/ai-edition/document/outputFormat";import {@@ -884,15 +884,12 @@ export function buildSceneDescription(// question, and it does not depend on the track.// Placed once: the projection below counts them, so a track after a pause lands whereconst removed = removedRawSpans(projectedClips, document.timeline.trimRanges);- // The take's pills, keyed by group. A voiceover is walked ONCE per pill and never per- // stored fragment: the document keeps one fragment per clip a take covers, so walking- // them separately would emit overlapping entries and `overlay_track_pcm` sums with `+=`- // at an absolute offset — the export would contain the take playing on top of itself.- const voiceoverPills = new Map(- collapseTracksToPills(document.audioTracks)- .filter((pill) => pill.kind === "voiceover" && !pill.loop)- .map((pill) => [trackGroupId(pill), pill]),- );+ // Every take is mixed ONCE, from its head fragment, as its whole pill — never per stored+ // fragment. The document keeps one fragment per clip a take covers: a voiceover walked per+ // fragment would play on top of itself (`overlay_track_pcm` sums with `+=`), and a bed laid+ // per fragment, each at its projected head with its ruler-advanced offset, went silent at the+ // end of a slowed clip and came back late (see `takesByHeadFragment`).+ const takes = takesByHeadFragment(document.audioTracks);// Click hits for the "mouse clicks" toggle, placed on the programme this export assembles.// Synchronous: the cues were read from the take's sidecar when the sound was switched on.// Its row lives under "Show cursor", like the click impact: a hidden cursor makes no sound.@@ -900,8 +897,9 @@ export function buildSceneDescription(clickSound: settings.cursorShow && settings.cursor.clickSound,clickSoundGainDb: settings.cursor.clickSoundGainDb,});- const audioTracks = document.audioTracks.flatMap((track) => {- if (track.muted) return [];+ const audioTracks = document.audioTracks.flatMap((fragment) => {+ const track = takes.get(fragment.id);+ if (!track || track.muted) return [];const asset = assetById.get(track.assetId);if (!asset?.originalPath) return [];const sourceDurationSec = asset.durationSec ?? track.durationSec;@@ -957,17 +955,12 @@ export function buildSceneDescription(// #560 refuses that combination outright, and inventing semantics for something// about to be banned would be the worse answer.if (track.kind === "voiceover" && !track.loop) {- const groupId = trackGroupId(track);- const pill = voiceoverPills.get(groupId);- // Emitted from the group's HEAD fragment only — every other fragment of the same- // take is already covered by the pill's own walk.- if (!pill || pill.id !== track.id) return [];- const rawSpanSec = Math.max(0, pill.endMs / 1000 - pill.startMs / 1000);+ const rawSpanSec = Math.max(0, track.endMs / 1000 - track.startMs / 1000);// Unprobed assets have no real duration to cap with; the RAW span is how much// file the take covers, which is the honest fallback once the cuts are taken out.const voWindowSec =sourceDurationSec > 0 ? Math.max(0, sourceDurationSec - offsetSec) : rawSpanSec;- const kept = takeProgramme(pill, removed)+ const kept = takeProgramme(track, removed).filter((piece) => piece.kind === "play").map((piece) => ({...base,