Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix(audio): reduce recording and export drift

webadderall6edc6d293d33
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index a090bbf5..a1e59a4e 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -1617,6 +1617,84 @@ async function probeMediaDurationSeconds(filePath: string): Promise<number> {
return 0
}
+type AudioSyncAdjustment = {
+ mode: 'none' | 'tempo' | 'delay'
+ delayMs: number
+ tempoRatio: number
+ durationDeltaMs: number
+}
+
+function buildAtempoFilters(tempoRatio: number) {
+ if (!Number.isFinite(tempoRatio) || tempoRatio <= 0) {
+ return []
+ }
+
+ const filters: string[] = []
+ let remaining = tempoRatio
+
+ while (remaining < 0.5) {
+ filters.push('atempo=0.5')
+ remaining /= 0.5
+ }
+
+ while (remaining > 2) {
+ filters.push('atempo=2.0')
+ remaining /= 2.0
+ }
+
+ if (Math.abs(remaining - 1) > 0.0005) {
+ filters.push(`atempo=${remaining.toFixed(6)}`)
+ }
+
+ return filters
+}
+
+function getAudioSyncAdjustment(videoDuration: number, audioDuration: number): AudioSyncAdjustment {
+ if (
+ !Number.isFinite(videoDuration) ||
+ !Number.isFinite(audioDuration) ||
+ videoDuration <= 0 ||
+ audioDuration <= 0
+ ) {
+ return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
+ }
+
+ const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000)
+ const absDeltaMs = Math.abs(durationDeltaMs)
+ if (absDeltaMs <= 50) {
+ return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs }
+ }
+
+ const tempoRatio = Math.max(0.5, Math.min(2, audioDuration / videoDuration))
+ const relativeDelta = absDeltaMs / Math.max(videoDuration * 1000, 1)
+
+ if (relativeDelta <= 0.03 || absDeltaMs <= 1500 || durationDeltaMs < 0) {
+ return { mode: 'tempo', delayMs: 0, tempoRatio, durationDeltaMs }
+ }
+
+ return { mode: 'delay', delayMs: durationDeltaMs, tempoRatio: 1, durationDeltaMs }
+}
+
+function appendSyncedAudioFilter(
+ filterParts: string[],
+ inputLabel: string,
+ outputLabel: string,
+ adjustment: AudioSyncAdjustment,
+) {
+ const filters: string[] = []
+
+ if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
+ filters.push(`adelay=${adjustment.delayMs}|${adjustment.delayMs}`)
+ }
+
+ if (adjustment.mode === 'tempo') {
+ filters.push(...buildAtempoFilters(adjustment.tempoRatio))
+ }
+
+ filters.push('aresample=async=1:first_pts=0', 'asetpts=PTS-STARTPTS')
+ filterParts.push(`${inputLabel}${filters.join(',')}[${outputLabel}]`)
+}
+
function sendWhisperModelDownloadProgress(
webContents: Electron.WebContents,
payload: { status: 'idle' | 'downloading' | 'downloaded' | 'error'; progress: number; path?: string | null; error?: string },
@@ -2551,27 +2629,35 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
if (audioInputs.length === 0) return
- // Probe durations to compute audio delay offsets.
- // If an audio file is shorter than the video it means the audio device
- // started delivering samples late (common with Bluetooth / iPhone mics).
+ // Match each audio track to the captured video duration.
+ // Small duration deltas are more often clock drift than a true late start,
+ // so prefer tempo correction there and reserve leading silence for larger gaps.
const videoDuration = await probeMediaDurationSeconds(videoPath)
- const audioDelays: Map<string, number> = new Map()
+ const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map()
if (videoDuration > 0) {
for (let i = 0; i < audioFilePaths.length; i++) {
const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
- const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
- audioDelays.set(audioInputs[i], delayMs)
- if (delayMs > 0) {
- console.log(`[mux-win] ${audioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
+ const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration)
+ audioAdjustments.set(audioInputs[i], adjustment)
+ if (adjustment.mode === 'tempo') {
+ console.log(
+ `[mux-win] ${audioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,
+ )
+ } else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
+ console.log(
+ `[mux-win] ${audioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,
+ )
}
}
}
const mixedOutputPath = `${videoPath}.muxed.mp4`
const normalizedPauseSegments = normalizePauseSegments(pauseSegments)
- const systemDelayMs = audioDelays.get('system') ?? 0
- const micDelayMs = audioDelays.get('mic') ?? 0
+ const systemAdjustment =
+ audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
+ const micAdjustment =
+ audioAdjustments.get('mic') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
if (audioInputs.length === 2) {
// Both system + mic audio: mix them
@@ -2589,19 +2675,9 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
const systemLabel = systemPauseFilter ? '[system_trimmed]' : '[1:a]'
const micLabel = micPauseFilter ? '[mic_trimmed]' : '[2:a]'
- // Apply delay to compensate for late audio start
- if (micDelayMs > 0) {
- filterParts.push(`${micLabel}adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
- } else {
- filterParts.push(`${micLabel}asetpts=PTS-STARTPTS[m]`)
- }
-
- if (systemDelayMs > 0) {
- filterParts.push(`${systemLabel}adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
- filterParts.push(`[s][m]amix=inputs=2:duration=longest:normalize=0[aout]`)
- } else {
- filterParts.push(`${systemLabel}[m]amix=inputs=2:duration=longest:normalize=0[aout]`)
- }
+ appendSyncedAudioFilter(filterParts, systemLabel, 's', systemAdjustment)
+ appendSyncedAudioFilter(filterParts, micLabel, 'm', micAdjustment)
+ filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]')
await execFileAsync(
ffmpegPath,
@@ -2621,19 +2697,17 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
)
} else {
// Single audio track
- const pauseFilter = buildPausedAudioFilter('1:a', 'aout', normalizedPauseSegments)
- const singleDelayMs = audioDelays.get(audioInputs[0]) ?? 0
+ const pauseFilter = buildPausedAudioFilter('1:a', 'trimmed_audio', normalizedPauseSegments)
+ const singleAdjustment =
+ audioAdjustments.get(audioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
- if (pauseFilter || singleDelayMs > 0) {
+ if (pauseFilter || singleAdjustment.mode !== 'none') {
const filterParts: string[] = []
if (pauseFilter) {
filterParts.push(pauseFilter)
}
- const srcLabel = pauseFilter ? '[aout]' : '[1:a]'
- if (singleDelayMs > 0) {
- filterParts.push(`${srcLabel}adelay=${singleDelayMs}|${singleDelayMs},asetpts=PTS-STARTPTS[delayed]`)
- }
- const outLabel = singleDelayMs > 0 ? '[delayed]' : '[aout]'
+ const srcLabel = pauseFilter ? '[trimmed_audio]' : '[1:a]'
+ appendSyncedAudioFilter(filterParts, srcLabel, 'aout', singleAdjustment)
await execFileAsync(
ffmpegPath,
@@ -2642,7 +2716,7 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
...inputs,
'-filter_complex', filterParts.join(';'),
'-map', '0:v:0',
- '-map', outLabel,
+ '-map', '[aout]',
'-c:v', 'copy',
'-c:a', 'aac',
'-b:a', '192k',
@@ -2880,39 +2954,40 @@ async function muxNativeMacRecordingWithAudio(
return
}
- // Probe durations — if audio is shorter than video it means the audio device
- // started late (e.g. iPhone mic over Continuity Camera). Add leading silence.
+ // Match each audio track to the captured video duration.
const videoDuration = await probeMediaDurationSeconds(videoPath)
- const audioDelays: Map<string, number> = new Map()
+ const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map()
if (videoDuration > 0) {
for (let i = 0; i < audioFilePaths.length; i++) {
const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
- const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
- audioDelays.set(availableAudioInputs[i], delayMs)
- if (delayMs > 0) {
- console.log(`[mux] ${availableAudioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
+ const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration)
+ audioAdjustments.set(availableAudioInputs[i], adjustment)
+ if (adjustment.mode === 'tempo') {
+ console.log(
+ `[mux] ${availableAudioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,
+ )
+ } else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
+ console.log(
+ `[mux] ${availableAudioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,
+ )
}
}
}
- const systemDelayMs = audioDelays.get('system') ?? 0
- const micDelayMs = audioDelays.get('microphone') ?? 0
- const needsFilter = systemDelayMs > 0 || micDelayMs > 0
+ const systemAdjustment =
+ audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
+ const micAdjustment =
+ audioAdjustments.get('microphone') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
+ const needsFilter = systemAdjustment.mode !== 'none' || micAdjustment.mode !== 'none'
let args: string[]
if (availableAudioInputs.length === 2) {
if (needsFilter) {
const filterParts: string[] = []
- if (systemDelayMs > 0) {
- filterParts.push(`[1:a]adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
- }
- if (micDelayMs > 0) {
- filterParts.push(`[2:a]adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
- }
- const sLabel = systemDelayMs > 0 ? '[s]' : '[1:a]'
- const mLabel = micDelayMs > 0 ? '[m]' : '[2:a]'
- filterParts.push(`${sLabel}${mLabel}amix=inputs=2:duration=longest:normalize=0[aout]`)
+ appendSyncedAudioFilter(filterParts, '[1:a]', 's', systemAdjustment)
+ appendSyncedAudioFilter(filterParts, '[2:a]', 'm', micAdjustment)
+ filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]')
args = [
'-y',
...inputs,
@@ -2940,12 +3015,15 @@ async function muxNativeMacRecordingWithAudio(
]
}
} else {
- if (needsFilter) {
- const delayMs = systemDelayMs || micDelayMs
+ const singleAdjustment =
+ audioAdjustments.get(availableAudioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
+ if (singleAdjustment.mode !== 'none') {
+ const filterParts: string[] = []
+ appendSyncedAudioFilter(filterParts, '[1:a]', 'aout', singleAdjustment)
args = [
'-y',
...inputs,
- '-filter_complex', `[1:a]adelay=${delayMs}|${delayMs},asetpts=PTS-STARTPTS[aout]`,
+ '-filter_complex', filterParts.join(';'),
'-map', '0:v:0',
'-map', '[aout]',
'-c:v', 'copy',
@@ -4030,12 +4108,20 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
ipcMain.handle('switch-to-editor', () => {
console.log('[switch-to-editor] Opening editor window')
recordingFinalizing = false
+ const sourceSelectorWin = getSourceSelectorWindow()
+ if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) {
+ sourceSelectorWin.close()
+ }
createEditorWindow()
})
ipcMain.handle('open-editor-early', () => {
console.log('[open-editor-early] Opening editor window while recording finalizes')
recordingFinalizing = true
+ const sourceSelectorWin = getSourceSelectorWindow()
+ if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) {
+ sourceSelectorWin.close()
+ }
createEditorWindow()
})
diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift
index d97cb304..1d17a64c 100644
--- a/electron/native/ScreenCaptureKitRecorder.swift
+++ b/electron/native/ScreenCaptureKitRecorder.swift
@@ -17,6 +17,7 @@ struct CaptureConfig: Codable {
}
let targetCaptureFPS = 60
+let maxInlineAudioTailExtension = CMTime(seconds: 2.0, preferredTimescale: 600)
final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
private let queue = DispatchQueue(label: "openscreen.screencapturekit.video")
@@ -33,6 +34,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
private var lastSampleBuffer: CMSampleBuffer?
private var lastVideoPresentationTime: CMTime = .zero
private var lastVideoDuration: CMTime = .zero
+ private var lastInlineAudioPresentationTime: CMTime = .invalid
+ private var lastInlineAudioDuration: CMTime = .zero
private var isRecording = false
private var isPaused = false
private var pauseStartedHostTime: CMTime?
@@ -360,7 +363,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
windowValidationTask = nil
trackedWindowId = nil
- isRecording = false
if let activeStream = stream {
do {
try await activeStream.stopCapture()
@@ -369,6 +371,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
}
}
stream = nil
+ isRecording = false
if let originalBuffer = lastSampleBuffer, let videoInput = videoInput {
let additionalTime = lastVideoPresentationTime + frameDuration(for: originalBuffer)
@@ -378,7 +381,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
}
}
- let endTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero)
+ let videoEndTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero)
+ let endTime = resolvedCaptureEndTime(videoEndTime: videoEndTime)
assetWriter?.endSession(atSourceTime: endTime)
videoInput?.markAsFinished()
inlineAudioInput?.markAsFinished()
@@ -408,6 +412,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
lastSampleBuffer = nil
lastVideoPresentationTime = .zero
lastVideoDuration = .zero
+ lastInlineAudioPresentationTime = .invalid
+ lastInlineAudioDuration = .zero
frameCount = 0
isPaused = false
pauseStartedHostTime = nil
@@ -466,6 +472,34 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
return CMTime(value: 1, timescale: CMTimeScale(targetCaptureFPS))
}
+ private func latestInlineAudioEndTime() -> CMTime {
+ guard lastInlineAudioPresentationTime.isValid else {
+ return .invalid
+ }
+
+ if lastInlineAudioDuration.isValid && lastInlineAudioDuration > .zero {
+ return lastInlineAudioPresentationTime + lastInlineAudioDuration
+ }
+
+ return lastInlineAudioPresentationTime
+ }
+
+ private func resolvedCaptureEndTime(videoEndTime: CMTime) -> CMTime {
+ let inlineAudioEndTime = latestInlineAudioEndTime()
+ guard inlineAudioEndTime.isValid else {
+ return videoEndTime
+ }
+
+ if CMTimeCompare(inlineAudioEndTime, videoEndTime) <= 0 {
+ return videoEndTime
+ }
+
+ // Prevent a stray inline-audio timestamp from forcing finishWriting
+ // to finalize an arbitrarily long tail.
+ let tailExtension = CMTimeSubtract(inlineAudioEndTime, videoEndTime)
+ return videoEndTime + CMTimeMinimum(tailExtension, maxInlineAudioTailExtension)
+ }
+
private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, firstSampleTime: inout CMTime?, presentationTime: CMTime) {
guard input.isReadyForMoreMediaData else { return }
@@ -477,7 +511,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
// (computed by adjustedPresentationTime), so use it directly.
let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp)
if let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) {
- input.append(retimedSampleBuffer)
+ let appended = input.append(retimedSampleBuffer)
+ if appended, input === inlineAudioInput {
+ lastInlineAudioPresentationTime = presentationTime
+ lastInlineAudioDuration = sampleBuffer.duration
+ }
}
}
diff --git a/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper b/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper
index fd928679..fcc0ca2c 100755
Binary files a/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper and b/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper differ
diff --git a/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper b/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper
index 8827155e..2ae827e4 100755
Binary files a/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper and b/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper differ
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index 7abab355..cea4d784 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -48,7 +48,7 @@ import {
VideoExporter,
} from "@/lib/exporter";
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
-import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
+import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from "@/lib/mediaTiming";
import { matchesShortcut } from "@/lib/shortcuts";
import { type AspectRatio, getAspectRatioValue } from "@/utils/aspectRatioUtils";
import { resolveAutoCaptionSourcePath } from "./autoCaptionSource";
@@ -2961,11 +2961,16 @@ export default function VideoEditor() {
// Sync audio playback with video currentTime and isPlaying state
useEffect(() => {
+ const currentTimeMs = currentTime * 1000;
+ const activeSpeedRegion = speedRegions.find(
+ (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
+ );
+ const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
+
for (const region of audioRegions) {
const audio = audioElementsRef.current.get(region.id);
if (!audio) continue;
- const currentTimeMs = currentTime * 1000;
const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs;
if (isPlaying && isInRegion) {
@@ -2974,6 +2979,14 @@ export default function VideoEditor() {
if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
audio.currentTime = audioOffset;
}
+ const syncedPlaybackRate = getMediaSyncPlaybackRate({
+ basePlaybackRate: targetPlaybackRate,
+ currentTime: audio.currentTime,
+ targetTime: audioOffset,
+ });
+ if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
+ audio.playbackRate = syncedPlaybackRate;
+ }
if (audio.paused) {
audio.play().catch(() => undefined);
}
@@ -2983,7 +2996,7 @@ export default function VideoEditor() {
}
}
}
- }, [isPlaying, currentTime, audioRegions]);
+ }, [isPlaying, currentTime, audioRegions, speedRegions]);
useEffect(() => {
if (sourceAudioFallbackPaths.length === 0) {
@@ -3001,14 +3014,8 @@ export default function VideoEditor() {
const driftThreshold = isPlaying ? 0.35 : 0.01;
for (const audio of sourceAudioElementsRef.current.values()) {
- const targetTime = clampMediaTimeToDuration(
- currentTime,
- Number.isFinite(audio.duration) ? audio.duration : null,
- );
-
- if (Math.abs(audio.playbackRate - targetPlaybackRate) > 0.001) {
- audio.playbackRate = targetPlaybackRate;
- }
+ const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
+ const targetTime = clampMediaTimeToDuration(currentTime, audioDuration);
if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
try {
@@ -3018,7 +3025,16 @@ export default function VideoEditor() {
}
}
- const atEnd = Number.isFinite(audio.duration) && targetTime >= audio.duration;
+ const syncedPlaybackRate = getMediaSyncPlaybackRate({
+ basePlaybackRate: targetPlaybackRate,
+ currentTime: audio.currentTime,
+ targetTime,
+ });
+ if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
+ audio.playbackRate = syncedPlaybackRate;
+ }
+
+ const atEnd = audioDuration !== null && targetTime >= audioDuration;
if (isPlaying && !atEnd) {
audio.play().catch(() => undefined);
} else if (!audio.paused) {
diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts
index cca0485d..f76ea975 100644
--- a/src/lib/exporter/audioEncoder.ts
+++ b/src/lib/exporter/audioEncoder.ts
@@ -1,7 +1,8 @@
import { WebDemuxer } from 'web-demuxer'
import type { SpeedRegion, TrimRegion, AudioRegion } from '@/components/video-editor/types'
-import type { VideoMuxer } from './muxer'
+import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from '@/lib/mediaTiming'
import { resolveMediaElementSource } from './localMediaSource'
+import type { VideoMuxer } from './muxer'
const AUDIO_BITRATE = 128_000
const DECODE_BACKPRESSURE_LIMIT = 20
@@ -44,19 +45,19 @@ export class AudioProcessor {
if (done || !chunk) break
if (passthroughTimestampOffsetUs === null) {
- passthroughTimestampOffsetUs = Math.min(0, chunk.timestamp)
- }
+ passthroughTimestampOffsetUs = chunk.timestamp
+ }
- const normalizedTimestamp = Math.max(
- 0,
- chunk.timestamp - passthroughTimestampOffsetUs,
- )
- const outputChunk = passthroughTimestampOffsetUs === 0
- ? chunk
- : this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp)
+ const normalizedTimestamp = Math.max(
+ 0,
+ chunk.timestamp - passthroughTimestampOffsetUs,
+ )
+ const outputChunk = passthroughTimestampOffsetUs === 0
+ ? chunk
+ : this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp)
await muxer.addAudioChunk(
- outputChunk,
+ outputChunk,
wroteAudio
? undefined
: {
@@ -212,16 +213,24 @@ export class AudioProcessor {
? demuxer.read('audio', 0, readEndSec)
: demuxer.read('audio')
+ let sourceTimestampOffsetUs: number | null = null
+
await this.transcodeAudioStream(
audioStream as ReadableStream<EncodedAudioChunk>,
audioConfig,
muxer,
{
+ observeChunkTimestampUs: (timestampUs) => {
+ if (sourceTimestampOffsetUs === null) {
+ sourceTimestampOffsetUs = timestampUs
+ }
+ },
shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims),
transformAudioData: (data) => {
const timestampMs = data.timestamp / 1000
const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims)
- const adjustedTimestampUs = data.timestamp - trimOffsetMs * 1000
+ const adjustedTimestampUs =
+ data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000
return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs))
},
},
@@ -233,6 +242,7 @@ export class AudioProcessor {
audioConfig: AudioDecoderConfig,
muxer: VideoMuxer,
options: {
+ observeChunkTimestampUs?: (timestampUs: number) => void
shouldSkipChunk?: (timestampMs: number) => boolean
transformAudioData?: (data: AudioData) => AudioData | null
} = {},
@@ -344,6 +354,7 @@ export class AudioProcessor {
const { done, value: chunk } = await reader.read()
if (done || !chunk) break
+ options.observeChunkTimestampUs?.(chunk.timestamp)
const timestampMs = chunk.timestamp / 1000
if (options.shouldSkipChunk?.(timestampMs)) continue
@@ -572,19 +583,10 @@ export class AudioProcessor {
for (const entry of sourceAudioElements) {
const audioEl = entry.media
- const targetTimeSec = Math.max(
- 0,
- Math.min(
- currentTimeMs / 1000,
- Number.isFinite(audioEl.duration) ? audioEl.duration : currentTimeMs / 1000,
- ),
- )
-
- if (Math.abs(audioEl.playbackRate - playbackRate) > 0.0001) {
- audioEl.playbackRate = playbackRate
- }
+ const audioDuration = Number.isFinite(audioEl.duration) ? audioEl.duration : null
+ const targetTimeSec = clampMediaTimeToDuration(currentTimeMs / 1000, audioDuration)
- const atEnd = Number.isFinite(audioEl.duration) && targetTimeSec >= audioEl.duration
+ const atEnd = audioDuration !== null && targetTimeSec >= audioDuration
if (atEnd) {
if (!audioEl.paused) {
audioEl.pause()
@@ -592,11 +594,22 @@ export class AudioProcessor {
continue
}
- if (audioEl.paused) {
+ if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
audioEl.currentTime = targetTimeSec
- audioEl.play().catch(() => {})
- } else if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
+ }
+
+ const syncedPlaybackRate = getMediaSyncPlaybackRate({
+ basePlaybackRate: playbackRate,
+ currentTime: audioEl.currentTime,
+ targetTime: targetTimeSec,
+ })
+ if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
+ audioEl.playbackRate = syncedPlaybackRate
+ }
+
+ if (audioEl.paused) {
audioEl.currentTime = targetTimeSec
+ audioEl.play().catch(() => undefined)
}
}
@@ -607,11 +620,22 @@ export class AudioProcessor {
if (isInRegion) {
const audioOffset = (currentTimeMs - region.startMs) / 1000
- if (audioEl.paused) {
+ if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
audioEl.currentTime = audioOffset
- audioEl.play().catch(() => {})
- } else if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
+ }
+
+ const syncedPlaybackRate = getMediaSyncPlaybackRate({
+ basePlaybackRate: playbackRate,
+ currentTime: audioEl.currentTime,
+ targetTime: audioOffset,
+ })
+ if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
+ audioEl.playbackRate = syncedPlaybackRate
+ }
+
+ if (audioEl.paused) {
audioEl.currentTime = audioOffset
+ audioEl.play().catch(() => undefined)
}
} else {
if (!audioEl.paused) {
diff --git a/src/lib/exporter/forwardFrameSource.ts b/src/lib/exporter/forwardFrameSource.ts
index c203075c..a88c6a5f 100644
--- a/src/lib/exporter/forwardFrameSource.ts
+++ b/src/lib/exporter/forwardFrameSource.ts
@@ -1,4 +1,5 @@
import { WebDemuxer } from "web-demuxer";
+import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
@@ -270,7 +271,16 @@ export class ForwardFrameSource {
throw new Error("Frame source not initialized");
}
- const clampedTargetTime = Math.max(0, Math.min(targetTimeSec, this.metadata.duration || targetTimeSec));
+ const clampedTargetTime = Math.max(
+ 0,
+ Math.min(
+ targetTimeSec,
+ getEffectiveVideoStreamDurationSeconds({
+ duration: this.metadata.duration,
+ streamDuration: this.metadata.streamDuration,
+ }) || targetTimeSec,
+ ),
+ );
if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) {
throw new Error("ForwardFrameSource only supports increasing timestamps");
}
diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts
index 83c906f8..471ff026 100644
--- a/src/lib/exporter/streamingDecoder.ts
+++ b/src/lib/exporter/streamingDecoder.ts
@@ -1,5 +1,6 @@
import { WebDemuxer } from 'web-demuxer';
import type { TrimRegion, SpeedRegion } from '@/components/video-editor/types';
+import { getEffectiveVideoStreamDurationSeconds } from '@/lib/mediaTiming';
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
@@ -178,8 +179,12 @@ export class StreamingVideoDecoder {
const decoderConfig = await this.demuxer.getDecoderConfig('video');
const codec = this.metadata.codec.toLowerCase();
const shouldPreferSoftwareDecode = codec.includes('av01') || codec.includes('av1');
+ const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
+ duration: this.metadata.duration,
+ streamDuration: this.metadata.streamDuration,
+ });
const segments = this.splitBySpeed(
- this.computeSegments(this.metadata.duration, trimRegions),
+ this.computeSegments(effectiveVideoDuration, trimRegions),
speedRegions
);
const segmentOutputFrameCounts = segments.map(segment =>
@@ -512,7 +517,13 @@ export class StreamingVideoDecoder {
getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number {
if (!this.metadata) throw new Error('Must call loadMetadata() first');
- const trimSegments = this.computeSegments(this.metadata.duration, trimRegions);
+ const trimSegments = this.computeSegments(
+ getEffectiveVideoStreamDurationSeconds({
+ duration: this.metadata.duration,
+ streamDuration: this.metadata.streamDuration,
+ }),
+ trimRegions,
+ );
const speedSegments = this.splitBySpeed(trimSegments, speedRegions);
return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0);
}
diff --git a/src/lib/mediaTiming.test.ts b/src/lib/mediaTiming.test.ts
index 762d65d0..f06be380 100644
--- a/src/lib/mediaTiming.test.ts
+++ b/src/lib/mediaTiming.test.ts
@@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest";
import {
clampMediaTimeToDuration,
+ getEffectiveVideoStreamDurationSeconds,
getEffectiveRecordingDurationMs,
+ getMediaSyncPlaybackRate,
} from "./mediaTiming";
describe("clampMediaTimeToDuration", () => {
@@ -38,4 +40,73 @@ describe("getEffectiveRecordingDurationMs", () => {
}),
).toBe(6_000);
});
-});
\ No newline at end of file
+});
+
+describe("getMediaSyncPlaybackRate", () => {
+ it("returns the base rate when drift is within tolerance", () => {
+ expect(
+ getMediaSyncPlaybackRate({
+ basePlaybackRate: 1,
+ currentTime: 10,
+ targetTime: 10.01,
+ }),
+ ).toBe(1);
+ });
+
+ it("nudges playback rate toward the target time", () => {
+ expect(
+ getMediaSyncPlaybackRate({
+ basePlaybackRate: 1,
+ currentTime: 10,
+ targetTime: 10.1,
+ }),
+ ).toBeCloseTo(1.05);
+
+ expect(
+ getMediaSyncPlaybackRate({
+ basePlaybackRate: 1,
+ currentTime: 10.1,
+ targetTime: 10,
+ }),
+ ).toBeCloseTo(0.95);
+ });
+
+ it("clamps oversized corrections", () => {
+ expect(
+ getMediaSyncPlaybackRate({
+ basePlaybackRate: 1,
+ currentTime: 10,
+ targetTime: 10.5,
+ }),
+ ).toBeCloseTo(1.08);
+ });
+});
+
+describe("getEffectiveVideoStreamDurationSeconds", () => {
+ it("prefers the video stream duration when present", () => {
+ expect(
+ getEffectiveVideoStreamDurationSeconds({
+ duration: 12,
+ streamDuration: 11.2,
+ }),
+ ).toBe(11.2);
+ });
+
+ it("falls back to the container duration when stream duration is missing", () => {
+ expect(
+ getEffectiveVideoStreamDurationSeconds({
+ duration: 12,
+ streamDuration: undefined,
+ }),
+ ).toBe(12);
+ });
+
+ it("returns zero when neither duration is usable", () => {
+ expect(
+ getEffectiveVideoStreamDurationSeconds({
+ duration: Number.NaN,
+ streamDuration: 0,
+ }),
+ ).toBe(0);
+ });
+});
diff --git a/src/lib/mediaTiming.ts b/src/lib/mediaTiming.ts
index 869ae43c..29583eaf 100644
--- a/src/lib/mediaTiming.ts
+++ b/src/lib/mediaTiming.ts
@@ -10,6 +10,63 @@ export function clampMediaTimeToDuration(
return Math.max(0, Math.min(safeTargetTime, Math.max(0, duration)));
}
+export function getMediaSyncPlaybackRate({
+ basePlaybackRate,
+ currentTime,
+ targetTime,
+ toleranceSeconds = 0.015,
+ correctionWindowSeconds = 2,
+ maxAdjustment = 0.08,
+}: {
+ basePlaybackRate: number;
+ currentTime: number;
+ targetTime: number;
+ toleranceSeconds?: number;
+ correctionWindowSeconds?: number;
+ maxAdjustment?: number;
+}): number {
+ const safeBasePlaybackRate =
+ Number.isFinite(basePlaybackRate) && basePlaybackRate > 0
+ ? basePlaybackRate
+ : 1;
+
+ if (!Number.isFinite(currentTime) || !Number.isFinite(targetTime)) {
+ return safeBasePlaybackRate;
+ }
+
+ const driftSeconds = targetTime - currentTime;
+ if (Math.abs(driftSeconds) <= toleranceSeconds) {
+ return safeBasePlaybackRate;
+ }
+
+ const safeCorrectionWindow = correctionWindowSeconds > 0 ? correctionWindowSeconds : 2;
+ const safeMaxAdjustment = Math.max(0, maxAdjustment);
+ const adjustment = Math.max(
+ -safeMaxAdjustment,
+ Math.min(safeMaxAdjustment, driftSeconds / safeCorrectionWindow),
+ );
+
+ return Math.max(0.1, safeBasePlaybackRate + adjustment);
+}
+
+export function getEffectiveVideoStreamDurationSeconds({
+ duration,
+ streamDuration,
+}: {
+ duration?: number | null;
+ streamDuration?: number | null;
+}): number {
+ if (Number.isFinite(streamDuration) && (streamDuration ?? 0) > 0) {
+ return Math.max(0, streamDuration ?? 0);
+ }
+
+ if (Number.isFinite(duration) && (duration ?? 0) > 0) {
+ return Math.max(0, duration ?? 0);
+ }
+
+ return 0;
+}
+
export function getEffectiveRecordingDurationMs({
startTimeMs,
endTimeMs,