Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix(recording): improve webcam audio sync and delay compensation

webadderalle90f8f654b07
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 2c91e0f8..c342abd4 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -1166,6 +1166,38 @@ function getFfmpegBinaryPath() {
return ffmpegStatic
}
+/** Probe the duration of a media file (in seconds) using ffmpeg. */
+async function probeMediaDurationSeconds(filePath: string): Promise<number> {
+ const ffmpegPath = getFfmpegBinaryPath()
+ try {
+ await execFileAsync(ffmpegPath, ['-i', filePath, '-f', 'null', '-'], { timeout: 30000, maxBuffer: 2 * 1024 * 1024 })
+ } catch (error) {
+ // ffmpeg reports info on stderr even on "success" — parse it from the error
+ const stderr = (error as NodeJS.ErrnoException & { stderr?: string })?.stderr ?? ''
+ // Match "Duration: HH:MM:SS.mm" or "time=HH:MM:SS.mm" (from progress output)
+ // Prefer the last "time=" value (actual decoded duration) over the container Duration header
+ const timeMatches = [...stderr.matchAll(/time=(\d{2}):(\d{2}):(\d{2})\.(\d{2,3})/g)]
+ if (timeMatches.length > 0) {
+ const last = timeMatches[timeMatches.length - 1]
+ const h = Number(last[1])
+ const m = Number(last[2])
+ const s = Number(last[3])
+ const frac = Number(last[4]) / (last[4].length === 3 ? 1000 : 100)
+ return h * 3600 + m * 60 + s + frac
+ }
+ // Fall back to Duration header
+ const durationMatch = stderr.match(/Duration:\s*(\d{2}):(\d{2}):(\d{2})\.(\d{2,3})/)
+ if (durationMatch) {
+ const h = Number(durationMatch[1])
+ const m = Number(durationMatch[2])
+ const s = Number(durationMatch[3])
+ const frac = Number(durationMatch[4]) / (durationMatch[4].length === 3 ? 1000 : 100)
+ return h * 3600 + m * 60 + s + frac
+ }
+ }
+ return 0
+}
+
function sendWhisperModelDownloadProgress(
webContents: Electron.WebContents,
payload: { status: 'idle' | 'downloading' | 'downloaded' | 'error'; progress: number; path?: string | null; error?: string },
@@ -2079,6 +2111,7 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
const ffmpegPath = getFfmpegBinaryPath()
const inputs: string[] = ['-i', videoPath]
const audioInputs: string[] = []
+ const audioFilePaths: string[] = []
for (const [label, audioPath] of [['system', systemAudioPath], ['mic', micAudioPath]] as const) {
if (!audioPath) continue
@@ -2091,6 +2124,7 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
}
inputs.push('-i', audioPath)
audioInputs.push(label)
+ audioFilePaths.push(audioPath)
} catch {
console.warn(`[mux-win] Skipping ${label} audio: file not accessible (${audioPath})`)
}
@@ -2098,8 +2132,27 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
if (audioInputs.length === 0) return
+ // Probe durations to compute audio delay offsets.
+ // If an audio file is shorter than the video it means the audio device
+ // started delivering samples late (common with Bluetooth / iPhone mics).
+ const videoDuration = await probeMediaDurationSeconds(videoPath)
+ const audioDelays: Map<string, number> = new Map()
+
+ if (videoDuration > 0) {
+ for (let i = 0; i < audioFilePaths.length; i++) {
+ const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
+ const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
+ audioDelays.set(audioInputs[i], delayMs)
+ if (delayMs > 0) {
+ console.log(`[mux-win] ${audioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
+ }
+ }
+ }
+
const mixedOutputPath = `${videoPath}.muxed.mp4`
const normalizedPauseSegments = normalizePauseSegments(pauseSegments)
+ const systemDelayMs = audioDelays.get('system') ?? 0
+ const micDelayMs = audioDelays.get('mic') ?? 0
if (audioInputs.length === 2) {
// Both system + mic audio: mix them
@@ -2114,8 +2167,22 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
filterParts.push(micPauseFilter)
}
- filterParts.push(`${micPauseFilter ? '[mic_trimmed]' : '[2:a]'}atrim=start=0.10,asetpts=PTS-STARTPTS[m]`)
- filterParts.push(`${systemPauseFilter ? '[system_trimmed]' : '[1:a]'}[m]amix=inputs=2:duration=longest:normalize=0[aout]`)
+ const systemLabel = systemPauseFilter ? '[system_trimmed]' : '[1:a]'
+ const micLabel = micPauseFilter ? '[mic_trimmed]' : '[2:a]'
+
+ // Apply delay to compensate for late audio start
+ if (micDelayMs > 0) {
+ filterParts.push(`${micLabel}adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
+ } else {
+ filterParts.push(`${micLabel}asetpts=PTS-STARTPTS[m]`)
+ }
+
+ if (systemDelayMs > 0) {
+ filterParts.push(`${systemLabel}adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
+ filterParts.push(`[s][m]amix=inputs=2:duration=longest:normalize=0[aout]`)
+ } else {
+ filterParts.push(`${systemLabel}[m]amix=inputs=2:duration=longest:normalize=0[aout]`)
+ }
await execFileAsync(
ffmpegPath,
@@ -2136,35 +2203,52 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
} else {
// Single audio track
const pauseFilter = buildPausedAudioFilter('1:a', 'aout', normalizedPauseSegments)
+ const singleDelayMs = audioDelays.get(audioInputs[0]) ?? 0
- await execFileAsync(
- ffmpegPath,
- pauseFilter
- ? [
- '-y',
- ...inputs,
- '-filter_complex', pauseFilter,
- '-map', '0:v:0',
- '-map', '[aout]',
- '-c:v', 'copy',
- '-c:a', 'aac',
- '-b:a', '192k',
- '-shortest',
- mixedOutputPath,
- ]
- : [
- '-y',
- ...inputs,
- '-map', '0:v:0',
- '-map', '1:a:0',
- '-c:v', 'copy',
- '-c:a', 'aac',
- '-b:a', '192k',
- '-shortest',
- mixedOutputPath,
- ],
- { timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
- )
+ if (pauseFilter || singleDelayMs > 0) {
+ const filterParts: string[] = []
+ if (pauseFilter) {
+ filterParts.push(pauseFilter)
+ }
+ const srcLabel = pauseFilter ? '[aout]' : '[1:a]'
+ if (singleDelayMs > 0) {
+ filterParts.push(`${srcLabel}adelay=${singleDelayMs}|${singleDelayMs},asetpts=PTS-STARTPTS[delayed]`)
+ }
+ const outLabel = singleDelayMs > 0 ? '[delayed]' : '[aout]'
+
+ await execFileAsync(
+ ffmpegPath,
+ [
+ '-y',
+ ...inputs,
+ '-filter_complex', filterParts.join(';'),
+ '-map', '0:v:0',
+ '-map', outLabel,
+ '-c:v', 'copy',
+ '-c:a', 'aac',
+ '-b:a', '192k',
+ '-shortest',
+ mixedOutputPath,
+ ],
+ { timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
+ )
+ } else {
+ await execFileAsync(
+ ffmpegPath,
+ [
+ '-y',
+ ...inputs,
+ '-map', '0:v:0',
+ '-map', '1:a:0',
+ '-c:v', 'copy',
+ '-c:a', 'aac',
+ '-b:a', '192k',
+ '-shortest',
+ mixedOutputPath,
+ ],
+ { timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
+ )
+ }
}
await moveFileWithOverwrite(mixedOutputPath, videoPath)
@@ -2353,6 +2437,7 @@ async function muxNativeMacRecordingWithAudio(
const inputs = ['-i', videoPath]
const availableAudioInputs: string[] = []
+ const audioFilePaths: string[] = []
for (const [label, audioPath] of [['system', systemAudioPath], ['microphone', microphonePath]] as const) {
if (!audioPath) continue
@@ -2365,6 +2450,7 @@ async function muxNativeMacRecordingWithAudio(
}
inputs.push('-i', audioPath)
availableAudioInputs.push(label)
+ audioFilePaths.push(audioPath)
} catch {
console.warn(`[mux] Skipping ${label} audio: file not accessible (${audioPath})`)
}
@@ -2375,8 +2461,53 @@ async function muxNativeMacRecordingWithAudio(
return
}
- const args = availableAudioInputs.length === 2
- ? [
+ // Probe durations — if audio is shorter than video it means the audio device
+ // started late (e.g. iPhone mic over Continuity Camera). Add leading silence.
+ const videoDuration = await probeMediaDurationSeconds(videoPath)
+ const audioDelays: Map<string, number> = new Map()
+
+ if (videoDuration > 0) {
+ for (let i = 0; i < audioFilePaths.length; i++) {
+ const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
+ const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
+ audioDelays.set(availableAudioInputs[i], delayMs)
+ if (delayMs > 0) {
+ console.log(`[mux] ${availableAudioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
+ }
+ }
+ }
+
+ const systemDelayMs = audioDelays.get('system') ?? 0
+ const micDelayMs = audioDelays.get('microphone') ?? 0
+ const needsFilter = systemDelayMs > 0 || micDelayMs > 0
+
+ let args: string[]
+ if (availableAudioInputs.length === 2) {
+ if (needsFilter) {
+ const filterParts: string[] = []
+ if (systemDelayMs > 0) {
+ filterParts.push(`[1:a]adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
+ }
+ if (micDelayMs > 0) {
+ filterParts.push(`[2:a]adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
+ }
+ const sLabel = systemDelayMs > 0 ? '[s]' : '[1:a]'
+ const mLabel = micDelayMs > 0 ? '[m]' : '[2:a]'
+ filterParts.push(`${sLabel}${mLabel}amix=inputs=2:duration=longest:normalize=0[aout]`)
+ args = [
+ '-y',
+ ...inputs,
+ '-filter_complex', filterParts.join(';'),
+ '-map', '0:v:0',
+ '-map', '[aout]',
+ '-c:v', 'copy',
+ '-c:a', 'aac',
+ '-b:a', '192k',
+ '-shortest',
+ mixedOutputPath,
+ ]
+ } else {
+ args = [
'-y',
...inputs,
'-filter_complex', '[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]',
@@ -2388,7 +2519,24 @@ async function muxNativeMacRecordingWithAudio(
'-shortest',
mixedOutputPath,
]
- : [
+ }
+ } else {
+ if (needsFilter) {
+ const delayMs = systemDelayMs || micDelayMs
+ args = [
+ '-y',
+ ...inputs,
+ '-filter_complex', `[1:a]adelay=${delayMs}|${delayMs},asetpts=PTS-STARTPTS[aout]`,
+ '-map', '0:v:0',
+ '-map', '[aout]',
+ '-c:v', 'copy',
+ '-c:a', 'aac',
+ '-b:a', '192k',
+ '-shortest',
+ mixedOutputPath,
+ ]
+ } else {
+ args = [
'-y',
...inputs,
'-map', '0:v:0',
@@ -2399,6 +2547,8 @@ async function muxNativeMacRecordingWithAudio(
'-shortest',
mixedOutputPath,
]
+ }
+ }
console.log('[mux] Running ffmpeg:', ffmpegPath, args.join(' '))
diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts
index a7407d67..a8246d6b 100644
--- a/src/hooks/useScreenRecorder.ts
+++ b/src/hooks/useScreenRecorder.ts
@@ -380,7 +380,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
return result.path;
}, [finalizeRecordingSession, stopWebcamRecorder]);
- const startWebcamRecorder = useCallback(async () => {
+ /**
+ * Acquire the webcam stream and prepare the MediaRecorder, but do NOT start
+ * recording yet. Call {@link beginWebcamCapture} after the main recording
+ * has started so both begin at approximately the same time.
+ */
+ const prepareWebcamRecorder = useCallback(async () => {
if (!webcamEnabled) {
resolvedWebcamPath.current = null;
pendingWebcamPathPromise.current = Promise.resolve(null);
@@ -462,9 +467,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
}
};
-
- webcamStartTime.current = Date.now();
- recorder.start(RECORDER_TIMESLICE_MS);
} catch (error) {
console.warn("Failed to start webcam recording; continuing without webcam layer:", error);
resolvedWebcamPath.current = null;
@@ -480,6 +482,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}
}, [getRecordingDurationMs, webcamDeviceId, webcamEnabled]);
+ /** Start the prepared webcam MediaRecorder. Call after main recording begins. */
+ const beginWebcamCapture = useCallback(() => {
+ const recorder = webcamRecorder.current;
+ if (recorder && recorder.state === "inactive") {
+ webcamStartTime.current = Date.now();
+ recorder.start(RECORDER_TIMESLICE_MS);
+ }
+ }, []);
+
const stopRecording = useRef(() => {
setPaused(false);
if (nativeScreenRecording.current) {
@@ -686,7 +697,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
recordingSessionTimestamp.current = Date.now();
resetRecordingClock(recordingSessionTimestamp.current);
- await startWebcamRecorder();
+ await prepareWebcamRecorder();
const platform = await window.electronAPI.getPlatform();
const useNativeMacScreenCapture =
@@ -766,6 +777,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
if (nativeResult.success) {
const mainStartedAt = Date.now();
+ beginWebcamCapture();
nativeScreenRecording.current = true;
nativeWindowsRecording.current = useNativeWindowsCapture;
resetRecordingClock(mainStartedAt);
@@ -1000,6 +1012,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
setRecording(false);
};
const mainStartedAt = Date.now();
+ beginWebcamCapture();
resetRecordingClock(mainStartedAt);
webcamTimeOffsetMs.current = webcamStartTime.current === null
? 0