Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

Fix companion audio sync for trimmed and long exports

webadderall3283ee921397
diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts
index 5012dbe5..1f88763b 100644
--- a/electron/electron-env.d.ts
+++ b/electron/electron-env.d.ts
@@ -159,6 +159,9 @@ interface Window {
storeMicrophoneSidecar: (
audioData: ArrayBuffer,
videoPath: string,
+ options?: {
+ startDelayMs?: number;
+ },
) => Promise<{ success: boolean; path?: string; error?: string }>;
getRecordedVideoPath: () => Promise<{ success: boolean; path?: string; message?: string }>;
listAssetDirectory: (relativeDir: string) => Promise<{
@@ -231,7 +234,12 @@ interface Window {
}>;
getVideoAudioFallbackPaths: (
videoPath: string,
- ) => Promise<{ success: boolean; paths: string[]; error?: string }>;
+ ) => Promise<{
+ success: boolean;
+ paths: string[];
+ startDelayMsByPath?: Record<string, number>;
+ error?: string;
+ }>;
setRecordingState: (recording: boolean) => Promise<void>;
getCursorTelemetry: (videoPath?: string) => Promise<{
success: boolean;
diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts
index b2a63085..9a775339 100644
--- a/electron/ipc/recording/diagnostics.test.ts
+++ b/electron/ipc/recording/diagnostics.test.ts
@@ -133,6 +133,39 @@ describe("getCompanionAudioFallbackPaths", () => {
]);
});
+ it("loads saved sidecar timing metadata alongside companion audio paths", async () => {
+ const videoPath = path.join(tempRoot, "recording.mp4");
+ const micPath = path.join(tempRoot, "recording.mic.webm");
+
+ await Promise.all([
+ fs.writeFile(videoPath, "video"),
+ fs.writeFile(micPath, "mic"),
+ fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: 2750 })),
+ ]);
+
+ execFileMock.mockImplementation(
+ (
+ _file: string,
+ _args: string[],
+ _options: Record<string, unknown>,
+ callback: ExecFileCallback,
+ ) => {
+ const error = new Error("ffmpeg probe failed") as Error & { stderr?: string };
+ error.stderr = "Stream #0:0: Video: h264";
+ callback(error, "", error.stderr);
+ },
+ );
+
+ const { getCompanionAudioFallbackInfo } = await import("./diagnostics");
+
+ await expect(getCompanionAudioFallbackInfo(videoPath)).resolves.toEqual({
+ paths: [micPath],
+ startDelayMsByPath: {
+ [micPath]: 2750,
+ },
+ });
+ });
+
it("rejects tiny MP4 container-only outputs before they reach the editor", async () => {
const videoPath = path.join(tempRoot, "recording-123.mp4");
await fs.writeFile(videoPath, Buffer.alloc(261));
diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts
index 1114414e..4588631e 100644
--- a/electron/ipc/recording/diagnostics.ts
+++ b/electron/ipc/recording/diagnostics.ts
@@ -9,6 +9,10 @@ import type { CompanionAudioCandidate, NativeCaptureDiagnostics } from "../types
const execFileAsync = promisify(execFile);
export const MIN_VALID_RECORDED_VIDEO_BYTES = 1024;
+type CompanionAudioTimingMetadata = {
+ startDelayMs?: number;
+};
+
export function recordNativeCaptureDiagnostics(
diagnostics: Omit<NativeCaptureDiagnostics, "timestamp">,
) {
@@ -99,6 +103,22 @@ export async function getUsableCompanionAudioCandidates(
return candidates;
}
+async function readCompanionAudioTimingMetadata(
+ companionPath: string,
+): Promise<CompanionAudioTimingMetadata | null> {
+ try {
+ const raw = await fs.readFile(`${companionPath}.json`, "utf8");
+ const parsed = JSON.parse(raw) as CompanionAudioTimingMetadata | null;
+ if (!parsed || typeof parsed !== "object") {
+ return null;
+ }
+
+ return parsed;
+ } catch {
+ return null;
+ }
+}
+
export async function hasEmbeddedAudioStream(videoPath: string) {
const ffmpegPath = getFfmpegBinaryPath();
let stderr = "";
@@ -118,11 +138,17 @@ export async function hasEmbeddedAudioStream(videoPath: string) {
}
export async function getCompanionAudioFallbackPaths(videoPath: string) {
+ const { paths } = await getCompanionAudioFallbackInfo(videoPath);
+ return paths;
+}
+
+export async function getCompanionAudioFallbackInfo(videoPath: string) {
const companionCandidates = await getUsableCompanionAudioCandidates(videoPath);
if (companionCandidates.length === 0) {
- return [];
+ return { paths: [], startDelayMsByPath: {} };
}
+ let paths: string[];
if (await hasEmbeddedAudioStream(videoPath)) {
const microphoneCompanionPaths = Array.from(
new Set(
@@ -134,13 +160,36 @@ export async function getCompanionAudioFallbackPaths(videoPath: string) {
),
);
if (microphoneCompanionPaths.length === 0) {
- return [];
+ return { paths: [], startDelayMsByPath: {} };
}
- return [videoPath, ...microphoneCompanionPaths];
+ paths = [videoPath, ...microphoneCompanionPaths];
+ } else {
+ paths = Array.from(
+ new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)),
+ );
}
- return Array.from(new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)));
+ const metadataEntries = await Promise.all(
+ paths.map(async (audioPath) => {
+ const metadata = await readCompanionAudioTimingMetadata(audioPath);
+ const startDelayMs = metadata?.startDelayMs;
+ if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) {
+ return null;
+ }
+
+ return [audioPath, Math.round(startDelayMs ?? 0)] as const;
+ }),
+ );
+
+ return {
+ paths,
+ startDelayMsByPath: Object.fromEntries(
+ metadataEntries.filter(
+ (entry): entry is readonly [string, number] => entry !== null,
+ ),
+ ),
+ };
}
export async function validateRecordedVideo(videoPath: string) {
diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts
index 1f19cb51..9286efab 100644
--- a/electron/ipc/register/recording.ts
+++ b/electron/ipc/register/recording.ts
@@ -35,7 +35,7 @@ import {
} from "../paths/binaries";
import { rememberApprovedLocalReadPath } from "../project/manager";
import {
- getCompanionAudioFallbackPaths,
+ getCompanionAudioFallbackInfo,
getFileSizeIfPresent,
recordNativeCaptureDiagnostics,
validateRecordedVideo,
@@ -1013,19 +1013,19 @@ export function registerRecordingHandlers(
ipcMain.handle("get-video-audio-fallback-paths", async (_event, videoPath: string) => {
if (!videoPath) {
- return { success: true, paths: [] };
+ return { success: true, paths: [], startDelayMsByPath: {} };
}
try {
- const paths = await getCompanionAudioFallbackPaths(videoPath);
+ const { paths, startDelayMsByPath } = await getCompanionAudioFallbackInfo(videoPath);
await Promise.all([
rememberApprovedLocalReadPath(videoPath),
...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)),
]);
- return { success: true, paths };
+ return { success: true, paths, startDelayMsByPath };
} catch (error) {
console.error("Failed to resolve companion audio fallback paths:", error);
- return { success: false, paths: [], error: String(error) };
+ return { success: false, paths: [], startDelayMsByPath: {}, error: String(error) };
}
});
@@ -1170,11 +1170,30 @@ export function registerRecordingHandlers(
ipcMain.handle(
"store-microphone-sidecar",
- async (_, audioData: ArrayBuffer, videoPath: string) => {
+ async (
+ _,
+ audioData: ArrayBuffer,
+ videoPath: string,
+ options?: { startDelayMs?: number },
+ ) => {
try {
const baseName = videoPath.replace(/\.[^.]+$/, "");
const sidecarPath = `${baseName}.mic.webm`;
await fs.writeFile(sidecarPath, Buffer.from(audioData));
+ const startDelayMs = options?.startDelayMs;
+ if (Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0) {
+ try {
+ await fs.writeFile(
+ `${sidecarPath}.json`,
+ JSON.stringify({ startDelayMs: Math.round(startDelayMs ?? 0) }),
+ );
+ } catch (metadataError) {
+ console.warn(
+ "Failed to store microphone sidecar timing metadata:",
+ metadataError,
+ );
+ }
+ }
return { success: true, path: sidecarPath };
} catch (error) {
console.error("Failed to store microphone sidecar:", error);
diff --git a/electron/preload.ts b/electron/preload.ts
index a8b0e67c..a47c13e5 100644
--- a/electron/preload.ts
+++ b/electron/preload.ts
@@ -268,8 +268,12 @@ contextBridge.exposeInMainWorld("electronAPI", {
storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => {
return ipcRenderer.invoke("store-recorded-video", videoData, fileName);
},
- storeMicrophoneSidecar: (audioData: ArrayBuffer, videoPath: string) => {
- return ipcRenderer.invoke("store-microphone-sidecar", audioData, videoPath);
+ storeMicrophoneSidecar: (
+ audioData: ArrayBuffer,
+ videoPath: string,
+ options?: { startDelayMs?: number },
+ ) => {
+ return ipcRenderer.invoke("store-microphone-sidecar", audioData, videoPath, options);
},
getRecordedVideoPath: () => {
return ipcRenderer.invoke("get-recorded-video-path");
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index a436aa25..6e4e7b98 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -596,6 +596,8 @@ export default function VideoEditor() {
const [showExportDropdown, setShowExportDropdown] = useState(false);
const [previewVolume, setPreviewVolume] = useState(1);
const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
+ const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] =
+ useState<Record<string, number>>({});
const [aspectRatio, setAspectRatio] = useState<AspectRatio>(
initialEditorPreferences.aspectRatio,
);
@@ -1219,6 +1221,7 @@ export default function VideoEditor() {
useEffect(() => {
let cancelled = false;
setSourceAudioFallbackPaths([]);
+ setSourceAudioFallbackStartDelayMsByPath({});
if (!currentSourcePath) {
return () => {
@@ -1235,6 +1238,7 @@ export default function VideoEditor() {
}
if (!result.success) {
setSourceAudioFallbackPaths([]);
+ setSourceAudioFallbackStartDelayMsByPath({});
toast.warning(
result.error
? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
@@ -1246,9 +1250,11 @@ export default function VideoEditor() {
toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
setSourceAudioFallbackPaths(result.paths ?? []);
+ setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {});
} catch (error) {
if (!cancelled) {
setSourceAudioFallbackPaths([]);
+ setSourceAudioFallbackStartDelayMsByPath({});
toast.warning(
`Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
@@ -3516,13 +3522,14 @@ export default function VideoEditor() {
}
}
- for (const audioPath of previewSourceAudioFallbackPaths) {
- let audio = existing.get(audioPath);
- if (!audio) {
- audio = new Audio();
- audio.preload = "auto";
- existing.set(audioPath, audio);
- }
+ for (const audioPath of previewSourceAudioFallbackPaths) {
+ let audio = existing.get(audioPath);
+ if (!audio) {
+ audio = new Audio();
+ audio.preload = "auto";
+ existing.set(audioPath, audio);
+ }
+ audio.dataset.sourceAudioPath = audioPath;
if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
audio.pause();
@@ -3665,6 +3672,7 @@ export default function VideoEditor() {
const startDelaySeconds = estimateCompanionAudioStartDelaySeconds(
duration,
audioDuration,
+ sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""],
);
const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
const targetTime = clampMediaTimeToDuration(
@@ -3698,7 +3706,14 @@ export default function VideoEditor() {
}
lastSourceAudioSyncTimeRef.current = currentTime;
- }, [currentTime, duration, isPlaying, previewSourceAudioFallbackPaths, speedRegions]);
+ }, [
+ currentTime,
+ duration,
+ isPlaying,
+ previewSourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
+ speedRegions,
+ ]);
const showExportSuccessToast = useCallback((filePath: string) => {
toast.success(`Exported successfully to ${filePath}`, {
@@ -4018,6 +4033,7 @@ export default function VideoEditor() {
frame,
audioRegions,
sourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
previewWidth,
previewHeight,
onProgress: (progress: ExportProgress) => {
@@ -4216,6 +4232,7 @@ export default function VideoEditor() {
cursorSway,
audioRegions,
sourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
exportEncodingMode,
exportBackendPreference,
exportPipelineModel,
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx
index 159e98fe..b4d3d351 100644
--- a/src/components/video-editor/timeline/TimelineEditor.tsx
+++ b/src/components/video-editor/timeline/TimelineEditor.tsx
@@ -44,7 +44,6 @@ import {
} from "@/utils/aspectRatioUtils";
import { formatShortcut } from "@/utils/platformUtils";
import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences";
-import { toFileUrl } from "../projectPersistence";
import type {
AnnotationRegion,
AudioRegion,
diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts
index 1b56dad9..cb41930e 100644
--- a/src/hooks/useScreenRecorder.ts
+++ b/src/hooks/useScreenRecorder.ts
@@ -152,6 +152,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const recordingFinalizationToastId = useRef<string | number | null>(null);
const micFallbackRecorder = useRef<MediaRecorder | null>(null);
const micFallbackChunks = useRef<Blob[]>([]);
+ const micFallbackStartDelayMs = useRef<number | null>(null);
const showRecordingFinalizationToast = useCallback((message = "Preparing recording...") => {
recordingFinalizationToastId.current = toast.loading(message, {
@@ -472,6 +473,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
async (
micFallbackBlobPromise: Promise<Blob | null> | null | undefined,
finalPath: string,
+ startDelayMs?: number | null,
) => {
const micFallbackBlob = await micFallbackBlobPromise;
if (!micFallbackBlob) {
@@ -483,6 +485,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const result = await window.electronAPI.storeMicrophoneSidecar(
arrayBuffer,
finalPath,
+ Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0
+ ? { startDelayMs: startDelayMs ?? 0 }
+ : undefined,
);
if (!result.success) {
const errorMessage =
@@ -531,7 +536,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
}, []);
const recoverNativeRecordingSession = useCallback(
- async (micFallbackBlobPromise?: Promise<Blob | null> | null) => {
+ async (
+ micFallbackBlobPromise?: Promise<Blob | null> | null,
+ startDelayMs?: number | null,
+ ) => {
if (typeof window.electronAPI?.recoverNativeScreenRecording !== "function") {
return null;
}
@@ -544,7 +552,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const resolvedMicFallbackBlobPromise =
micFallbackBlobPromise ?? stopMicFallbackRecorder();
const webcamPath = await stopWebcamRecorder();
- await storeMicrophoneSidecar(resolvedMicFallbackBlobPromise, result.path);
+ await storeMicrophoneSidecar(resolvedMicFallbackBlobPromise, result.path, startDelayMs);
await finalizeRecordingSession(result.path, webcamPath);
return result.path;
},
@@ -685,6 +693,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
void (async () => {
showRecordingFinalizationToast();
+ const fallbackStartDelayMs = micFallbackStartDelayMs.current;
const micFallbackBlobPromise = stopMicFallbackRecorder();
const webcamPath = await stopWebcamRecorder();
const isNativeWindows = nativeWindowsRecording.current;
@@ -702,8 +711,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
);
void logNativeCaptureDiagnostics("stop-native-screen-recording");
try {
- const recoveredPath =
- await recoverNativeRecordingSession(micFallbackBlobPromise);
+ const recoveredPath = await recoverNativeRecordingSession(
+ micFallbackBlobPromise,
+ fallbackStartDelayMs,
+ );
if (recoveredPath) {
return;
}
@@ -750,7 +761,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
finalPath = muxResult.path;
}
- await storeMicrophoneSidecar(micFallbackBlobPromise, finalPath);
+ await storeMicrophoneSidecar(
+ micFallbackBlobPromise,
+ finalPath,
+ fallbackStartDelayMs,
+ );
await finalizeRecordingSession(finalPath, webcamPath);
})();
@@ -1025,6 +1040,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
if (nativeResult.success) {
const mainStartedAt = Date.now();
+ micFallbackStartDelayMs.current = null;
beginWebcamCapture();
nativeScreenRecording.current = true;
nativeWindowsRecording.current = useNativeWindowsCapture;
@@ -1067,9 +1083,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
micFallbackChunks.current.push(event.data);
}
};
+ micFallbackStartDelayMs.current = Math.max(
+ 0,
+ Date.now() - mainStartedAt,
+ );
recorder.start(RECORDER_TIMESLICE_MS);
micFallbackRecorder.current = recorder;
} catch (micError) {
+ micFallbackStartDelayMs.current = null;
console.warn("Browser microphone fallback failed:", micError);
const permissionDenied =
micError instanceof DOMException &&
diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts
index a39e9358..9e9deeec 100644
--- a/src/lib/exporter/audioEncoder.test.ts
+++ b/src/lib/exporter/audioEncoder.test.ts
@@ -12,6 +12,7 @@ type OfflineRenderTestHarness = AudioProcessor & {
speedRegions: never[],
audioRegions: never[],
sourceAudioFallbackPaths: string[],
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
): Promise<{
mainBuffer: AudioBuffer | null;
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>;
@@ -22,6 +23,7 @@ type OfflineRenderTestHarness = AudioProcessor & {
speedRegions: never[],
audioRegions: never[],
sourceAudioFallbackPaths: string[],
+ sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
muxer: unknown,
): Promise<void>;
};
@@ -82,4 +84,54 @@ describe("AudioProcessor offline render preparation", () => {
expect(loadAudioFileDemuxer).not.toHaveBeenCalled();
expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled();
});
+
+ it("uses recorded companion start-delay metadata instead of inferring from duration gap", async () => {
+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
+ const mainBuffer = { duration: 600, numberOfChannels: 2 } as AudioBuffer;
+ const micBuffer = { duration: 565, numberOfChannels: 1 } as AudioBuffer;
+
+ vi.spyOn(processor, "decodeAudioFromUrl").mockImplementation(async (url: string) => {
+ if (url === "file:///tmp/recording.mp4") {
+ return mainBuffer;
+ }
+ if (url === "/tmp/recording.mic.webm") {
+ return micBuffer;
+ }
+ return null;
+ });
+
+ const prepared = await processor.prepareOfflineRender(
+ "file:///tmp/recording.mp4",
+ [],
+ [],
+ [],
+ ["/tmp/recording.mic.webm"],
+ { "/tmp/recording.mic.webm": 3_500 },
+ );
+
+ expect(prepared.companionEntries[0]?.startDelaySec).toBeCloseTo(3.5);
+ });
+
+ it("avoids the single-sidecar fast path when companion timing metadata is present", async () => {
+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
+ const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer");
+ const renderAndMuxOfflineAudio = vi
+ .spyOn(processor, "renderAndMuxOfflineAudio")
+ .mockResolvedValue();
+
+ await processor.process(
+ null,
+ {} as never,
+ "file:///tmp/recording.mp4",
+ [],
+ [],
+ undefined,
+ [],
+ ["/tmp/recording.mic.webm"],
+ { "/tmp/recording.mic.webm": 2_000 },
+ );
+
+ expect(loadAudioFileDemuxer).not.toHaveBeenCalled();
+ expect(renderAndMuxOfflineAudio).toHaveBeenCalled();
+ });
});
diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts
index 150fbfea..d78d2cbc 100644
--- a/src/lib/exporter/audioEncoder.ts
+++ b/src/lib/exporter/audioEncoder.ts
@@ -143,6 +143,7 @@ export class AudioProcessor {
readEndSec?: number,
audioRegions?: AudioRegion[],
sourceAudioFallbackPaths?: string[],
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
): Promise<void> {
const sortedTrims = trimRegions
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
@@ -164,9 +165,13 @@ export class AudioProcessor {
videoUrl,
sortedSourceAudioFallbackPaths,
);
+ const hasTimedCompanionAudio = externalAudioPaths.some(
+ (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,
+ );
const needsSourceAudioMixing =
externalAudioPaths.length > 1 ||
- (hasEmbeddedSourceAudio && externalAudioPaths.length > 0);
+ (hasEmbeddedSourceAudio && externalAudioPaths.length > 0) ||
+ hasTimedCompanionAudio;
// When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline.
if (
@@ -180,6 +185,7 @@ export class AudioProcessor {
sortedSpeedRegions,
sortedAudioRegions,
sortedSourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
muxer,
);
return;
@@ -210,6 +216,7 @@ export class AudioProcessor {
[],
[],
externalAudioPaths,
+ sourceAudioFallbackStartDelayMsByPath,
muxer,
);
return;
@@ -255,6 +262,7 @@ export class AudioProcessor {
speedRegions?: SpeedRegion[],
audioRegions?: AudioRegion[],
sourceAudioFallbackPaths?: string[],
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
): Promise<Blob> {
const sortedTrims = trimRegions
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
@@ -279,6 +287,7 @@ export class AudioProcessor {
sortedSpeedRegions,
sortedAudioRegions,
sortedSourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
);
return this.renderToWavBlobChunked(prepared);
}
@@ -528,6 +537,7 @@ export class AudioProcessor {
speedRegions: SpeedRegion[],
audioRegions: AudioRegion[],
sourceAudioFallbackPaths: string[],
+ sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
muxer: VideoMuxer,
): Promise<void> {
const prepared = await this.prepareOfflineRender(
@@ -536,6 +546,7 @@ export class AudioProcessor {
speedRegions,
audioRegions,
sourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
);
if (this.cancelled) return;
await this.renderAndEncodeChunked(prepared, muxer);
@@ -547,6 +558,7 @@ export class AudioProcessor {
speedRegions: SpeedRegion[],
audioRegions: AudioRegion[],
sourceAudioFallbackPaths: string[],
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
): Promise<PreparedOfflineRender> {
if (this.cancelled) throw new Error("Export cancelled");
this.onProgress?.(0);
@@ -562,17 +574,20 @@ export class AudioProcessor {
// Decode companion / sidecar audio files
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = [];
+ const refDuration =
+ mainBuffer?.duration ??
+ (externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
for (const audioPath of externalAudioPaths) {
if (this.cancelled) throw new Error("Export cancelled");
const buffer = await this.decodeAudioFromUrl(audioPath);
if (!buffer) continue;
- const refDuration = mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl));
companionEntries.push({
buffer,
startDelaySec: estimateCompanionAudioStartDelaySeconds(
refDuration,
buffer.duration,
+ sourceAudioFallbackStartDelayMsByPath?.[audioPath],
),
});
}
diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts
index 7b798d74..52b38441 100644
--- a/src/lib/exporter/modernVideoExporter.ts
+++ b/src/lib/exporter/modernVideoExporter.ts
@@ -91,6 +91,7 @@ interface VideoExporterConfig extends ExportConfig {
frame?: string | null;
audioRegions?: AudioRegion[];
sourceAudioFallbackPaths?: string[];
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
previewWidth?: number;
previewHeight?: number;
onProgress?: (progress: ExportProgress) => void;
@@ -497,6 +498,7 @@ export class ModernVideoExporter {
undefined,
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
+ this.config.sourceAudioFallbackStartDelayMsByPath,
),
"audio processing",
"audio",
@@ -728,6 +730,10 @@ export class ModernVideoExporter {
const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter(
(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
);
+ const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some(
+ (audioPath) =>
+ (this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,
+ );
const localVideoSourcePath = this.getNativeVideoSourcePath();
const primaryAudioSourcePath =
(videoInfo.hasAudio ? localVideoSourcePath : null) ??
@@ -745,7 +751,8 @@ export class ModernVideoExporter {
if (
speedRegions.length > 0 ||
audioRegions.length > 0 ||
- sourceAudioFallbackPaths.length > 1
+ sourceAudioFallbackPaths.length > 1 ||
+ hasTimedSourceAudioFallback
) {
const sourceDurationMs = Math.max(
0,
@@ -1026,6 +1033,7 @@ export class ModernVideoExporter {
this.config.speedRegions,
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
+ this.config.sourceAudioFallbackStartDelayMsByPath,
),
`${NATIVE_EXPORT_ENGINE_NAME} edited audio rendering`,
"audio",
@@ -1137,6 +1145,7 @@ export class ModernVideoExporter {
this.config.speedRegions,
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
+ this.config.sourceAudioFallbackStartDelayMsByPath,
),
"FFmpeg edited audio rendering",
"audio",
diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts
index 85df6690..937f34ec 100644
--- a/src/lib/exporter/types.ts
+++ b/src/lib/exporter/types.ts
@@ -11,6 +11,7 @@ export interface ExportConfig {
maxDecodeQueue?: number;
maxPendingFrames?: number;
maxInFlightNativeWrites?: number;
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
}
export type ExportRenderBackend = "webgpu" | "webgl";
diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts
index 7a6c05f5..452d64ab 100644
--- a/src/lib/exporter/videoExporter.ts
+++ b/src/lib/exporter/videoExporter.ts
@@ -79,6 +79,7 @@ interface VideoExporterConfig extends ExportConfig {
frame?: string | null;
audioRegions?: AudioRegion[];
sourceAudioFallbackPaths?: string[];
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
previewWidth?: number;
previewHeight?: number;
onProgress?: (progress: ExportProgress) => void;
@@ -369,6 +370,7 @@ export class VideoExporter {
undefined,
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
+ this.config.sourceAudioFallbackStartDelayMsByPath,
),
"audio processing",
"audio",
@@ -495,6 +497,10 @@ export class VideoExporter {
const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter(
(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
);
+ const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some(
+ (audioPath) =>
+ (this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,
+ );
const localVideoSourcePath = this.getNativeVideoSourcePath();
const primaryAudioSourcePath =
(videoInfo.hasAudio ? localVideoSourcePath : null) ??
@@ -512,7 +518,8 @@ export class VideoExporter {
if (
speedRegions.length > 0 ||
audioRegions.length > 0 ||
- sourceAudioFallbackPaths.length > 1
+ sourceAudioFallbackPaths.length > 1 ||
+ hasTimedSourceAudioFallback
) {
const sourceDurationMs = Math.max(
0,
@@ -786,6 +793,7 @@ export class VideoExporter {
this.config.speedRegions,
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
+ this.config.sourceAudioFallbackStartDelayMsByPath,
),
"native edited audio rendering",
"audio",
@@ -884,6 +892,7 @@ export class VideoExporter {
this.config.speedRegions,
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
+ this.config.sourceAudioFallbackStartDelayMsByPath,
),
"ffmpeg edited audio rendering",
"audio",
diff --git a/src/lib/mediaTiming.test.ts b/src/lib/mediaTiming.test.ts
index 62885863..a6f20479 100644
--- a/src/lib/mediaTiming.test.ts
+++ b/src/lib/mediaTiming.test.ts
@@ -21,14 +21,20 @@ describe("clampMediaTimeToDuration", () => {
});
describe("estimateCompanionAudioStartDelaySeconds", () => {
- it("returns the positive tail gap when companion audio is shorter", () => {
+ it("keeps small inferred offsets when the companion audio is only slightly shorter", () => {
expect(estimateCompanionAudioStartDelaySeconds(10, 9.6)).toBeCloseTo(0.4);
expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBeCloseTo(0.03);
});
- it("ignores tiny or negative differences", () => {
+ it("prefers an explicitly recorded start delay", () => {
+ expect(estimateCompanionAudioStartDelaySeconds(10, 2, 3_500)).toBeCloseTo(3.5);
+ expect(estimateCompanionAudioStartDelaySeconds(10, 2, 0)).toBe(0);
+ });
+
+ it("ignores tiny, negative, or suspiciously large inferred differences", () => {
expect(estimateCompanionAudioStartDelaySeconds(10, 9.99)).toBe(0);
expect(estimateCompanionAudioStartDelaySeconds(10, 10.5)).toBe(0);
+ expect(estimateCompanionAudioStartDelaySeconds(600, 565)).toBe(0);
});
});
diff --git a/src/lib/mediaTiming.ts b/src/lib/mediaTiming.ts
index ed3c0ff4..d9dcf673 100644
--- a/src/lib/mediaTiming.ts
+++ b/src/lib/mediaTiming.ts
@@ -7,10 +7,18 @@ export function clampMediaTimeToDuration(targetTime: number, duration?: number |
return Math.max(0, Math.min(safeTargetTime, Math.max(0, duration)));
}
+const MIN_COMPANION_AUDIO_DELAY_SECONDS = 0.025;
+const MAX_INFERRED_COMPANION_AUDIO_DELAY_SECONDS = 0.5;
+
export function estimateCompanionAudioStartDelaySeconds(
timelineDuration?: number | null,
audioDuration?: number | null,
+ recordedStartDelayMs?: number | null,
): number {
+ if (Number.isFinite(recordedStartDelayMs) && (recordedStartDelayMs ?? 0) >= 0) {
+ return Math.max(0, recordedStartDelayMs ?? 0) / 1000;
+ }
+
if (!Number.isFinite(timelineDuration) || !Number.isFinite(audioDuration)) {
return 0;
}
@@ -19,7 +27,14 @@ export function estimateCompanionAudioStartDelaySeconds(
const safeAudioDuration = Math.max(0, audioDuration ?? 0);
const estimatedDelaySeconds = safeTimelineDuration - safeAudioDuration;
- return estimatedDelaySeconds > 0.025 ? estimatedDelaySeconds : 0;
+ if (
+ estimatedDelaySeconds <= MIN_COMPANION_AUDIO_DELAY_SECONDS ||
+ estimatedDelaySeconds > MAX_INFERRED_COMPANION_AUDIO_DELAY_SECONDS
+ ) {
+ return 0;
+ }
+
+ return estimatedDelaySeconds;
}
export function getMediaSyncPlaybackRate({