Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix: preserve embedded desktop audio in browser exports with mic sidecars

Matt Van Horn32becb66a292
diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts
index f3e8b8f6..7c73b687 100644
--- a/src/lib/exporter/audioEncoder.test.ts
+++ b/src/lib/exporter/audioEncoder.test.ts
@@ -6,6 +6,7 @@ type OfflineRenderTestHarness = AudioProcessor & {
decodeAudioFromUrl(url: string): Promise<AudioBuffer | null>;
getMediaDurationSec(url: string): Promise<number>;
loadAudioFileDemuxer(audioPath: string): Promise<unknown>;
+ processTrimOnlyAudio(demuxer: unknown, muxer: unknown, trimRegions: never[]): Promise<void>;
prepareOfflineRender(
videoUrl: string,
trimRegions: never[],
@@ -55,14 +56,37 @@ function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer {
describe("AudioProcessor offline render preparation", () => {
it("routes a muted full-track clip through offline audio rendering", async () => {
const processor = new AudioProcessor();
- const render = vi.spyOn(processor as unknown as OfflineRenderTestHarness,
- "renderAndMuxOfflineAudio").mockResolvedValue();
- const clips = [{ id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }];
+ const render = vi
+ .spyOn(processor as unknown as OfflineRenderTestHarness, "renderAndMuxOfflineAudio")
+ .mockResolvedValue();
+ const clips = [
+ { id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true },
+ ];
const muxer = {} as never;
- await processor.process(null, muxer, "recording.mp4", [], [], undefined,
- [], [], undefined, undefined, clips);
- expect(render).toHaveBeenCalledWith("recording.mp4", [], [], [], [],
- undefined, undefined, clips, muxer);
+ await processor.process(
+ null,
+ muxer,
+ "recording.mp4",
+ [],
+ [],
+ undefined,
+ [],
+ [],
+ undefined,
+ undefined,
+ clips,
+ );
+ expect(render).toHaveBeenCalledWith(
+ "recording.mp4",
+ [],
+ [],
+ [],
+ [],
+ undefined,
+ undefined,
+ clips,
+ muxer,
+ );
});
it("rejects a cancelled chunked render instead of returning a partial WAV", async () => {
@@ -208,6 +232,134 @@ describe("AudioProcessor offline render preparation", () => {
expect(renderAndMuxOfflineAudio).toHaveBeenCalled();
});
+ it("mixes embedded desktop audio with a WAV microphone sidecar instead of demuxing only the sidecar", async () => {
+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
+ const videoUrl = "file:///C:/recordly/recording.mp4";
+ const videoPath = "C:/recordly/recording.mp4";
+ const micPath = "C:\\recordly\\recording.mic.wav";
+ const muxer = {} as never;
+ const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer");
+ const renderAndMuxOfflineAudio = vi
+ .spyOn(processor, "renderAndMuxOfflineAudio")
+ .mockResolvedValue();
+
+ await processor.process(null, muxer, videoUrl, [], [], undefined, [], [videoPath, micPath]);
+
+ expect(loadAudioFileDemuxer).not.toHaveBeenCalled();
+ expect(renderAndMuxOfflineAudio).toHaveBeenCalledWith(
+ videoUrl,
+ [],
+ [],
+ [],
+ [videoPath, micPath],
+ undefined,
+ undefined,
+ undefined,
+ muxer,
+ );
+
+ renderAndMuxOfflineAudio.mockRestore();
+
+ const mainBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer;
+ const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer;
+ const decodeAudioFromUrl = vi
+ .spyOn(processor, "decodeAudioFromUrl")
+ .mockImplementation(async (url: string) => {
+ if (url === videoUrl) {
+ return mainBuffer;
+ }
+ if (url === micPath) {
+ return micBuffer;
+ }
+ return null;
+ });
+ vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10);
+
+ const prepared = await processor.prepareOfflineRender(
+ videoUrl,
+ [],
+ [],
+ [],
+ [videoPath, micPath],
+ );
+
+ expect(prepared.mainBufferEntry?.buffer).toBe(mainBuffer);
+ expect(prepared.companionEntries).toHaveLength(1);
+ expect(prepared.companionEntries[0]?.buffer).toBe(micBuffer);
+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(videoUrl);
+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(micPath);
+ expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoPath);
+ });
+
+ it("keeps a single microphone sidecar on the direct demux path when the video has no embedded audio", async () => {
+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
+ const micPath = "C:\\recordly\\recording.mic.wav";
+ const loadAudioFileDemuxer = vi
+ .spyOn(processor, "loadAudioFileDemuxer")
+ .mockResolvedValue({ destroy: vi.fn() });
+ const processTrimOnlyAudio = vi
+ .spyOn(processor, "processTrimOnlyAudio")
+ .mockResolvedValue();
+ const renderAndMuxOfflineAudio = vi
+ .spyOn(processor, "renderAndMuxOfflineAudio")
+ .mockResolvedValue();
+
+ await processor.process(
+ null,
+ {} as never,
+ "file:///C:/recordly/recording.mp4",
+ [],
+ [],
+ undefined,
+ [],
+ [micPath],
+ );
+
+ expect(loadAudioFileDemuxer).toHaveBeenCalledWith(micPath);
+ expect(processTrimOnlyAudio).toHaveBeenCalled();
+ expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled();
+ });
+
+ it("does not mix an embedded copy when dedicated system and microphone sidecars are present", async () => {
+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
+ const videoUrl = "file:///C:/recordly/recording.mp4";
+ const videoPath = "C:/recordly/recording.mp4";
+ const systemPath = "C:\\recordly\\recording.system.wav";
+ const micPath = "C:\\recordly\\recording.mic.wav";
+ const systemBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer;
+ const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer;
+ const decodeAudioFromUrl = vi
+ .spyOn(processor, "decodeAudioFromUrl")
+ .mockImplementation(async (url: string) => {
+ if (url === systemPath) {
+ return systemBuffer;
+ }
+ if (url === micPath) {
+ return micBuffer;
+ }
+ return null;
+ });
+ vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10);
+
+ const prepared = await processor.prepareOfflineRender(
+ videoUrl,
+ [],
+ [],
+ [],
+ [videoPath, systemPath, micPath],
+ );
+
+ expect(prepared.mainBufferEntry).toBeNull();
+ expect(prepared.companionEntries.map((entry) => entry.buffer)).toEqual([
+ systemBuffer,
+ micBuffer,
+ ]);
+ expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoUrl);
+ expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoPath);
+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(systemPath);
+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(micPath);
+ });
+
it("soft-limits mixed peaks before encoding or WAV conversion", () => {
const samples = new Float32Array([
-1.6,
diff --git a/src/lib/exporter/modernVideoExporter.fallback.test.ts b/src/lib/exporter/modernVideoExporter.fallback.test.ts
index f4d291ab..00334a3a 100644
--- a/src/lib/exporter/modernVideoExporter.fallback.test.ts
+++ b/src/lib/exporter/modernVideoExporter.fallback.test.ts
@@ -1,4 +1,5 @@
-import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { AudioProcessor } from "./audioEncoder";
import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter";
const mocks = vi.hoisted(() => {
@@ -519,4 +520,222 @@ describe("ModernVideoExporter native fallback routing", () => {
}),
);
});
+
+ describe("browser source audio routing", () => {
+ const videoUrl = "file:///C:/recordly/recording.mp4";
+ const micPath = "C:\\recordly\\recording.mic.wav";
+ const browserExportConfig = {
+ videoUrl,
+ width: 1920,
+ height: 1080,
+ frameRate: 30,
+ bitrate: 8_000_000,
+ wallpaper: "#101010",
+ padding: 0,
+ borderRadius: 0,
+ backgroundBlur: 0,
+ shadowIntensity: 0,
+ showShadow: false,
+ cropRegion: { x: 0, y: 0, width: 1, height: 1 },
+ backendPreference: "webcodecs",
+ sourceAudioFallbackPaths: [micPath],
+ } as never;
+
+ let processAudio: ReturnType<typeof vi.spyOn>;
+
+ beforeEach(() => {
+ vi.stubGlobal("AudioEncoder", {
+ isConfigSupported: vi.fn(async () => ({ supported: true })),
+ });
+ mocks.streamingDecoderGetEffectiveDuration.mockReturnValue(1);
+ processAudio = vi.spyOn(AudioProcessor.prototype, "process").mockResolvedValue();
+ });
+
+ afterEach(() => {
+ processAudio.mockRestore();
+ mocks.streamingDecoderLoadMetadata.mockImplementation(async () => mocks.videoInfo);
+ mocks.streamingDecoderGetDemuxer.mockReturnValue(null);
+ });
+
+ function stubEmbeddedDesktopAudio() {
+ mocks.streamingDecoderLoadMetadata.mockResolvedValue({
+ ...mocks.videoInfo,
+ hasAudio: true,
+ audioCodec: "aac",
+ audioSampleRate: 48_000,
+ });
+ }
+
+ function createBrowserExporter(overrides: Record<string, unknown> = {}) {
+ return new ModernVideoExporter({
+ ...browserExportConfig,
+ ...overrides,
+ } as never) as unknown as {
+ export: () => Promise<{ success: boolean; blob?: Blob; error?: string }>;
+ initializeEncoder: () => Promise<unknown>;
+ tryStartNativeVideoExport: () => Promise<boolean>;
+ finishNativeVideoExport: () => Promise<unknown>;
+ };
+ }
+
+ async function exportWithWebCodecs(
+ overrides: Record<string, unknown> = {},
+ exporter = createBrowserExporter(overrides),
+ ) {
+ vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({
+ codec: "avc1.640034",
+ hardwareAcceleration: "prefer-hardware",
+ });
+ return { exporter, result: await exporter.export() };
+ }
+
+ it("passes embedded desktop audio and a WAV microphone sidecar into browser export mixing", async () => {
+ stubEmbeddedDesktopAudio();
+
+ const { result } = await exportWithWebCodecs();
+
+ expect(result.success).toBe(true);
+ expect(processAudio).toHaveBeenCalledTimes(1);
+ expect(processAudio).toHaveBeenCalledWith(
+ null,
+ expect.anything(),
+ videoUrl,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ [expect.stringMatching(/recording\.mp4$/i), micPath],
+ undefined,
+ undefined,
+ undefined,
+ );
+ });
+
+ it("preserves companion delay, edits, and source settings when normalizing browser audio sources", async () => {
+ stubEmbeddedDesktopAudio();
+ const trimRegions = [{ id: "trim-1", startMs: 1_000, endMs: 2_000 }];
+ const speedRegions = [{ id: "speed-1", startMs: 3_000, endMs: 4_000, speed: 1.5 }];
+ const sourceAudioFallbackStartDelayMsByPath = { [micPath]: 250 };
+ const sourceAudioTrackSettings = {
+ mic: { volume: 0.8, normalize: false },
+ system: { volume: 1, normalize: false },
+ };
+ const clipRegions = [
+ { id: "clip", startMs: 0, endMs: 1_000, sourceStartMs: 0, speed: 1, muted: true },
+ ];
+
+ const { result } = await exportWithWebCodecs({
+ trimRegions,
+ speedRegions,
+ sourceAudioFallbackStartDelayMsByPath,
+ sourceAudioTrackSettings,
+ clipRegions,
+ });
+
+ expect(result.success).toBe(true);
+ expect(processAudio).toHaveBeenCalledTimes(1);
+ expect(processAudio).toHaveBeenCalledWith(
+ null,
+ expect.anything(),
+ videoUrl,
+ trimRegions,
+ speedRegions,
+ undefined,
+ undefined,
+ [expect.stringMatching(/recording\.mp4$/i), micPath],
+ sourceAudioFallbackStartDelayMsByPath,
+ sourceAudioTrackSettings,
+ clipRegions,
+ );
+ });
+
+ it("retries native export once in the browser without dropping the embedded desktop source", async () => {
+ vi.stubGlobal("navigator", { platform: "Win32" });
+ stubEmbeddedDesktopAudio();
+ const log = vi.spyOn(console, "error").mockImplementation(() => {});
+ const exporter = createBrowserExporter({ backendPreference: "auto" });
+ const startNative = vi
+ .spyOn(exporter, "tryStartNativeVideoExport")
+ .mockResolvedValue(true);
+ const initializeEncoder = vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({
+ codec: "avc1.640034",
+ hardwareAcceleration: "prefer-hardware",
+ });
+ vi.spyOn(exporter, "finishNativeVideoExport").mockResolvedValue({
+ success: false,
+ error: "Native finish failed",
+ });
+
+ const result = await exporter.export();
+
+ expect(result.success).toBe(true);
+ expect(startNative).toHaveBeenCalledTimes(1);
+ expect(initializeEncoder).toHaveBeenCalledTimes(1);
+ expect(processAudio).toHaveBeenCalledTimes(1);
+ expect(processAudio).toHaveBeenCalledWith(
+ null,
+ expect.anything(),
+ videoUrl,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ [expect.stringMatching(/recording\.mp4$/i), micPath],
+ undefined,
+ undefined,
+ undefined,
+ );
+ expect(log).toHaveBeenCalledWith(
+ expect.stringContaining("restarting once with WebCodecs"),
+ );
+ });
+
+ it("does not add the local video twice when it is already present as a Windows path or file URL", async () => {
+ stubEmbeddedDesktopAudio();
+
+ const { result } = await exportWithWebCodecs({
+ sourceAudioFallbackPaths: [
+ "C:\\recordly\\recording.mp4",
+ "file:///C:/recordly/recording.mp4",
+ micPath,
+ ],
+ });
+
+ expect(result.success).toBe(true);
+ expect(processAudio.mock.calls[0]?.[7]).toEqual([
+ expect.stringMatching(/recording\.mp4$/i),
+ micPath,
+ ]);
+ });
+
+ it("keeps a microphone-only sidecar list when the source video has no embedded audio", async () => {
+ const { result } = await exportWithWebCodecs();
+
+ expect(result.success).toBe(true);
+ expect(processAudio).toHaveBeenCalledTimes(1);
+ expect(processAudio.mock.calls[0]?.[7]).toEqual([micPath]);
+ });
+
+ it("keeps embedded-only browser export on the source demuxer without inventing companion paths", async () => {
+ stubEmbeddedDesktopAudio();
+ mocks.streamingDecoderGetDemuxer.mockReturnValue({});
+
+ const { result } = await exportWithWebCodecs({
+ sourceAudioFallbackPaths: undefined,
+ });
+
+ expect(result.success).toBe(true);
+ expect(processAudio).toHaveBeenCalledTimes(1);
+ expect(processAudio.mock.calls[0]?.[7]).toEqual([]);
+ });
+
+ it("skips browser audio processing for a genuinely silent source video", async () => {
+ const { result } = await exportWithWebCodecs({
+ sourceAudioFallbackPaths: undefined,
+ });
+
+ expect(result.success).toBe(true);
+ expect(processAudio).not.toHaveBeenCalled();
+ });
+ });
});
diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts
index 81137b26..2f945ece 100644
--- a/src/lib/exporter/modernVideoExporter.ts
+++ b/src/lib/exporter/modernVideoExporter.ts
@@ -815,10 +815,12 @@ export class ModernVideoExporter {
!this.cancelled
) {
const demuxer = this.streamingDecoder.getDemuxer();
+ const sourceAudioFallbackPaths =
+ this.getNormalizedAudioFallbackPaths(videoInfo);
if (
demuxer ||
(this.config.audioRegions ?? []).length > 0 ||
- (this.config.sourceAudioFallbackPaths ?? []).length > 0
+ sourceAudioFallbackPaths.length > 0
) {
this.audioProcessor = new AudioProcessor();
this.audioProcessor.setOnProgress((progress) => {
@@ -835,7 +837,7 @@ export class ModernVideoExporter {
this.config.speedRegions,
undefined,
this.config.audioRegions,
- this.config.sourceAudioFallbackPaths,
+ sourceAudioFallbackPaths,
this.config.sourceAudioFallbackStartDelayMsByPath,
this.config.sourceAudioTrackSettings,
this.config.clipRegions,
@@ -1416,7 +1418,7 @@ export class ModernVideoExporter {
return buildNativeStaticLayoutTimelineSegments(sourceSegments);
}
- private getNativeAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] {
+ private getNormalizedAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] {
const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter(
(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
);
@@ -1455,7 +1457,7 @@ export class ModernVideoExporter {
private buildNativeAudioPlan(videoInfo: DecodedVideoInfo): NativeAudioPlan {
const speedRegions = this.config.speedRegions ?? [];
const audioRegions = this.config.audioRegions ?? [];
- const sourceAudioFallbackPaths = this.getNativeAudioFallbackPaths(videoInfo);
+ const sourceAudioFallbackPaths = this.getNormalizedAudioFallbackPaths(videoInfo);
const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some(
(audioPath) =>
(this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,