Create polished demo videos without editing skills. Mac/Windows/Linux
fix: preserve embedded desktop audio in browser exports with mic sidecars
Matt Van Horn32becb66a292
diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.tsindex f3e8b8f6..7c73b687 100644--- a/src/lib/exporter/audioEncoder.test.ts+++ b/src/lib/exporter/audioEncoder.test.ts@@ -6,6 +6,7 @@ type OfflineRenderTestHarness = AudioProcessor & {decodeAudioFromUrl(url: string): Promise<AudioBuffer | null>;getMediaDurationSec(url: string): Promise<number>;loadAudioFileDemuxer(audioPath: string): Promise<unknown>;+ processTrimOnlyAudio(demuxer: unknown, muxer: unknown, trimRegions: never[]): Promise<void>;prepareOfflineRender(videoUrl: string,trimRegions: never[],@@ -55,14 +56,37 @@ function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer {describe("AudioProcessor offline render preparation", () => {it("routes a muted full-track clip through offline audio rendering", async () => {const processor = new AudioProcessor();- const render = vi.spyOn(processor as unknown as OfflineRenderTestHarness,- "renderAndMuxOfflineAudio").mockResolvedValue();- const clips = [{ id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }];+ const render = vi+ .spyOn(processor as unknown as OfflineRenderTestHarness, "renderAndMuxOfflineAudio")+ .mockResolvedValue();+ const clips = [+ { id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true },+ ];const muxer = {} as never;- await processor.process(null, muxer, "recording.mp4", [], [], undefined,- [], [], undefined, undefined, clips);- expect(render).toHaveBeenCalledWith("recording.mp4", [], [], [], [],- undefined, undefined, clips, muxer);+ await processor.process(+ null,+ muxer,+ "recording.mp4",+ [],+ [],+ undefined,+ [],+ [],+ undefined,+ undefined,+ clips,+ );+ expect(render).toHaveBeenCalledWith(+ "recording.mp4",+ [],+ [],+ [],+ [],+ undefined,+ undefined,+ clips,+ muxer,+ );});it("rejects a cancelled chunked render instead of returning a partial WAV", async () => {@@ -208,6 +232,134 @@ describe("AudioProcessor offline render preparation", () => {expect(renderAndMuxOfflineAudio).toHaveBeenCalled();});+ it("mixes embedded desktop audio with a WAV microphone sidecar instead of demuxing only the sidecar", async () => {+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;+ const videoUrl = "file:///C:/recordly/recording.mp4";+ const videoPath = "C:/recordly/recording.mp4";+ const micPath = "C:\\recordly\\recording.mic.wav";+ const muxer = {} as never;+ const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer");+ const renderAndMuxOfflineAudio = vi+ .spyOn(processor, "renderAndMuxOfflineAudio")+ .mockResolvedValue();++ await processor.process(null, muxer, videoUrl, [], [], undefined, [], [videoPath, micPath]);++ expect(loadAudioFileDemuxer).not.toHaveBeenCalled();+ expect(renderAndMuxOfflineAudio).toHaveBeenCalledWith(+ videoUrl,+ [],+ [],+ [],+ [videoPath, micPath],+ undefined,+ undefined,+ undefined,+ muxer,+ );++ renderAndMuxOfflineAudio.mockRestore();++ const mainBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer;+ const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer;+ const decodeAudioFromUrl = vi+ .spyOn(processor, "decodeAudioFromUrl")+ .mockImplementation(async (url: string) => {+ if (url === videoUrl) {+ return mainBuffer;+ }+ if (url === micPath) {+ return micBuffer;+ }+ return null;+ });+ vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10);++ const prepared = await processor.prepareOfflineRender(+ videoUrl,+ [],+ [],+ [],+ [videoPath, micPath],+ );++ expect(prepared.mainBufferEntry?.buffer).toBe(mainBuffer);+ expect(prepared.companionEntries).toHaveLength(1);+ expect(prepared.companionEntries[0]?.buffer).toBe(micBuffer);+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(videoUrl);+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(micPath);+ expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoPath);+ });++ it("keeps a single microphone sidecar on the direct demux path when the video has no embedded audio", async () => {+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;+ const micPath = "C:\\recordly\\recording.mic.wav";+ const loadAudioFileDemuxer = vi+ .spyOn(processor, "loadAudioFileDemuxer")+ .mockResolvedValue({ destroy: vi.fn() });+ const processTrimOnlyAudio = vi+ .spyOn(processor, "processTrimOnlyAudio")+ .mockResolvedValue();+ const renderAndMuxOfflineAudio = vi+ .spyOn(processor, "renderAndMuxOfflineAudio")+ .mockResolvedValue();++ await processor.process(+ null,+ {} as never,+ "file:///C:/recordly/recording.mp4",+ [],+ [],+ undefined,+ [],+ [micPath],+ );++ expect(loadAudioFileDemuxer).toHaveBeenCalledWith(micPath);+ expect(processTrimOnlyAudio).toHaveBeenCalled();+ expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled();+ });++ it("does not mix an embedded copy when dedicated system and microphone sidecars are present", async () => {+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;+ const videoUrl = "file:///C:/recordly/recording.mp4";+ const videoPath = "C:/recordly/recording.mp4";+ const systemPath = "C:\\recordly\\recording.system.wav";+ const micPath = "C:\\recordly\\recording.mic.wav";+ const systemBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer;+ const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer;+ const decodeAudioFromUrl = vi+ .spyOn(processor, "decodeAudioFromUrl")+ .mockImplementation(async (url: string) => {+ if (url === systemPath) {+ return systemBuffer;+ }+ if (url === micPath) {+ return micBuffer;+ }+ return null;+ });+ vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10);++ const prepared = await processor.prepareOfflineRender(+ videoUrl,+ [],+ [],+ [],+ [videoPath, systemPath, micPath],+ );++ expect(prepared.mainBufferEntry).toBeNull();+ expect(prepared.companionEntries.map((entry) => entry.buffer)).toEqual([+ systemBuffer,+ micBuffer,+ ]);+ expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoUrl);+ expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoPath);+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(systemPath);+ expect(decodeAudioFromUrl).toHaveBeenCalledWith(micPath);+ });+it("soft-limits mixed peaks before encoding or WAV conversion", () => {const samples = new Float32Array([-1.6,diff --git a/src/lib/exporter/modernVideoExporter.fallback.test.ts b/src/lib/exporter/modernVideoExporter.fallback.test.tsindex f4d291ab..00334a3a 100644--- a/src/lib/exporter/modernVideoExporter.fallback.test.ts+++ b/src/lib/exporter/modernVideoExporter.fallback.test.ts@@ -1,4 +1,5 @@-import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { AudioProcessor } from "./audioEncoder";import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter";const mocks = vi.hoisted(() => {@@ -519,4 +520,222 @@ describe("ModernVideoExporter native fallback routing", () => {}),);});++ describe("browser source audio routing", () => {+ const videoUrl = "file:///C:/recordly/recording.mp4";+ const micPath = "C:\\recordly\\recording.mic.wav";+ const browserExportConfig = {+ videoUrl,+ width: 1920,+ height: 1080,+ frameRate: 30,+ bitrate: 8_000_000,+ wallpaper: "#101010",+ padding: 0,+ borderRadius: 0,+ backgroundBlur: 0,+ shadowIntensity: 0,+ showShadow: false,+ cropRegion: { x: 0, y: 0, width: 1, height: 1 },+ backendPreference: "webcodecs",+ sourceAudioFallbackPaths: [micPath],+ } as never;++ let processAudio: ReturnType<typeof vi.spyOn>;++ beforeEach(() => {+ vi.stubGlobal("AudioEncoder", {+ isConfigSupported: vi.fn(async () => ({ supported: true })),+ });+ mocks.streamingDecoderGetEffectiveDuration.mockReturnValue(1);+ processAudio = vi.spyOn(AudioProcessor.prototype, "process").mockResolvedValue();+ });++ afterEach(() => {+ processAudio.mockRestore();+ mocks.streamingDecoderLoadMetadata.mockImplementation(async () => mocks.videoInfo);+ mocks.streamingDecoderGetDemuxer.mockReturnValue(null);+ });++ function stubEmbeddedDesktopAudio() {+ mocks.streamingDecoderLoadMetadata.mockResolvedValue({+ ...mocks.videoInfo,+ hasAudio: true,+ audioCodec: "aac",+ audioSampleRate: 48_000,+ });+ }++ function createBrowserExporter(overrides: Record<string, unknown> = {}) {+ return new ModernVideoExporter({+ ...browserExportConfig,+ ...overrides,+ } as never) as unknown as {+ export: () => Promise<{ success: boolean; blob?: Blob; error?: string }>;+ initializeEncoder: () => Promise<unknown>;+ tryStartNativeVideoExport: () => Promise<boolean>;+ finishNativeVideoExport: () => Promise<unknown>;+ };+ }++ async function exportWithWebCodecs(+ overrides: Record<string, unknown> = {},+ exporter = createBrowserExporter(overrides),+ ) {+ vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({+ codec: "avc1.640034",+ hardwareAcceleration: "prefer-hardware",+ });+ return { exporter, result: await exporter.export() };+ }++ it("passes embedded desktop audio and a WAV microphone sidecar into browser export mixing", async () => {+ stubEmbeddedDesktopAudio();++ const { result } = await exportWithWebCodecs();++ expect(result.success).toBe(true);+ expect(processAudio).toHaveBeenCalledTimes(1);+ expect(processAudio).toHaveBeenCalledWith(+ null,+ expect.anything(),+ videoUrl,+ undefined,+ undefined,+ undefined,+ undefined,+ [expect.stringMatching(/recording\.mp4$/i), micPath],+ undefined,+ undefined,+ undefined,+ );+ });++ it("preserves companion delay, edits, and source settings when normalizing browser audio sources", async () => {+ stubEmbeddedDesktopAudio();+ const trimRegions = [{ id: "trim-1", startMs: 1_000, endMs: 2_000 }];+ const speedRegions = [{ id: "speed-1", startMs: 3_000, endMs: 4_000, speed: 1.5 }];+ const sourceAudioFallbackStartDelayMsByPath = { [micPath]: 250 };+ const sourceAudioTrackSettings = {+ mic: { volume: 0.8, normalize: false },+ system: { volume: 1, normalize: false },+ };+ const clipRegions = [+ { id: "clip", startMs: 0, endMs: 1_000, sourceStartMs: 0, speed: 1, muted: true },+ ];++ const { result } = await exportWithWebCodecs({+ trimRegions,+ speedRegions,+ sourceAudioFallbackStartDelayMsByPath,+ sourceAudioTrackSettings,+ clipRegions,+ });++ expect(result.success).toBe(true);+ expect(processAudio).toHaveBeenCalledTimes(1);+ expect(processAudio).toHaveBeenCalledWith(+ null,+ expect.anything(),+ videoUrl,+ trimRegions,+ speedRegions,+ undefined,+ undefined,+ [expect.stringMatching(/recording\.mp4$/i), micPath],+ sourceAudioFallbackStartDelayMsByPath,+ sourceAudioTrackSettings,+ clipRegions,+ );+ });++ it("retries native export once in the browser without dropping the embedded desktop source", async () => {+ vi.stubGlobal("navigator", { platform: "Win32" });+ stubEmbeddedDesktopAudio();+ const log = vi.spyOn(console, "error").mockImplementation(() => {});+ const exporter = createBrowserExporter({ backendPreference: "auto" });+ const startNative = vi+ .spyOn(exporter, "tryStartNativeVideoExport")+ .mockResolvedValue(true);+ const initializeEncoder = vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({+ codec: "avc1.640034",+ hardwareAcceleration: "prefer-hardware",+ });+ vi.spyOn(exporter, "finishNativeVideoExport").mockResolvedValue({+ success: false,+ error: "Native finish failed",+ });++ const result = await exporter.export();++ expect(result.success).toBe(true);+ expect(startNative).toHaveBeenCalledTimes(1);+ expect(initializeEncoder).toHaveBeenCalledTimes(1);+ expect(processAudio).toHaveBeenCalledTimes(1);+ expect(processAudio).toHaveBeenCalledWith(+ null,+ expect.anything(),+ videoUrl,+ undefined,+ undefined,+ undefined,+ undefined,+ [expect.stringMatching(/recording\.mp4$/i), micPath],+ undefined,+ undefined,+ undefined,+ );+ expect(log).toHaveBeenCalledWith(+ expect.stringContaining("restarting once with WebCodecs"),+ );+ });++ it("does not add the local video twice when it is already present as a Windows path or file URL", async () => {+ stubEmbeddedDesktopAudio();++ const { result } = await exportWithWebCodecs({+ sourceAudioFallbackPaths: [+ "C:\\recordly\\recording.mp4",+ "file:///C:/recordly/recording.mp4",+ micPath,+ ],+ });++ expect(result.success).toBe(true);+ expect(processAudio.mock.calls[0]?.[7]).toEqual([+ expect.stringMatching(/recording\.mp4$/i),+ micPath,+ ]);+ });++ it("keeps a microphone-only sidecar list when the source video has no embedded audio", async () => {+ const { result } = await exportWithWebCodecs();++ expect(result.success).toBe(true);+ expect(processAudio).toHaveBeenCalledTimes(1);+ expect(processAudio.mock.calls[0]?.[7]).toEqual([micPath]);+ });++ it("keeps embedded-only browser export on the source demuxer without inventing companion paths", async () => {+ stubEmbeddedDesktopAudio();+ mocks.streamingDecoderGetDemuxer.mockReturnValue({});++ const { result } = await exportWithWebCodecs({+ sourceAudioFallbackPaths: undefined,+ });++ expect(result.success).toBe(true);+ expect(processAudio).toHaveBeenCalledTimes(1);+ expect(processAudio.mock.calls[0]?.[7]).toEqual([]);+ });++ it("skips browser audio processing for a genuinely silent source video", async () => {+ const { result } = await exportWithWebCodecs({+ sourceAudioFallbackPaths: undefined,+ });++ expect(result.success).toBe(true);+ expect(processAudio).not.toHaveBeenCalled();+ });+ });});diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.tsindex 81137b26..2f945ece 100644--- a/src/lib/exporter/modernVideoExporter.ts+++ b/src/lib/exporter/modernVideoExporter.ts@@ -815,10 +815,12 @@ export class ModernVideoExporter {!this.cancelled) {const demuxer = this.streamingDecoder.getDemuxer();+ const sourceAudioFallbackPaths =+ this.getNormalizedAudioFallbackPaths(videoInfo);if (demuxer ||(this.config.audioRegions ?? []).length > 0 ||- (this.config.sourceAudioFallbackPaths ?? []).length > 0+ sourceAudioFallbackPaths.length > 0) {this.audioProcessor = new AudioProcessor();this.audioProcessor.setOnProgress((progress) => {@@ -835,7 +837,7 @@ export class ModernVideoExporter {this.config.speedRegions,undefined,this.config.audioRegions,- this.config.sourceAudioFallbackPaths,+ sourceAudioFallbackPaths,this.config.sourceAudioFallbackStartDelayMsByPath,this.config.sourceAudioTrackSettings,this.config.clipRegions,@@ -1416,7 +1418,7 @@ export class ModernVideoExporter {return buildNativeStaticLayoutTimelineSegments(sourceSegments);}- private getNativeAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] {+ private getNormalizedAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] {const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter((audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,);@@ -1455,7 +1457,7 @@ export class ModernVideoExporter {private buildNativeAudioPlan(videoInfo: DecodedVideoInfo): NativeAudioPlan {const speedRegions = this.config.speedRegions ?? [];const audioRegions = this.config.audioRegions ?? [];- const sourceAudioFallbackPaths = this.getNativeAudioFallbackPaths(videoInfo);+ const sourceAudioFallbackPaths = this.getNormalizedAudioFallbackPaths(videoInfo);const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some((audioPath) =>(this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,