Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix: preserve mac audio sidecar tracks

wiiiii123f66e2c90b326
diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts
index 09dd5aa7..01951345 100644
--- a/electron/ipc/recording/mac.ts
+++ b/electron/ipc/recording/mac.ts
@@ -30,9 +30,9 @@ import {
validateRecordedVideo,
} from "./diagnostics";
import { emitRecordingInterrupted } from "./events";
+import { getFinalMacCompanionAudioPath } from "./macCompanionAudio";
import { pruneAutoRecordings } from "./prune";
-
export function waitForNativeCaptureStart(process: ChildProcessWithoutNullStreams) {
return new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
@@ -119,13 +119,11 @@ export async function muxNativeMacRecordingWithAudio(
microphonePath?: string | null,
) {
console.log("[mac-mux] Optimization active: keeping tracks separate.");
-
- const videoPathWithoutExt = videoPath.replace(/\.[^.]+$/u, "");
// Optimization: instead of heavy FFmpeg muxing, we ensure audio sidecars
// are available alongside the video for the editor.
if (systemAudioPath) {
- const finalSystemPath = `${videoPathWithoutExt}.system.wav`;
+ const finalSystemPath = getFinalMacCompanionAudioPath(videoPath, systemAudioPath, "system");
try {
const stat = await fs.stat(systemAudioPath);
if (stat.size > 0 && systemAudioPath !== finalSystemPath) {
@@ -137,7 +135,7 @@ export async function muxNativeMacRecordingWithAudio(
}
if (microphonePath) {
- const finalMicPath = `${videoPathWithoutExt}.mic.wav`;
+ const finalMicPath = getFinalMacCompanionAudioPath(videoPath, microphonePath, "mic");
try {
const stat = await fs.stat(microphonePath);
if (stat.size > 0 && microphonePath !== finalMicPath) {
diff --git a/electron/ipc/recording/macCompanionAudio.test.ts b/electron/ipc/recording/macCompanionAudio.test.ts
new file mode 100644
index 00000000..269a7662
--- /dev/null
+++ b/electron/ipc/recording/macCompanionAudio.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+
+import { getFinalMacCompanionAudioPath } from "./macCompanionAudio";
+
+describe("mac companion audio paths", () => {
+ it("preserves the helper's AAC container extension", () => {
+ expect(
+ getFinalMacCompanionAudioPath(
+ "/Users/egg/Recordly/recording-1.mp4",
+ "/Users/egg/Recordly/recording-1.mic.m4a",
+ "mic",
+ ),
+ ).toBe("/Users/egg/Recordly/recording-1.mic.m4a");
+ });
+
+ it("preserves legacy sidecar extensions instead of renaming bytes", () => {
+ expect(
+ getFinalMacCompanionAudioPath(
+ "/Users/egg/Recordly/recording-1.mp4",
+ "/tmp/recordly-native.system.webm",
+ "system",
+ ),
+ ).toBe("/Users/egg/Recordly/recording-1.system.webm");
+ });
+
+ it("keeps dotted directories when the video path has no extension", () => {
+ expect(
+ getFinalMacCompanionAudioPath(
+ "/Users/egg/Recordly.videos/recording-1",
+ "/tmp/recordly-native.mic.m4a",
+ "mic",
+ ),
+ ).toBe("/Users/egg/Recordly.videos/recording-1.mic.m4a");
+ });
+});
diff --git a/electron/ipc/recording/macCompanionAudio.ts b/electron/ipc/recording/macCompanionAudio.ts
new file mode 100644
index 00000000..fdfc9e26
--- /dev/null
+++ b/electron/ipc/recording/macCompanionAudio.ts
@@ -0,0 +1,16 @@
+import path from "node:path";
+
+export type MacCompanionAudioSuffix = "system" | "mic";
+
+export function getFinalMacCompanionAudioPath(
+ videoPath: string,
+ sourceAudioPath: string,
+ suffix: MacCompanionAudioSuffix,
+) {
+ const separatorIndex = Math.max(videoPath.lastIndexOf("/"), videoPath.lastIndexOf("\\"));
+ const videoDirectory = separatorIndex >= 0 ? videoPath.slice(0, separatorIndex + 1) : "";
+ const videoFileName = separatorIndex >= 0 ? videoPath.slice(separatorIndex + 1) : videoPath;
+ const videoPathWithoutExt = `${videoDirectory}${path.parse(videoFileName).name}`;
+ const sourceExtension = path.extname(sourceAudioPath).toLowerCase() || ".m4a";
+ return `${videoPathWithoutExt}.${suffix}${sourceExtension}`;
+}
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx
index 02b8b9b7..041bc318 100644
--- a/src/components/video-editor/timeline/TimelineEditor.tsx
+++ b/src/components/video-editor/timeline/TimelineEditor.tsx
@@ -1,20 +1,13 @@
-import type { Span } from "dnd-timeline";
import { Plus } from "@phosphor-icons/react";
-import {
- forwardRef,
- useEffect,
- useMemo,
- useRef,
- useState,
-} from "react";
-import { useScopedT } from "@/contexts/I18nContext";
-import { useShortcuts } from "@/contexts/ShortcutsContext";
-import { fromFileUrl } from "../projectPersistence";
+import type { Span } from "dnd-timeline";
+import { forwardRef, useEffect, useMemo, useRef, useState } from "react";
import type {
SourceAudioTrackMeta,
SourceAudioTrackSettings,
- SourceAudioTrackWithPeaks,
} from "@/components/video-editor/audio/audioTypes";
+import { useScopedT } from "@/contexts/I18nContext";
+import { useShortcuts } from "@/contexts/ShortcutsContext";
+import { fromFileUrl } from "../projectPersistence";
import type {
AnnotationRegion,
AudioRegion,
@@ -26,12 +19,16 @@ import type {
ZoomRegion,
} from "../types";
import KeyframeMarkers from "./components/markers/KeyframeMarkers";
+import TimelineCanvas from "./components/viewport/TimelineCanvas";
import TimelineWrapper from "./components/wrapper/TimelineWrapper";
-import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks";
import { calculateTimelineScale } from "./core/time";
+import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks";
import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime";
import { useTimelineRange } from "./hooks/useTimelineRange";
-import TimelineCanvas from "./components/viewport/TimelineCanvas";
+import {
+ buildSourceSidecarPathCandidates,
+ buildTimelineSourceAudioTracks,
+} from "./sourceAudioTracks";
export interface TimelineEditorProps {
videoDuration: number;
@@ -77,9 +74,7 @@ export interface TimelineEditorProps {
showSourceAudioTrack?: boolean;
onSourceAudioAvailabilityChange?: (available: boolean) => void;
sourceAudioTrackSettings?: SourceAudioTrackSettings;
- getSourceAudioTrackSettingsForClip?: (
- clipId: string | null,
- ) => SourceAudioTrackSettings;
+ getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings;
onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void;
}
@@ -98,16 +93,6 @@ function extractLocalPathFromMediaServerUrl(input: string | null | undefined): s
}
}
-function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string {
- const normalized = source.replace(/\\/g, "/");
- const lastSlash = normalized.lastIndexOf("/");
- const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
- const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
- const dotIndex = fileName.lastIndexOf(".");
- const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
- return `${dir}${baseName}.${suffix}.wav`;
-}
-
export interface TimelineEditorHandle {
addZoom: () => void;
suggestZooms: () => void;
@@ -117,7 +102,6 @@ export interface TimelineEditorHandle {
keyframes: { id: string; time: number }[];
}
-
const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
function TimelineEditor(
{
@@ -209,9 +193,7 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
...(newStart > oldClip.startMs
? [{ startMs: oldClip.startMs, endMs: newStart }]
: []),
- ...(newEnd < oldClip.endMs
- ? [{ startMs: newEnd, endMs: oldClip.endMs }]
- : []),
+ ...(newEnd < oldClip.endMs ? [{ startMs: newEnd, endMs: oldClip.endMs }] : []),
];
const startDelta = newStart - oldClip.startMs;
@@ -245,9 +227,8 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
return { previewSpans, hiddenZoomIds };
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
- const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } = useTimelineAudioPeaks(videoPath, {
- enableSourceSidecarFallback: true,
- });
+ const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } =
+ useTimelineAudioPeaks(videoPath);
const localSourcePath = useMemo(() => {
if (!videoPath) return null;
return (
@@ -255,56 +236,67 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
(/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath)
);
}, [videoPath]);
- const micSidecarPath = useMemo(
- () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null),
+ const micSidecarPaths = useMemo(
+ () => (localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "mic") : []),
[localSourcePath],
);
- const systemSidecarPath = useMemo(
- () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null),
+ const micSidecarFallbackPaths = useMemo(() => micSidecarPaths.slice(1), [micSidecarPaths]);
+ const systemSidecarPaths = useMemo(
+ () =>
+ localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "system") : [],
[localSourcePath],
);
- const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks(micSidecarPath);
- const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks(systemSidecarPath);
- const sourceAudioTracks = useMemo<SourceAudioTrackWithPeaks[]>(() => {
- if (systemSidecarPeaks || micSidecarPeaks) {
- const tracks: SourceAudioTrackWithPeaks[] = [];
- if (systemSidecarPeaks)
- tracks.push({
- id: "system",
- label: t("audio.systemLabel", "Source System"),
- peaks: systemSidecarPeaks,
- });
- if (micSidecarPeaks)
- tracks.push({
- id: "mic",
- label: t("audio.micLabel", "Source Mic"),
- peaks: micSidecarPeaks,
- });
- return tracks;
- }
- return sourceAudioPeaks
- ? [
- {
- id: "mixed",
- label: t("audio.mixedLabel", "Source"),
- peaks: sourceAudioPeaks,
- },
- ]
- : [];
- }, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]);
+ const systemSidecarFallbackPaths = useMemo(
+ () => systemSidecarPaths.slice(1),
+ [systemSidecarPaths],
+ );
+ const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks(
+ micSidecarPaths[0] ?? null,
+ { fallbackResources: micSidecarFallbackPaths },
+ );
+ const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks(
+ systemSidecarPaths[0] ?? null,
+ {
+ fallbackResources: systemSidecarFallbackPaths,
+ },
+ );
+ const sourceAudioTracks = useMemo(
+ () =>
+ buildTimelineSourceAudioTracks({
+ sourceAudioPeaks,
+ micSidecarPeaks,
+ systemSidecarPeaks,
+ labels: {
+ system: t("audio.systemLabel", "Source System"),
+ mic: t("audio.micLabel", "Source Mic"),
+ mixed: t("audio.mixedLabel", "Source"),
+ },
+ }),
+ [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t],
+ );
const isLoading = useMemo(() => {
// If we are still actively trying to load audio peaks (main or sidecars)
- if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) return true;
+ if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading))
+ return true;
// Robust telemetry loading detection:
// If a source path is set but telemetry hasn't arrived (or failed/retried) for it yet.
if (videoSourcePath && cursorTelemetrySourcePath !== videoSourcePath) return true;
return false;
- }, [videoPath, videoSourcePath, cursorTelemetrySourcePath, sourceAudioLoading, micSidecarLoading, systemSidecarLoading]);
+ }, [
+ videoPath,
+ videoSourcePath,
+ cursorTelemetrySourcePath,
+ sourceAudioLoading,
+ micSidecarLoading,
+ systemSidecarLoading,
+ ]);
useEffect(() => {
- onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })));
+ onSourceAudioTracksMetaChange?.(
+ sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })),
+ );
}, [onSourceAudioTracksMetaChange, sourceAudioTracks]);
void sourceAudioTrackSettings;
useEffect(() => {
diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
index 87ebc357..cf5ca393 100644
--- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
+++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
@@ -1,10 +1,12 @@
import { useEffect, useRef, useState } from "react";
import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
-import { fromFileUrl } from "../../projectPersistence";
import { waveformGenerator } from "../../audio/waveform/WaveformGenerator";
+import { fromFileUrl } from "../../projectPersistence";
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants";
import type { AudioPeaksData } from "../core/timelineTypes";
+const EMPTY_FALLBACK_RESOURCES: string[] = [];
+
function buildSidecarAudioCandidates(sourcePath: string): string[] {
const normalized = sourcePath.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
@@ -18,6 +20,8 @@ function buildSidecarAudioCandidates(sourcePath: string): string[] {
`${dir}${baseName}.mic.wav`,
`${dir}${baseName}.system.m4a`,
`${dir}${baseName}.mic.m4a`,
+ `${dir}${baseName}.system.webm`,
+ `${dir}${baseName}.mic.webm`,
];
}
@@ -37,6 +41,7 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null {
interface TimelineAudioPeaksOptions {
enableSourceSidecarFallback?: boolean;
+ fallbackResources?: string[];
peakCount?: number;
}
@@ -53,6 +58,7 @@ export function useTimelineAudioPeaks(
const [loading, setLoading] = useState(false);
const sourceRef = useRef(mediaResource);
const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;
+ const fallbackResources = options.fallbackResources ?? EMPTY_FALLBACK_RESOURCES;
const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT;
useEffect(() => {
@@ -83,25 +89,29 @@ export function useTimelineAudioPeaks(
// fallthrough
}
- if (!enableSourceSidecarFallback) {
+ if (!enableSourceSidecarFallback && fallbackResources.length === 0) {
if (!cancelled && sourceRef.current === mediaResource) {
setLoading(false);
}
return;
}
- const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
- const localSourcePath =
- localPathFromServer ||
- (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);
- if (!localSourcePath) {
- if (!cancelled && sourceRef.current === mediaResource) {
- setLoading(false);
+ let sourceSidecarCandidates: string[] = [];
+ if (enableSourceSidecarFallback) {
+ const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
+ const localSourcePath =
+ localPathFromServer ||
+ (/^file:\/\//i.test(mediaResource)
+ ? fromFileUrl(mediaResource)
+ : mediaResource);
+ if (localSourcePath) {
+ sourceSidecarCandidates = buildSidecarAudioCandidates(localSourcePath);
}
- return;
}
- const candidates = buildSidecarAudioCandidates(localSourcePath);
+ const candidates = Array.from(
+ new Set([...fallbackResources, ...sourceSidecarCandidates]),
+ );
for (const candidate of candidates) {
try {
const result = await tryGenerate(candidate);
@@ -125,7 +135,7 @@ export function useTimelineAudioPeaks(
return () => {
cancelled = true;
};
- }, [mediaResource, enableSourceSidecarFallback, peakCount]);
+ }, [mediaResource, enableSourceSidecarFallback, fallbackResources, peakCount]);
return { peaks, loading };
}
diff --git a/src/components/video-editor/timeline/sourceAudioTracks.test.ts b/src/components/video-editor/timeline/sourceAudioTracks.test.ts
new file mode 100644
index 00000000..f7c809e6
--- /dev/null
+++ b/src/components/video-editor/timeline/sourceAudioTracks.test.ts
@@ -0,0 +1,90 @@
+import { describe, expect, it } from "vitest";
+import type { AudioPeaksData } from "./core/timelineTypes";
+import {
+ buildSourceSidecarPathCandidates,
+ buildTimelineSourceAudioTracks,
+} from "./sourceAudioTracks";
+
+function peaks(id: number): AudioPeaksData {
+ return {
+ durationMs: 1000,
+ peaks: new Float32Array([id]),
+ };
+}
+
+const labels = {
+ system: "Source System",
+ mic: "Source Mic",
+ mixed: "Source",
+};
+
+describe("timeline source audio tracks", () => {
+ it("builds candidates for Windows and macOS sidecar containers", () => {
+ expect(buildSourceSidecarPathCandidates("C:\\Recordly\\recording-1.mp4", "mic")).toEqual([
+ "C:/Recordly/recording-1.mic.wav",
+ "C:/Recordly/recording-1.mic.m4a",
+ "C:/Recordly/recording-1.mic.webm",
+ ]);
+ });
+
+ it("keeps embedded system audio controllable when mic is a sidecar", () => {
+ const source = peaks(1);
+ const mic = peaks(2);
+
+ expect(
+ buildTimelineSourceAudioTracks({
+ sourceAudioPeaks: source,
+ micSidecarPeaks: mic,
+ systemSidecarPeaks: null,
+ labels,
+ }),
+ ).toEqual([
+ { id: "system", label: "Source System", peaks: source },
+ { id: "mic", label: "Source Mic", peaks: mic },
+ ]);
+ });
+
+ it("does not invent a system track when only the mic sidecar exists", () => {
+ const mic = peaks(2);
+
+ expect(
+ buildTimelineSourceAudioTracks({
+ sourceAudioPeaks: null,
+ micSidecarPeaks: mic,
+ systemSidecarPeaks: null,
+ labels,
+ }),
+ ).toEqual([{ id: "mic", label: "Source Mic", peaks: mic }]);
+ });
+
+ it("uses dedicated sidecars over the embedded track when both source tracks exist", () => {
+ const source = peaks(1);
+ const system = peaks(2);
+ const mic = peaks(3);
+
+ expect(
+ buildTimelineSourceAudioTracks({
+ sourceAudioPeaks: source,
+ micSidecarPeaks: mic,
+ systemSidecarPeaks: system,
+ labels,
+ }),
+ ).toEqual([
+ { id: "system", label: "Source System", peaks: system },
+ { id: "mic", label: "Source Mic", peaks: mic },
+ ]);
+ });
+
+ it("falls back to one mixed source track when no dedicated sidecar exists", () => {
+ const source = peaks(1);
+
+ expect(
+ buildTimelineSourceAudioTracks({
+ sourceAudioPeaks: source,
+ micSidecarPeaks: null,
+ systemSidecarPeaks: null,
+ labels,
+ }),
+ ).toEqual([{ id: "mixed", label: "Source", peaks: source }]);
+ });
+});
diff --git a/src/components/video-editor/timeline/sourceAudioTracks.ts b/src/components/video-editor/timeline/sourceAudioTracks.ts
new file mode 100644
index 00000000..46a089ee
--- /dev/null
+++ b/src/components/video-editor/timeline/sourceAudioTracks.ts
@@ -0,0 +1,68 @@
+import type { SourceAudioTrackWithPeaks } from "@/components/video-editor/audio/audioTypes";
+import type { AudioPeaksData } from "./core/timelineTypes";
+
+const SOURCE_SIDECAR_EXTENSIONS = [".wav", ".m4a", ".webm"] as const;
+
+export function buildSourceSidecarPathCandidates(
+ source: string,
+ suffix: "mic" | "system",
+): string[] {
+ const normalized = source.replace(/\\/g, "/");
+ const lastSlash = normalized.lastIndexOf("/");
+ const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
+ const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
+ const dotIndex = fileName.lastIndexOf(".");
+ const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
+ return SOURCE_SIDECAR_EXTENSIONS.map((extension) => `${dir}${baseName}.${suffix}${extension}`);
+}
+
+export function buildTimelineSourceAudioTracks({
+ sourceAudioPeaks,
+ micSidecarPeaks,
+ systemSidecarPeaks,
+ labels,
+}: {
+ sourceAudioPeaks: AudioPeaksData | null;
+ micSidecarPeaks: AudioPeaksData | null;
+ systemSidecarPeaks: AudioPeaksData | null;
+ labels: {
+ system: string;
+ mic: string;
+ mixed: string;
+ };
+}): SourceAudioTrackWithPeaks[] {
+ if (systemSidecarPeaks || micSidecarPeaks) {
+ const tracks: SourceAudioTrackWithPeaks[] = [];
+ if (systemSidecarPeaks) {
+ tracks.push({
+ id: "system",
+ label: labels.system,
+ peaks: systemSidecarPeaks,
+ });
+ } else if (micSidecarPeaks && sourceAudioPeaks) {
+ tracks.push({
+ id: "system",
+ label: labels.system,
+ peaks: sourceAudioPeaks,
+ });
+ }
+ if (micSidecarPeaks) {
+ tracks.push({
+ id: "mic",
+ label: labels.mic,
+ peaks: micSidecarPeaks,
+ });
+ }
+ return tracks;
+ }
+
+ return sourceAudioPeaks
+ ? [
+ {
+ id: "mixed",
+ label: labels.mixed,
+ peaks: sourceAudioPeaks,
+ },
+ ]
+ : [];
+}
diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts
index 309ff8fe..6047b548 100644
--- a/src/lib/exporter/audioEncoder.test.ts
+++ b/src/lib/exporter/audioEncoder.test.ts
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
-import { AudioProcessor } from "./audioEncoder";
+import { AudioProcessor, softLimitOfflineMixPeaksInPlace } from "./audioEncoder";
type OfflineRenderTestHarness = AudioProcessor & {
decodeAudioFromUrl(url: string): Promise<AudioBuffer | null>;
@@ -26,8 +26,32 @@ type OfflineRenderTestHarness = AudioProcessor & {
sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
muxer: unknown,
): Promise<void>;
+ renderChunked(
+ prepared: {
+ mainBufferEntry: null;
+ companionEntries: [];
+ regionEntries: [];
+ mutedSourceOutputRangesSec: [];
+ slices: [];
+ outputDurationMs: number;
+ numChannels: number;
+ },
+ totalOutputSec: number,
+ onChunk: (
+ rendered: AudioBuffer,
+ outputOffsetSec: number,
+ chunkIndex: number,
+ ) => Promise<void>,
+ ): Promise<void>;
};
+function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer {
+ return {
+ numberOfChannels: channels.length,
+ getChannelData: (channel: number) => channels[channel],
+ } as AudioBuffer;
+}
+
describe("AudioProcessor offline render preparation", () => {
it("keeps embedded source audio separate from external companion sidecars", async () => {
const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
@@ -136,4 +160,71 @@ describe("AudioProcessor offline render preparation", () => {
expect(loadAudioFileDemuxer).not.toHaveBeenCalled();
expect(renderAndMuxOfflineAudio).toHaveBeenCalled();
});
+
+ it("soft-limits mixed peaks before encoding or WAV conversion", () => {
+ const samples = new Float32Array([
+ -1.6,
+ -0.5,
+ Number.NEGATIVE_INFINITY,
+ Number.NaN,
+ 0,
+ 0.5,
+ 0.95,
+ Number.POSITIVE_INFINITY,
+ 1.6,
+ ]);
+ const changed = softLimitOfflineMixPeaksInPlace(fakeAudioBuffer([samples]));
+
+ expect(changed).toBe(true);
+ expect(samples[0]).toBeGreaterThanOrEqual(-0.986);
+ expect(samples[1]).toBe(-0.5);
+ expect(samples[2]).toBe(0);
+ expect(samples[3]).toBe(0);
+ expect(samples[5]).toBe(0.5);
+ expect(samples[6]).toBeLessThan(0.95);
+ expect(samples[6]).toBeGreaterThan(0.9);
+ expect(samples[7]).toBe(0);
+ expect(samples[8]).toBeLessThanOrEqual(0.986);
+ });
+
+ it("runs the offline mix limiter for every rendered chunk", async () => {
+ const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
+ const renderedSamples = new Float32Array([1.4]);
+ const renderedBuffer = fakeAudioBuffer([renderedSamples]);
+ const originalOfflineAudioContext = globalThis.OfflineAudioContext;
+ (
+ globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext }
+ ).OfflineAudioContext = class {
+ constructor() {}
+
+ startRendering() {
+ return Promise.resolve(renderedBuffer);
+ }
+ } as unknown as typeof OfflineAudioContext;
+
+ try {
+ let observedPeak = Number.POSITIVE_INFINITY;
+ await processor.renderChunked(
+ {
+ mainBufferEntry: null,
+ companionEntries: [],
+ regionEntries: [],
+ mutedSourceOutputRangesSec: [],
+ slices: [],
+ outputDurationMs: 100,
+ numChannels: 1,
+ },
+ 0.1,
+ async (rendered) => {
+ observedPeak = rendered.getChannelData(0)[0] ?? 0;
+ },
+ );
+
+ expect(observedPeak).toBeLessThanOrEqual(0.986);
+ } finally {
+ (
+ globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext }
+ ).OfflineAudioContext = originalOfflineAudioContext;
+ }
+ });
});
diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts
index fba88b6b..d66e0bea 100644
--- a/src/lib/exporter/audioEncoder.ts
+++ b/src/lib/exporter/audioEncoder.ts
@@ -1,20 +1,17 @@
import { WebDemuxer } from "web-demuxer";
+import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
import type {
AudioRegion,
ClipRegion,
- SpeedRegion,
SourceAudioTrackSettings,
+ SpeedRegion,
TrimRegion,
} from "@/components/video-editor/types";
-import {
- buildResolvedAudioPlan,
- SourceTrackId,
-} from "@/lib/exporter/audioRoutingEngine";
+import { buildResolvedAudioPlan, SourceTrackId } from "@/lib/exporter/audioRoutingEngine";
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
import { resolveMediaElementSource } from "./localMediaSource";
import type { VideoMuxer } from "./muxer";
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
-import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
const AUDIO_BITRATE = 128_000;
const DECODE_BACKPRESSURE_LIMIT = 20;
@@ -24,6 +21,44 @@ const MP4_AUDIO_CODEC = "mp4a.40.2";
const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
const OFFLINE_CHUNK_DURATION_SEC = 30;
+const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9;
+const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985;
+
+function softLimitSample(sample: number): number {
+ const magnitude = Math.abs(sample);
+ if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) {
+ return sample;
+ }
+
+ const sign = sample < 0 ? -1 : 1;
+ const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD;
+ const limitedMagnitude =
+ OFFLINE_MIX_SOFT_LIMITER_THRESHOLD +
+ kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange);
+ return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude);
+}
+
+export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean {
+ let changed = false;
+ for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) {
+ const data = buffer.getChannelData(channel);
+ for (let index = 0; index < data.length; index += 1) {
+ const sample = data[index];
+ if (!Number.isFinite(sample)) {
+ data[index] = 0;
+ changed = true;
+ continue;
+ }
+
+ const limited = softLimitSample(sample);
+ if (limited !== sample) {
+ data[index] = limited;
+ changed = true;
+ }
+ }
+ }
+ return changed;
+}
function resolveSourceTrackGain(
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
@@ -698,8 +733,11 @@ export class AudioProcessor {
if (this.cancelled) throw new Error("Export cancelled");
// Decode companion / sidecar audio files
- const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> =
- [];
+ const companionEntries: Array<{
+ buffer: AudioBuffer;
+ startDelaySec: number;
+ gain: number;
+ }> = [];
const refDuration =
mainBuffer?.duration ??
(resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
@@ -947,6 +985,7 @@ export class AudioProcessor {
const rendered = await offlineCtx.startRendering();
if (this.cancelled) break;
+ softLimitOfflineMixPeaksInPlace(rendered);
await onChunk(rendered, outputOffsetSec, i);
@@ -1459,7 +1498,9 @@ export class AudioProcessor {
{
startSec: localOutputStartSec + chunkOutputStartSec,
endSec:
- localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed,
+ localOutputStartSec +
+ chunkOutputStartSec +
+ effectiveSourceDurationSec / slice.speed,
},
];
for (const mutedRange of mutedOutputRangesSec) {
@@ -1491,7 +1532,8 @@ export class AudioProcessor {
const sourceOffsetSec =
effectiveBufferStartSec +
- (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed;
+ (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) *
+ slice.speed;
const localStartSec = audibleRange.startSec - chunkOutputStartSec;
const sourceDurationSec = audibleDurationSec * slice.speed;
@@ -1543,7 +1585,9 @@ export class AudioProcessor {
if (copyLength > 0) {
for (let c = 0; c < channels; c++) {
outBuffer.copyToChannel(
- originalBuffer.getChannelData(c).subarray(startSample, startSample + copyLength),
+ originalBuffer
+ .getChannelData(c)
+ .subarray(startSample, startSample + copyLength),
c,
);
}
@@ -1559,7 +1603,7 @@ export class AudioProcessor {
const workStartIn = Math.max(0, startSample - paddingInSamples);
const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples);
-
+
const actualPaddingInStart = startSample - workStartIn;
// We expect the output offset for the requested start to be roughly:
const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed);
@@ -1572,8 +1616,12 @@ export class AudioProcessor {
const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2;
const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate);
- const inDataByChannel = Array.from({ length: channels }, (_, c) => originalBuffer.getChannelData(c));
- const workOutDataByChannel = Array.from({ length: channels }, (_, c) => workOutBuffer.getChannelData(c));
+ const inDataByChannel = Array.from({ length: channels }, (_, c) =>
+ originalBuffer.getChannelData(c),
+ );
+ const workOutDataByChannel = Array.from({ length: channels }, (_, c) =>
+ workOutBuffer.getChannelData(c),
+ );
const window = new Float32Array(windowSize);
for (let i = 0; i < windowSize; i++) {
@@ -1587,7 +1635,8 @@ export class AudioProcessor {
for (let i = 0; i < windowSize; i++) {
if (inOffset + i < workEndIn && outOffset + i < workOutSamples) {
for (let c = 0; c < channels; c++) {
- workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][inOffset + i] * window[i];
+ workOutDataByChannel[c][outOffset + i] +=
+ inDataByChannel[c][inOffset + i] * window[i];
}
}
}
@@ -1609,7 +1658,9 @@ export class AudioProcessor {
for (let i = 0; i < hopOut; i += 4) {
if (outOffset + i < workOutSamples && testOffset + i < workEndIn) {
for (let c = 0; c < channels; c++) {
- corr += workOutDataByChannel[c][outOffset + i] * inDataByChannel[c][testOffset + i];
+ corr +=
+ workOutDataByChannel[c][outOffset + i] *
+ inDataByChannel[c][testOffset + i];
}
}
}
@@ -1624,7 +1675,8 @@ export class AudioProcessor {
for (let i = 0; i < windowSize; i++) {
if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) {
for (let c = 0; c < channels; c++) {
- workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][bestOffset + i] * window[i];
+ workOutDataByChannel[c][outOffset + i] +=
+ inDataByChannel[c][bestOffset + i] * window[i];
}
}
}