Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix(export): fast-path simple edited audio tracks

wiiiii123c8431c1f268f
diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts
index f37eae2b..5012dbe5 100644
--- a/electron/electron-env.d.ts
+++ b/electron/electron-env.d.ts
@@ -194,7 +194,10 @@ interface Window {
options?: {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
+ audioSourceSampleRate?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
+ editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
+ editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
editedAudioData?: ArrayBuffer;
editedAudioMimeType?: string | null;
},
@@ -213,7 +216,10 @@ interface Window {
options?: {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
+ audioSourceSampleRate?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
+ editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
+ editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
editedAudioData?: ArrayBuffer;
editedAudioMimeType?: string | null;
},
diff --git a/electron/ipc/export/native-video.ts b/electron/ipc/export/native-video.ts
index 77552cbb..a9205369 100644
--- a/electron/ipc/export/native-video.ts
+++ b/electron/ipc/export/native-video.ts
@@ -14,6 +14,7 @@ import type {
NativeVideoExportFinishOptions,
} from "../nativeVideoExport";
import {
+ buildEditedTrackSourceAudioFilter,
buildNativeVideoExportArgs,
buildTrimmedSourceAudioFilter,
getEditedAudioExtension,
@@ -387,8 +388,10 @@ export async function muxNativeVideoExportAudio(
const metrics: NativeVideoAudioMuxMetrics = {};
const tempArtifacts: string[] = [];
let audioInputPath = options.audioSourcePath ?? null;
+ const useEditedTrackFiltergraph =
+ audioMode === "edited-track" && options.editedTrackStrategy === "filtergraph-fast-path";
- if (audioMode === "edited-track") {
+ if (audioMode === "edited-track" && !useEditedTrackFiltergraph) {
if (!options.editedAudioData) {
throw new Error("Edited audio data is missing for native export");
}
@@ -435,6 +438,15 @@ export async function muxNativeVideoExportAudio(
} else {
args.push("-map", "0:v:0", "-map", "1:a:0");
}
+ } else if (useEditedTrackFiltergraph) {
+ const filter = buildEditedTrackSourceAudioFilter(
+ options.editedTrackSegments ?? [],
+ options.audioSourceSampleRate ?? 0,
+ );
+ if (!filter) {
+ throw new Error("Edited-track filtergraph inputs are incomplete for native export");
+ }
+ args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]");
} else {
args.push("-map", "0:v:0", "-map", "1:a:0");
}
diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts
new file mode 100644
index 00000000..1d0630d9
--- /dev/null
+++ b/electron/ipc/nativeVideoExport.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vitest";
+import {
+ buildEditedTrackSourceAudioFilter,
+ buildTrimmedSourceAudioFilter,
+} from "./nativeVideoExport";
+
+describe("buildTrimmedSourceAudioFilter", () => {
+ it("concatenates trimmed source segments into a single output label", () => {
+ expect(
+ buildTrimmedSourceAudioFilter([
+ { startMs: 0, endMs: 2_000 },
+ { startMs: 4_000, endMs: 6_000 },
+ ]),
+ ).toContain("concat=n=2:v=0:a=1[aout]");
+ });
+});
+
+describe("buildEditedTrackSourceAudioFilter", () => {
+ it("builds a concat filtergraph that pitch-shifts via asetrate for speed changes", () => {
+ const filter = buildEditedTrackSourceAudioFilter(
+ [
+ { startMs: 0, endMs: 2_000, speed: 1 },
+ { startMs: 2_000, endMs: 6_000, speed: 1.5 },
+ ],
+ 44_100,
+ );
+
+ expect(filter).toContain(
+ "[1:a]atrim=start=2.000:end=6.000,asetpts=PTS-STARTPTS,asetrate=66150,aresample=44100[edited_audio_1]",
+ );
+ expect(filter).toContain("concat=n=2:v=0:a=1[aout]");
+ });
+
+ it("returns null when the edited-track filtergraph inputs are incomplete", () => {
+ expect(buildEditedTrackSourceAudioFilter([], 44_100)).toBeNull();
+ expect(
+ buildEditedTrackSourceAudioFilter(
+ [{ startMs: 0, endMs: 2_000, speed: 1.5 }],
+ Number.NaN,
+ ),
+ ).toBeNull();
+ });
+});
diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts
index 58dbf5cc..fd7435fa 100644
--- a/electron/ipc/nativeVideoExport.ts
+++ b/electron/ipc/nativeVideoExport.ts
@@ -3,6 +3,9 @@ const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4;
export type NativeExportEncodingMode = "fast" | "balanced" | "quality";
export type NativeVideoExportAudioMode = "none" | "copy-source" | "trim-source" | "edited-track";
+export type NativeVideoExportEditedTrackStrategy =
+ | "filtergraph-fast-path"
+ | "offline-render-fallback";
export interface NativeVideoExportStartOptions {
width: number;
@@ -18,10 +21,17 @@ export interface NativeVideoExportAudioSegment {
endMs: number;
}
+export interface NativeVideoExportEditedTrackSegment extends NativeVideoExportAudioSegment {
+ speed: number;
+}
+
export interface NativeVideoExportFinishOptions {
audioMode?: NativeVideoExportAudioMode;
audioSourcePath?: string | null;
+ audioSourceSampleRate?: number;
trimSegments?: NativeVideoExportAudioSegment[];
+ editedTrackStrategy?: NativeVideoExportEditedTrackStrategy;
+ editedTrackSegments?: NativeVideoExportEditedTrackSegment[];
editedAudioData?: ArrayBuffer;
editedAudioMimeType?: string | null;
}
@@ -162,6 +172,54 @@ export function buildTrimmedSourceAudioFilter(
return filterParts.join(";");
}
+export function buildEditedTrackSourceAudioFilter(
+ segments: NativeVideoExportEditedTrackSegment[],
+ sourceSampleRate: number,
+): string | null {
+ if (segments.length === 0 || !Number.isFinite(sourceSampleRate) || sourceSampleRate <= 0) {
+ return null;
+ }
+
+ const normalizedSourceSampleRate = Math.round(sourceSampleRate);
+ const filterParts: string[] = [];
+ const segmentLabels: string[] = [];
+
+ segments.forEach((segment, index) => {
+ if (segment.endMs - segment.startMs <= 0.5) {
+ return;
+ }
+
+ const label = `edited_audio_${index}`;
+ const speed = Number.isFinite(segment.speed) && segment.speed > 0 ? segment.speed : 1;
+ const segmentFilter = [
+ `[1:a]atrim=start=${formatFfmpegSeconds(segment.startMs)}:end=${formatFfmpegSeconds(segment.endMs)}`,
+ "asetpts=PTS-STARTPTS",
+ ];
+
+ if (Math.abs(speed - 1) > 0.0001) {
+ segmentFilter.push(
+ `asetrate=${Math.max(1, Math.round(normalizedSourceSampleRate * speed))}`,
+ `aresample=${normalizedSourceSampleRate}`,
+ );
+ }
+
+ filterParts.push(`${segmentFilter.join(",")}[${label}]`);
+ segmentLabels.push(`[${label}]`);
+ });
+
+ if (segmentLabels.length === 0) {
+ return null;
+ }
+
+ if (segmentLabels.length === 1) {
+ filterParts.push(`${segmentLabels[0]}anull[aout]`);
+ } else {
+ filterParts.push(`${segmentLabels.join("")}concat=n=${segmentLabels.length}:v=0:a=1[aout]`);
+ }
+
+ return filterParts.join(";");
+}
+
/**
* Builds FFmpeg arguments for a zero-copy H.264 stream export.
* FFmpeg receives a pre-encoded Annex B H.264 stream on stdin (produced by the
diff --git a/electron/preload.ts b/electron/preload.ts
index 98562522..a8b0e67c 100644
--- a/electron/preload.ts
+++ b/electron/preload.ts
@@ -143,7 +143,10 @@ contextBridge.exposeInMainWorld("electronAPI", {
options?: {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
+ audioSourceSampleRate?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
+ editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
+ editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
editedAudioData?: ArrayBuffer;
editedAudioMimeType?: string | null;
},
@@ -186,7 +189,10 @@ contextBridge.exposeInMainWorld("electronAPI", {
options?: {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
+ audioSourceSampleRate?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
+ editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
+ editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
editedAudioData?: ArrayBuffer;
editedAudioMimeType?: string | null;
},
diff --git a/src/lib/exporter/editedTrackStrategy.test.ts b/src/lib/exporter/editedTrackStrategy.test.ts
new file mode 100644
index 00000000..d70dd83f
--- /dev/null
+++ b/src/lib/exporter/editedTrackStrategy.test.ts
@@ -0,0 +1,121 @@
+import { describe, expect, it } from "vitest";
+import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types";
+import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
+
+const SOURCE_DURATION_MS = 20_000;
+const EMPTY_TRIMS: TrimRegion[] = [];
+
+describe("editedTrackStrategy", () => {
+ it("marks single-source speed-only edits as filtergraph candidates", () => {
+ const speedRegions: SpeedRegion[] = [
+ { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 },
+ ];
+
+ expect(
+ classifyEditedTrackStrategy({
+ primaryAudioSourcePath: "recording.mp4",
+ sourceDurationMs: SOURCE_DURATION_MS,
+ trimRegions: EMPTY_TRIMS,
+ speedRegions,
+ audioRegions: [],
+ sourceAudioFallbackPaths: [],
+ }),
+ ).toBe("filtergraph-fast-path");
+ });
+
+ it("falls back when the edited track depends on sidecar sources", () => {
+ const speedRegions: SpeedRegion[] = [
+ { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.25 },
+ ];
+
+ expect(
+ classifyEditedTrackStrategy({
+ primaryAudioSourcePath: "mic.m4a",
+ sourceDurationMs: SOURCE_DURATION_MS,
+ trimRegions: EMPTY_TRIMS,
+ speedRegions,
+ audioRegions: [],
+ sourceAudioFallbackPaths: ["mic.m4a"],
+ }),
+ ).toBe("offline-render-fallback");
+ });
+
+ it("falls back for multi-source mixes", () => {
+ const speedRegions: SpeedRegion[] = [
+ { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 },
+ ];
+
+ expect(
+ classifyEditedTrackStrategy({
+ primaryAudioSourcePath: "recording.mp4",
+ sourceDurationMs: SOURCE_DURATION_MS,
+ trimRegions: EMPTY_TRIMS,
+ speedRegions,
+ audioRegions: [],
+ sourceAudioFallbackPaths: ["mic.m4a"],
+ }),
+ ).toBe("offline-render-fallback");
+ });
+
+ it("falls back when audio regions require overlay mixing", () => {
+ const speedRegions: SpeedRegion[] = [
+ { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 1.5 },
+ ];
+ const audioRegions: AudioRegion[] = [
+ {
+ id: "audio-1",
+ audioPath: "overlay.wav",
+ startMs: 1_000,
+ endMs: 4_000,
+ volume: 0.8,
+ },
+ ];
+
+ expect(
+ classifyEditedTrackStrategy({
+ primaryAudioSourcePath: "recording.mp4",
+ sourceDurationMs: SOURCE_DURATION_MS,
+ trimRegions: EMPTY_TRIMS,
+ speedRegions,
+ audioRegions,
+ sourceAudioFallbackPaths: [],
+ }),
+ ).toBe("offline-render-fallback");
+ });
+
+ it("falls back for speeds outside the conservative filtergraph window", () => {
+ const speedRegions = [
+ { id: "speed-1", startMs: 2_000, endMs: 10_000, speed: 2.5 },
+ ] as SpeedRegion[];
+
+ expect(
+ classifyEditedTrackStrategy({
+ primaryAudioSourcePath: "recording.mp4",
+ sourceDurationMs: SOURCE_DURATION_MS,
+ trimRegions: EMPTY_TRIMS,
+ speedRegions,
+ audioRegions: [],
+ sourceAudioFallbackPaths: [],
+ }),
+ ).toBe("offline-render-fallback");
+ });
+
+ it("builds source segments that preserve trims and speed boundaries", () => {
+ const trimRegions: TrimRegion[] = [{ id: "trim-1", startMs: 10_000, endMs: 12_000 }];
+ const speedRegions: SpeedRegion[] = [
+ { id: "speed-1", startMs: 2_000, endMs: 6_000, speed: 1.5 },
+ { id: "speed-2", startMs: 14_000, endMs: 18_000, speed: 0.75 },
+ ];
+
+ expect(
+ buildEditedTrackSourceSegments(SOURCE_DURATION_MS, trimRegions, speedRegions),
+ ).toEqual([
+ { startMs: 0, endMs: 2_000, speed: 1 },
+ { startMs: 2_000, endMs: 6_000, speed: 1.5 },
+ { startMs: 6_000, endMs: 10_000, speed: 1 },
+ { startMs: 12_000, endMs: 14_000, speed: 1 },
+ { startMs: 14_000, endMs: 18_000, speed: 0.75 },
+ { startMs: 18_000, endMs: 20_000, speed: 1 },
+ ]);
+ });
+});
diff --git a/src/lib/exporter/editedTrackStrategy.ts b/src/lib/exporter/editedTrackStrategy.ts
new file mode 100644
index 00000000..52a10783
--- /dev/null
+++ b/src/lib/exporter/editedTrackStrategy.ts
@@ -0,0 +1,141 @@
+import type { AudioRegion, SpeedRegion, TrimRegion } from "@/components/video-editor/types";
+
+const MIN_FILTERGRAPH_SPEED = 0.5;
+const MAX_FILTERGRAPH_SPEED = 2;
+
+export type EditedTrackStrategy = "filtergraph-fast-path" | "offline-render-fallback";
+
+export interface EditedTrackSourceSegment {
+ startMs: number;
+ endMs: number;
+ speed: number;
+}
+
+export interface EditedTrackStrategyInput {
+ primaryAudioSourcePath: string | null;
+ sourceDurationMs: number;
+ trimRegions: TrimRegion[];
+ speedRegions: SpeedRegion[];
+ audioRegions: AudioRegion[];
+ sourceAudioFallbackPaths: string[];
+}
+
+function isSafeFiltergraphSpeed(speed: number): boolean {
+ return (
+ Number.isFinite(speed) && speed >= MIN_FILTERGRAPH_SPEED && speed <= MAX_FILTERGRAPH_SPEED
+ );
+}
+
+function hasSafeFiltergraphSpeedRegions(speedRegions: SpeedRegion[]): boolean {
+ return speedRegions.every((region) => isSafeFiltergraphSpeed(region.speed));
+}
+
+function buildKeptRanges(
+ sourceDurationMs: number,
+ trimRegions: TrimRegion[],
+): Array<{ startMs: number; endMs: number }> {
+ const sortedTrimRegions = [...trimRegions].sort((left, right) => left.startMs - right.startMs);
+ const keptRanges: Array<{ startMs: number; endMs: number }> = [];
+ let cursorMs = 0;
+
+ for (const trimRegion of sortedTrimRegions) {
+ const startMs = Math.max(0, trimRegion.startMs);
+ const endMs = Math.min(sourceDurationMs, trimRegion.endMs);
+ if (endMs <= startMs) {
+ continue;
+ }
+
+ if (startMs > cursorMs) {
+ keptRanges.push({ startMs: cursorMs, endMs: startMs });
+ }
+
+ cursorMs = Math.max(cursorMs, endMs);
+ }
+
+ if (cursorMs < sourceDurationMs) {
+ keptRanges.push({ startMs: cursorMs, endMs: sourceDurationMs });
+ }
+
+ return keptRanges;
+}
+
+export function buildEditedTrackSourceSegments(
+ sourceDurationMs: number,
+ trimRegions: TrimRegion[],
+ speedRegions: SpeedRegion[],
+): EditedTrackSourceSegment[] {
+ const segments: EditedTrackSourceSegment[] = [];
+ const keptRanges = buildKeptRanges(sourceDurationMs, trimRegions);
+
+ for (const keptRange of keptRanges) {
+ const boundaries = new Set<number>([keptRange.startMs, keptRange.endMs]);
+ for (const speedRegion of speedRegions) {
+ const startMs = Math.max(keptRange.startMs, speedRegion.startMs);
+ const endMs = Math.min(keptRange.endMs, speedRegion.endMs);
+ if (endMs > startMs) {
+ boundaries.add(startMs);
+ boundaries.add(endMs);
+ }
+ }
+
+ const orderedBoundaries = [...boundaries].sort((left, right) => left - right);
+ for (let index = 0; index < orderedBoundaries.length - 1; index += 1) {
+ const startMs = orderedBoundaries[index] ?? 0;
+ const endMs = orderedBoundaries[index + 1] ?? 0;
+ if (endMs - startMs <= 0.5) {
+ continue;
+ }
+
+ const midpointMs = startMs + (endMs - startMs) / 2;
+ const speedRegion = speedRegions.find(
+ (region) => midpointMs >= region.startMs && midpointMs < region.endMs,
+ );
+ const speed = speedRegion?.speed ?? 1;
+ if (!isSafeFiltergraphSpeed(speed)) {
+ return [];
+ }
+
+ segments.push({
+ startMs,
+ endMs,
+ speed,
+ });
+ }
+ }
+
+ return segments;
+}
+
+export function classifyEditedTrackStrategy(input: EditedTrackStrategyInput): EditedTrackStrategy {
+ if (!input.primaryAudioSourcePath) {
+ return "offline-render-fallback";
+ }
+
+ if (input.audioRegions.length > 0) {
+ return "offline-render-fallback";
+ }
+
+ if (input.speedRegions.length === 0) {
+ return "offline-render-fallback";
+ }
+
+ if (!hasSafeFiltergraphSpeedRegions(input.speedRegions)) {
+ return "offline-render-fallback";
+ }
+
+ if (input.sourceAudioFallbackPaths.length > 0) {
+ return "offline-render-fallback";
+ }
+
+ if (
+ buildEditedTrackSourceSegments(
+ input.sourceDurationMs,
+ input.trimRegions,
+ input.speedRegions,
+ ).length === 0
+ ) {
+ return "offline-render-fallback";
+ }
+
+ return "filtergraph-fast-path";
+}
diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts
index 843c56ef..15f71a58 100644
--- a/src/lib/exporter/modernVideoExporter.ts
+++ b/src/lib/exporter/modernVideoExporter.ts
@@ -15,6 +15,7 @@ import type {
import { extensionHost } from "@/lib/extensions";
import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder";
import { normalizeLightningRuntimePlatform } from "./backendPolicy";
+import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
import {
type ExportBackpressureProfile,
getExportBackpressureProfile,
@@ -106,6 +107,14 @@ type NativeAudioPlan =
}
| {
audioMode: "edited-track";
+ strategy: "offline-render-fallback";
+ }
+ | {
+ audioMode: "edited-track";
+ strategy: "filtergraph-fast-path";
+ audioSourcePath: string;
+ audioSourceSampleRate: number;
+ editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>;
};
const NATIVE_EXPORT_ENGINE_NAME = "Breeze";
@@ -737,11 +746,62 @@ export class ModernVideoExporter {
audioRegions.length > 0 ||
sourceAudioFallbackPaths.length > 1
) {
- return { audioMode: "edited-track" };
+ const sourceDurationMs = Math.max(
+ 0,
+ Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000),
+ );
+ const trimRegions = this.config.trimRegions ?? [];
+ const strategy =
+ videoInfo.hasAudio &&
+ localVideoSourcePath &&
+ sourceAudioFallbackPaths.length === 0 &&
+ typeof videoInfo.audioSampleRate === "number" &&
+ Number.isFinite(videoInfo.audioSampleRate) &&
+ videoInfo.audioSampleRate > 0
+ ? classifyEditedTrackStrategy({
+ primaryAudioSourcePath,
+ sourceDurationMs,
+ trimRegions,
+ speedRegions,
+ audioRegions,
+ sourceAudioFallbackPaths,
+ })
+ : "offline-render-fallback";
+
+ if (strategy === "filtergraph-fast-path") {
+ const audioSourcePath = localVideoSourcePath;
+ const audioSourceSampleRate = videoInfo.audioSampleRate;
+ const editedTrackSegments = buildEditedTrackSourceSegments(
+ sourceDurationMs,
+ trimRegions,
+ speedRegions,
+ );
+ if (
+ audioSourcePath &&
+ typeof audioSourceSampleRate === "number" &&
+ editedTrackSegments.length > 0
+ ) {
+ return {
+ audioMode: "edited-track",
+ strategy,
+ audioSourcePath,
+ audioSourceSampleRate,
+ editedTrackSegments,
+ };
+ }
+ }
+
+ return {
+ audioMode: "edited-track",
+ strategy: "offline-render-fallback",
+ };
}
if (!primaryAudioSourcePath) {
- return { audioMode: "edited-track" };
+ return {
+ audioMode: "edited-track",
+ strategy: "offline-render-fallback",
+ };
}
if ((this.config.trimRegions ?? []).length > 0) {
@@ -949,7 +1009,10 @@ export class ModernVideoExporter {
let editedAudioBuffer: ArrayBuffer | undefined;
let editedAudioMimeType: string | null = null;
- if (audioPlan.audioMode === "edited-track") {
+ if (
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "offline-render-fallback"
+ ) {
this.audioProcessor = new AudioProcessor();
this.audioProcessor.setOnProgress((progress) => {
this.reportFinalizingProgress(this.processedFrameCount, 99, progress);
@@ -976,6 +1039,8 @@ export class ModernVideoExporter {
console.log(`[VideoExporter] Finalizing ${NATIVE_EXPORT_ENGINE_NAME} export`, {
sessionId,
audioMode: audioPlan.audioMode,
+ editedTrackStrategy:
+ audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
encoderName: this.encoderName ?? "unknown",
});
@@ -987,11 +1052,25 @@ export class ModernVideoExporter {
audioMode: audioPlan.audioMode,
audioSourcePath:
audioPlan.audioMode === "copy-source" ||
- audioPlan.audioMode === "trim-source"
+ audioPlan.audioMode === "trim-source" ||
+ (audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path")
? audioPlan.audioSourcePath
: null,
trimSegments:
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
+ editedTrackStrategy:
+ audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
+ editedTrackSegments:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.editedTrackSegments
+ : undefined,
+ audioSourceSampleRate:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.audioSourceSampleRate
+ : undefined,
editedAudioData: editedAudioBuffer,
editedAudioMimeType,
}),
@@ -1041,7 +1120,10 @@ export class ModernVideoExporter {
let editedAudioBuffer: ArrayBuffer | undefined;
let editedAudioMimeType: string | null = null;
- if (audioPlan.audioMode === "edited-track") {
+ if (
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "offline-render-fallback"
+ ) {
this.audioProcessor = new AudioProcessor();
this.audioProcessor.setOnProgress((progress) => {
this.reportFinalizingProgress(this.processedFrameCount, 99, progress);
@@ -1071,11 +1153,25 @@ export class ModernVideoExporter {
audioMode: audioPlan.audioMode,
audioSourcePath:
audioPlan.audioMode === "copy-source" ||
- audioPlan.audioMode === "trim-source"
+ audioPlan.audioMode === "trim-source" ||
+ (audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path")
? audioPlan.audioSourcePath
: null,
trimSegments:
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
+ editedTrackStrategy:
+ audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
+ editedTrackSegments:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.editedTrackSegments
+ : undefined,
+ audioSourceSampleRate:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.audioSourceSampleRate
+ : undefined,
editedAudioData: editedAudioBuffer,
editedAudioMimeType,
}),
diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts
index d13ff888..6463dca2 100644
--- a/src/lib/exporter/streamingDecoder.ts
+++ b/src/lib/exporter/streamingDecoder.ts
@@ -17,6 +17,7 @@ export interface DecodedVideoInfo {
codec: string;
hasAudio: boolean;
audioCodec?: string;
+ audioSampleRate?: number;
}
/** Caller must close the VideoFrame after use. */
@@ -205,6 +206,10 @@ export class StreamingVideoDecoder {
codec: videoStream?.codec_string || "unknown",
hasAudio: !!audioStream,
audioCodec: audioStream?.codec_string,
+ audioSampleRate:
+ typeof audioStream?.sample_rate === "string"
+ ? Number.parseInt(audioStream.sample_rate, 10)
+ : undefined,
};
return this.metadata;
diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts
index 42f4995a..8f6e772e 100644
--- a/src/lib/exporter/videoExporter.ts
+++ b/src/lib/exporter/videoExporter.ts
@@ -13,6 +13,7 @@ import type {
ZoomTransitionEasing,
} from "@/components/video-editor/types";
import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder";
+import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy";
import {
advanceFinalizationProgress,
type FinalizationProgressWatchdog,
@@ -94,6 +95,14 @@ type NativeAudioPlan =
}
| {
audioMode: "edited-track";
+ strategy: "offline-render-fallback";
+ }
+ | {
+ audioMode: "edited-track";
+ strategy: "filtergraph-fast-path";
+ audioSourcePath: string;
+ audioSourceSampleRate: number;
+ editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>;
};
export class VideoExporter {
@@ -504,11 +513,62 @@ export class VideoExporter {
audioRegions.length > 0 ||
sourceAudioFallbackPaths.length > 1
) {
- return { audioMode: "edited-track" };
+ const sourceDurationMs = Math.max(
+ 0,
+ Math.round((videoInfo.streamDuration ?? videoInfo.duration) * 1000),
+ );
+ const trimRegions = this.config.trimRegions ?? [];
+ const strategy =
+ videoInfo.hasAudio &&
+ localVideoSourcePath &&
+ sourceAudioFallbackPaths.length === 0 &&
+ typeof videoInfo.audioSampleRate === "number" &&
+ Number.isFinite(videoInfo.audioSampleRate) &&
+ videoInfo.audioSampleRate > 0
+ ? classifyEditedTrackStrategy({
+ primaryAudioSourcePath,
+ sourceDurationMs,
+ trimRegions,
+ speedRegions,
+ audioRegions,
+ sourceAudioFallbackPaths,
+ })
+ : "offline-render-fallback";
+
+ if (strategy === "filtergraph-fast-path") {
+ const audioSourcePath = localVideoSourcePath;
+ const audioSourceSampleRate = videoInfo.audioSampleRate;
+ const editedTrackSegments = buildEditedTrackSourceSegments(
+ sourceDurationMs,
+ trimRegions,
+ speedRegions,
+ );
+ if (
+ audioSourcePath &&
+ typeof audioSourceSampleRate === "number" &&
+ editedTrackSegments.length > 0
+ ) {
+ return {
+ audioMode: "edited-track",
+ strategy,
+ audioSourcePath,
+ audioSourceSampleRate,
+ editedTrackSegments,
+ };
+ }
+ }
+
+ return {
+ audioMode: "edited-track",
+ strategy: "offline-render-fallback",
+ };
}
if (!primaryAudioSourcePath) {
- return { audioMode: "edited-track" };
+ return {
+ audioMode: "edited-track",
+ strategy: "offline-render-fallback",
+ };
}
if ((this.config.trimRegions ?? []).length > 0) {
@@ -709,7 +769,10 @@ export class VideoExporter {
let editedAudioBuffer: ArrayBuffer | undefined;
let editedAudioMimeType: string | null = null;
- if (audioPlan.audioMode === "edited-track") {
+ if (
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "offline-render-fallback"
+ ) {
this.audioProcessor = new AudioProcessor();
this.audioProcessor.setOnProgress((progress) => {
this.reportFinalizingProgress(totalFrames, 99, progress);
@@ -741,11 +804,25 @@ export class VideoExporter {
audioMode: audioPlan.audioMode,
audioSourcePath:
audioPlan.audioMode === "copy-source" ||
- audioPlan.audioMode === "trim-source"
+ audioPlan.audioMode === "trim-source" ||
+ (audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path")
? audioPlan.audioSourcePath
: null,
trimSegments:
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
+ editedTrackStrategy:
+ audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
+ editedTrackSegments:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.editedTrackSegments
+ : undefined,
+ audioSourceSampleRate:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.audioSourceSampleRate
+ : undefined,
editedAudioData: editedAudioBuffer,
editedAudioMimeType,
}),
@@ -790,7 +867,10 @@ export class VideoExporter {
let editedAudioBuffer: ArrayBuffer | undefined;
let editedAudioMimeType: string | null = null;
- if (audioPlan.audioMode === "edited-track") {
+ if (
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "offline-render-fallback"
+ ) {
this.audioProcessor = new AudioProcessor();
this.audioProcessor.setOnProgress((progress) => {
this.reportFinalizingProgress(totalFrames, 99, progress);
@@ -820,11 +900,25 @@ export class VideoExporter {
audioMode: audioPlan.audioMode,
audioSourcePath:
audioPlan.audioMode === "copy-source" ||
- audioPlan.audioMode === "trim-source"
+ audioPlan.audioMode === "trim-source" ||
+ (audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path")
? audioPlan.audioSourcePath
: null,
trimSegments:
audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined,
+ editedTrackStrategy:
+ audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined,
+ editedTrackSegments:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.editedTrackSegments
+ : undefined,
+ audioSourceSampleRate:
+ audioPlan.audioMode === "edited-track" &&
+ audioPlan.strategy === "filtergraph-fast-path"
+ ? audioPlan.audioSourceSampleRate
+ : undefined,
editedAudioData: editedAudioBuffer,
editedAudioMimeType,
}),