Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix(export): stabilize Lightning webcam rendering and round Windows icon

webadderall37f95ae65265
diff --git a/icons/icons/png/1024x1024.png b/icons/icons/png/1024x1024.png
index c194c3ff..7cc41c02 100644
Binary files a/icons/icons/png/1024x1024.png and b/icons/icons/png/1024x1024.png differ
diff --git a/icons/icons/png/128x128.png b/icons/icons/png/128x128.png
index 8dc2b50f..82b456e9 100644
Binary files a/icons/icons/png/128x128.png and b/icons/icons/png/128x128.png differ
diff --git a/icons/icons/png/16x16.png b/icons/icons/png/16x16.png
index 66ea3f13..e1f45c1f 100644
Binary files a/icons/icons/png/16x16.png and b/icons/icons/png/16x16.png differ
diff --git a/icons/icons/png/24x24.png b/icons/icons/png/24x24.png
index b56ffe89..e146b337 100644
Binary files a/icons/icons/png/24x24.png and b/icons/icons/png/24x24.png differ
diff --git a/icons/icons/png/256x256.png b/icons/icons/png/256x256.png
index 6d6eb25e..0592161a 100644
Binary files a/icons/icons/png/256x256.png and b/icons/icons/png/256x256.png differ
diff --git a/icons/icons/png/32x32.png b/icons/icons/png/32x32.png
index dee1d807..6fb056f8 100644
Binary files a/icons/icons/png/32x32.png and b/icons/icons/png/32x32.png differ
diff --git a/icons/icons/png/48x48.png b/icons/icons/png/48x48.png
index 78725c5a..48af277b 100644
Binary files a/icons/icons/png/48x48.png and b/icons/icons/png/48x48.png differ
diff --git a/icons/icons/png/512x512.png b/icons/icons/png/512x512.png
index 1d1a8fa6..4016ff59 100644
Binary files a/icons/icons/png/512x512.png and b/icons/icons/png/512x512.png differ
diff --git a/icons/icons/png/64x64.png b/icons/icons/png/64x64.png
index fc3c7a67..e132d69a 100644
Binary files a/icons/icons/png/64x64.png and b/icons/icons/png/64x64.png differ
diff --git a/icons/icons/win/icon.ico b/icons/icons/win/icon.ico
index 6a327f3d..52e47c89 100644
Binary files a/icons/icons/win/icon.ico and b/icons/icons/win/icon.ico differ
diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts
index 7df341f9..a362c126 100644
--- a/src/lib/exporter/modernFrameRenderer.test.ts
+++ b/src/lib/exporter/modernFrameRenderer.test.ts
@@ -304,6 +304,28 @@ describe("ModernFrameRenderer blur export path", () => {
});
describe("ModernFrameRenderer webcam frame cache", () => {
+ it("stages webcam video frames on WebGPU instead of using retained frame uploads", () => {
+ const renderer = createRenderer() as any;
+ renderer.rendererBackend = "webgpu";
+
+ const frame = {
+ displayWidth: 320,
+ displayHeight: 180,
+ timestamp: 0,
+ } as VideoFrame;
+
+ const result = renderer.stageVideoFrameForTexture(frame, "webcam", 640, 360);
+
+ expect(result).toBe(renderer.webcamVideoFrameStagingCanvas);
+ expect(renderer.webcamVideoFrameStagingCtx.drawImage).toHaveBeenCalledWith(
+ frame,
+ 0,
+ 0,
+ 320,
+ 180,
+ );
+ });
+
it("uses staging canvas instead of recursing when WebGPU frame retention fails", () => {
const renderer = createRenderer() as any;
const originalVideoFrame = (globalThis as any).VideoFrame;
@@ -355,7 +377,12 @@ describe("ModernFrameRenderer webcam frame cache", () => {
it("bypasses the refresh throttle for cropped webcam regions", () => {
const renderer = createRenderer() as any;
- renderer.config.webcam.cropRegion = { x: 0.25, y: 0, width: 0.5, height: 1 };
+ renderer.config.webcam.cropRegion = {
+ x: 0.25,
+ y: 0,
+ width: 0.5,
+ height: 1,
+ };
renderer.webcamFrameCacheCanvas = { width: 640, height: 720 };
renderer.lastWebcamCacheRefreshTime = 10;
renderer.currentVideoTime = 10.1;
@@ -506,4 +533,221 @@ describe("ModernFrameRenderer webcam export fallback", () => {
vi.useRealTimers();
}
});
+
+ it("keeps the webcam live when sync uses an offset timeline", () => {
+ const renderer = createRenderer() as any;
+ renderer.config.webcam = {
+ ...DEFAULT_WEBCAM_OVERLAY,
+ enabled: true,
+ timeOffsetMs: 500,
+ };
+ renderer.currentVideoTime = 10;
+ renderer.lastSyncedWebcamTime = 9.5;
+ renderer.webcamVideoElement = {
+ readyState: 2,
+ seeking: false,
+ videoWidth: 640,
+ videoHeight: 360,
+ duration: Number.NaN,
+ };
+ renderer.webcamRootContainer = {
+ visible: false,
+ position: { set: vi.fn() },
+ };
+ renderer.webcamContainer = {
+ addChildAt: vi.fn(),
+ };
+ renderer.webcamMaskGraphics = {
+ clear: vi.fn(),
+ moveTo: vi.fn(),
+ lineTo: vi.fn(),
+ closePath: vi.fn(),
+ fill: vi.fn(),
+ };
+ renderer.webcamShadowLayers = [];
+ renderer.animationState = {
+ appliedScale: 1,
+ };
+
+ renderer.updateWebcamOverlay();
+
+ expect(renderer.webcamRootContainer.visible).toBe(true);
+ expect(renderer.webcamSprite).toBeTruthy();
+ });
+
+ it("keeps the webcam live when the media element time is current but lastSyncedWebcamTime is stale", () => {
+ const renderer = createRenderer() as any;
+ renderer.config.webcam = {
+ ...DEFAULT_WEBCAM_OVERLAY,
+ enabled: true,
+ timeOffsetMs: 500,
+ };
+ renderer.currentVideoTime = 10;
+ renderer.lastSyncedWebcamTime = 8;
+ renderer.webcamVideoElement = {
+ currentTime: 9.5,
+ readyState: 2,
+ seeking: false,
+ videoWidth: 640,
+ videoHeight: 360,
+ duration: Number.NaN,
+ };
+ renderer.webcamRootContainer = {
+ visible: false,
+ position: { set: vi.fn() },
+ };
+ renderer.webcamContainer = {
+ addChildAt: vi.fn(),
+ };
+ renderer.webcamMaskGraphics = {
+ clear: vi.fn(),
+ moveTo: vi.fn(),
+ lineTo: vi.fn(),
+ closePath: vi.fn(),
+ fill: vi.fn(),
+ };
+ renderer.webcamShadowLayers = [];
+ renderer.animationState = {
+ appliedScale: 1,
+ };
+
+ renderer.updateWebcamOverlay();
+
+ expect(renderer.webcamRootContainer.visible).toBe(true);
+ expect(renderer.webcamSprite).toBeTruthy();
+ });
+
+ it("snapshots media-element webcam frames into the cache before rendering", () => {
+ const renderer = createRenderer() as any;
+ renderer.config.webcam = {
+ ...DEFAULT_WEBCAM_OVERLAY,
+ enabled: true,
+ cropRegion: { x: 0, y: 0, width: 1, height: 1 },
+ };
+ renderer.currentVideoTime = 4;
+ const previousHtmlVideoElement = (
+ globalThis as typeof globalThis & { HTMLVideoElement?: unknown }
+ ).HTMLVideoElement;
+ class MockHtmlVideoElement {
+ currentTime = 4;
+ readyState = 2;
+ seeking = false;
+ videoWidth = 640;
+ videoHeight = 360;
+ duration = Number.NaN;
+ }
+ Object.assign(globalThis, {
+ HTMLVideoElement: MockHtmlVideoElement,
+ });
+
+ const webcamVideoElement = new MockHtmlVideoElement();
+
+ try {
+ const renderableSource = renderer.resolveRenderableWebcamSource(
+ webcamVideoElement,
+ 640,
+ 360,
+ true,
+ );
+
+ expect(renderableSource?.source).toBe(renderer.webcamFrameCacheCanvas);
+ expect(renderer.webcamFrameCacheCtx.drawImage).toHaveBeenCalledWith(
+ webcamVideoElement,
+ 0,
+ 0,
+ 640,
+ 360,
+ 0,
+ 0,
+ 640,
+ 360,
+ );
+ } finally {
+ Object.assign(globalThis, {
+ HTMLVideoElement: previousHtmlVideoElement,
+ });
+ }
+ });
+
+ it("renders decoder-backed webcam frames directly for the default crop region", () => {
+ const renderer = createRenderer() as any;
+ renderer.config.webcam = {
+ ...DEFAULT_WEBCAM_OVERLAY,
+ enabled: true,
+ cropRegion: { x: 0, y: 0, width: 1, height: 1 },
+ };
+
+ const previousVideoFrame = (globalThis as typeof globalThis & { VideoFrame?: unknown })
+ .VideoFrame;
+ class MockVideoFrame {}
+ Object.assign(globalThis, {
+ VideoFrame: MockVideoFrame,
+ });
+
+ try {
+ const webcamFrame = new MockVideoFrame();
+ const renderableSource = renderer.resolveRenderableWebcamSource(
+ webcamFrame,
+ 640,
+ 360,
+ true,
+ );
+
+ expect(renderableSource?.source).toBe(webcamFrame);
+ expect(renderer.webcamFrameCacheCanvas).toBeNull();
+ } finally {
+ Object.assign(globalThis, {
+ VideoFrame: previousVideoFrame,
+ });
+ }
+ });
+});
+
+describe("ModernFrameRenderer temporal webcam sync", () => {
+ it("pins webcam sync to the frame center during temporal blur sampling", async () => {
+ const renderer = createRenderer() as any;
+ renderer.config.annotationRegions = [];
+ renderer.config.zoomTemporalMotionBlur = 1;
+ renderer.config.zoomMotionBlurSampleCount = 3;
+ renderer.config.zoomMotionBlurShutterFraction = 0.5;
+ renderer.app = { canvas: createMockCanvas() };
+ renderer.updateCaptionLayer = vi.fn();
+ renderer.renderSceneSample = vi.fn(async (sampleTimestamp: number) => ({
+ timeMs: sampleTimestamp / 1000,
+ cursorTimeMs: sampleTimestamp / 1000,
+ backgroundTimelineTimeMs: sampleTimestamp / 1000,
+ sceneTransform: { scale: 1, x: 0, y: 0 },
+ zoom: { scale: 1, focusX: 0.5, focusY: 0.5, progress: 0 },
+ }));
+
+ await renderer.renderTemporalMotionBlurFrame(
+ 1_000_000,
+ 1_000_000,
+ 1_000_000,
+ 33_333,
+ {
+ stageSize: { width: 1920, height: 1080 },
+ videoSize: { width: 1920, height: 1080 },
+ baseScale: 1,
+ baseOffset: { x: 0, y: 0 },
+ maskRect: {
+ x: 0,
+ y: 0,
+ width: 1920,
+ height: 1080,
+ sourceCrop: { x: 0, y: 0, width: 1, height: 1 },
+ },
+ },
+ );
+
+ expect(renderer.renderSceneSample).toHaveBeenCalledTimes(3);
+ expect(renderer.renderSceneSample.mock.calls.map((call: unknown[]) => call[6])).toEqual([
+ 1,
+ 1,
+ 1,
+ ]);
+ expect(
+ new Set(renderer.renderSceneSample.mock.calls.map((call: unknown[]) => call[0])).size,
+ ).toBeGreaterThan(1);
+ });
});
diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts
index b71efe81..a6819f6c 100644
--- a/src/lib/exporter/modernFrameRenderer.ts
+++ b/src/lib/exporter/modernFrameRenderer.ts
@@ -450,9 +450,15 @@ export class FrameRenderer {
private captionRenderKey: string | null = null;
private frameSprite: Sprite | null = null;
private frameImage: HTMLImageElement | null = null;
- private frameDraw: ((ctx: CanvasRenderingContext2D, width: number, height: number) => void) | null =
- null;
- private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null;
+ private frameDraw:
+ | ((ctx: CanvasRenderingContext2D, width: number, height: number) => void)
+ | null = null;
+ private frameInsets: {
+ top: number;
+ right: number;
+ bottom: number;
+ left: number;
+ } | null = null;
private frameRasterCanvas: HTMLCanvasElement | null = null;
private frameRasterWidth = 0;
private frameRasterHeight = 0;
@@ -534,7 +540,9 @@ export class FrameRenderer {
canvas.height = this.config.height;
try {
- const exportCanvas = canvas as HTMLCanvasElement & { colorSpace?: string };
+ const exportCanvas = canvas as HTMLCanvasElement & {
+ colorSpace?: string;
+ };
if ("colorSpace" in exportCanvas) {
exportCanvas.colorSpace = "srgb";
}
@@ -628,7 +636,10 @@ export class FrameRenderer {
this.setupCaptionResources();
if (this.shouldUseZoomMotionBlur()) {
- this.zoomBlurFilter = new ZoomBlurFilter({ strength: 0, maxKernelSize: 13 });
+ this.zoomBlurFilter = new ZoomBlurFilter({
+ strength: 0,
+ maxKernelSize: 13,
+ });
this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0);
}
@@ -732,7 +743,11 @@ export class FrameRenderer {
private createShadowLayers(
parent: Container,
- configs: ReadonlyArray<{ offsetScale: number; alphaScale: number; blurScale: number }>,
+ configs: ReadonlyArray<{
+ offsetScale: number;
+ alphaScale: number;
+ blurScale: number;
+ }>,
): ShadowLayer[] {
return configs.map((config) => {
const container = new Container();
@@ -1050,6 +1065,13 @@ export class FrameRenderer {
fallbackWidth: number,
fallbackHeight: number,
): CanvasImageSource | VideoFrame {
+ // Keep webcam uploads on the older canvas-staged path. The newer
+ // retained-VideoFrame upload path is fine for the main scene/background,
+ // but it has produced unstable webcam overlays in Lightning exports.
+ if (kind === "webcam") {
+ return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);
+ }
+
if (this.rendererBackend === "webgpu") {
return this.resolveRetainedVideoFrameSource(frame, kind, fallbackWidth, fallbackHeight);
}
@@ -2004,8 +2026,12 @@ export class FrameRenderer {
}
};
- video.addEventListener("seeked", waitForPresentedFrame, { once: true });
- video.addEventListener("loadeddata", handleMediaReady, { once: true });
+ video.addEventListener("seeked", waitForPresentedFrame, {
+ once: true,
+ });
+ video.addEventListener("loadeddata", handleMediaReady, {
+ once: true,
+ });
video.addEventListener("canplay", handleMediaReady, { once: true });
video.addEventListener("error", finish, { once: true });
@@ -2111,7 +2137,11 @@ export class FrameRenderer {
await new Promise<void>((resolve, reject) => {
image.onload = () => resolve();
image.onerror = () =>
- reject(new Error(`[ModernFrameRenderer] Failed to load device frame image: ${frameId}`));
+ reject(
+ new Error(
+ `[ModernFrameRenderer] Failed to load device frame image: ${frameId}`,
+ ),
+ );
image.src = frame.filePath;
});
this.frameImage = image;
@@ -2474,6 +2504,7 @@ export class FrameRenderer {
source: CanvasImageSource | VideoFrame,
width: number,
height: number,
+ referenceTimeSeconds = this.currentVideoTime,
): boolean {
const sourceRect = getWebcamCropSourceRect(this.config.webcam?.cropRegion, width, height);
if (!this.ensureWebcamFrameCache(sourceRect.sw, sourceRect.sh)) {
@@ -2501,7 +2532,7 @@ export class FrameRenderer {
this.webcamFrameCacheCanvas.width,
this.webcamFrameCacheCanvas.height,
);
- this.lastWebcamCacheRefreshTime = this.currentVideoTime;
+ this.lastWebcamCacheRefreshTime = referenceTimeSeconds;
return true;
}
@@ -2527,18 +2558,29 @@ export class FrameRenderer {
liveSourceWidth: number,
liveSourceHeight: number,
canUseLiveSource: boolean,
+ referenceTimeSeconds = this.currentVideoTime,
): WebcamRenderSource | null {
if (canUseLiveSource && liveSource && liveSourceWidth > 0 && liveSourceHeight > 0) {
- if (this.shouldRefreshWebcamFrameCache(liveSourceWidth, liveSourceHeight)) {
- this.refreshWebcamFrameCache(liveSource, liveSourceWidth, liveSourceHeight);
- }
- if (!isWebcamCropRegionDefault(this.config.webcam?.cropRegion)) {
+ const usesDefaultCropRegion = isWebcamCropRegionDefault(this.config.webcam?.cropRegion);
+ const needsCacheBackedSource =
+ !usesDefaultCropRegion ||
+ (typeof HTMLVideoElement !== "undefined" &&
+ liveSource instanceof HTMLVideoElement);
+
+ if (needsCacheBackedSource) {
+ this.refreshWebcamFrameCache(
+ liveSource,
+ liveSourceWidth,
+ liveSourceHeight,
+ referenceTimeSeconds,
+ );
const cachedSource = this.getCachedWebcamRenderSource();
if (cachedSource) {
this.setWebcamRenderMode("live");
return cachedSource;
}
}
+
this.setWebcamRenderMode("live");
return {
source: liveSource,
@@ -2636,14 +2678,18 @@ export class FrameRenderer {
this.webcamLayoutCache = { ...nextLayout };
}
- private async syncWebcamFrame(targetTime: number): Promise<void> {
- const webcamTargetTime = getWebcamMediaTargetTimeSeconds({
+ private getExpectedWebcamTargetTimeSeconds(targetTime: number): number {
+ return getWebcamMediaTargetTimeSeconds({
currentTime: targetTime,
webcamDuration: Number.isFinite(this.webcamVideoElement?.duration)
? this.webcamVideoElement?.duration
: null,
timeOffsetMs: this.config.webcam?.timeOffsetMs,
});
+ }
+
+ private async syncWebcamFrame(targetTime: number): Promise<void> {
+ const webcamTargetTime = this.getExpectedWebcamTargetTimeSeconds(targetTime);
if (this.webcamForwardFrameSource) {
const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null);
@@ -2779,8 +2825,12 @@ export class FrameRenderer {
}
};
- webcamVideo.addEventListener("seeked", waitForPresentedFrame, { once: true });
- webcamVideo.addEventListener("loadeddata", handleMediaReady, { once: true });
+ webcamVideo.addEventListener("seeked", waitForPresentedFrame, {
+ once: true,
+ });
+ webcamVideo.addEventListener("loadeddata", handleMediaReady, {
+ once: true,
+ });
webcamVideo.addEventListener("canplay", handleMediaReady, { once: true });
webcamVideo.addEventListener("error", finish, { once: true });
@@ -2811,7 +2861,7 @@ export class FrameRenderer {
}
}
- private updateWebcamOverlay(): void {
+ private updateWebcamOverlay(referenceTimeSeconds = this.currentVideoTime): void {
const webcam = this.config.webcam;
if (!webcam?.enabled || !this.webcamRootContainer || !this.webcamMaskGraphics) {
if (this.webcamRootContainer) {
@@ -2828,10 +2878,16 @@ export class FrameRenderer {
: { width: 0, height: 0 };
const activeWebcamVideoElement =
webcamSource === this.webcamVideoElement ? this.webcamVideoElement : null;
+ const expectedWebcamTargetTime =
+ this.getExpectedWebcamTargetTimeSeconds(referenceTimeSeconds);
+ const measuredWebcamTime =
+ activeWebcamVideoElement && Number.isFinite(activeWebcamVideoElement.currentTime)
+ ? activeWebcamVideoElement.currentTime
+ : this.lastSyncedWebcamTime;
const webcamTimeDrift =
- this.lastSyncedWebcamTime === null
+ measuredWebcamTime === null
? 0
- : Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime);
+ : Math.abs(measuredWebcamTime - expectedWebcamTargetTime);
const canUseLiveSource =
!!webcamSource &&
liveSourceDimensions.width > 0 &&
@@ -2845,6 +2901,7 @@ export class FrameRenderer {
liveSourceDimensions.width,
liveSourceDimensions.height,
canUseLiveSource,
+ referenceTimeSeconds,
);
if (!renderableWebcamSource) {
@@ -2910,15 +2967,20 @@ export class FrameRenderer {
layoutCache: LayoutCache,
useVelocityMotionBlur: boolean,
includeOverlayLayers = true,
+ webcamTimeSecondsOverride?: number,
): Promise<RenderSnapshot> {
if (!this.app || !this.cameraContainer || !this.videoMaskGraphics) {
throw new Error("Renderer not initialized");
}
this.currentVideoTime = timestamp / 1_000_000;
+ const webcamRenderTimeSeconds = Math.max(
+ 0,
+ webcamTimeSecondsOverride ?? this.currentVideoTime,
+ );
if (this.webcamForwardFrameSource || this.webcamVideoElement) {
- await this.syncWebcamFrame(Math.max(0, this.currentVideoTime));
+ await this.syncWebcamFrame(webcamRenderTimeSeconds);
}
if (this.backgroundForwardFrameSource || this.backgroundVideoElement) {
@@ -2966,7 +3028,7 @@ export class FrameRenderer {
this.updateAnnotationLayer(timeMs);
this.updateCaptionLayer(timeMs);
}
- this.updateWebcamOverlay();
+ this.updateWebcamOverlay(webcamRenderTimeSeconds);
const annotationContainerVisible = this.annotationContainer?.visible ?? true;
const captionContainerVisible = this.captionContainer?.visible ?? true;
@@ -3033,6 +3095,7 @@ export class FrameRenderer {
}
const samplePlan = buildTemporalSamplePlanUs(frameDurationUs, blurConfig);
+ const webcamReferenceTimeSeconds = Math.max(0, timestamp / 1_000_000);
compositeState.context.clearRect(
0,
@@ -3058,6 +3121,7 @@ export class FrameRenderer {
layoutCache,
false,
false,
+ webcamReferenceTimeSeconds,
);
lastSnapshot = snapshot;
if (Math.abs(sampleOffsetUs) < 0.0001) {
diff --git a/src/lib/exporter/modernVideoExporter.fallback.test.ts b/src/lib/exporter/modernVideoExporter.fallback.test.ts
index 80125266..95f06a99 100644
--- a/src/lib/exporter/modernVideoExporter.fallback.test.ts
+++ b/src/lib/exporter/modernVideoExporter.fallback.test.ts
@@ -117,4 +117,57 @@ describe("ModernVideoExporter native fallback routing", () => {
expect(initializeEncoder).toHaveBeenCalledTimes(1);
expect(mocks.muxerFinalize).toHaveBeenCalledTimes(1);
}, 15_000);
+
+ it("retries the main decode path once with a readable file-backed source", async () => {
+ const { ModernVideoExporter } = await import("./modernVideoExporter");
+ mocks.streamingDecoderGetEffectiveDuration.mockReturnValue(1);
+ mocks.streamingDecoderDecodeAll
+ .mockRejectedValueOnce(
+ new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
+ )
+ .mockResolvedValueOnce(undefined);
+
+ const exporter = new ModernVideoExporter({
+ videoUrl: "file:///recording.mp4",
+ width: 1920,
+ height: 1080,
+ frameRate: 30,
+ bitrate: 8_000_000,
+ wallpaper: "#101010",
+ padding: 0,
+ borderRadius: 0,
+ backgroundBlur: 0,
+ shadowIntensity: 0,
+ showShadow: false,
+ cropRegion: { x: 0, y: 0, width: 1, height: 1 },
+ backendPreference: "webcodecs",
+ } as never) as unknown as {
+ export: () => Promise<{ success: boolean; blob?: Blob; error?: string }>;
+ initializeEncoder: () => Promise<unknown>;
+ };
+
+ vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({
+ codec: "avc1.640034",
+ hardwareAcceleration: "prefer-hardware",
+ });
+
+ const result = await exporter.export();
+
+ expect(result.success).toBe(true);
+ expect(mocks.streamingDecoderLoadMetadata).toHaveBeenCalledTimes(2);
+ expect(mocks.streamingDecoderLoadMetadata.mock.calls[0]).toEqual([
+ "file:///recording.mp4",
+ {
+ forceReadableFileSource: false,
+ },
+ ]);
+ expect(mocks.streamingDecoderLoadMetadata.mock.calls[1]).toEqual([
+ "file:///recording.mp4",
+ {
+ forceReadableFileSource: true,
+ },
+ ]);
+ expect(mocks.streamingDecoderDecodeAll).toHaveBeenCalledTimes(2);
+ expect(mocks.muxerFinalize).toHaveBeenCalledTimes(1);
+ });
});
diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts
index 55f47923..679c9a9d 100644
--- a/src/lib/exporter/modernVideoExporter.ts
+++ b/src/lib/exporter/modernVideoExporter.ts
@@ -167,7 +167,11 @@ type NativeAudioPlan =
audioSourcePath: string;
audioSourceCodec?: string;
audioSourceSampleRate: number;
- editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>;
+ editedTrackSegments: Array<{
+ startMs: number;
+ endMs: number;
+ speed: number;
+ }>;
};
const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000;
@@ -271,6 +275,13 @@ type NativeStaticLayoutZoomSample = {
};
const NATIVE_EXPORT_ENGINE_NAME = "Breeze";
+const READABLE_SOURCE_RETRY_ERROR_TOKENS = [
+ "readavpacket",
+ "get_media_info",
+ "avfoundation",
+ "failed after 3 attempts",
+ "pipeline failed",
+];
const LIGHTNING_PIPELINE_NAME = "Lightning (Beta)";
const STATIC_LAYOUT_CHUNK_DURATION_SEC = 120;
const MISSING_NATIVE_WALLPAPER_FALLBACK_COLOR = "#ffffff";
@@ -347,128 +358,177 @@ export class ModernVideoExporter {
}
async export(): Promise<ExportResult> {
- try {
- this.cleanup();
- this.cancelled = false;
- this.encoderError = null;
- this.nativeEncoderError = null;
- this.nativeStaticLayoutSkipReason = null;
- this.nativeStaticLayoutSkipReasons = [];
- this.nativeStaticLayoutBackgroundSkipReason = null;
- this.totalExportStartTimeMs = this.getNowMs();
- const backendPreference = this.config.backendPreference ?? "auto";
- const runtimePlatform = this.getRuntimePlatform();
- let useNativeEncoder = false;
- let triedNativeStaticLayoutWithProbe = false;
- let shouldDeferNativeEncoderStart = backendPreference === "breeze";
- this.lastNativeExportError = null;
-
- let stageStartedAt = this.getNowMs();
- if (backendPreference === "breeze") {
- // Defer the streaming native encoder until after metadata is known.
- // Static-layout exports can then use the faster Windows D3D compositor
- // instead of unnecessarily rendering every frame through JS first.
- } else if (
- backendPreference === "auto" &&
- shouldPreferNativeAutoBackend(runtimePlatform)
- ) {
- stageStartedAt = this.getNowMs();
- useNativeEncoder = await this.tryStartNativeVideoExport();
- this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
+ let preferReadableFileSource = false;
+ let retriedWithReadableFileSource = false;
- if (!useNativeEncoder) {
- console.warn(
- `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} auto-preferred native export was unavailable; falling back to WebCodecs.`,
- this.lastNativeExportError,
- );
+ while (true) {
+ let shouldRetryWithReadableFileSource = false;
+ try {
+ this.cleanup();
+ this.cancelled = false;
+ this.encoderError = null;
+ this.nativeEncoderError = null;
+ this.nativeStaticLayoutSkipReason = null;
+ this.nativeStaticLayoutSkipReasons = [];
+ this.nativeStaticLayoutBackgroundSkipReason = null;
+ this.totalExportStartTimeMs = this.getNowMs();
+ const backendPreference = this.config.backendPreference ?? "auto";
+ const runtimePlatform = this.getRuntimePlatform();
+ let useNativeEncoder = false;
+ let triedNativeStaticLayoutWithProbe = false;
+ let shouldDeferNativeEncoderStart = backendPreference === "breeze";
+ this.lastNativeExportError = null;
+
+ let stageStartedAt = this.getNowMs();
+ if (backendPreference === "breeze") {
+ // Defer the streaming native encoder until after metadata is known.
+ // Static-layout exports can then use the faster Windows D3D compositor
+ // instead of unnecessarily rendering every frame through JS first.
+ } else if (
+ backendPreference === "auto" &&
+ shouldPreferNativeAutoBackend(runtimePlatform)
+ ) {
stageStartedAt = this.getNowMs();
- await this.initializeEncoder();
- }
- } else {
- try {
- const configuredWebCodecsPath = await this.initializeEncoder();
- if (
- backendPreference === "auto" &&
- configuredWebCodecsPath.hardwareAcceleration === "prefer-software"
- ) {
+ useNativeEncoder = await this.tryStartNativeVideoExport();
+ this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
+
+ if (!useNativeEncoder) {
console.warn(
- "[VideoExporter] Auto backend resolved to a software WebCodecs encoder; trying Breeze native export instead.",
+ `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} auto-preferred native export was unavailable; falling back to WebCodecs.`,
+ this.lastNativeExportError,
);
+ stageStartedAt = this.getNowMs();
+ await this.initializeEncoder();
+ }
+ } else {
+ try {
+ const configuredWebCodecsPath = await this.initializeEncoder();
+ if (
+ backendPreference === "auto" &&
+ configuredWebCodecsPath.hardwareAcceleration === "prefer-software"
+ ) {
+ console.warn(
+ "[VideoExporter] Auto backend resolved to a software WebCodecs encoder; trying Breeze native export instead.",
+ );
+ stageStartedAt = this.getNowMs();
+ useNativeEncoder = await this.tryStartNativeVideoExport();
+ this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
+ if (useNativeEncoder) {
+ this.disposeEncoder();
+ }
+ }
+ } catch (error) {
+ const webCodecsError =
+ error instanceof Error ? error : new Error(String(error));
+ if (backendPreference === "webcodecs") {
+ throw webCodecsError;
+ }
+
+ console.warn(
+ `[VideoExporter] WebCodecs encoder unavailable, trying ${NATIVE_EXPORT_ENGINE_NAME} native export fallback`,
+ webCodecsError,
+ );
+ this.disposeEncoder();
+
stageStartedAt = this.getNowMs();
useNativeEncoder = await this.tryStartNativeVideoExport();
this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
- if (useNativeEncoder) {
- this.disposeEncoder();
+
+ if (!useNativeEncoder) {
+ throw webCodecsError;
}
}
- } catch (error) {
- const webCodecsError =
- error instanceof Error ? error : new Error(String(error));
- if (backendPreference === "webcodecs") {
- throw webCodecsError;
- }
-
- console.warn(
- `[VideoExporter] WebCodecs encoder unavailable, trying ${NATIVE_EXPORT_ENGINE_NAME} native export fallback`,
- webCodecsError,
- );
- this.disposeEncoder();
+ }
- stageStartedAt = this.getNowMs();
- useNativeEncoder = await this.tryStartNativeVideoExport();
- this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
+ this.backpressureProfile = getExportBackpressureProfile({
+ encodeBackend:
+ shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs",
+ width: this.config.width,
+ height: this.config.height,
+ frameRate: this.config.frameRate,
+ encodingMode: this.config.encodingMode,
+ });
+ this.maxNativeWriteInFlight = useNativeEncoder
+ ? Math.max(
+ 1,
+ Math.floor(
+ this.config.maxInFlightNativeWrites ??
+ this.backpressureProfile.maxInFlightNativeWrites,
+ ),
+ )
+ : 1;
+
+ console.log("[VideoExporter] Backpressure profile", {
+ profile: this.backpressureProfile.name,
+ encodeBackend:
+ shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs",
+ maxEncodeQueue:
+ this.config.maxEncodeQueue ?? this.backpressureProfile.maxEncodeQueue,
+ maxDecodeQueue:
+ this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue,
+ maxPendingFrames:
+ this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames,
+ maxInFlightNativeWrites: this.maxNativeWriteInFlight,
+ });
- if (!useNativeEncoder) {
- throw webCodecsError;
+ if (
+ (backendPreference === "auto" || backendPreference === "breeze") &&
+ !useNativeEncoder
+ ) {
+ const nativeVideoInfo = await this.loadNativeStaticLayoutVideoInfo();
+ if (nativeVideoInfo) {
+ triedNativeStaticLayoutWithProbe = true;
+ const nativeAudioPlan = this.buildNativeAudioPlan(nativeVideoInfo);
+ const effectiveDuration =
+ this.getNativeStaticLayoutEffectiveDuration(nativeVideoInfo);
+ this.effectiveDurationSec = effectiveDuration;
+ const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
+ const staticLayoutResult = await this.tryExportNativeStaticLayout(
+ nativeVideoInfo,
+ nativeAudioPlan,
+ effectiveDuration,
+ totalFrames,
+ );
+ if (staticLayoutResult) {
+ this.disposeEncoder();
+ return staticLayoutResult;
+ }
}
}
- }
- this.backpressureProfile = getExportBackpressureProfile({
- encodeBackend:
- shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs",
- width: this.config.width,
- height: this.config.height,
- frameRate: this.config.frameRate,
- encodingMode: this.config.encodingMode,
- });
- this.maxNativeWriteInFlight = useNativeEncoder
- ? Math.max(
- 1,
- Math.floor(
- this.config.maxInFlightNativeWrites ??
- this.backpressureProfile.maxInFlightNativeWrites,
- ),
- )
- : 1;
-
- console.log("[VideoExporter] Backpressure profile", {
- profile: this.backpressureProfile.name,
- encodeBackend:
- shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs",
- maxEncodeQueue:
- this.config.maxEncodeQueue ?? this.backpressureProfile.maxEncodeQueue,
- maxDecodeQueue:
- this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue,
- maxPendingFrames:
- this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames,
- maxInFlightNativeWrites: this.maxNativeWriteInFlight,
- });
+ this.streamingDecoder = new StreamingVideoDecoder({
+ maxDecodeQueue:
+ this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue,
+ maxPendingFrames:
+ this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames,
+ });
+ stageStartedAt = this.getNowMs();
+ const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl, {
+ forceReadableFileSource: preferReadableFileSource,
+ });
+ this.metadataLoadTimeMs = this.getNowMs() - stageStartedAt;
+ const nativeAudioPlan = this.buildNativeAudioPlan(videoInfo);
+ const shouldUsePitchPreservingFfmpegAudio =
+ nativeAudioPlan.audioMode === "edited-track" &&
+ nativeAudioPlan.strategy === "filtergraph-fast-path";
+ const shouldUseFfmpegAudioFallback =
+ !useNativeEncoder &&
+ nativeAudioPlan.audioMode !== "none" &&
+ (shouldUsePitchPreservingFfmpegAudio || !(await isAacAudioEncodingSupported()));
+ const effectiveDuration = this.streamingDecoder.getEffectiveDuration(
+ this.config.trimRegions,
+ this.config.speedRegions,
+ );
+ this.effectiveDurationSec = effectiveDuration;
+ const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
- if (
- (backendPreference === "auto" || backendPreference === "breeze") &&
- !useNativeEncoder
- ) {
- const nativeVideoInfo = await this.loadNativeStaticLayoutVideoInfo();
- if (nativeVideoInfo) {
- triedNativeStaticLayoutWithProbe = true;
- const nativeAudioPlan = this.buildNativeAudioPlan(nativeVideoInfo);
- const effectiveDuration =
- this.getNativeStaticLayoutEffectiveDuration(nativeVideoInfo);
- this.effectiveDurationSec = effectiveDuration;
- const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
+ if (
+ (backendPreference === "auto" || backendPreference === "breeze") &&
+ !useNativeEncoder &&
+ !triedNativeStaticLayoutWithProbe
+ ) {
const staticLayoutResult = await this.tryExportNativeStaticLayout(
- nativeVideoInfo,
+ videoInfo,
nativeAudioPlan,
effectiveDuration,
totalFrames,
@@ -478,382 +538,390 @@ export class ModernVideoExporter {
return staticLayoutResult;
}
}
- }
- this.streamingDecoder = new StreamingVideoDecoder({
- maxDecodeQueue:
- this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue,
- maxPendingFrames:
- this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames,
- });
- stageStartedAt = this.getNowMs();
- const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl);
- this.metadataLoadTimeMs = this.getNowMs() - stageStartedAt;
- const nativeAudioPlan = this.buildNativeAudioPlan(videoInfo);
- const shouldUsePitchPreservingFfmpegAudio =
- nativeAudioPlan.audioMode === "edited-track" &&
- nativeAudioPlan.strategy === "filtergraph-fast-path";
- const shouldUseFfmpegAudioFallback =
- !useNativeEncoder &&
- nativeAudioPlan.audioMode !== "none" &&
- (shouldUsePitchPreservingFfmpegAudio || !(await isAacAudioEncodingSupported()));
- const effectiveDuration = this.streamingDecoder.getEffectiveDuration(
- this.config.trimRegions,
- this.config.speedRegions,
- );
- this.effectiveDurationSec = effectiveDuration;
- const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate);
-
- if (
- (backendPreference === "auto" || backendPreference === "breeze") &&
- !useNativeEncoder &&
- !triedNativeStaticLayoutWithProbe
- ) {
- const staticLayoutResult = await this.tryExportNativeStaticLayout(
- videoInfo,
- nativeAudioPlan,
- effectiveDuration,
- totalFrames,
- );
- if (staticLayoutResult) {
- this.disposeEncoder();
- return staticLayoutResult;
+ if (shouldDeferNativeEncoderStart && !useNativeEncoder) {
+ stageStartedAt = this.getNowMs();
+ useNativeEncoder = await this.tryStartNativeVideoExport();
+ this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
+ if (!useNativeEncoder) {
+ const nativeFailure =
+ this.lastNativeExportError ??
+ `${NATIVE_EXPORT_ENGINE_NAME} export is unavailable for this output profile on this system.`;
+ console.warn(
+ `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} native export unavailable after static-layout fallback; falling back to WebCodecs.`,
+ nativeFailure,
+ );
+ shouldDeferNativeEncoderStart = false;
+ this.backpressureProfile = getExportBackpressureProfile({
+ encodeBackend: "webcodecs",
+ width: this.config.width,
+ height: this.config.height,
+ frameRate: this.config.frameRate,
+ encodingMode: this.config.encodingMode,
+ });
+ this.maxNativeWriteInFlight = 1;
+ await this.initializeEncoder();
+ }
}
- }
- if (shouldDeferNativeEncoderStart && !useNativeEncoder) {
stageStartedAt = this.getNowMs();
- useNativeEncoder = await this.tryStartNativeVideoExport();
- this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt;
+ this.renderer = new ModernFrameRenderer({
+ width: this.config.width,
+ height: this.config.height,
+ preferredRenderBackend: undefined,
+ wallpaper: this.config.wallpaper,
+ zoomRegions: this.config.zoomRegions,
+ showShadow: this.config.showShadow,
+ shadowIntensity: this.config.shadowIntensity,
+ backgroundBlur: this.config.backgroundBlur,
+ zoomMotionBlur: this.config.zoomMotionBlur,
+ zoomMotionBlurTuning: this.config.zoomMotionBlurTuning,
+ zoomTemporalMotionBlur: this.config.zoomTemporalMotionBlur,
+ zoomMotionBlurSampleCount: this.config.zoomMotionBlurSampleCount,
+ zoomMotionBlurShutterFraction: this.config.zoomMotionBlurShutterFraction,
+ connectZooms: this.config.connectZooms,
+ zoomInDurationMs: this.config.zoomInDurationMs,
+ zoomInOverlapMs: this.config.zoomInOverlapMs,
+ zoomOutDurationMs: this.config.zoomOutDurationMs,
+ connectedZoomGapMs: this.config.connectedZoomGapMs,
+ connectedZoomDurationMs: this.config.connectedZoomDurationMs,
+ zoomInEasing: this.config.zoomInEasing,
+ zoomOutEasing: this.config.zoomOutEasing,
+ connectedZoomEasing: this.config.connectedZoomEasing,
+ borderRadius: this.config.borderRadius,
+ padding: this.config.padding,
+ cropRegion: this.config.cropRegion,
+ webcam: this.config.webcam,
+ webcamUrl: this.config.webcamUrl,
+ videoWidth: videoInfo.width,
+ videoHeight: videoInfo.height,
+ annotationRegions: this.config.annotationRegions,
+ autoCaptions: this.config.autoCaptions,
+ autoCaptionSettings: this.config.autoCaptionSettings,
+ speedRegions: this.config.speedRegions,
+ previewWidth: this.config.previewWidth,
+ previewHeight: this.config.previewHeight,
+ cursorTelemetry: this.config.cursorTelemetry,
+ showCursor: this.config.showCursor,
+ cursorStyle: this.config.cursorStyle,
+ cursorSize: this.config.cursorSize,
+ cursorSmoothing: this.config.cursorSmoothing,
+ cursorSpringStiffnessMultiplier: this.config.cursorSpringStiffnessMultiplier,
+ cursorSpringDampingMultiplier: this.config.cursorSpringDampingMultiplier,
+ cursorSpringMassMultiplier: this.config.cursorSpringMassMultiplier,
+ cameraSpringStiffnessMultiplier: this.config.cameraSpringStiffnessMultiplier,
+ cameraSpringDampingMultiplier: this.config.cameraSpringDampingMultiplier,
+ cameraSpringMassMultiplier: this.config.cameraSpringMassMultiplier,
+ cursorMotionBlur: this.config.cursorMotionBlur,
+ cursorClickBounce: this.config.cursorClickBounce,
+ cursorClickBounceDuration: this.config.cursorClickBounceDuration,
+ cursorSway: this.config.cursorSway,
+ zoomSmoothness: this.config.zoomSmoothness,
+ zoomClassicMode: this.config.zoomClassicMode,
+ frame: this.config.frame,
+ });
+ await this.renderer.initialize();
+ this.rendererInitTimeMs = this.getNowMs() - stageStartedAt;
+ this.renderBackend = this.renderer.getRendererBackend();
+ console.log(`[VideoExporter] Using ${this.renderBackend} render backend`);
+
if (!useNativeEncoder) {
- const nativeFailure =
- this.lastNativeExportError ??
- `${NATIVE_EXPORT_ENGINE_NAME} export is unavailable for this output profile on this system.`;
- console.warn(
- `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} native export unavailable after static-layout fallback; falling back to WebCodecs.`,
- nativeFailure,
+ const hasAudio = nativeAudioPlan.audioMode !== "none";
+ this.muxer = new VideoMuxer(
+ this.config,
+ hasAudio && !shouldUseFfmpegAudioFallback,
);
- shouldDeferNativeEncoderStart = false;
- this.backpressureProfile = getExportBackpressureProfile({
- encodeBackend: "webcodecs",
- width: this.config.width,
- height: this.config.height,
- frameRate: this.config.frameRate,
- encodingMode: this.config.encodingMode,
- });
- this.maxNativeWriteInFlight = 1;
- await this.initializeEncoder();
+ await this.muxer.initialize();
}
- }
-
- stageStartedAt = this.getNowMs();
- this.renderer = new ModernFrameRenderer({
- width: this.config.width,
- height: this.config.height,
- preferredRenderBackend: undefined,
- wallpaper: this.config.wallpaper,
- zoomRegions: this.config.zoomRegions,
- showShadow: this.config.showShadow,
- shadowIntensity: this.config.shadowIntensity,
- backgroundBlur: this.config.backgroundBlur,
- zoomMotionBlur: this.config.zoomMotionBlur,
- zoomMotionBlurTuning: this.config.zoomMotionBlurTuning,
- zoomTemporalMotionBlur: this.config.zoomTemporalMotionBlur,
- zoomMotionBlurSampleCount: this.config.zoomMotionBlurSampleCount,
- zoomMotionBlurShutterFraction: this.config.zoomMotionBlurShutterFraction,
- connectZooms: this.config.connectZooms,
- zoomInDurationMs: this.config.zoomInDurationMs,
- zoomInOverlapMs: this.config.zoomInOverlapMs,
- zoomOutDurationMs: this.config.zoomOutDurationMs,
- connectedZoomGapMs: this.config.connectedZoomGapMs,
- connectedZoomDurationMs: this.config.connectedZoomDurationMs,
- zoomInEasing: this.config.zoomInEasing,
- zoomOutEasing: this.config.zoomOutEasing,
- connectedZoomEasing: this.config.connectedZoomEasing,
- borderRadius: this.config.borderRadius,
- padding: this.config.padding,
- cropRegion: this.config.cropRegion,
- webcam: this.config.webcam,
- webcamUrl: this.config.webcamUrl,
- videoWidth: videoInfo.width,
- videoHeight: videoInfo.height,
- annotationRegions: this.config.annotationRegions,
- autoCaptions: this.config.autoCaptions,
- autoCaptionSettings: this.config.autoCaptionSettings,
- speedRegions: this.config.speedRegions,
- previewWidth: this.config.previewWidth,
- previewHeight: this.config.previewHeight,
- cursorTelemetry: this.config.cursorTelemetry,
- showCursor: this.config.showCursor,
- cursorStyle: this.config.cursorStyle,
- cursorSize: this.config.cursorSize,
- cursorSmoothing: this.config.cursorSmoothing,
- cursorSpringStiffnessMultiplier: this.config.cursorSpringStiffnessMultiplier,
- cursorSpringDampingMultiplier: this.config.cursorSpringDampingMultiplier,
- cursorSpringMassMultiplier: this.config.cursorSpringMassMultiplier,
- cameraSpringStiffnessMultiplier: this.config.cameraSpringStiffnessMultiplier,
- cameraSpringDampingMultiplier: this.config.cameraSpringDampingMultiplier,
- cameraSpringMassMultiplier: this.config.cameraSpringMassMultiplier,
- cursorMotionBlur: this.config.cursorMotionBlur,
- cursorClickBounce: this.config.cursorClickBounce,
- cursorClickBounceDuration: this.config.cursorClickBounceDuration,
- cursorSway: this.config.cursorSway,
- zoomSmoothness: this.config.zoomSmoothness,
- zoomClassicMode: this.config.zoomClassicMode,
- frame: this.config.frame,
- });
- await this.renderer.initialize();
- this.rendererInitTimeMs = this.getNowMs() - stageStartedAt;
- this.renderBackend = this.renderer.getRendererBackend();
- console.log(`[VideoExporter] Using ${this.renderBackend} render backend`);
-
- if (!useNativeEncoder) {
- const hasAudio = nativeAudioPlan.audioMode !== "none";
- this.muxer = new VideoMuxer(this.config, hasAudio && !shouldUseFfmpegAudioFallback);
- await this.muxer.initialize();
- }
- console.log("[VideoExporter] Original duration:", videoInfo.duration, "s");
- console.log("[VideoExporter] Effective duration:", effectiveDuration, "s");
- console.log("[VideoExporter] Total frames to export:", totalFrames);
- console.log(
- `[VideoExporter] Using ${useNativeEncoder ? `${NATIVE_EXPORT_ENGINE_NAME} native` : "WebCodecs"} encode path`,
- );
-
- const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds
- let frameIndex = 0;
- this.exportStartTimeMs = this.getNowMs();
- this.lastThroughputLogTimeMs = this.exportStartTimeMs;
- this.lastProgressSampleTimeMs = this.exportStartTimeMs;
- this.lastProgressSampleFrame = 0;
- this.displayedRenderFps = 0;
- const decodeLoopStartedAt = this.getNowMs();
-
- await this.streamingDecoder.decodeAll(
- this.config.frameRate,
- this.config.trimRegions,
- this.config.speedRegions,
- async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => {
- const callbackStartedAt = this.getNowMs();
- if (this.cancelled) {
- return;
- }
+ console.log("[VideoExporter] Original duration:", videoInfo.duration, "s");
+ console.log("[VideoExporter] Effective duration:", effectiveDuration, "s");
+ console.log("[VideoExporter] Total frames to export:", totalFrames);
+ console.log(
+ `[VideoExporter] Using ${useNativeEncoder ? `${NATIVE_EXPORT_ENGINE_NAME} native` : "WebCodecs"} encode path`,
+ );
- const timestamp = frameIndex * frameDuration;
- const sourceTimestampUs = sourceTimestampMs * 1000;
- const cursorTimestampUs = cursorTimestampMs * 1000;
- const renderStartedAt = this.getNowMs();
- await this.renderer!.renderFrame(
+ const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds
+ let frameIndex = 0;
+ this.exportStartTimeMs = this.getNowMs();
+ this.lastThroughputLogTimeMs = this.exportStartTimeMs;
+ this.lastProgressSampleTimeMs = this.exportStartTimeMs;
+ this.lastProgressSampleFrame = 0;
+ this.displayedRenderFps = 0;
+ const decodeLoopStartedAt = this.getNowMs();
+
+ await this.streamingDecoder.decodeAll(
+ this.config.frameRate,
+ this.config.trimRegions,
+ this.config.speedRegions,
+ async (
videoFrame,
- sourceTimestampUs,
- cursorTimestampUs,
- frameDuration,
- timestamp,
- );
- this.renderFrameTimeMs += this.getNowMs() - renderStartedAt;
+ _exportTimestampUs,
+ sourceTimestampMs,
+ cursorTimestampMs,
+ ) => {
+ const callbackStartedAt = this.getNowMs();
+ if (this.cancelled) {
+ return;
+ }
- if (this.cancelled) {
- return;
- }
+ const timestamp = frameIndex * frameDuration;
+ const sourceTimestampUs = sourceTimestampMs * 1000;
+ const cursorTimestampUs = cursorTimestampMs * 1000;
+ const renderStartedAt = this.getNowMs();
+ await this.renderer!.renderFrame(
+ videoFrame,
+ sourceTimestampUs,
+ cursorTimestampUs,
+ frameDuration,
+ timestamp,
+ );
+ this.renderFrameTimeMs += this.getNowMs() - renderStartedAt;
- if (useNativeEncoder) {
- await this.encodeRenderedFrameNative(timestamp, frameDuration, frameIndex);
- } else {
- await this.encodeRenderedFrame(timestamp, frameDuration, frameIndex);
+ if (this.cancelled) {
+ return;
+ }
+
+ if (useNativeEncoder) {
+ await this.encodeRenderedFrameNative(
+ timestamp,
+ frameDuration,
+ frameIndex,
+ );
+ } else {
+ await this.encodeRenderedFrame(timestamp, frameDuration, frameIndex);
+ }
+ this.frameCallbackTimeMs += this.getNowMs() - callbackStartedAt;
+ frameIndex++;
+ this.processedFrameCount = frameIndex;
+ this.reportProgress(frameIndex, totalFrames, "extracting");
+ extensionHost.emitEvent({
+ type: "export:frame",
+ data: { frameIndex, totalFrames },
+ });
+ },
+ );
+ this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt;
+
+ if (this.cancelled) {
+ if (this.encoderError) {
+ return {
+ success: false,
+ error: this.buildLightningExportError(this.encoderError),
+ metrics: this.buildExportMetrics(),
+ };
}
- this.frameCallbackTimeMs += this.getNowMs() - callbackStartedAt;
- frameIndex++;
- this.processedFrameCount = frameIndex;
- this.reportProgress(frameIndex, totalFrames, "extracting");
- extensionHost.emitEvent({
- type: "export:frame",
- data: { frameIndex, totalFrames },
- });
- },
- );
- this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt;
- if (this.cancelled) {
- if (this.encoderError) {
return {
success: false,
- error: this.buildLightningExportError(this.encoderError),
+ error: "Export cancelled",
metrics: this.buildExportMetrics(),
};
}
- return {
- success: false,
- error: "Export cancelled",
- metrics: this.buildExportMetrics(),
- };
- }
+ this.reportFinalizingProgress(totalFrames, 96);
- this.reportFinalizingProgress(totalFrames, 96);
+ if (useNativeEncoder) {
+ stageStartedAt = this.getNowMs();
+ this.reportFinalizingProgress(totalFrames, 99);
+ if (this.nativeH264Encoder) {
+ await this.measureFinalizationStage("nativeEncoderFlushMs", async () => {
+ await this.nativeH264Encoder!.flush();
+ });
+ }
+ const finishResult = await this.finishNativeVideoExport(nativeAudioPlan);
+ this.finalizationTimeMs = this.getNowMs() - stageStartedAt;
+ if (
+ !finishResult.success ||
+ (!finishResult.tempFilePath && !finishResult.blob)
+ ) {
+ return {
+ success: false,
+ error:
+ finishResult.error || `${NATIVE_EXPORT_ENGINE_NAME} export failed`,
+ metrics: this.buildExportMetrics(),
+ };
+ }
- if (useNativeEncoder) {
- stageStartedAt = this.getNowMs();
- this.reportFinalizingProgress(totalFrames, 99);
- if (this.nativeH264Encoder) {
- await this.measureFinalizationStage("nativeEncoderFlushMs", async () => {
- await this.nativeH264Encoder!.flush();
- });
- }
- const finishResult = await this.finishNativeVideoExport(nativeAudioPlan);
- this.finalizationTimeMs = this.getNowMs() - stageStartedAt;
- if (!finishResult.success || (!finishResult.tempFilePath && !finishResult.blob)) {
return {
- success: false,
- error: finishResult.error || `${NATIVE_EXPORT_ENGINE_NAME} export failed`,
+ success: true,
+ tempFilePath: finishResult.tempFilePath,
+ blob: finishResult.blob,
metrics: this.buildExportMetrics(),
};
}
- return {
- success: true,
- tempFilePath: finishResult.tempFilePath,
- blob: finishResult.blob,
- metrics: this.buildExportMetrics(),
- };
- }
+ stageStartedAt = this.getNowMs();
+ if (this.encoder && this.encoder.state === "configured") {
+ this.reportFinalizingProgress(totalFrames, 97);
+ await this.measureFinalizationStage("encoderFlushMs", async () => {
+ await this.awaitWithFinalizationTimeout(
+ this.encoder!.flush(),
+ "encoder flush",
+ );
+ });
+ }
- stageStartedAt = this.getNowMs();
- if (this.encoder && this.encoder.state === "configured") {
- this.reportFinalizingProgress(totalFrames, 97);
- await this.measureFinalizationStage("encoderFlushMs", async () => {
- await this.awaitWithFinalizationTimeout(this.encoder!.flush(), "encoder flush");
+ this.reportFinalizingProgress(totalFrames, 98);
+ await this.measureFinalizationStage("queuedMuxingMs", async () => {
+ await this.awaitWithFinalizationTimeout(
+ this.pendingMuxing,
+ "muxing queued video chunks",
+ );
});
- }
-
- this.reportFinalizingProgress(totalFrames, 98);
- await this.measureFinalizationStage("queuedMuxingMs", async () => {
- await this.awaitWithFinalizationTimeout(
- this.pendingMuxing,
- "muxing queued video chunks",
- );
- });
- // Surface muxing errors before proceeding with finalization
- if (this.encoderError) {
- throw this.encoderError;
- }
+ // Surface muxing errors before proceeding with finalization
+ if (this.encoderError) {
+ throw this.encoderError;
+ }
- if (
- nativeAudioPlan.audioMode !== "none" &&
- !shouldUseFfmpegAudioFallback &&
- !this.cancelled
- ) {
- const demuxer = this.streamingDecoder.getDemuxer();
if (
- demuxer ||
- (this.config.audioRegions ?? []).length > 0 ||
- (this.config.sourceAudioFallbackPaths ?? []).length > 0
+ nativeAudioPlan.audioMode !== "none" &&
+ !shouldUseFfmpegAudioFallback &&
+ !this.cancelled
) {
- this.audioProcessor = new AudioProcessor();
- this.audioProcessor.setOnProgress((progress) => {
- this.reportFinalizingProgress(totalFrames, 99, progress);
- });
- this.reportFinalizingProgress(totalFrames, 99);
- await this.measureFinalizationStage("audioProcessingMs", async () => {
- await this.awaitWithFinalizationTimeout(
- this.audioProcessor!.process(
- demuxer,
- this.muxer!,
- this.config.videoUrl,
- this.config.trimRegions,
- this.config.speedRegions,
- undefined,
- this.config.audioRegions,
- this.config.sourceAudioFallbackPaths,
- this.config.sourceAudioFallbackStartDelayMsByPath,
- this.config.sourceAudioTrackSettings,
- this.config.clipRegions,
- ),
- "audio processing",
- "audio",
- true,
- );
- });
+ const demuxer = this.streamingDecoder.getDemuxer();
+ if (
+ demuxer ||
+ (this.config.audioRegions ?? []).length > 0 ||
+ (this.config.sourceAudioFallbackPaths ?? []).length > 0
+ ) {
+ this.audioProcessor = new AudioProcessor();
+ this.audioProcessor.setOnProgress((progress) => {
+ this.reportFinalizingProgress(totalFrames, 99, progress);
+ });
+ this.reportFinalizingProgress(totalFrames, 99);
+ await this.measureFinalizationStage("audioProcessingMs", async () => {
+ await this.awaitWithFinalizationTimeout(
+ this.audioProcessor!.process(
+ demuxer,
+ this.muxer!,
+ this.config.videoUrl,
+ this.config.trimRegions,
+ this.config.speedRegions,
+ undefined,
+ this.config.audioRegions,
+ this.config.sourceAudioFallbackPaths,
+ this.config.sourceAudioFallbackStartDelayMsByPath,
+ this.config.sourceAudioTrackSettings,
+ this.config.clipRegions,
+ ),
+ "audio processing",
+ "audio",
+ true,
+ );
+ });
+ }
}
- }
-
- this.reportFinalizingProgress(totalFrames, 99);
- const muxerResult = await this.measureFinalizationStage("muxerFinalizeMs", async () =>
- this.awaitWithFinalizationTimeout(
- this.muxer!.finalize(),
- "muxer finalization",
- nativeAudioPlan.audioMode !== "none" && !shouldUseFfmpegAudioFallback
- ? "audio"
- : "default",
- ),
- );
- if (shouldUseFfmpegAudioFallback) {
- console.warn(
- shouldUsePitchPreservingFfmpegAudio
- ? "[VideoExporter] Using FFmpeg audio muxing for pitch-preserving speed edits."
- : "[VideoExporter] Browser AAC encoding is unavailable; falling back to FFmpeg audio muxing.",
- );
- const muxedResult = await this.finalizeExportWithFfmpegAudio(
- muxerResult,
- nativeAudioPlan,
+ this.reportFinalizingProgress(totalFrames, 99);
+ const muxerResult = await this.measureFinalizationStage(
+ "muxerFinalizeMs",
+ async () =>
+ this.awaitWithFinalizationTimeout(
+ this.muxer!.finalize(),
+ "muxer finalization",
+ nativeAudioPlan.audioMode !== "none" && !shouldUseFfmpegAudioFallback
+ ? "audio"
+ : "default",
+ ),
);
+
+ if (shouldUseFfmpegAudioFallback) {
+ console.warn(
+ shouldUsePitchPreservingFfmpegAudio
+ ? "[VideoExporter] Using FFmpeg audio muxing for pitch-preserving speed edits."
+ : "[VideoExporter] Browser AAC encoding is unavailable; falling back to FFmpeg audio muxing.",
+ );
+ const muxedResult = await this.finalizeExportWithFfmpegAudio(
+ muxerResult,
+ nativeAudioPlan,
+ );
+ this.finalizationTimeMs = this.getNowMs() - stageStartedAt;
+ if (!muxedResult.success || (!muxedResult.blob && !muxedResult.tempFilePath)) {
+ return {
+ success: false,
+ error: muxedResult.error || "Failed to mux audio with FFmpeg",
+ metrics: this.buildExportMetrics(),
+ };
+ }
+
+ return {
+ success: true,
+ blob: muxedResult.blob,
+ tempFilePath: muxedResult.tempFilePath,
+ metrics: muxedResult.metrics ?? this.buildExportMetrics(),
+ };
+ }
+
this.finalizationTimeMs = this.getNowMs() - stageStartedAt;
- if (!muxedResult.success || (!muxedResult.blob && !muxedResult.tempFilePath)) {
+ if (muxerResult.mode === "stream") {
return {
- success: false,
- error: muxedResult.error || "Failed to mux audio with FFmpeg",
+ success: true,
+ tempFilePath: muxerResult.tempFilePath,
metrics: this.buildExportMetrics(),
};
}
-
return {
success: true,
- blob: muxedResult.blob,
- tempFilePath: muxedResult.tempFilePath,
- metrics: muxedResult.metrics ?? this.buildExportMetrics(),
- };
- }
-
- this.finalizationTimeMs = this.getNowMs() - stageStartedAt;
- if (muxerResult.mode === "stream") {
- return {
- success: true,
- tempFilePath: muxerResult.tempFilePath,
- metrics: this.buildExportMetrics(),
- };
- }
- return {
- success: true,
- blob: muxerResult.blob,
- metrics: this.buildExportMetrics(),
- };
- } catch (error) {
- if (this.cancelled && !this.encoderError) {
- return {
- success: false,
- error: "Export cancelled",
+ blob: muxerResult.blob,
metrics: this.buildExportMetrics(),
};
+ } catch (error) {
+ if (
+ !preferReadableFileSource &&
+ !retriedWithReadableFileSource &&
+ this.shouldRetryWithReadableFileSource(error)
+ ) {
+ retriedWithReadableFileSource = true;
+ preferReadableFileSource = true;
+ shouldRetryWithReadableFileSource = true;
+ console.warn(
+ "[VideoExporter] Primary decode path failed; retrying export once with a readable file-backed media source.",
+ error,
+ );
+ } else {
+ if (this.cancelled && !this.encoderError) {
+ return {
+ success: false,
+ error: "Export cancelled",
+ metrics: this.buildExportMetrics(),
+ };
+ }
+
+ const resolvedError = this.encoderError ?? error;
+ console.error("Export error:", error);
+ return {
+ success: false,
+ error: this.buildLightningExportError(resolvedError),
+ metrics: this.buildExportMetrics(),
+ };
+ }
+ } finally {
+ if (!shouldRetryWithReadableFileSource && this.totalExportStartTimeMs > 0) {
+ console.log(
+ `[VideoExporter] Final metrics ${JSON.stringify(this.buildExportMetrics())}`,
+ );
+ }
+ this.cleanup();
}
- const resolvedError = this.encoderError ?? error;
- console.error("Export error:", error);
- return {
- success: false,
- error: this.buildLightningExportError(resolvedError),
- metrics: this.buildExportMetrics(),
- };
- } finally {
- if (this.totalExportStartTimeMs > 0) {
- console.log(
- `[VideoExporter] Final metrics ${JSON.stringify(this.buildExportMetrics())}`,
- );
+ if (shouldRetryWithReadableFileSource) {
+ continue;
}
- this.cleanup();
}
}
+ private shouldRetryWithReadableFileSource(error: unknown): boolean {
+ const resolvedError = this.encoderError ?? error;
+ const message =
+ resolvedError instanceof Error ? resolvedError.message : String(resolvedError);
+ const normalizedMessage = message.toLowerCase();
+ return READABLE_SOURCE_RETRY_ERROR_TOKENS.some((token) =>
+ normalizedMessage.includes(token),
+ );
+ }
+
private getPlatformLabel(): string {
switch (this.getRuntimePlatform()) {
case "win32":
@@ -1100,7 +1168,11 @@ export class ModernVideoExporter {
speed: region.speed,
}))
.filter((region) => region.endMs - region.startMs > 0.5);
- const sourceSegments: Array<{ startMs: number; endMs: number; speed: number }> = [];
+ const sourceSegments: Array<{
+ startMs: number;
+ endMs: number;
+ speed: number;
+ }> = [];
for (const keptRange of this.buildNativeTrimSegments(sourceDurationMs)) {
const boundaries = new Set<number>([keptRange.startMs, keptRange.endMs]);
@@ -2635,7 +2707,10 @@ export class ModernVideoExporter {
if (this.nativeEncoderError) throw this.nativeEncoderError;
}
const canvas = this.renderer!.getCanvas();
- const frame = new VideoFrame(canvas, { timestamp, duration: frameDuration });
+ const frame = new VideoFrame(canvas, {
+ timestamp,
+ duration: frameDuration,
+ });
this.nativeH264Encoder.encode(frame, { keyFrame: frameIndex % 300 === 0 });
frame.close();
}
diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts
index e0f5a18c..861d6720 100644
--- a/src/lib/exporter/streamingDecoder.ts
+++ b/src/lib/exporter/streamingDecoder.ts
@@ -1,10 +1,7 @@
import { WebDemuxer } from "web-demuxer";
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
-import {
- createReadableMediaResourceFile,
- resolveMediaResourceUrl,
-} from "./localMediaSource";
+import { createReadableMediaResourceFile, resolveMediaResourceUrl } from "./localMediaSource";
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
@@ -26,6 +23,10 @@ export interface DecodedVideoInfo {
audioSampleRate?: number;
}
+interface StreamingVideoDecoderLoadOptions {
+ forceReadableFileSource?: boolean;
+}
+
/** Decoder retains ownership of the VideoFrame and closes it after use. */
type OnFrameCallback = (
frame: VideoFrame,
@@ -76,7 +77,10 @@ export class StreamingVideoDecoder {
private readonly maxDecodeQueue: number;
private readonly maxPendingFrames: number;
- constructor(options?: { maxDecodeQueue?: number; maxPendingFrames?: number }) {
+ constructor(options?: {
+ maxDecodeQueue?: number;
+ maxPendingFrames?: number;
+ }) {
this.maxDecodeQueue = Math.max(
1,
Math.floor(options?.maxDecodeQueue ?? DEFAULT_MAX_DECODE_QUEUE),
@@ -87,7 +91,10 @@ export class StreamingVideoDecoder {
);
}
- async loadMetadata(videoUrl: string): Promise<DecodedVideoInfo> {
+ async loadMetadata(
+ videoUrl: string,
+ options: StreamingVideoDecoderLoadOptions = {},
+ ): Promise<DecodedVideoInfo> {
if (this.decoder) {
try {
if (this.decoder.state === "configured") {
@@ -119,22 +126,26 @@ export class StreamingVideoDecoder {
};
let mediaInfo;
- try {
- mediaInfo = await loadMediaInfo(resourceUrl);
- } catch (error) {
- console.warn(
- "[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:",
- error,
- );
- const currentDemuxer = this.demuxer;
- if (currentDemuxer) {
- try {
- (currentDemuxer as unknown as { destroy: () => void }).destroy();
- } catch {
- // Ignore cleanup errors before fallback re-init.
+ if (options.forceReadableFileSource) {
+ mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
+ } else {
+ try {
+ mediaInfo = await loadMediaInfo(resourceUrl);
+ } catch (error) {
+ console.warn(
+ "[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:",
+ error,
+ );
+ const currentDemuxer = this.demuxer;
+ if (currentDemuxer) {
+ try {
+ (currentDemuxer as unknown as { destroy: () => void }).destroy();
+ } catch {
+ // Ignore cleanup errors before fallback re-init.
+ }
}
+ mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
}
- mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl));
}
const videoStream = mediaInfo.streams.find((s) => s.codec_type_string === "video");
@@ -628,7 +639,11 @@ export class StreamingVideoDecoder {
}
const effectiveStart = Math.max(cursor, srStart);
if (srEnd > effectiveStart) {
- result.push({ startSec: effectiveStart, endSec: srEnd, speed: sr.speed });
+ result.push({
+ startSec: effectiveStart,
+ endSec: srEnd,
+ speed: sr.speed,
+ });
}
cursor = Math.max(cursor, srEnd);
}