Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix(export): fallback webcam packet streaming failures

wiiiii12387968870eb2a
diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts
index f8b95fd5..c0c0927e 100644
--- a/src/lib/exporter/modernFrameRenderer.test.ts
+++ b/src/lib/exporter/modernFrameRenderer.test.ts
@@ -1,7 +1,16 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types";
-const { initializeForwardFrameSourceMock, resolveMediaElementSourceMock } = vi.hoisted(() => ({
+const {
+ cancelForwardFrameSourceMock,
+ destroyForwardFrameSourceMock,
+ getForwardFrameAtTimeMock,
+ initializeForwardFrameSourceMock,
+ resolveMediaElementSourceMock,
+} = vi.hoisted(() => ({
+ cancelForwardFrameSourceMock: vi.fn(),
+ destroyForwardFrameSourceMock: vi.fn(async () => undefined),
+ getForwardFrameAtTimeMock: vi.fn(async () => null),
initializeForwardFrameSourceMock: vi.fn(async () => undefined),
resolveMediaElementSourceMock: vi.fn(async () => ({
src: "blob:background",
@@ -82,6 +91,9 @@ vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({
vi.mock("./forwardFrameSource", () => ({
ForwardFrameSource: class {
+ cancel = cancelForwardFrameSourceMock;
+ destroy = destroyForwardFrameSourceMock;
+ getFrameAtTime = getForwardFrameAtTimeMock;
initialize = initializeForwardFrameSourceMock;
},
}));
@@ -248,6 +260,43 @@ describe("ModernFrameRenderer blur export path", () => {
});
describe("ModernFrameRenderer webcam frame cache", () => {
+ it("uses staging canvas instead of recursing when WebGPU frame retention fails", () => {
+ const renderer = createRenderer() as any;
+ const originalVideoFrame = (globalThis as any).VideoFrame;
+
+ (globalThis as any).VideoFrame = class {
+ constructor() {
+ throw new Error("retain failed");
+ }
+ };
+
+ try {
+ renderer.rendererBackend = "webgpu";
+ const frame = {
+ displayWidth: 320,
+ displayHeight: 180,
+ timestamp: 0,
+ } as VideoFrame;
+
+ const result = renderer.stageVideoFrameForTexture(frame, "webcam", 640, 360);
+
+ expect(result).toBe(renderer.webcamVideoFrameStagingCanvas);
+ expect(renderer.webcamVideoFrameStagingCtx.drawImage).toHaveBeenCalledWith(
+ frame,
+ 0,
+ 0,
+ 320,
+ 180,
+ );
+ } finally {
+ if (originalVideoFrame === undefined) {
+ delete (globalThis as any).VideoFrame;
+ } else {
+ (globalThis as any).VideoFrame = originalVideoFrame;
+ }
+ }
+ });
+
it("keeps the refresh throttle for default crop regions", () => {
const renderer = createRenderer() as any;
@@ -270,3 +319,147 @@ describe("ModernFrameRenderer webcam frame cache", () => {
expect(renderer.shouldRefreshWebcamFrameCache(1280, 720)).toBe(true);
});
});
+
+describe("ModernFrameRenderer webcam export fallback", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ initializeForwardFrameSourceMock.mockResolvedValue(undefined);
+ getForwardFrameAtTimeMock.mockResolvedValue(null);
+ resolveMediaElementSourceMock.mockResolvedValue({
+ src: "blob:webcam",
+ revoke: vi.fn(),
+ });
+
+ Object.assign(globalThis, {
+ window: {
+ clearTimeout,
+ setTimeout,
+ },
+ HTMLMediaElement: {
+ HAVE_CURRENT_DATA: 2,
+ },
+ cancelAnimationFrame: vi.fn(),
+ requestAnimationFrame: vi.fn((callback: FrameRequestCallback) => {
+ callback(0);
+ return 1;
+ }),
+ document: {
+ createElement: vi.fn((tag: string) => {
+ if (tag === "video") {
+ return {
+ duration: 5,
+ readyState: 2,
+ videoWidth: 640,
+ videoHeight: 360,
+ muted: true,
+ loop: true,
+ playsInline: true,
+ preload: "auto",
+ src: "",
+ currentTime: 0,
+ seeking: false,
+ load: vi.fn(),
+ pause: vi.fn(),
+ addEventListener: vi.fn(),
+ removeEventListener: vi.fn(),
+ };
+ }
+ if (tag !== "canvas") {
+ throw new Error(`Unexpected element requested in test: ${tag}`);
+ }
+
+ return createMockCanvas();
+ }),
+ },
+ });
+ });
+
+ it("falls back to media-element webcam sync when packet streaming fails after initialize", async () => {
+ getForwardFrameAtTimeMock.mockRejectedValueOnce(
+ new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
+ );
+ const renderer = createRenderer() as any;
+ renderer.config.webcam = {
+ ...DEFAULT_WEBCAM_OVERLAY,
+ enabled: true,
+ };
+ renderer.config.webcamUrl = "file:///tmp/webcam.webm";
+
+ await renderer.setupWebcamSource();
+ await expect(renderer.syncWebcamFrame(1)).resolves.toBeUndefined();
+
+ expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
+ expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
+ expect(resolveMediaElementSourceMock).toHaveBeenCalledWith("file:///tmp/webcam.webm");
+ expect(renderer.webcamForwardFrameSource).toBeNull();
+ expect(renderer.webcamVideoElement).toBeTruthy();
+ });
+
+ it("tears down the media-element fallback when readiness times out", async () => {
+ vi.useFakeTimers();
+ const originalCreateElement = (globalThis as any).document.createElement;
+ const revoke = vi.fn();
+ getForwardFrameAtTimeMock.mockRejectedValueOnce(
+ new Error("readAVPacket pipeline failed: Failed after 3 attempts"),
+ );
+ resolveMediaElementSourceMock.mockResolvedValueOnce({
+ src: "blob:webcam-timeout",
+ revoke,
+ });
+ Object.assign((globalThis as any).window, {
+ clearTimeout,
+ setTimeout,
+ });
+
+ (globalThis as any).document.createElement = vi.fn((tag: string) => {
+ if (tag === "video") {
+ return {
+ duration: Number.NaN,
+ readyState: 0,
+ videoWidth: 0,
+ videoHeight: 0,
+ muted: true,
+ loop: true,
+ playsInline: true,
+ preload: "auto",
+ src: "",
+ currentTime: 0,
+ seeking: false,
+ load: vi.fn(),
+ pause: vi.fn(),
+ addEventListener: vi.fn(),
+ removeEventListener: vi.fn(),
+ };
+ }
+ if (tag !== "canvas") {
+ throw new Error(`Unexpected element requested in test: ${tag}`);
+ }
+
+ return createMockCanvas();
+ });
+
+ try {
+ const renderer = createRenderer() as any;
+ renderer.config.webcam = {
+ ...DEFAULT_WEBCAM_OVERLAY,
+ enabled: true,
+ };
+ renderer.config.webcamUrl = "file:///tmp/webcam.webm";
+
+ await renderer.setupWebcamSource();
+ const syncPromise = renderer.syncWebcamFrame(1);
+
+ await vi.advanceTimersByTimeAsync(5_001);
+ await expect(syncPromise).resolves.toBeUndefined();
+
+ expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
+ expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
+ expect(revoke).toHaveBeenCalled();
+ expect(renderer.webcamForwardFrameSource).toBeNull();
+ expect(renderer.webcamVideoElement).toBeNull();
+ } finally {
+ (globalThis as any).document.createElement = originalCreateElement;
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts
index c6830315..02e648aa 100644
--- a/src/lib/exporter/modernFrameRenderer.ts
+++ b/src/lib/exporter/modernFrameRenderer.ts
@@ -246,10 +246,14 @@ type PixiRendererAttempt = {
const CANVAS_RENDERER_NOT_IMPLEMENTED_HINT = "CanvasRenderer is not yet implemented";
const NO_RENDERER_HINT = "no available renderer";
const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;
+const WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS = 5_000;
function isCanvasRenderer(application: Application): boolean {
const rendererName = application?.renderer?.constructor?.name?.toLowerCase();
- return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")));
+ return Boolean(
+ rendererName &&
+ (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")),
+ );
}
function toErrorMessage(error: unknown): string {
@@ -681,7 +685,8 @@ export class FrameRenderer {
backend,
);
const elapsed = Math.round(
- (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
+ (typeof performance === "undefined" ? Date.now() : performance.now()) -
+ initStarted,
);
if (isCanvasRenderer(app)) {
throw new Error(
@@ -691,7 +696,8 @@ export class FrameRenderer {
return { app, backend };
} catch (error) {
const elapsed = Math.round(
- (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
+ (typeof performance === "undefined" ? Date.now() : performance.now()) -
+ initStarted,
);
failures.push({
backend,
@@ -894,7 +900,9 @@ export class FrameRenderer {
}
const cachedTimestamp =
- kind === "scene" ? this.retainedSceneBitmapTimestamp : this.retainedBackgroundBitmapTimestamp;
+ kind === "scene"
+ ? this.retainedSceneBitmapTimestamp
+ : this.retainedBackgroundBitmapTimestamp;
const cachedBitmap =
kind === "scene" ? this.retainedSceneBitmap : this.retainedBackgroundBitmap;
if (cachedTimestamp === frame.timestamp && cachedBitmap) {
@@ -924,8 +932,8 @@ export class FrameRenderer {
private resolveRetainedVideoFrameSource(
frame: VideoFrame,
kind: "scene" | "background" | "webcam",
- _fallbackWidth: number,
- _fallbackHeight: number,
+ fallbackWidth: number,
+ fallbackHeight: number,
): CanvasImageSource | VideoFrame {
if (this.rendererBackend !== "webgpu") {
return frame;
@@ -948,12 +956,7 @@ export class FrameRenderer {
`[ModernFrameRenderer] Failed to retain ${kind} VideoFrame, falling back to staging canvas:`,
error,
);
- return this.stageVideoFrameForTexture(
- frame,
- "scene",
- this.config.videoWidth,
- this.config.videoHeight,
- );
+ return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);
}
}
@@ -1010,21 +1013,12 @@ export class FrameRenderer {
return { canvas, context };
}
- private stageVideoFrameForTexture(
+ private stageVideoFrameOnCanvas(
frame: VideoFrame,
kind: "scene" | "background" | "webcam",
fallbackWidth: number,
fallbackHeight: number,
): CanvasImageSource | VideoFrame {
- if (this.rendererBackend === "webgpu") {
- return this.resolveRetainedVideoFrameSource(
- frame,
- kind,
- fallbackWidth,
- fallbackHeight,
- );
- }
-
const width = Math.max(1, frame.displayWidth || fallbackWidth);
const height = Math.max(1, frame.displayHeight || fallbackHeight);
const staging = this.ensureVideoFrameStagingCanvas(kind, width, height);
@@ -1037,6 +1031,19 @@ export class FrameRenderer {
return staging.canvas;
}
+ private stageVideoFrameForTexture(
+ frame: VideoFrame,
+ kind: "scene" | "background" | "webcam",
+ fallbackWidth: number,
+ fallbackHeight: number,
+ ): CanvasImageSource | VideoFrame {
+ if (this.rendererBackend === "webgpu") {
+ return this.resolveRetainedVideoFrameSource(frame, kind, fallbackWidth, fallbackHeight);
+ }
+
+ return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);
+ }
+
private replaceSpriteTexture(
sprite: Sprite,
source: CanvasImageSource | VideoFrame,
@@ -2085,54 +2092,37 @@ export class FrameRenderer {
return getRenderableAssetUrl(wallpaperAsset);
}
- private async setupWebcamSource(): Promise<void> {
- const webcamUrl = this.config.webcamUrl;
- if (!this.config.webcam?.enabled || !webcamUrl) {
- this.webcamForwardFrameSource?.cancel();
- void this.webcamForwardFrameSource?.destroy();
- this.webcamForwardFrameSource = null;
- this.closeWebcamDecodedFrame();
- this.cleanupWebcamSource?.();
- this.cleanupWebcamSource = null;
- this.webcamVideoElement = null;
- this.webcamFrameCacheCanvas = null;
- this.webcamFrameCacheCtx = null;
- this.lastSyncedWebcamTime = null;
- this.lastWebcamCacheRefreshTime = null;
- this.webcamLayoutCache = null;
- this.webcamRenderMode = "hidden";
- return;
+ private disposeWebcamMediaElement(video: HTMLVideoElement): void {
+ try {
+ video.pause();
+ video.src = "";
+ video.load();
+ } catch {
+ // Ignore media element teardown errors during export fallback.
}
+ }
- this.webcamForwardFrameSource?.cancel();
- void this.webcamForwardFrameSource?.destroy();
- this.webcamForwardFrameSource = null;
- this.closeWebcamDecodedFrame();
+ private clearWebcamMediaElement(): void {
+ if (this.webcamVideoElement) {
+ this.disposeWebcamMediaElement(this.webcamVideoElement);
+ }
+
+ this.webcamVideoElement = null;
+ this.webcamSeekPromise = null;
this.cleanupWebcamSource?.();
this.cleanupWebcamSource = null;
- this.webcamFrameCacheCanvas = null;
- this.webcamFrameCacheCtx = null;
- this.lastWebcamCacheRefreshTime = null;
- this.webcamLayoutCache = null;
- this.webcamRenderMode = "hidden";
+ }
+ private async loadWebcamMediaElementSource(webcamUrl: string): Promise<boolean> {
+ this.clearWebcamMediaElement();
+
+ let webcamSource: Awaited<ReturnType<typeof resolveMediaElementSource>>;
try {
- const frameSource = new ForwardFrameSource();
- await frameSource.initialize(webcamUrl);
- this.webcamForwardFrameSource = frameSource;
- this.webcamVideoElement = null;
- this.webcamSeekPromise = null;
- this.lastSyncedWebcamTime = null;
- this.lastWebcamCacheRefreshTime = null;
- return;
+ webcamSource = await resolveMediaElementSource(webcamUrl);
} catch (error) {
- console.warn(
- "[FrameRenderer] Decoder-backed webcam source unavailable during export; falling back to media element sync:",
- error,
- );
+ console.warn("[FrameRenderer] Unable to resolve webcam media element source:", error);
+ return false;
}
-
- const webcamSource = await resolveMediaElementSource(webcamUrl);
this.cleanupWebcamSource = webcamSource.revoke;
const video = document.createElement("video");
@@ -2142,27 +2132,32 @@ export class FrameRenderer {
video.playsInline = true;
video.load();
- await new Promise<void>((resolve, reject) => {
+ const ready = await new Promise<boolean>((resolve) => {
+ let readyTimeout: number | null = null;
const onReady = () => {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
return;
}
cleanup();
- resolve();
+ resolve(true);
};
const onError = () => {
cleanup();
- reject(new Error("Failed to load webcam source for export"));
+ resolve(false);
};
const cleanup = () => {
video.removeEventListener("loadeddata", onReady);
video.removeEventListener("canplay", onReady);
video.removeEventListener("canplaythrough", onReady);
video.removeEventListener("error", onError);
+ if (readyTimeout !== null) {
+ window.clearTimeout(readyTimeout);
+ readyTimeout = null;
+ }
};
if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
- resolve();
+ resolve(true);
return;
}
@@ -2170,17 +2165,71 @@ export class FrameRenderer {
video.addEventListener("canplay", onReady, { once: true });
video.addEventListener("canplaythrough", onReady, { once: true });
video.addEventListener("error", onError, { once: true });
- }).catch((error) => {
- console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error);
- this.webcamVideoElement = null;
+ readyTimeout = window.setTimeout(() => {
+ console.warn(
+ `[FrameRenderer] Webcam media element fallback did not become ready within ${WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS}ms`,
+ );
+ onError();
+ }, WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS);
});
- if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
+ if (ready && video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
this.webcamVideoElement = video;
+ this.lastSyncedWebcamTime = null;
+ return true;
+ }
+
+ console.warn("[FrameRenderer] Webcam overlay unavailable during export");
+ this.disposeWebcamMediaElement(video);
+ this.clearWebcamMediaElement();
+ return false;
+ }
+
+ private async setupWebcamSource(): Promise<void> {
+ const webcamUrl = this.config.webcamUrl;
+ if (!this.config.webcam?.enabled || !webcamUrl) {
+ this.webcamForwardFrameSource?.cancel();
+ void this.webcamForwardFrameSource?.destroy();
+ this.webcamForwardFrameSource = null;
+ this.closeWebcamDecodedFrame();
+ this.clearWebcamMediaElement();
+ this.webcamFrameCacheCanvas = null;
+ this.webcamFrameCacheCtx = null;
+ this.lastSyncedWebcamTime = null;
+ this.lastWebcamCacheRefreshTime = null;
+ this.webcamLayoutCache = null;
+ this.webcamRenderMode = "hidden";
return;
}
- this.webcamVideoElement = null;
+ this.webcamForwardFrameSource?.cancel();
+ void this.webcamForwardFrameSource?.destroy();
+ this.webcamForwardFrameSource = null;
+ this.closeWebcamDecodedFrame();
+ this.clearWebcamMediaElement();
+ this.webcamFrameCacheCanvas = null;
+ this.webcamFrameCacheCtx = null;
+ this.lastWebcamCacheRefreshTime = null;
+ this.webcamLayoutCache = null;
+ this.webcamRenderMode = "hidden";
+
+ try {
+ const frameSource = new ForwardFrameSource();
+ await frameSource.initialize(webcamUrl);
+ this.webcamForwardFrameSource = frameSource;
+ this.webcamVideoElement = null;
+ this.webcamSeekPromise = null;
+ this.lastSyncedWebcamTime = null;
+ this.lastWebcamCacheRefreshTime = null;
+ return;
+ } catch (error) {
+ console.warn(
+ "[FrameRenderer] Decoder-backed webcam source unavailable during export; falling back to media element sync:",
+ error,
+ );
+ }
+
+ await this.loadWebcamMediaElementSource(webcamUrl);
this.lastSyncedWebcamTime = null;
}
@@ -2457,13 +2506,30 @@ export class FrameRenderer {
if (this.webcamForwardFrameSource) {
const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null);
- const decodedFrame = await this.webcamForwardFrameSource.getFrameAtTime(clampedTime);
- this.closeWebcamDecodedFrame();
- this.webcamDecodedFrame = decodedFrame;
- if (decodedFrame) {
- this.lastSyncedWebcamTime = clampedTime;
+ try {
+ const decodedFrame =
+ await this.webcamForwardFrameSource.getFrameAtTime(clampedTime);
+ this.closeWebcamDecodedFrame();
+ this.webcamDecodedFrame = decodedFrame;
+ if (decodedFrame) {
+ this.lastSyncedWebcamTime = clampedTime;
+ }
+ return;
+ } catch (error) {
+ console.warn(
+ "[FrameRenderer] Decoder-backed webcam source failed during export; falling back to media element sync:",
+ error,
+ );
+ this.webcamForwardFrameSource.cancel();
+ void this.webcamForwardFrameSource.destroy();
+ this.webcamForwardFrameSource = null;
+ this.closeWebcamDecodedFrame();
+ this.lastSyncedWebcamTime = null;
+ const webcamUrl = this.config.webcamUrl;
+ if (!webcamUrl || !(await this.loadWebcamMediaElementSource(webcamUrl))) {
+ return;
+ }
}
- return;
}
const webcamVideo = this.webcamVideoElement;