Create polished demo videos without editing skills. Mac/Windows/Linux
fix(export): fallback webcam packet streaming failures
wiiiii12387968870eb2a
diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.tsindex f8b95fd5..c0c0927e 100644--- a/src/lib/exporter/modernFrameRenderer.test.ts+++ b/src/lib/exporter/modernFrameRenderer.test.ts@@ -1,7 +1,16 @@import { beforeEach, describe, expect, it, vi } from "vitest";import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types";-const { initializeForwardFrameSourceMock, resolveMediaElementSourceMock } = vi.hoisted(() => ({+const {+ cancelForwardFrameSourceMock,+ destroyForwardFrameSourceMock,+ getForwardFrameAtTimeMock,+ initializeForwardFrameSourceMock,+ resolveMediaElementSourceMock,+} = vi.hoisted(() => ({+ cancelForwardFrameSourceMock: vi.fn(),+ destroyForwardFrameSourceMock: vi.fn(async () => undefined),+ getForwardFrameAtTimeMock: vi.fn(async () => null),initializeForwardFrameSourceMock: vi.fn(async () => undefined),resolveMediaElementSourceMock: vi.fn(async () => ({src: "blob:background",@@ -82,6 +91,9 @@ vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({vi.mock("./forwardFrameSource", () => ({ForwardFrameSource: class {+ cancel = cancelForwardFrameSourceMock;+ destroy = destroyForwardFrameSourceMock;+ getFrameAtTime = getForwardFrameAtTimeMock;initialize = initializeForwardFrameSourceMock;},}));@@ -248,6 +260,43 @@ describe("ModernFrameRenderer blur export path", () => {});describe("ModernFrameRenderer webcam frame cache", () => {+ it("uses staging canvas instead of recursing when WebGPU frame retention fails", () => {+ const renderer = createRenderer() as any;+ const originalVideoFrame = (globalThis as any).VideoFrame;++ (globalThis as any).VideoFrame = class {+ constructor() {+ throw new Error("retain failed");+ }+ };++ try {+ renderer.rendererBackend = "webgpu";+ const frame = {+ displayWidth: 320,+ displayHeight: 180,+ timestamp: 0,+ } as VideoFrame;++ const result = renderer.stageVideoFrameForTexture(frame, "webcam", 640, 360);++ expect(result).toBe(renderer.webcamVideoFrameStagingCanvas);+ expect(renderer.webcamVideoFrameStagingCtx.drawImage).toHaveBeenCalledWith(+ frame,+ 0,+ 0,+ 320,+ 180,+ );+ } finally {+ if (originalVideoFrame === undefined) {+ delete (globalThis as any).VideoFrame;+ } else {+ (globalThis as any).VideoFrame = originalVideoFrame;+ }+ }+ });+it("keeps the refresh throttle for default crop regions", () => {const renderer = createRenderer() as any;@@ -270,3 +319,147 @@ describe("ModernFrameRenderer webcam frame cache", () => {expect(renderer.shouldRefreshWebcamFrameCache(1280, 720)).toBe(true);});});++describe("ModernFrameRenderer webcam export fallback", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ initializeForwardFrameSourceMock.mockResolvedValue(undefined);+ getForwardFrameAtTimeMock.mockResolvedValue(null);+ resolveMediaElementSourceMock.mockResolvedValue({+ src: "blob:webcam",+ revoke: vi.fn(),+ });++ Object.assign(globalThis, {+ window: {+ clearTimeout,+ setTimeout,+ },+ HTMLMediaElement: {+ HAVE_CURRENT_DATA: 2,+ },+ cancelAnimationFrame: vi.fn(),+ requestAnimationFrame: vi.fn((callback: FrameRequestCallback) => {+ callback(0);+ return 1;+ }),+ document: {+ createElement: vi.fn((tag: string) => {+ if (tag === "video") {+ return {+ duration: 5,+ readyState: 2,+ videoWidth: 640,+ videoHeight: 360,+ muted: true,+ loop: true,+ playsInline: true,+ preload: "auto",+ src: "",+ currentTime: 0,+ seeking: false,+ load: vi.fn(),+ pause: vi.fn(),+ addEventListener: vi.fn(),+ removeEventListener: vi.fn(),+ };+ }+ if (tag !== "canvas") {+ throw new Error(`Unexpected element requested in test: ${tag}`);+ }++ return createMockCanvas();+ }),+ },+ });+ });++ it("falls back to media-element webcam sync when packet streaming fails after initialize", async () => {+ getForwardFrameAtTimeMock.mockRejectedValueOnce(+ new Error("readAVPacket pipeline failed: Failed after 3 attempts"),+ );+ const renderer = createRenderer() as any;+ renderer.config.webcam = {+ ...DEFAULT_WEBCAM_OVERLAY,+ enabled: true,+ };+ renderer.config.webcamUrl = "file:///tmp/webcam.webm";++ await renderer.setupWebcamSource();+ await expect(renderer.syncWebcamFrame(1)).resolves.toBeUndefined();++ expect(cancelForwardFrameSourceMock).toHaveBeenCalled();+ expect(destroyForwardFrameSourceMock).toHaveBeenCalled();+ expect(resolveMediaElementSourceMock).toHaveBeenCalledWith("file:///tmp/webcam.webm");+ expect(renderer.webcamForwardFrameSource).toBeNull();+ expect(renderer.webcamVideoElement).toBeTruthy();+ });++ it("tears down the media-element fallback when readiness times out", async () => {+ vi.useFakeTimers();+ const originalCreateElement = (globalThis as any).document.createElement;+ const revoke = vi.fn();+ getForwardFrameAtTimeMock.mockRejectedValueOnce(+ new Error("readAVPacket pipeline failed: Failed after 3 attempts"),+ );+ resolveMediaElementSourceMock.mockResolvedValueOnce({+ src: "blob:webcam-timeout",+ revoke,+ });+ Object.assign((globalThis as any).window, {+ clearTimeout,+ setTimeout,+ });++ (globalThis as any).document.createElement = vi.fn((tag: string) => {+ if (tag === "video") {+ return {+ duration: Number.NaN,+ readyState: 0,+ videoWidth: 0,+ videoHeight: 0,+ muted: true,+ loop: true,+ playsInline: true,+ preload: "auto",+ src: "",+ currentTime: 0,+ seeking: false,+ load: vi.fn(),+ pause: vi.fn(),+ addEventListener: vi.fn(),+ removeEventListener: vi.fn(),+ };+ }+ if (tag !== "canvas") {+ throw new Error(`Unexpected element requested in test: ${tag}`);+ }++ return createMockCanvas();+ });++ try {+ const renderer = createRenderer() as any;+ renderer.config.webcam = {+ ...DEFAULT_WEBCAM_OVERLAY,+ enabled: true,+ };+ renderer.config.webcamUrl = "file:///tmp/webcam.webm";++ await renderer.setupWebcamSource();+ const syncPromise = renderer.syncWebcamFrame(1);++ await vi.advanceTimersByTimeAsync(5_001);+ await expect(syncPromise).resolves.toBeUndefined();++ expect(cancelForwardFrameSourceMock).toHaveBeenCalled();+ expect(destroyForwardFrameSourceMock).toHaveBeenCalled();+ expect(revoke).toHaveBeenCalled();+ expect(renderer.webcamForwardFrameSource).toBeNull();+ expect(renderer.webcamVideoElement).toBeNull();+ } finally {+ (globalThis as any).document.createElement = originalCreateElement;+ vi.useRealTimers();+ }+ });+});diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.tsindex c6830315..02e648aa 100644--- a/src/lib/exporter/modernFrameRenderer.ts+++ b/src/lib/exporter/modernFrameRenderer.ts@@ -246,10 +246,14 @@ type PixiRendererAttempt = {const CANVAS_RENDERER_NOT_IMPLEMENTED_HINT = "CanvasRenderer is not yet implemented";const NO_RENDERER_HINT = "no available renderer";const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;+const WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS = 5_000;function isCanvasRenderer(application: Application): boolean {const rendererName = application?.renderer?.constructor?.name?.toLowerCase();- return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")));+ return Boolean(+ rendererName &&+ (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")),+ );}function toErrorMessage(error: unknown): string {@@ -681,7 +685,8 @@ export class FrameRenderer {backend,);const elapsed = Math.round(- (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,+ (typeof performance === "undefined" ? Date.now() : performance.now()) -+ initStarted,);if (isCanvasRenderer(app)) {throw new Error(@@ -691,7 +696,8 @@ export class FrameRenderer {return { app, backend };} catch (error) {const elapsed = Math.round(- (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,+ (typeof performance === "undefined" ? Date.now() : performance.now()) -+ initStarted,);failures.push({backend,@@ -894,7 +900,9 @@ export class FrameRenderer {}const cachedTimestamp =- kind === "scene" ? this.retainedSceneBitmapTimestamp : this.retainedBackgroundBitmapTimestamp;+ kind === "scene"+ ? this.retainedSceneBitmapTimestamp+ : this.retainedBackgroundBitmapTimestamp;const cachedBitmap =kind === "scene" ? this.retainedSceneBitmap : this.retainedBackgroundBitmap;if (cachedTimestamp === frame.timestamp && cachedBitmap) {@@ -924,8 +932,8 @@ export class FrameRenderer {private resolveRetainedVideoFrameSource(frame: VideoFrame,kind: "scene" | "background" | "webcam",- _fallbackWidth: number,- _fallbackHeight: number,+ fallbackWidth: number,+ fallbackHeight: number,): CanvasImageSource | VideoFrame {if (this.rendererBackend !== "webgpu") {return frame;@@ -948,12 +956,7 @@ export class FrameRenderer {`[ModernFrameRenderer] Failed to retain ${kind} VideoFrame, falling back to staging canvas:`,error,);- return this.stageVideoFrameForTexture(- frame,- "scene",- this.config.videoWidth,- this.config.videoHeight,- );+ return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);}}@@ -1010,21 +1013,12 @@ export class FrameRenderer {return { canvas, context };}- private stageVideoFrameForTexture(+ private stageVideoFrameOnCanvas(frame: VideoFrame,kind: "scene" | "background" | "webcam",fallbackWidth: number,fallbackHeight: number,): CanvasImageSource | VideoFrame {- if (this.rendererBackend === "webgpu") {- return this.resolveRetainedVideoFrameSource(- frame,- kind,- fallbackWidth,- fallbackHeight,- );- }-const width = Math.max(1, frame.displayWidth || fallbackWidth);const height = Math.max(1, frame.displayHeight || fallbackHeight);const staging = this.ensureVideoFrameStagingCanvas(kind, width, height);@@ -1037,6 +1031,19 @@ export class FrameRenderer {return staging.canvas;}+ private stageVideoFrameForTexture(+ frame: VideoFrame,+ kind: "scene" | "background" | "webcam",+ fallbackWidth: number,+ fallbackHeight: number,+ ): CanvasImageSource | VideoFrame {+ if (this.rendererBackend === "webgpu") {+ return this.resolveRetainedVideoFrameSource(frame, kind, fallbackWidth, fallbackHeight);+ }++ return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight);+ }+private replaceSpriteTexture(sprite: Sprite,source: CanvasImageSource | VideoFrame,@@ -2085,54 +2092,37 @@ export class FrameRenderer {return getRenderableAssetUrl(wallpaperAsset);}- private async setupWebcamSource(): Promise<void> {- const webcamUrl = this.config.webcamUrl;- if (!this.config.webcam?.enabled || !webcamUrl) {- this.webcamForwardFrameSource?.cancel();- void this.webcamForwardFrameSource?.destroy();- this.webcamForwardFrameSource = null;- this.closeWebcamDecodedFrame();- this.cleanupWebcamSource?.();- this.cleanupWebcamSource = null;- this.webcamVideoElement = null;- this.webcamFrameCacheCanvas = null;- this.webcamFrameCacheCtx = null;- this.lastSyncedWebcamTime = null;- this.lastWebcamCacheRefreshTime = null;- this.webcamLayoutCache = null;- this.webcamRenderMode = "hidden";- return;+ private disposeWebcamMediaElement(video: HTMLVideoElement): void {+ try {+ video.pause();+ video.src = "";+ video.load();+ } catch {+ // Ignore media element teardown errors during export fallback.}+ }- this.webcamForwardFrameSource?.cancel();- void this.webcamForwardFrameSource?.destroy();- this.webcamForwardFrameSource = null;- this.closeWebcamDecodedFrame();+ private clearWebcamMediaElement(): void {+ if (this.webcamVideoElement) {+ this.disposeWebcamMediaElement(this.webcamVideoElement);+ }++ this.webcamVideoElement = null;+ this.webcamSeekPromise = null;this.cleanupWebcamSource?.();this.cleanupWebcamSource = null;- this.webcamFrameCacheCanvas = null;- this.webcamFrameCacheCtx = null;- this.lastWebcamCacheRefreshTime = null;- this.webcamLayoutCache = null;- this.webcamRenderMode = "hidden";+ }+ private async loadWebcamMediaElementSource(webcamUrl: string): Promise<boolean> {+ this.clearWebcamMediaElement();++ let webcamSource: Awaited<ReturnType<typeof resolveMediaElementSource>>;try {- const frameSource = new ForwardFrameSource();- await frameSource.initialize(webcamUrl);- this.webcamForwardFrameSource = frameSource;- this.webcamVideoElement = null;- this.webcamSeekPromise = null;- this.lastSyncedWebcamTime = null;- this.lastWebcamCacheRefreshTime = null;- return;+ webcamSource = await resolveMediaElementSource(webcamUrl);} catch (error) {- console.warn(- "[FrameRenderer] Decoder-backed webcam source unavailable during export; falling back to media element sync:",- error,- );+ console.warn("[FrameRenderer] Unable to resolve webcam media element source:", error);+ return false;}-- const webcamSource = await resolveMediaElementSource(webcamUrl);this.cleanupWebcamSource = webcamSource.revoke;const video = document.createElement("video");@@ -2142,27 +2132,32 @@ export class FrameRenderer {video.playsInline = true;video.load();- await new Promise<void>((resolve, reject) => {+ const ready = await new Promise<boolean>((resolve) => {+ let readyTimeout: number | null = null;const onReady = () => {if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {return;}cleanup();- resolve();+ resolve(true);};const onError = () => {cleanup();- reject(new Error("Failed to load webcam source for export"));+ resolve(false);};const cleanup = () => {video.removeEventListener("loadeddata", onReady);video.removeEventListener("canplay", onReady);video.removeEventListener("canplaythrough", onReady);video.removeEventListener("error", onError);+ if (readyTimeout !== null) {+ window.clearTimeout(readyTimeout);+ readyTimeout = null;+ }};if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {- resolve();+ resolve(true);return;}@@ -2170,17 +2165,71 @@ export class FrameRenderer {video.addEventListener("canplay", onReady, { once: true });video.addEventListener("canplaythrough", onReady, { once: true });video.addEventListener("error", onError, { once: true });- }).catch((error) => {- console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error);- this.webcamVideoElement = null;+ readyTimeout = window.setTimeout(() => {+ console.warn(+ `[FrameRenderer] Webcam media element fallback did not become ready within ${WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS}ms`,+ );+ onError();+ }, WEBCAM_MEDIA_ELEMENT_READY_TIMEOUT_MS);});- if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {+ if (ready && video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {this.webcamVideoElement = video;+ this.lastSyncedWebcamTime = null;+ return true;+ }++ console.warn("[FrameRenderer] Webcam overlay unavailable during export");+ this.disposeWebcamMediaElement(video);+ this.clearWebcamMediaElement();+ return false;+ }++ private async setupWebcamSource(): Promise<void> {+ const webcamUrl = this.config.webcamUrl;+ if (!this.config.webcam?.enabled || !webcamUrl) {+ this.webcamForwardFrameSource?.cancel();+ void this.webcamForwardFrameSource?.destroy();+ this.webcamForwardFrameSource = null;+ this.closeWebcamDecodedFrame();+ this.clearWebcamMediaElement();+ this.webcamFrameCacheCanvas = null;+ this.webcamFrameCacheCtx = null;+ this.lastSyncedWebcamTime = null;+ this.lastWebcamCacheRefreshTime = null;+ this.webcamLayoutCache = null;+ this.webcamRenderMode = "hidden";return;}- this.webcamVideoElement = null;+ this.webcamForwardFrameSource?.cancel();+ void this.webcamForwardFrameSource?.destroy();+ this.webcamForwardFrameSource = null;+ this.closeWebcamDecodedFrame();+ this.clearWebcamMediaElement();+ this.webcamFrameCacheCanvas = null;+ this.webcamFrameCacheCtx = null;+ this.lastWebcamCacheRefreshTime = null;+ this.webcamLayoutCache = null;+ this.webcamRenderMode = "hidden";++ try {+ const frameSource = new ForwardFrameSource();+ await frameSource.initialize(webcamUrl);+ this.webcamForwardFrameSource = frameSource;+ this.webcamVideoElement = null;+ this.webcamSeekPromise = null;+ this.lastSyncedWebcamTime = null;+ this.lastWebcamCacheRefreshTime = null;+ return;+ } catch (error) {+ console.warn(+ "[FrameRenderer] Decoder-backed webcam source unavailable during export; falling back to media element sync:",+ error,+ );+ }++ await this.loadWebcamMediaElementSource(webcamUrl);this.lastSyncedWebcamTime = null;}@@ -2457,13 +2506,30 @@ export class FrameRenderer {if (this.webcamForwardFrameSource) {const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null);- const decodedFrame = await this.webcamForwardFrameSource.getFrameAtTime(clampedTime);- this.closeWebcamDecodedFrame();- this.webcamDecodedFrame = decodedFrame;- if (decodedFrame) {- this.lastSyncedWebcamTime = clampedTime;+ try {+ const decodedFrame =+ await this.webcamForwardFrameSource.getFrameAtTime(clampedTime);+ this.closeWebcamDecodedFrame();+ this.webcamDecodedFrame = decodedFrame;+ if (decodedFrame) {+ this.lastSyncedWebcamTime = clampedTime;+ }+ return;+ } catch (error) {+ console.warn(+ "[FrameRenderer] Decoder-backed webcam source failed during export; falling back to media element sync:",+ error,+ );+ this.webcamForwardFrameSource.cancel();+ void this.webcamForwardFrameSource.destroy();+ this.webcamForwardFrameSource = null;+ this.closeWebcamDecodedFrame();+ this.lastSyncedWebcamTime = null;+ const webcamUrl = this.config.webcamUrl;+ if (!webcamUrl || !(await this.loadWebcamMediaElementSource(webcamUrl))) {+ return;+ }}- return;}const webcamVideo = this.webcamVideoElement;