Create polished demo videos without editing skills. Mac/Windows/Linux
Detach Lightning frames for stable WebGPU export
webadderall30e6ef992ba3
diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.tsindex c5a26cb1..7c03221c 100644--- a/src/lib/exporter/modernFrameRenderer.ts+++ b/src/lib/exporter/modernFrameRenderer.ts@@ -330,8 +330,6 @@ function areNearlyEqual(first: number, second: number, epsilon = 0.01): booleanreturn Math.abs(first - second) <= epsilon;}-const VIDEO_FRAME_STARTUP_STAGING_WINDOW_SEC = 2.25;-// Renders video frames with all effects directly into a GPU-backed Pixi scene for export.export class FrameRenderer {private app: Application | null = null;@@ -411,6 +409,14 @@ export class FrameRenderer {private webcamTextureUsesStartupStaging = false;private retainedSceneSourceFrame: VideoFrame | null = null;private retainedSceneTextureFrame: VideoFrame | null = null;+ private retainedBackgroundSourceFrame: VideoFrame | null = null;+ private retainedBackgroundTextureFrame: VideoFrame | null = null;+ private retainedWebcamSourceFrame: VideoFrame | null = null;+ private retainedWebcamTextureFrame: VideoFrame | null = null;+ private retainedSceneBitmapTimestamp: number | null = null;+ private retainedSceneBitmap: ImageBitmap | null = null;+ private retainedBackgroundBitmapTimestamp: number | null = null;+ private retainedBackgroundBitmap: ImageBitmap | null = null;private compositeCanvas: HTMLCanvasElement | null = null;private compositeCtx: CanvasRenderingContext2D | null = null;private lastEmittedClickTimeMs = -1;@@ -751,44 +757,135 @@ export class FrameRenderer {layer.container.visible = true;}- private shouldUseVideoFrameStaging(kind: "scene" | "background" | "webcam"): boolean {- if (this.rendererBackend !== "webgpu") {- return false;+ private getRetainedVideoFrameState(kind: "scene" | "background" | "webcam") {+ if (kind === "scene") {+ return {+ sourceFrame: this.retainedSceneSourceFrame,+ textureFrame: this.retainedSceneTextureFrame,+ };+ }++ if (kind === "background") {+ return {+ sourceFrame: this.retainedBackgroundSourceFrame,+ textureFrame: this.retainedBackgroundTextureFrame,+ };+ }++ return {+ sourceFrame: this.retainedWebcamSourceFrame,+ textureFrame: this.retainedWebcamTextureFrame,+ };+ }++ private setRetainedVideoFrameState(+ kind: "scene" | "background" | "webcam",+ sourceFrame: VideoFrame | null,+ textureFrame: VideoFrame | null,+ ): void {+ if (kind === "scene") {+ this.retainedSceneSourceFrame = sourceFrame;+ this.retainedSceneTextureFrame = textureFrame;+ return;+ }++ if (kind === "background") {+ this.retainedBackgroundSourceFrame = sourceFrame;+ this.retainedBackgroundTextureFrame = textureFrame;+ return;+ }++ this.retainedWebcamSourceFrame = sourceFrame;+ this.retainedWebcamTextureFrame = textureFrame;+ }++ private closeRetainedVideoFrame(kind: "scene" | "background" | "webcam"): void {+ const state = this.getRetainedVideoFrameState(kind);+ if (!state.textureFrame) {+ this.setRetainedVideoFrameState(kind, null, null);+ return;}- return kind !== "scene" && this.currentVideoTime < VIDEO_FRAME_STARTUP_STAGING_WINDOW_SEC;+ state.textureFrame.close();+ this.setRetainedVideoFrameState(kind, null, null);}- private closeRetainedSceneTextureFrame(): void {- if (!this.retainedSceneTextureFrame) {+ private closeRetainedBitmap(kind: "scene" | "background"): void {+ if (kind === "scene") {+ this.retainedSceneBitmap?.close();+ this.retainedSceneBitmap = null;+ this.retainedSceneBitmapTimestamp = null;return;}- this.retainedSceneTextureFrame.close();- this.retainedSceneTextureFrame = null;- this.retainedSceneSourceFrame = null;+ this.retainedBackgroundBitmap?.close();+ this.retainedBackgroundBitmap = null;+ this.retainedBackgroundBitmapTimestamp = null;+ }++ private async resolveDetachedVideoFrameSource(+ frame: VideoFrame,+ kind: "scene" | "background",+ fallbackWidth: number,+ fallbackHeight: number,+ ): Promise<CanvasImageSource | VideoFrame> {+ if (this.rendererBackend !== "webgpu" || typeof createImageBitmap !== "function") {+ return this.stageVideoFrameForTexture(frame, kind, fallbackWidth, fallbackHeight);+ }++ const cachedTimestamp =+ kind === "scene" ? this.retainedSceneBitmapTimestamp : this.retainedBackgroundBitmapTimestamp;+ const cachedBitmap =+ kind === "scene" ? this.retainedSceneBitmap : this.retainedBackgroundBitmap;+ if (cachedTimestamp === frame.timestamp && cachedBitmap) {+ return cachedBitmap;+ }++ try {+ const bitmap = await createImageBitmap(frame);+ this.closeRetainedBitmap(kind);+ if (kind === "scene") {+ this.retainedSceneBitmap = bitmap;+ this.retainedSceneBitmapTimestamp = frame.timestamp;+ } else {+ this.retainedBackgroundBitmap = bitmap;+ this.retainedBackgroundBitmapTimestamp = frame.timestamp;+ }+ return bitmap;+ } catch (error) {+ console.warn(+ `[ModernFrameRenderer] Failed to detach ${kind} VideoFrame to ImageBitmap, falling back to retained VideoFrame:`,+ error,+ );+ return this.stageVideoFrameForTexture(frame, kind, fallbackWidth, fallbackHeight);+ }}- private resolveSceneVideoFrameSource(frame: VideoFrame): CanvasImageSource | VideoFrame {+ private resolveRetainedVideoFrameSource(+ frame: VideoFrame,+ kind: "scene" | "background" | "webcam",+ _fallbackWidth: number,+ _fallbackHeight: number,+ ): CanvasImageSource | VideoFrame {if (this.rendererBackend !== "webgpu") {return frame;}- if (this.retainedSceneSourceFrame === frame && this.retainedSceneTextureFrame) {- return this.retainedSceneTextureFrame;+ const state = this.getRetainedVideoFrameState(kind);+ if (state.sourceFrame === frame && state.textureFrame) {+ return state.textureFrame;}try {const retainedFrame = new VideoFrame(frame, {timestamp: frame.timestamp,});- this.closeRetainedSceneTextureFrame();- this.retainedSceneSourceFrame = frame;- this.retainedSceneTextureFrame = retainedFrame;+ this.closeRetainedVideoFrame(kind);+ this.setRetainedVideoFrameState(kind, frame, retainedFrame);return retainedFrame;} catch (error) {console.warn(- "[ModernFrameRenderer] Failed to retain scene VideoFrame, falling back to staging canvas:",+ `[ModernFrameRenderer] Failed to retain ${kind} VideoFrame, falling back to staging canvas:`,error,);return this.stageVideoFrameForTexture(@@ -859,8 +956,13 @@ export class FrameRenderer {fallbackWidth: number,fallbackHeight: number,): CanvasImageSource | VideoFrame {- if (!this.shouldUseVideoFrameStaging(kind)) {- return frame;+ if (this.rendererBackend === "webgpu") {+ return this.resolveRetainedVideoFrameSource(+ frame,+ kind,+ fallbackWidth,+ fallbackHeight,+ );}const width = Math.max(1, frame.displayWidth || fallbackWidth);@@ -1152,18 +1254,23 @@ export class FrameRenderer {}}- private ensureBackgroundSprite(+ private async ensureBackgroundSprite(source: CanvasImageSource | VideoFrame,sourceWidth: number,sourceHeight: number,- ): void {+ ): Promise<void> {if (!this.backgroundContainer) {return;}const resolvedSource =typeof VideoFrame !== "undefined" && source instanceof VideoFrame- ? this.stageVideoFrameForTexture(source, "background", sourceWidth, sourceHeight)+ ? await this.resolveDetachedVideoFrameSource(+ source,+ "background",+ sourceWidth,+ sourceHeight,+ ): typeof HTMLVideoElement !== "undefined" &&source instanceof HTMLVideoElement &&sourceWidth > 0 &&@@ -1711,7 +1818,7 @@ export class FrameRenderer {restartedFrame.displayWidth,restartedFrame.displayHeight,);- this.ensureBackgroundSprite(+ await this.ensureBackgroundSprite(resolvedBackgroundSource,restartedFrame.displayWidth,restartedFrame.displayHeight,@@ -1735,7 +1842,7 @@ export class FrameRenderer {decodedFrame.displayWidth,decodedFrame.displayHeight,);- this.ensureBackgroundSprite(+ await this.ensureBackgroundSprite(resolvedBackgroundSource,decodedFrame.displayWidth,decodedFrame.displayHeight,@@ -2729,7 +2836,12 @@ export class FrameRenderer {this.currentVideoTime = timestamp / 1_000_000;- const resolvedVideoSource = this.resolveSceneVideoFrameSource(videoFrame);+ const resolvedVideoSource = await this.resolveDetachedVideoFrameSource(+ videoFrame,+ "scene",+ this.config.videoWidth,+ this.config.videoHeight,+ );const usesStartupStaging = resolvedVideoSource !== videoFrame;if (!this.videoSprite) {@@ -3492,7 +3604,11 @@ export class FrameRenderer {this.webcamVideoFrameStagingCtx = null;this.videoTextureUsesStartupStaging = false;this.webcamTextureUsesStartupStaging = false;- this.closeRetainedSceneTextureFrame();+ this.closeRetainedVideoFrame("scene");+ this.closeRetainedVideoFrame("background");+ this.closeRetainedVideoFrame("webcam");+ this.closeRetainedBitmap("scene");+ this.closeRetainedBitmap("background");this.captionCanvas = null;this.captionCtx = null;