Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

Detach Lightning frames for stable WebGPU export

webadderall30e6ef992ba3
diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts
index c5a26cb1..7c03221c 100644
--- a/src/lib/exporter/modernFrameRenderer.ts
+++ b/src/lib/exporter/modernFrameRenderer.ts
@@ -330,8 +330,6 @@ function areNearlyEqual(first: number, second: number, epsilon = 0.01): boolean
return Math.abs(first - second) <= epsilon;
}
-const VIDEO_FRAME_STARTUP_STAGING_WINDOW_SEC = 2.25;
-
// Renders video frames with all effects directly into a GPU-backed Pixi scene for export.
export class FrameRenderer {
private app: Application | null = null;
@@ -411,6 +409,14 @@ export class FrameRenderer {
private webcamTextureUsesStartupStaging = false;
private retainedSceneSourceFrame: VideoFrame | null = null;
private retainedSceneTextureFrame: VideoFrame | null = null;
+ private retainedBackgroundSourceFrame: VideoFrame | null = null;
+ private retainedBackgroundTextureFrame: VideoFrame | null = null;
+ private retainedWebcamSourceFrame: VideoFrame | null = null;
+ private retainedWebcamTextureFrame: VideoFrame | null = null;
+ private retainedSceneBitmapTimestamp: number | null = null;
+ private retainedSceneBitmap: ImageBitmap | null = null;
+ private retainedBackgroundBitmapTimestamp: number | null = null;
+ private retainedBackgroundBitmap: ImageBitmap | null = null;
private compositeCanvas: HTMLCanvasElement | null = null;
private compositeCtx: CanvasRenderingContext2D | null = null;
private lastEmittedClickTimeMs = -1;
@@ -751,44 +757,135 @@ export class FrameRenderer {
layer.container.visible = true;
}
- private shouldUseVideoFrameStaging(kind: "scene" | "background" | "webcam"): boolean {
- if (this.rendererBackend !== "webgpu") {
- return false;
+ private getRetainedVideoFrameState(kind: "scene" | "background" | "webcam") {
+ if (kind === "scene") {
+ return {
+ sourceFrame: this.retainedSceneSourceFrame,
+ textureFrame: this.retainedSceneTextureFrame,
+ };
+ }
+
+ if (kind === "background") {
+ return {
+ sourceFrame: this.retainedBackgroundSourceFrame,
+ textureFrame: this.retainedBackgroundTextureFrame,
+ };
+ }
+
+ return {
+ sourceFrame: this.retainedWebcamSourceFrame,
+ textureFrame: this.retainedWebcamTextureFrame,
+ };
+ }
+
+ private setRetainedVideoFrameState(
+ kind: "scene" | "background" | "webcam",
+ sourceFrame: VideoFrame | null,
+ textureFrame: VideoFrame | null,
+ ): void {
+ if (kind === "scene") {
+ this.retainedSceneSourceFrame = sourceFrame;
+ this.retainedSceneTextureFrame = textureFrame;
+ return;
+ }
+
+ if (kind === "background") {
+ this.retainedBackgroundSourceFrame = sourceFrame;
+ this.retainedBackgroundTextureFrame = textureFrame;
+ return;
+ }
+
+ this.retainedWebcamSourceFrame = sourceFrame;
+ this.retainedWebcamTextureFrame = textureFrame;
+ }
+
+ private closeRetainedVideoFrame(kind: "scene" | "background" | "webcam"): void {
+ const state = this.getRetainedVideoFrameState(kind);
+ if (!state.textureFrame) {
+ this.setRetainedVideoFrameState(kind, null, null);
+ return;
}
- return kind !== "scene" && this.currentVideoTime < VIDEO_FRAME_STARTUP_STAGING_WINDOW_SEC;
+ state.textureFrame.close();
+ this.setRetainedVideoFrameState(kind, null, null);
}
- private closeRetainedSceneTextureFrame(): void {
- if (!this.retainedSceneTextureFrame) {
+ private closeRetainedBitmap(kind: "scene" | "background"): void {
+ if (kind === "scene") {
+ this.retainedSceneBitmap?.close();
+ this.retainedSceneBitmap = null;
+ this.retainedSceneBitmapTimestamp = null;
return;
}
- this.retainedSceneTextureFrame.close();
- this.retainedSceneTextureFrame = null;
- this.retainedSceneSourceFrame = null;
+ this.retainedBackgroundBitmap?.close();
+ this.retainedBackgroundBitmap = null;
+ this.retainedBackgroundBitmapTimestamp = null;
+ }
+
+ private async resolveDetachedVideoFrameSource(
+ frame: VideoFrame,
+ kind: "scene" | "background",
+ fallbackWidth: number,
+ fallbackHeight: number,
+ ): Promise<CanvasImageSource | VideoFrame> {
+ if (this.rendererBackend !== "webgpu" || typeof createImageBitmap !== "function") {
+ return this.stageVideoFrameForTexture(frame, kind, fallbackWidth, fallbackHeight);
+ }
+
+ const cachedTimestamp =
+ kind === "scene" ? this.retainedSceneBitmapTimestamp : this.retainedBackgroundBitmapTimestamp;
+ const cachedBitmap =
+ kind === "scene" ? this.retainedSceneBitmap : this.retainedBackgroundBitmap;
+ if (cachedTimestamp === frame.timestamp && cachedBitmap) {
+ return cachedBitmap;
+ }
+
+ try {
+ const bitmap = await createImageBitmap(frame);
+ this.closeRetainedBitmap(kind);
+ if (kind === "scene") {
+ this.retainedSceneBitmap = bitmap;
+ this.retainedSceneBitmapTimestamp = frame.timestamp;
+ } else {
+ this.retainedBackgroundBitmap = bitmap;
+ this.retainedBackgroundBitmapTimestamp = frame.timestamp;
+ }
+ return bitmap;
+ } catch (error) {
+ console.warn(
+ `[ModernFrameRenderer] Failed to detach ${kind} VideoFrame to ImageBitmap, falling back to retained VideoFrame:`,
+ error,
+ );
+ return this.stageVideoFrameForTexture(frame, kind, fallbackWidth, fallbackHeight);
+ }
}
- private resolveSceneVideoFrameSource(frame: VideoFrame): CanvasImageSource | VideoFrame {
+ private resolveRetainedVideoFrameSource(
+ frame: VideoFrame,
+ kind: "scene" | "background" | "webcam",
+ _fallbackWidth: number,
+ _fallbackHeight: number,
+ ): CanvasImageSource | VideoFrame {
if (this.rendererBackend !== "webgpu") {
return frame;
}
- if (this.retainedSceneSourceFrame === frame && this.retainedSceneTextureFrame) {
- return this.retainedSceneTextureFrame;
+ const state = this.getRetainedVideoFrameState(kind);
+ if (state.sourceFrame === frame && state.textureFrame) {
+ return state.textureFrame;
}
try {
const retainedFrame = new VideoFrame(frame, {
timestamp: frame.timestamp,
});
- this.closeRetainedSceneTextureFrame();
- this.retainedSceneSourceFrame = frame;
- this.retainedSceneTextureFrame = retainedFrame;
+ this.closeRetainedVideoFrame(kind);
+ this.setRetainedVideoFrameState(kind, frame, retainedFrame);
return retainedFrame;
} catch (error) {
console.warn(
- "[ModernFrameRenderer] Failed to retain scene VideoFrame, falling back to staging canvas:",
+ `[ModernFrameRenderer] Failed to retain ${kind} VideoFrame, falling back to staging canvas:`,
error,
);
return this.stageVideoFrameForTexture(
@@ -859,8 +956,13 @@ export class FrameRenderer {
fallbackWidth: number,
fallbackHeight: number,
): CanvasImageSource | VideoFrame {
- if (!this.shouldUseVideoFrameStaging(kind)) {
- return frame;
+ if (this.rendererBackend === "webgpu") {
+ return this.resolveRetainedVideoFrameSource(
+ frame,
+ kind,
+ fallbackWidth,
+ fallbackHeight,
+ );
}
const width = Math.max(1, frame.displayWidth || fallbackWidth);
@@ -1152,18 +1254,23 @@ export class FrameRenderer {
}
}
- private ensureBackgroundSprite(
+ private async ensureBackgroundSprite(
source: CanvasImageSource | VideoFrame,
sourceWidth: number,
sourceHeight: number,
- ): void {
+ ): Promise<void> {
if (!this.backgroundContainer) {
return;
}
const resolvedSource =
typeof VideoFrame !== "undefined" && source instanceof VideoFrame
- ? this.stageVideoFrameForTexture(source, "background", sourceWidth, sourceHeight)
+ ? await this.resolveDetachedVideoFrameSource(
+ source,
+ "background",
+ sourceWidth,
+ sourceHeight,
+ )
: typeof HTMLVideoElement !== "undefined" &&
source instanceof HTMLVideoElement &&
sourceWidth > 0 &&
@@ -1711,7 +1818,7 @@ export class FrameRenderer {
restartedFrame.displayWidth,
restartedFrame.displayHeight,
);
- this.ensureBackgroundSprite(
+ await this.ensureBackgroundSprite(
resolvedBackgroundSource,
restartedFrame.displayWidth,
restartedFrame.displayHeight,
@@ -1735,7 +1842,7 @@ export class FrameRenderer {
decodedFrame.displayWidth,
decodedFrame.displayHeight,
);
- this.ensureBackgroundSprite(
+ await this.ensureBackgroundSprite(
resolvedBackgroundSource,
decodedFrame.displayWidth,
decodedFrame.displayHeight,
@@ -2729,7 +2836,12 @@ export class FrameRenderer {
this.currentVideoTime = timestamp / 1_000_000;
- const resolvedVideoSource = this.resolveSceneVideoFrameSource(videoFrame);
+ const resolvedVideoSource = await this.resolveDetachedVideoFrameSource(
+ videoFrame,
+ "scene",
+ this.config.videoWidth,
+ this.config.videoHeight,
+ );
const usesStartupStaging = resolvedVideoSource !== videoFrame;
if (!this.videoSprite) {
@@ -3492,7 +3604,11 @@ export class FrameRenderer {
this.webcamVideoFrameStagingCtx = null;
this.videoTextureUsesStartupStaging = false;
this.webcamTextureUsesStartupStaging = false;
- this.closeRetainedSceneTextureFrame();
+ this.closeRetainedVideoFrame("scene");
+ this.closeRetainedVideoFrame("background");
+ this.closeRetainedVideoFrame("webcam");
+ this.closeRetainedBitmap("scene");
+ this.closeRetainedBitmap("background");
this.captionCanvas = null;
this.captionCtx = null;