Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix: decode webcam frames during export

webadderall17668090ce54
diff --git a/src/lib/exporter/forwardFrameSource.ts b/src/lib/exporter/forwardFrameSource.ts
new file mode 100644
index 00000000..226b28ff
--- /dev/null
+++ b/src/lib/exporter/forwardFrameSource.ts
@@ -0,0 +1,345 @@
+import { WebDemuxer } from "web-demuxer";
+
+export interface ForwardFrameSourceMetadata {
+ width: number;
+ height: number;
+ duration: number;
+ codec: string;
+}
+
+/**
+ * Forward-only decoded frame source for monotonic timestamp access.
+ *
+ * This avoids per-frame HTMLVideoElement seeking during export and returns
+ * the nearest decoded frame for increasing target timestamps.
+ */
+export class ForwardFrameSource {
+ private demuxer: WebDemuxer | null = null;
+ private decoder: VideoDecoder | null = null;
+ private cancelled = false;
+ private metadata: ForwardFrameSourceMetadata | null = null;
+ private pendingFrames: VideoFrame[] = [];
+ private frameResolve: ((frame: VideoFrame | null) => void) | null = null;
+ private decodeError: Error | null = null;
+ private decodeDone = false;
+ private feedPromise: Promise<void> | null = null;
+ private reader: ReadableStreamDefaultReader<EncodedVideoChunk> | null = null;
+ private heldFrame: VideoFrame | null = null;
+ private heldFrameSec = 0;
+ private lastTargetTimeSec = 0;
+
+ private toLocalFilePath(resourceUrl: string): string | null {
+ if (!resourceUrl.startsWith("file:")) {
+ return null;
+ }
+
+ try {
+ const url = new URL(resourceUrl);
+ let filePath = decodeURIComponent(url.pathname);
+ if (/^\/[A-Za-z]:/.test(filePath)) {
+ filePath = filePath.slice(1);
+ }
+ return filePath;
+ } catch {
+ return resourceUrl.replace(/^file:\/\//, "");
+ }
+ }
+
+ private inferMimeType(fileName: string): string {
+ const extension = fileName.split(".").pop()?.toLowerCase();
+ switch (extension) {
+ case "mov":
+ return "video/quicktime";
+ case "webm":
+ return "video/webm";
+ case "mkv":
+ return "video/x-matroska";
+ case "avi":
+ return "video/x-msvideo";
+ case "mp4":
+ default:
+ return "video/mp4";
+ }
+ }
+
+ private async loadVideoFile(resourceUrl: string): Promise<File> {
+ const filename = resourceUrl.split("/").pop() || "video";
+ const localFilePath = this.toLocalFilePath(resourceUrl);
+
+ if (localFilePath) {
+ const result = await window.electronAPI.readLocalFile(localFilePath);
+ if (!result.success || !result.data) {
+ throw new Error(result.error || "Failed to read local video file");
+ }
+
+ const bytes =
+ result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);
+ const arrayBuffer = bytes.buffer.slice(
+ bytes.byteOffset,
+ bytes.byteOffset + bytes.byteLength,
+ ) as ArrayBuffer;
+ return new File([arrayBuffer], filename, {
+ type: this.inferMimeType(filename),
+ });
+ }
+
+ const response = await fetch(resourceUrl);
+ if (!response.ok) {
+ throw new Error(
+ `Failed to load video resource: ${response.status} ${response.statusText}`,
+ );
+ }
+
+ const blob = await response.blob();
+ return new File([blob], filename, {
+ type: blob.type || this.inferMimeType(filename),
+ });
+ }
+
+ private resolveVideoResourceUrl(videoUrl: string): string {
+ if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) {
+ return videoUrl;
+ }
+
+ if (videoUrl.startsWith("/")) {
+ return `file://${encodeURI(videoUrl)}`;
+ }
+
+ return videoUrl;
+ }
+
+ async initialize(videoUrl: string): Promise<ForwardFrameSourceMetadata> {
+ const resourceUrl = this.resolveVideoResourceUrl(videoUrl);
+ const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
+ this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
+
+ const file = await this.loadVideoFile(resourceUrl);
+ await this.demuxer.load(file);
+
+ const mediaInfo = await this.demuxer.getMediaInfo();
+ const videoStream = mediaInfo.streams.find(
+ (stream) => stream.codec_type_string === "video",
+ );
+
+ this.metadata = {
+ width: videoStream?.width || 0,
+ height: videoStream?.height || 0,
+ duration: mediaInfo.duration,
+ codec: videoStream?.codec_string || "unknown",
+ };
+
+ await this.startDecoder();
+ return this.metadata;
+ }
+
+ private async startDecoder(): Promise<void> {
+ if (!this.demuxer || !this.metadata) {
+ throw new Error("Must call initialize() before starting decoder");
+ }
+
+ const decoderConfig = await this.demuxer.getDecoderConfig("video");
+ const codec = this.metadata.codec.toLowerCase();
+ const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
+
+ this.decoder = new VideoDecoder({
+ output: (frame: VideoFrame) => {
+ if (this.frameResolve) {
+ const resolve = this.frameResolve;
+ this.frameResolve = null;
+ resolve(frame);
+ } else {
+ this.pendingFrames.push(frame);
+ }
+ },
+ error: (error: DOMException) => {
+ this.decodeError = new Error(`VideoDecoder error: ${error.message}`);
+ if (this.frameResolve) {
+ const resolve = this.frameResolve;
+ this.frameResolve = null;
+ resolve(null);
+ }
+ },
+ });
+
+ const preferredDecoderConfig = shouldPreferSoftwareDecode
+ ? {
+ ...decoderConfig,
+ hardwareAcceleration: "prefer-software" as const,
+ }
+ : decoderConfig;
+
+ try {
+ this.decoder.configure(preferredDecoderConfig);
+ } catch (error) {
+ if (!shouldPreferSoftwareDecode) {
+ throw error;
+ }
+ this.decoder.configure(decoderConfig);
+ }
+
+ const readEndSec = Math.max(this.metadata.duration, 0) + 0.5;
+ this.reader = this.demuxer.read("video", 0, readEndSec).getReader();
+
+ this.feedPromise = (async () => {
+ try {
+ while (!this.cancelled) {
+ const { done, value: chunk } = await this.reader!.read();
+ if (done || !chunk) {
+ break;
+ }
+
+ while (
+ (this.decoder!.decodeQueueSize > 10 || this.pendingFrames.length > 24) &&
+ !this.cancelled
+ ) {
+ await new Promise((resolve) => setTimeout(resolve, 1));
+ }
+
+ if (this.cancelled) {
+ break;
+ }
+
+ this.decoder!.decode(chunk);
+ }
+
+ if (!this.cancelled && this.decoder?.state === "configured") {
+ await this.decoder.flush();
+ }
+ } catch (error) {
+ this.decodeError =
+ error instanceof Error ? error : new Error(String(error));
+ } finally {
+ this.decodeDone = true;
+ if (this.frameResolve) {
+ const resolve = this.frameResolve;
+ this.frameResolve = null;
+ resolve(null);
+ }
+ }
+ })();
+ }
+
+ private getNextFrame(): Promise<VideoFrame | null> {
+ if (this.decodeError) {
+ throw this.decodeError;
+ }
+
+ if (this.pendingFrames.length > 0) {
+ return Promise.resolve(this.pendingFrames.shift()!);
+ }
+
+ if (this.decodeDone) {
+ return Promise.resolve(null);
+ }
+
+ return new Promise((resolve) => {
+ this.frameResolve = resolve;
+ });
+ }
+
+ async getFrameAtTime(targetTimeSec: number): Promise<VideoFrame | null> {
+ if (!this.metadata) {
+ throw new Error("Frame source not initialized");
+ }
+
+ const clampedTargetTime = Math.max(0, Math.min(targetTimeSec, this.metadata.duration || targetTimeSec));
+ if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) {
+ throw new Error("ForwardFrameSource only supports increasing timestamps");
+ }
+ this.lastTargetTimeSec = clampedTargetTime;
+
+ if (!this.heldFrame) {
+ const firstFrame = await this.getNextFrame();
+ if (!firstFrame) {
+ return null;
+ }
+ this.heldFrame = firstFrame;
+ this.heldFrameSec = firstFrame.timestamp / 1_000_000;
+ }
+
+ while (!this.cancelled) {
+ const nextFrame = await this.getNextFrame();
+ if (!nextFrame) {
+ return new VideoFrame(this.heldFrame, {
+ timestamp: this.heldFrame.timestamp,
+ });
+ }
+
+ const nextFrameSec = nextFrame.timestamp / 1_000_000;
+ const handoffBoundarySec = (this.heldFrameSec + nextFrameSec) / 2;
+ if (clampedTargetTime <= handoffBoundarySec) {
+ this.pendingFrames.unshift(nextFrame);
+ return new VideoFrame(this.heldFrame, {
+ timestamp: this.heldFrame.timestamp,
+ });
+ }
+
+ this.heldFrame.close();
+ this.heldFrame = nextFrame;
+ this.heldFrameSec = nextFrameSec;
+ }
+
+ return null;
+ }
+
+ cancel(): void {
+ this.cancelled = true;
+ }
+
+ async destroy(): Promise<void> {
+ this.cancelled = true;
+
+ if (this.reader) {
+ try {
+ await this.reader.cancel();
+ } catch {
+ // Ignore cancellation errors during shutdown.
+ }
+ this.reader = null;
+ }
+
+ if (this.feedPromise) {
+ try {
+ await this.feedPromise;
+ } catch {
+ // Decoder errors are already surfaced through getFrameAtTime.
+ }
+ this.feedPromise = null;
+ }
+
+ if (this.heldFrame) {
+ this.heldFrame.close();
+ this.heldFrame = null;
+ }
+
+ for (const frame of this.pendingFrames) {
+ frame.close();
+ }
+ this.pendingFrames = [];
+
+ if (this.decoder) {
+ try {
+ if (this.decoder.state === "configured") {
+ this.decoder.close();
+ }
+ } catch {
+ // Ignore decoder shutdown errors.
+ }
+ this.decoder = null;
+ }
+
+ if (this.demuxer) {
+ try {
+ this.demuxer.destroy();
+ } catch {
+ // Ignore demuxer shutdown errors.
+ }
+ this.demuxer = null;
+ }
+
+ this.metadata = null;
+ this.decodeDone = false;
+ this.decodeError = null;
+ this.lastTargetTimeSec = 0;
+ }
+}
\ No newline at end of file
diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts
index a8cff4d1..b024b6e1 100644
--- a/src/lib/exporter/frameRenderer.ts
+++ b/src/lib/exporter/frameRenderer.ts
@@ -37,6 +37,7 @@ import {
preloadCursorAssets,
} from "@/components/video-editor/videoPlayback/cursorRenderer";
import { getWebcamOverlaySizePx } from "@/components/video-editor/webcamOverlay";
+import { ForwardFrameSource } from "./forwardFrameSource";
interface FrameRenderConfig {
width: number;
@@ -113,11 +114,12 @@ export class FrameRenderer {
private currentVideoTime = 0;
private lastMotionVector = { x: 0, y: 0 };
private cursorOverlay: PixiCursorOverlay | null = null;
- private webcamVideoElement: HTMLVideoElement | null = null;
- private webcamSeekPromise: Promise<void> | null = null;
+ private webcamFrameSource: ForwardFrameSource | null = null;
+ private webcamCurrentFrame: VideoFrame | null = null;
private webcamFrameCacheCanvas: HTMLCanvasElement | null = null;
private webcamFrameCacheCtx: CanvasRenderingContext2D | null = null;
- private lastSyncedWebcamTime: number | null = null;
+ private webcamFrameWidth = 0;
+ private webcamFrameHeight = 0;
constructor(config: FrameRenderConfig) {
this.config = config;
@@ -422,157 +424,71 @@ export class FrameRenderer {
private async setupWebcamSource(): Promise<void> {
const webcamUrl = this.config.webcamUrl;
if (!this.config.webcam?.enabled || !webcamUrl) {
- this.webcamVideoElement = null;
+ this.webcamFrameSource = null;
+ this.webcamCurrentFrame?.close();
+ this.webcamCurrentFrame = null;
this.webcamFrameCacheCanvas = null;
this.webcamFrameCacheCtx = null;
- this.lastSyncedWebcamTime = null;
+ this.webcamFrameWidth = 0;
+ this.webcamFrameHeight = 0;
return;
}
- const video = document.createElement("video");
- video.src = webcamUrl;
- video.muted = true;
- video.preload = "auto";
- video.playsInline = true;
- video.load();
-
- await new Promise<void>((resolve, reject) => {
- const onReady = () => {
- if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
- return;
- }
- cleanup();
- resolve();
- };
- const onError = () => {
- cleanup();
- reject(new Error("Failed to load webcam source for export"));
- };
- const cleanup = () => {
- video.removeEventListener("loadeddata", onReady);
- video.removeEventListener("canplay", onReady);
- video.removeEventListener("canplaythrough", onReady);
- video.removeEventListener("error", onError);
- };
- if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
- resolve();
- return;
- }
- video.addEventListener("loadeddata", onReady, { once: true });
- video.addEventListener("canplay", onReady, { once: true });
- video.addEventListener("canplaythrough", onReady, { once: true });
- video.addEventListener("error", onError, { once: true });
- }).catch((error) => {
- console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error);
- this.webcamVideoElement = null;
- });
+ const frameSource = new ForwardFrameSource();
- if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
- this.webcamVideoElement = video;
- return;
+ try {
+ await frameSource.initialize(webcamUrl);
+ this.webcamFrameSource = frameSource;
+ } catch (error) {
+ console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error);
+ await frameSource.destroy();
+ this.webcamFrameSource = null;
+ this.webcamCurrentFrame?.close();
+ this.webcamCurrentFrame = null;
+ this.webcamFrameCacheCanvas = null;
+ this.webcamFrameCacheCtx = null;
+ this.webcamFrameWidth = 0;
+ this.webcamFrameHeight = 0;
}
-
- this.webcamVideoElement = null;
- this.webcamFrameCacheCanvas = null;
- this.webcamFrameCacheCtx = null;
- this.lastSyncedWebcamTime = null;
}
private async syncWebcamFrame(targetTime: number): Promise<void> {
- const webcamVideo = this.webcamVideoElement;
- if (!webcamVideo) {
+ const webcamFrameSource = this.webcamFrameSource;
+ if (!webcamFrameSource) {
return;
}
- const duration = Number.isFinite(webcamVideo.duration)
- ? webcamVideo.duration
- : targetTime;
- const clampedTime = Math.max(0, Math.min(targetTime, duration || targetTime));
-
- if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.008) {
- this.lastSyncedWebcamTime = clampedTime;
+ const nextFrame = await webcamFrameSource.getFrameAtTime(targetTime);
+ if (!nextFrame) {
return;
}
- if (this.webcamSeekPromise) {
- await this.webcamSeekPromise;
- }
+ this.webcamCurrentFrame?.close();
+ this.webcamCurrentFrame = nextFrame;
- this.webcamSeekPromise = new Promise<void>((resolve) => {
- let settled = false;
- let fallbackTimeout: number | null = null;
- const waitForPresentedFrame = () => {
- requestAnimationFrame(() => {
- finish();
- });
- };
- const finish = () => {
- if (settled) {
- return;
- }
- settled = true;
- if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.02) {
- this.lastSyncedWebcamTime = clampedTime;
- }
- cleanup();
- resolve();
- };
- const handleMediaReady = () => {
- if (
- !webcamVideo.seeking &&
- Math.abs(webcamVideo.currentTime - clampedTime) <= 0.01 &&
- webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA
- ) {
- waitForPresentedFrame();
- }
- };
- const cleanup = () => {
- webcamVideo.removeEventListener("seeked", waitForPresentedFrame);
- webcamVideo.removeEventListener("loadeddata", handleMediaReady);
- webcamVideo.removeEventListener("canplay", handleMediaReady);
- webcamVideo.removeEventListener("error", finish);
- if (fallbackTimeout !== null) {
- window.clearTimeout(fallbackTimeout);
- }
- };
-
- webcamVideo.addEventListener("seeked", waitForPresentedFrame, {
- once: true,
- });
- webcamVideo.addEventListener("loadeddata", handleMediaReady, {
- once: true,
- });
- webcamVideo.addEventListener("canplay", handleMediaReady, {
- once: true,
- });
- webcamVideo.addEventListener("error", finish, {
- once: true,
- });
- fallbackTimeout = window.setTimeout(() => {
- finish();
- }, 250);
-
- try {
- webcamVideo.currentTime = clampedTime;
- } catch {
- finish();
- return;
- }
-
- if (
- !webcamVideo.seeking &&
- Math.abs(webcamVideo.currentTime - clampedTime) <= 0.001 &&
- webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA
- ) {
- waitForPresentedFrame();
- }
- });
+ const frameWidth =
+ nextFrame.displayWidth || nextFrame.codedWidth || this.webcamFrameWidth;
+ const frameHeight =
+ nextFrame.displayHeight || nextFrame.codedHeight || this.webcamFrameHeight;
+ if (frameWidth <= 0 || frameHeight <= 0) {
+ return;
+ }
- try {
- await this.webcamSeekPromise;
- } finally {
- this.webcamSeekPromise = null;
+ if (
+ !this.webcamFrameCacheCanvas ||
+ this.webcamFrameWidth !== frameWidth ||
+ this.webcamFrameHeight !== frameHeight
+ ) {
+ this.webcamFrameCacheCanvas = document.createElement("canvas");
+ this.webcamFrameCacheCanvas.width = frameWidth;
+ this.webcamFrameCacheCanvas.height = frameHeight;
+ this.webcamFrameCacheCtx = this.webcamFrameCacheCanvas.getContext("2d");
+ this.webcamFrameWidth = frameWidth;
+ this.webcamFrameHeight = frameHeight;
}
+
+ this.webcamFrameCacheCtx?.clearRect(0, 0, frameWidth, frameHeight);
+ this.webcamFrameCacheCtx?.drawImage(nextFrame, 0, 0, frameWidth, frameHeight);
}
async renderFrame(videoFrame: VideoFrame, timestamp: number): Promise<void> {
@@ -582,7 +498,7 @@ export class FrameRenderer {
this.currentVideoTime = timestamp / 1000000;
- if (this.webcamVideoElement) {
+ if (this.webcamFrameSource) {
const targetTime = Math.max(0, this.currentVideoTime);
await this.syncWebcamFrame(targetTime);
}
@@ -944,8 +860,7 @@ export class FrameRenderer {
height: number,
): void {
const webcam = this.config.webcam;
- const webcamVideo = this.webcamVideoElement;
- if (!webcam?.enabled || !webcamVideo || webcamVideo.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
+ if (!webcam?.enabled || !this.webcamFrameCacheCanvas) {
return;
}
@@ -970,45 +885,7 @@ export class FrameRenderer {
return;
}
- const canRefreshCache =
- webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA &&
- !webcamVideo.seeking &&
- this.lastSyncedWebcamTime !== null &&
- Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime) <= 0.02 &&
- Math.abs(webcamVideo.currentTime - this.currentVideoTime) <= 0.02 &&
- webcamVideo.videoWidth > 0 &&
- webcamVideo.videoHeight > 0;
-
- if (canRefreshCache) {
- if (
- !this.webcamFrameCacheCanvas ||
- this.webcamFrameCacheCanvas.width !== webcamVideo.videoWidth ||
- this.webcamFrameCacheCanvas.height !== webcamVideo.videoHeight
- ) {
- this.webcamFrameCacheCanvas = document.createElement("canvas");
- this.webcamFrameCacheCanvas.width = webcamVideo.videoWidth;
- this.webcamFrameCacheCanvas.height = webcamVideo.videoHeight;
- this.webcamFrameCacheCtx = this.webcamFrameCacheCanvas.getContext("2d");
- }
-
- this.webcamFrameCacheCtx?.clearRect(
- 0,
- 0,
- this.webcamFrameCacheCanvas!.width,
- this.webcamFrameCacheCanvas!.height,
- );
- this.webcamFrameCacheCtx?.drawImage(
- webcamVideo,
- 0,
- 0,
- this.webcamFrameCacheCanvas!.width,
- this.webcamFrameCacheCanvas!.height,
- );
- }
-
- const webcamFrameSource = canRefreshCache
- ? webcamVideo
- : this.webcamFrameCacheCanvas;
+ const webcamFrameSource = this.webcamFrameCacheCanvas;
if (!webcamFrameSource) {
return;
}
@@ -1084,18 +961,19 @@ export class FrameRenderer {
this.cursorOverlay.destroy();
this.cursorOverlay = null;
}
+ this.webcamCurrentFrame?.close();
+ this.webcamCurrentFrame = null;
+ if (this.webcamFrameSource) {
+ void this.webcamFrameSource.destroy();
+ this.webcamFrameSource = null;
+ }
this.shadowCanvas = null;
this.shadowCtx = null;
this.compositeCanvas = null;
this.compositeCtx = null;
- if (this.webcamVideoElement) {
- this.webcamVideoElement.pause();
- this.webcamVideoElement.src = "";
- this.webcamVideoElement.load();
- this.webcamVideoElement = null;
- }
this.webcamFrameCacheCanvas = null;
this.webcamFrameCacheCtx = null;
- this.lastSyncedWebcamTime = null;
+ this.webcamFrameWidth = 0;
+ this.webcamFrameHeight = 0;
}
}