Create polished demo videos without editing skills. Mac/Windows/Linux
fix: decode webcam frames during export
webadderall17668090ce54
diff --git a/src/lib/exporter/forwardFrameSource.ts b/src/lib/exporter/forwardFrameSource.tsnew file mode 100644index 00000000..226b28ff--- /dev/null+++ b/src/lib/exporter/forwardFrameSource.ts@@ -0,0 +1,345 @@+import { WebDemuxer } from "web-demuxer";++export interface ForwardFrameSourceMetadata {+ width: number;+ height: number;+ duration: number;+ codec: string;+}++/**+ * Forward-only decoded frame source for monotonic timestamp access.+ *+ * This avoids per-frame HTMLVideoElement seeking during export and returns+ * the nearest decoded frame for increasing target timestamps.+ */+export class ForwardFrameSource {+ private demuxer: WebDemuxer | null = null;+ private decoder: VideoDecoder | null = null;+ private cancelled = false;+ private metadata: ForwardFrameSourceMetadata | null = null;+ private pendingFrames: VideoFrame[] = [];+ private frameResolve: ((frame: VideoFrame | null) => void) | null = null;+ private decodeError: Error | null = null;+ private decodeDone = false;+ private feedPromise: Promise<void> | null = null;+ private reader: ReadableStreamDefaultReader<EncodedVideoChunk> | null = null;+ private heldFrame: VideoFrame | null = null;+ private heldFrameSec = 0;+ private lastTargetTimeSec = 0;++ private toLocalFilePath(resourceUrl: string): string | null {+ if (!resourceUrl.startsWith("file:")) {+ return null;+ }++ try {+ const url = new URL(resourceUrl);+ let filePath = decodeURIComponent(url.pathname);+ if (/^\/[A-Za-z]:/.test(filePath)) {+ filePath = filePath.slice(1);+ }+ return filePath;+ } catch {+ return resourceUrl.replace(/^file:\/\//, "");+ }+ }++ private inferMimeType(fileName: string): string {+ const extension = fileName.split(".").pop()?.toLowerCase();+ switch (extension) {+ case "mov":+ return "video/quicktime";+ case "webm":+ return "video/webm";+ case "mkv":+ return "video/x-matroska";+ case "avi":+ return "video/x-msvideo";+ case "mp4":+ default:+ return "video/mp4";+ }+ }++ private async loadVideoFile(resourceUrl: string): Promise<File> {+ const filename = resourceUrl.split("/").pop() || "video";+ const localFilePath = this.toLocalFilePath(resourceUrl);++ if (localFilePath) {+ const result = await window.electronAPI.readLocalFile(localFilePath);+ if (!result.success || !result.data) {+ throw new Error(result.error || "Failed to read local video file");+ }++ const bytes =+ result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data);+ const arrayBuffer = bytes.buffer.slice(+ bytes.byteOffset,+ bytes.byteOffset + bytes.byteLength,+ ) as ArrayBuffer;+ return new File([arrayBuffer], filename, {+ type: this.inferMimeType(filename),+ });+ }++ const response = await fetch(resourceUrl);+ if (!response.ok) {+ throw new Error(+ `Failed to load video resource: ${response.status} ${response.statusText}`,+ );+ }++ const blob = await response.blob();+ return new File([blob], filename, {+ type: blob.type || this.inferMimeType(filename),+ });+ }++ private resolveVideoResourceUrl(videoUrl: string): string {+ if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) {+ return videoUrl;+ }++ if (videoUrl.startsWith("/")) {+ return `file://${encodeURI(videoUrl)}`;+ }++ return videoUrl;+ }++ async initialize(videoUrl: string): Promise<ForwardFrameSourceMetadata> {+ const resourceUrl = this.resolveVideoResourceUrl(videoUrl);+ const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;+ this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });++ const file = await this.loadVideoFile(resourceUrl);+ await this.demuxer.load(file);++ const mediaInfo = await this.demuxer.getMediaInfo();+ const videoStream = mediaInfo.streams.find(+ (stream) => stream.codec_type_string === "video",+ );++ this.metadata = {+ width: videoStream?.width || 0,+ height: videoStream?.height || 0,+ duration: mediaInfo.duration,+ codec: videoStream?.codec_string || "unknown",+ };++ await this.startDecoder();+ return this.metadata;+ }++ private async startDecoder(): Promise<void> {+ if (!this.demuxer || !this.metadata) {+ throw new Error("Must call initialize() before starting decoder");+ }++ const decoderConfig = await this.demuxer.getDecoderConfig("video");+ const codec = this.metadata.codec.toLowerCase();+ const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");++ this.decoder = new VideoDecoder({+ output: (frame: VideoFrame) => {+ if (this.frameResolve) {+ const resolve = this.frameResolve;+ this.frameResolve = null;+ resolve(frame);+ } else {+ this.pendingFrames.push(frame);+ }+ },+ error: (error: DOMException) => {+ this.decodeError = new Error(`VideoDecoder error: ${error.message}`);+ if (this.frameResolve) {+ const resolve = this.frameResolve;+ this.frameResolve = null;+ resolve(null);+ }+ },+ });++ const preferredDecoderConfig = shouldPreferSoftwareDecode+ ? {+ ...decoderConfig,+ hardwareAcceleration: "prefer-software" as const,+ }+ : decoderConfig;++ try {+ this.decoder.configure(preferredDecoderConfig);+ } catch (error) {+ if (!shouldPreferSoftwareDecode) {+ throw error;+ }+ this.decoder.configure(decoderConfig);+ }++ const readEndSec = Math.max(this.metadata.duration, 0) + 0.5;+ this.reader = this.demuxer.read("video", 0, readEndSec).getReader();++ this.feedPromise = (async () => {+ try {+ while (!this.cancelled) {+ const { done, value: chunk } = await this.reader!.read();+ if (done || !chunk) {+ break;+ }++ while (+ (this.decoder!.decodeQueueSize > 10 || this.pendingFrames.length > 24) &&+ !this.cancelled+ ) {+ await new Promise((resolve) => setTimeout(resolve, 1));+ }++ if (this.cancelled) {+ break;+ }++ this.decoder!.decode(chunk);+ }++ if (!this.cancelled && this.decoder?.state === "configured") {+ await this.decoder.flush();+ }+ } catch (error) {+ this.decodeError =+ error instanceof Error ? error : new Error(String(error));+ } finally {+ this.decodeDone = true;+ if (this.frameResolve) {+ const resolve = this.frameResolve;+ this.frameResolve = null;+ resolve(null);+ }+ }+ })();+ }++ private getNextFrame(): Promise<VideoFrame | null> {+ if (this.decodeError) {+ throw this.decodeError;+ }++ if (this.pendingFrames.length > 0) {+ return Promise.resolve(this.pendingFrames.shift()!);+ }++ if (this.decodeDone) {+ return Promise.resolve(null);+ }++ return new Promise((resolve) => {+ this.frameResolve = resolve;+ });+ }++ async getFrameAtTime(targetTimeSec: number): Promise<VideoFrame | null> {+ if (!this.metadata) {+ throw new Error("Frame source not initialized");+ }++ const clampedTargetTime = Math.max(0, Math.min(targetTimeSec, this.metadata.duration || targetTimeSec));+ if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) {+ throw new Error("ForwardFrameSource only supports increasing timestamps");+ }+ this.lastTargetTimeSec = clampedTargetTime;++ if (!this.heldFrame) {+ const firstFrame = await this.getNextFrame();+ if (!firstFrame) {+ return null;+ }+ this.heldFrame = firstFrame;+ this.heldFrameSec = firstFrame.timestamp / 1_000_000;+ }++ while (!this.cancelled) {+ const nextFrame = await this.getNextFrame();+ if (!nextFrame) {+ return new VideoFrame(this.heldFrame, {+ timestamp: this.heldFrame.timestamp,+ });+ }++ const nextFrameSec = nextFrame.timestamp / 1_000_000;+ const handoffBoundarySec = (this.heldFrameSec + nextFrameSec) / 2;+ if (clampedTargetTime <= handoffBoundarySec) {+ this.pendingFrames.unshift(nextFrame);+ return new VideoFrame(this.heldFrame, {+ timestamp: this.heldFrame.timestamp,+ });+ }++ this.heldFrame.close();+ this.heldFrame = nextFrame;+ this.heldFrameSec = nextFrameSec;+ }++ return null;+ }++ cancel(): void {+ this.cancelled = true;+ }++ async destroy(): Promise<void> {+ this.cancelled = true;++ if (this.reader) {+ try {+ await this.reader.cancel();+ } catch {+ // Ignore cancellation errors during shutdown.+ }+ this.reader = null;+ }++ if (this.feedPromise) {+ try {+ await this.feedPromise;+ } catch {+ // Decoder errors are already surfaced through getFrameAtTime.+ }+ this.feedPromise = null;+ }++ if (this.heldFrame) {+ this.heldFrame.close();+ this.heldFrame = null;+ }++ for (const frame of this.pendingFrames) {+ frame.close();+ }+ this.pendingFrames = [];++ if (this.decoder) {+ try {+ if (this.decoder.state === "configured") {+ this.decoder.close();+ }+ } catch {+ // Ignore decoder shutdown errors.+ }+ this.decoder = null;+ }++ if (this.demuxer) {+ try {+ this.demuxer.destroy();+ } catch {+ // Ignore demuxer shutdown errors.+ }+ this.demuxer = null;+ }++ this.metadata = null;+ this.decodeDone = false;+ this.decodeError = null;+ this.lastTargetTimeSec = 0;+ }+}\ No newline at end of filediff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.tsindex a8cff4d1..b024b6e1 100644--- a/src/lib/exporter/frameRenderer.ts+++ b/src/lib/exporter/frameRenderer.ts@@ -37,6 +37,7 @@ import {preloadCursorAssets,} from "@/components/video-editor/videoPlayback/cursorRenderer";import { getWebcamOverlaySizePx } from "@/components/video-editor/webcamOverlay";+import { ForwardFrameSource } from "./forwardFrameSource";interface FrameRenderConfig {width: number;@@ -113,11 +114,12 @@ export class FrameRenderer {private currentVideoTime = 0;private lastMotionVector = { x: 0, y: 0 };private cursorOverlay: PixiCursorOverlay | null = null;- private webcamVideoElement: HTMLVideoElement | null = null;- private webcamSeekPromise: Promise<void> | null = null;+ private webcamFrameSource: ForwardFrameSource | null = null;+ private webcamCurrentFrame: VideoFrame | null = null;private webcamFrameCacheCanvas: HTMLCanvasElement | null = null;private webcamFrameCacheCtx: CanvasRenderingContext2D | null = null;- private lastSyncedWebcamTime: number | null = null;+ private webcamFrameWidth = 0;+ private webcamFrameHeight = 0;constructor(config: FrameRenderConfig) {this.config = config;@@ -422,157 +424,71 @@ export class FrameRenderer {private async setupWebcamSource(): Promise<void> {const webcamUrl = this.config.webcamUrl;if (!this.config.webcam?.enabled || !webcamUrl) {- this.webcamVideoElement = null;+ this.webcamFrameSource = null;+ this.webcamCurrentFrame?.close();+ this.webcamCurrentFrame = null;this.webcamFrameCacheCanvas = null;this.webcamFrameCacheCtx = null;- this.lastSyncedWebcamTime = null;+ this.webcamFrameWidth = 0;+ this.webcamFrameHeight = 0;return;}- const video = document.createElement("video");- video.src = webcamUrl;- video.muted = true;- video.preload = "auto";- video.playsInline = true;- video.load();-- await new Promise<void>((resolve, reject) => {- const onReady = () => {- if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {- return;- }- cleanup();- resolve();- };- const onError = () => {- cleanup();- reject(new Error("Failed to load webcam source for export"));- };- const cleanup = () => {- video.removeEventListener("loadeddata", onReady);- video.removeEventListener("canplay", onReady);- video.removeEventListener("canplaythrough", onReady);- video.removeEventListener("error", onError);- };- if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {- resolve();- return;- }- video.addEventListener("loadeddata", onReady, { once: true });- video.addEventListener("canplay", onReady, { once: true });- video.addEventListener("canplaythrough", onReady, { once: true });- video.addEventListener("error", onError, { once: true });- }).catch((error) => {- console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error);- this.webcamVideoElement = null;- });+ const frameSource = new ForwardFrameSource();- if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {- this.webcamVideoElement = video;- return;+ try {+ await frameSource.initialize(webcamUrl);+ this.webcamFrameSource = frameSource;+ } catch (error) {+ console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error);+ await frameSource.destroy();+ this.webcamFrameSource = null;+ this.webcamCurrentFrame?.close();+ this.webcamCurrentFrame = null;+ this.webcamFrameCacheCanvas = null;+ this.webcamFrameCacheCtx = null;+ this.webcamFrameWidth = 0;+ this.webcamFrameHeight = 0;}-- this.webcamVideoElement = null;- this.webcamFrameCacheCanvas = null;- this.webcamFrameCacheCtx = null;- this.lastSyncedWebcamTime = null;}private async syncWebcamFrame(targetTime: number): Promise<void> {- const webcamVideo = this.webcamVideoElement;- if (!webcamVideo) {+ const webcamFrameSource = this.webcamFrameSource;+ if (!webcamFrameSource) {return;}- const duration = Number.isFinite(webcamVideo.duration)- ? webcamVideo.duration- : targetTime;- const clampedTime = Math.max(0, Math.min(targetTime, duration || targetTime));-- if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.008) {- this.lastSyncedWebcamTime = clampedTime;+ const nextFrame = await webcamFrameSource.getFrameAtTime(targetTime);+ if (!nextFrame) {return;}- if (this.webcamSeekPromise) {- await this.webcamSeekPromise;- }+ this.webcamCurrentFrame?.close();+ this.webcamCurrentFrame = nextFrame;- this.webcamSeekPromise = new Promise<void>((resolve) => {- let settled = false;- let fallbackTimeout: number | null = null;- const waitForPresentedFrame = () => {- requestAnimationFrame(() => {- finish();- });- };- const finish = () => {- if (settled) {- return;- }- settled = true;- if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.02) {- this.lastSyncedWebcamTime = clampedTime;- }- cleanup();- resolve();- };- const handleMediaReady = () => {- if (- !webcamVideo.seeking &&- Math.abs(webcamVideo.currentTime - clampedTime) <= 0.01 &&- webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA- ) {- waitForPresentedFrame();- }- };- const cleanup = () => {- webcamVideo.removeEventListener("seeked", waitForPresentedFrame);- webcamVideo.removeEventListener("loadeddata", handleMediaReady);- webcamVideo.removeEventListener("canplay", handleMediaReady);- webcamVideo.removeEventListener("error", finish);- if (fallbackTimeout !== null) {- window.clearTimeout(fallbackTimeout);- }- };-- webcamVideo.addEventListener("seeked", waitForPresentedFrame, {- once: true,- });- webcamVideo.addEventListener("loadeddata", handleMediaReady, {- once: true,- });- webcamVideo.addEventListener("canplay", handleMediaReady, {- once: true,- });- webcamVideo.addEventListener("error", finish, {- once: true,- });- fallbackTimeout = window.setTimeout(() => {- finish();- }, 250);-- try {- webcamVideo.currentTime = clampedTime;- } catch {- finish();- return;- }-- if (- !webcamVideo.seeking &&- Math.abs(webcamVideo.currentTime - clampedTime) <= 0.001 &&- webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA- ) {- waitForPresentedFrame();- }- });+ const frameWidth =+ nextFrame.displayWidth || nextFrame.codedWidth || this.webcamFrameWidth;+ const frameHeight =+ nextFrame.displayHeight || nextFrame.codedHeight || this.webcamFrameHeight;+ if (frameWidth <= 0 || frameHeight <= 0) {+ return;+ }- try {- await this.webcamSeekPromise;- } finally {- this.webcamSeekPromise = null;+ if (+ !this.webcamFrameCacheCanvas ||+ this.webcamFrameWidth !== frameWidth ||+ this.webcamFrameHeight !== frameHeight+ ) {+ this.webcamFrameCacheCanvas = document.createElement("canvas");+ this.webcamFrameCacheCanvas.width = frameWidth;+ this.webcamFrameCacheCanvas.height = frameHeight;+ this.webcamFrameCacheCtx = this.webcamFrameCacheCanvas.getContext("2d");+ this.webcamFrameWidth = frameWidth;+ this.webcamFrameHeight = frameHeight;}++ this.webcamFrameCacheCtx?.clearRect(0, 0, frameWidth, frameHeight);+ this.webcamFrameCacheCtx?.drawImage(nextFrame, 0, 0, frameWidth, frameHeight);}async renderFrame(videoFrame: VideoFrame, timestamp: number): Promise<void> {@@ -582,7 +498,7 @@ export class FrameRenderer {this.currentVideoTime = timestamp / 1000000;- if (this.webcamVideoElement) {+ if (this.webcamFrameSource) {const targetTime = Math.max(0, this.currentVideoTime);await this.syncWebcamFrame(targetTime);}@@ -944,8 +860,7 @@ export class FrameRenderer {height: number,): void {const webcam = this.config.webcam;- const webcamVideo = this.webcamVideoElement;- if (!webcam?.enabled || !webcamVideo || webcamVideo.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {+ if (!webcam?.enabled || !this.webcamFrameCacheCanvas) {return;}@@ -970,45 +885,7 @@ export class FrameRenderer {return;}- const canRefreshCache =- webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA &&- !webcamVideo.seeking &&- this.lastSyncedWebcamTime !== null &&- Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime) <= 0.02 &&- Math.abs(webcamVideo.currentTime - this.currentVideoTime) <= 0.02 &&- webcamVideo.videoWidth > 0 &&- webcamVideo.videoHeight > 0;-- if (canRefreshCache) {- if (- !this.webcamFrameCacheCanvas ||- this.webcamFrameCacheCanvas.width !== webcamVideo.videoWidth ||- this.webcamFrameCacheCanvas.height !== webcamVideo.videoHeight- ) {- this.webcamFrameCacheCanvas = document.createElement("canvas");- this.webcamFrameCacheCanvas.width = webcamVideo.videoWidth;- this.webcamFrameCacheCanvas.height = webcamVideo.videoHeight;- this.webcamFrameCacheCtx = this.webcamFrameCacheCanvas.getContext("2d");- }-- this.webcamFrameCacheCtx?.clearRect(- 0,- 0,- this.webcamFrameCacheCanvas!.width,- this.webcamFrameCacheCanvas!.height,- );- this.webcamFrameCacheCtx?.drawImage(- webcamVideo,- 0,- 0,- this.webcamFrameCacheCanvas!.width,- this.webcamFrameCacheCanvas!.height,- );- }-- const webcamFrameSource = canRefreshCache- ? webcamVideo- : this.webcamFrameCacheCanvas;+ const webcamFrameSource = this.webcamFrameCacheCanvas;if (!webcamFrameSource) {return;}@@ -1084,18 +961,19 @@ export class FrameRenderer {this.cursorOverlay.destroy();this.cursorOverlay = null;}+ this.webcamCurrentFrame?.close();+ this.webcamCurrentFrame = null;+ if (this.webcamFrameSource) {+ void this.webcamFrameSource.destroy();+ this.webcamFrameSource = null;+ }this.shadowCanvas = null;this.shadowCtx = null;this.compositeCanvas = null;this.compositeCtx = null;- if (this.webcamVideoElement) {- this.webcamVideoElement.pause();- this.webcamVideoElement.src = "";- this.webcamVideoElement.load();- this.webcamVideoElement = null;- }this.webcamFrameCacheCanvas = null;this.webcamFrameCacheCtx = null;- this.lastSyncedWebcamTime = null;+ this.webcamFrameWidth = 0;+ this.webcamFrameHeight = 0;}}