Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

feat(export): add extension render hooks to export pipeline

webadderall5bef938dc15e
diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts
index 104df89a..9cc5684c 100644
--- a/src/lib/exporter/audioEncoder.ts
+++ b/src/lib/exporter/audioEncoder.ts
@@ -562,7 +562,7 @@ export class AudioProcessor {
}
const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream)
- let rafId: number | null = null
+ let tickTimerId: ReturnType<typeof setTimeout> | null = null
try {
if (audioContext.state === 'suspended') {
@@ -577,9 +577,9 @@ export class AudioProcessor {
await new Promise<void>((resolve, reject) => {
const cleanup = () => {
- if (rafId !== null) {
- cancelAnimationFrame(rafId)
- rafId = null
+ if (tickTimerId !== null) {
+ clearTimeout(tickTimerId)
+ tickTimerId = null
}
timelineMedia.removeEventListener('error', onError)
timelineMedia.removeEventListener('ended', onEnded)
@@ -701,7 +701,7 @@ export class AudioProcessor {
}
if (!timelineMedia.paused && !timelineMedia.ended) {
- rafId = requestAnimationFrame(tick)
+ tickTimerId = setTimeout(tick, 16)
} else {
cleanup()
resolve()
@@ -710,11 +710,11 @@ export class AudioProcessor {
timelineMedia.addEventListener('error', onError, { once: true })
timelineMedia.addEventListener('ended', onEnded, { once: true })
- rafId = requestAnimationFrame(tick)
+ tickTimerId = setTimeout(tick, 16)
})
} finally {
- if (rafId !== null) {
- cancelAnimationFrame(rafId)
+ if (tickTimerId !== null) {
+ clearTimeout(tickTimerId)
}
timelineMedia.pause()
timelineAudioSourceNode?.disconnect()
@@ -856,6 +856,8 @@ export class AudioProcessor {
}
return new Promise<void>((resolve, reject) => {
+ let timeoutId: ReturnType<typeof setTimeout> | null = null
+
const onLoaded = () => {
cleanup()
resolve()
@@ -864,11 +866,20 @@ export class AudioProcessor {
cleanup()
reject(new Error('Failed to load media metadata for speed-adjusted audio'))
}
+ const onTimeout = () => {
+ cleanup()
+ reject(new Error('Timed out waiting for media metadata (30s)'))
+ }
const cleanup = () => {
+ if (timeoutId !== null) {
+ clearTimeout(timeoutId)
+ timeoutId = null
+ }
media.removeEventListener('loadedmetadata', onLoaded)
media.removeEventListener('error', onError)
}
+ timeoutId = setTimeout(onTimeout, 30_000)
media.addEventListener('loadedmetadata', onLoaded)
media.addEventListener('error', onError, { once: true })
})
@@ -880,6 +891,8 @@ export class AudioProcessor {
}
return new Promise<void>((resolve, reject) => {
+ let timeoutId: ReturnType<typeof setTimeout> | null = null
+
const onSeeked = () => {
cleanup()
resolve()
@@ -888,11 +901,20 @@ export class AudioProcessor {
cleanup()
reject(new Error('Failed to seek media for speed-adjusted audio'))
}
+ const onTimeout = () => {
+ cleanup()
+ reject(new Error('Timed out waiting for media seek (30s)'))
+ }
const cleanup = () => {
+ if (timeoutId !== null) {
+ clearTimeout(timeoutId)
+ timeoutId = null
+ }
media.removeEventListener('seeked', onSeeked)
media.removeEventListener('error', onError)
}
+ timeoutId = setTimeout(onTimeout, 30_000)
media.addEventListener('seeked', onSeeked, { once: true })
media.addEventListener('error', onError, { once: true })
media.currentTime = targetSec
diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts
index 808aee0b..c22872a9 100644
--- a/src/lib/exporter/frameRenderer.ts
+++ b/src/lib/exporter/frameRenderer.ts
@@ -23,6 +23,7 @@ import type {
import { ZOOM_DEPTH_SCALES, BASE_PREVIEW_WIDTH, BASE_PREVIEW_HEIGHT } from "@/components/video-editor/types";
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
+import { extensionHost } from "@/lib/extensions";
import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils";
import {
type CursorFollowCameraState,
@@ -49,6 +50,16 @@ import {
} from "@/components/video-editor/videoPlayback/motionSmoothing";
import { renderAnnotations } from "./annotationRenderer";
import { renderCaptions } from "./captionRenderer";
+import {
+ executeExtensionRenderHooks,
+ executeExtensionCursorEffects,
+ notifyCursorInteraction,
+} from "@/lib/extensions/renderHooks";
+import {
+ mapCursorToCanvasNormalized,
+ mapSmoothedCursorToCanvasNormalized,
+} from "@/lib/extensions/cursorCoordinates";
+import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
import {
PixiCursorOverlay,
DEFAULT_CURSOR_CONFIG,
@@ -104,6 +115,7 @@ interface FrameRenderConfig {
cursorClickBounce?: number;
cursorClickBounceDuration?: number;
cursorSway?: number;
+ frame?: string | null;
}
interface AnimationState {
@@ -183,6 +195,9 @@ export class FrameRenderer {
private webcamBubbleCtx: CanvasRenderingContext2D | null = null;
private lastSyncedWebcamTime: number | null = null;
private cleanupWebcamSource: (() => void) | null = null;
+ private frameImage: HTMLImageElement | null = null;
+ private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null;
+ private frameDraw: ((ctx: CanvasRenderingContext2D, w: number, h: number) => void) | null = null;
constructor(config: FrameRenderConfig) {
this.config = config;
@@ -271,6 +286,7 @@ export class FrameRenderer {
// Setup background (render separately, not in PixiJS)
await this.setupBackground();
await this.setupWebcamSource();
+ await this.setupFrame();
// Setup blur filter for video container
this.blurFilter = new BlurFilter();
@@ -826,6 +842,37 @@ export class FrameRenderer {
}
}
+ private async setupFrame(): Promise<void> {
+ const frameId = this.config.frame;
+ if (!frameId) return;
+
+ const { extensionHost } = await import("@/lib/extensions/extensionHost");
+ const frames = extensionHost.getFrames();
+ const frame = frames.find((f) => f.id === frameId);
+ if (!frame) {
+ console.warn(`[FrameRenderer] Device frame "${frameId}" not found`);
+ return;
+ }
+
+ this.frameInsets = frame.screenInsets;
+
+ if (frame.draw) {
+ // Prefer draw function — renders at export resolution, no bitmap scaling
+ this.frameDraw = frame.draw;
+ return;
+ }
+
+ const img = new Image();
+ img.crossOrigin = "anonymous";
+ await new Promise<void>((resolve, reject) => {
+ img.onload = () => resolve();
+ img.onerror = () => reject(new Error(`Failed to load device frame image: ${frameId}`));
+ img.src = frame.filePath;
+ });
+
+ this.frameImage = img;
+ }
+
async renderFrame(
videoFrame: VideoFrame,
timestamp: number,
@@ -881,6 +928,25 @@ export class FrameRenderer {
);
}
+ const smoothedCursor = mapSmoothedCursorToCanvasNormalized(
+ this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null,
+ {
+ maskRect: this.layoutCache.maskRect,
+ canvasWidth: this.config.width,
+ canvasHeight: this.config.height,
+ },
+ );
+ extensionHost.setSmoothedCursor(
+ smoothedCursor
+ ? {
+ timeMs,
+ cx: smoothedCursor.cx,
+ cy: smoothedCursor.cy,
+ trail: smoothedCursor.trail,
+ }
+ : null,
+ );
+
const TICKS_PER_FRAME = 1;
let maxMotionIntensity = 0;
@@ -919,6 +985,9 @@ export class FrameRenderer {
// Composite with shadows to final output canvas
this.compositeWithShadows();
+ // Draw device frame overlay on top of video content
+ this.drawFrame();
+
// Render annotations on top if present
if (
this.config.annotationRegions &&
@@ -956,6 +1025,140 @@ export class FrameRenderer {
timeMs,
);
}
+
+ // Extension render hooks — run after all built-in rendering
+ if (this.compositeCtx) {
+ const maskRect = this.layoutCache?.maskRect;
+ const hookParams = {
+ width: this.config.width,
+ height: this.config.height,
+ timeMs,
+ durationMs: 0,
+ cursor: smoothedCursor
+ ? {
+ cx: smoothedCursor.cx,
+ cy: smoothedCursor.cy,
+ interactionType: this.getCursorPosition(cursorTimeMs)?.interactionType,
+ }
+ : this.getCursorPosition(cursorTimeMs),
+ smoothedCursor,
+ videoLayout: maskRect ? {
+ maskRect: { x: maskRect.x, y: maskRect.y, width: maskRect.width, height: maskRect.height },
+ borderRadius: this.config.borderRadius ?? 0,
+ padding: this.config.padding ?? 0,
+ } : undefined,
+ zoom: {
+ scale: this.animationState.scale,
+ focusX: this.animationState.focusX,
+ focusY: this.animationState.focusY,
+ progress: this.animationState.progress,
+ },
+ shadow: {
+ enabled: this.config.showShadow,
+ intensity: this.config.shadowIntensity,
+ },
+ sceneTransform: {
+ scale: this.animationState.appliedScale,
+ x: this.animationState.x,
+ y: this.animationState.y,
+ },
+ };
+
+ this.compositeCtx.save();
+ applyCanvasSceneTransform(this.compositeCtx, {
+ scale: this.animationState.appliedScale,
+ x: this.animationState.x,
+ y: this.animationState.y,
+ });
+ executeExtensionRenderHooks('post-video', this.compositeCtx, hookParams);
+ executeExtensionRenderHooks('post-zoom', this.compositeCtx, hookParams);
+ executeExtensionRenderHooks('post-cursor', this.compositeCtx, hookParams);
+
+ // Cursor click effects
+ this.emitCursorInteractions(cursorTimeMs);
+ executeExtensionCursorEffects(
+ this.compositeCtx,
+ timeMs,
+ this.config.width,
+ this.config.height,
+ {
+ zoom: hookParams.zoom,
+ sceneTransform: hookParams.sceneTransform,
+ videoLayout: hookParams.videoLayout,
+ },
+ );
+ this.compositeCtx.restore();
+
+ executeExtensionRenderHooks('post-webcam', this.compositeCtx, hookParams);
+ executeExtensionRenderHooks('post-annotations', this.compositeCtx, hookParams);
+
+ executeExtensionRenderHooks('final', this.compositeCtx, hookParams);
+ }
+ }
+
+ /**
+ * Get the cursor position (normalized 0-1) at the given time.
+ */
+ private getCursorPosition(timeMs: number): { cx: number; cy: number; interactionType?: string } | null {
+ const telemetry = this.config.cursorTelemetry;
+ if (!telemetry || telemetry.length === 0) return null;
+
+ // Find the closest telemetry point
+ let closest = telemetry[0];
+ let minDist = Math.abs(telemetry[0].timeMs - timeMs);
+ for (let i = 1; i < telemetry.length; i++) {
+ const dist = Math.abs(telemetry[i].timeMs - timeMs);
+ if (dist < minDist) {
+ minDist = dist;
+ closest = telemetry[i];
+ }
+ if (telemetry[i].timeMs > timeMs) break;
+ }
+
+ return mapCursorToCanvasNormalized(
+ { cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType },
+ {
+ maskRect: this.layoutCache?.maskRect,
+ canvasWidth: this.config.width,
+ canvasHeight: this.config.height,
+ },
+ );
+ }
+
+ /**
+ * Emit cursor interaction events for extensions based on telemetry clicks.
+ */
+ private lastEmittedClickTimeMs = -1;
+
+ private emitCursorInteractions(timeMs: number): void {
+ const telemetry = this.config.cursorTelemetry;
+ if (!telemetry || telemetry.length === 0) return;
+
+ // Find click events near this time
+ for (const point of telemetry) {
+ if (point.timeMs > timeMs) break;
+ if (point.timeMs < timeMs - 100) continue;
+ if (!point.interactionType || point.interactionType === 'move') continue;
+ if (point.timeMs === this.lastEmittedClickTimeMs) continue;
+
+ const mappedCursor = mapCursorToCanvasNormalized(
+ { cx: point.cx, cy: point.cy, interactionType: point.interactionType },
+ {
+ maskRect: this.layoutCache?.maskRect,
+ canvasWidth: this.config.width,
+ canvasHeight: this.config.height,
+ },
+ );
+ if (!mappedCursor) continue;
+
+ this.lastEmittedClickTimeMs = point.timeMs;
+ notifyCursorInteraction(
+ point.timeMs,
+ mappedCursor.cx,
+ mappedCursor.cy,
+ point.interactionType,
+ );
+ }
}
private updateLayout(): void {
@@ -984,9 +1187,17 @@ export class FrameRenderer {
const paddingScale = 1.0 - (padding / 100) * 0.4;
const viewportWidth = width * paddingScale;
const viewportHeight = height * paddingScale;
+
+ // When a device frame is active, scale to fit the ENTIRE frame (video + bezels)
+ const insets = this.frameInsets;
+ const screenFracW = insets ? (1 - insets.left - insets.right) : 1;
+ const screenFracH = insets ? (1 - insets.top - insets.bottom) : 1;
+ const fullFrameVideoW = croppedVideoWidth / screenFracW;
+ const fullFrameVideoH = croppedVideoHeight / screenFracH;
+
const scale = Math.min(
- viewportWidth / croppedVideoWidth,
- viewportHeight / croppedVideoHeight,
+ viewportWidth / fullFrameVideoW,
+ viewportHeight / fullFrameVideoH,
);
this.videoSprite.scale.set(scale);
@@ -995,8 +1206,18 @@ export class FrameRenderer {
const fullVideoDisplayHeight = videoHeight * scale;
const croppedDisplayWidth = croppedVideoWidth * scale;
const croppedDisplayHeight = croppedVideoHeight * scale;
- const centerOffsetX = (width - croppedDisplayWidth) / 2;
- const centerOffsetY = (height - croppedDisplayHeight) / 2;
+
+ // Center the full frame (video + bezels) in the output canvas
+ const fullFrameDisplayW = fullFrameVideoW * scale;
+ const fullFrameDisplayH = fullFrameVideoH * scale;
+ const frameCenterX = (width - fullFrameDisplayW) / 2;
+ const frameCenterY = (height - fullFrameDisplayH) / 2;
+ const centerOffsetX = insets
+ ? frameCenterX + insets.left * fullFrameDisplayW
+ : (width - croppedDisplayWidth) / 2;
+ const centerOffsetY = insets
+ ? frameCenterY + insets.top * fullFrameDisplayH
+ : (height - croppedDisplayHeight) / 2;
const spriteX = centerOffsetX - cropRegion.x * fullVideoDisplayWidth;
const spriteY = centerOffsetY - cropRegion.y * fullVideoDisplayHeight;
@@ -1236,6 +1457,55 @@ export class FrameRenderer {
this.drawWebcamOverlay(ctx, w, h);
}
+ private drawFrame(): void {
+ if ((!this.frameImage && !this.frameDraw) || !this.compositeCtx || !this.layoutCache) return;
+
+ const ctx = this.compositeCtx;
+ const maskRect = this.layoutCache.maskRect;
+ const insets = this.frameInsets;
+
+ if (!insets) {
+ // No insets: draw frame spanning entire mask area
+ if (this.frameDraw) {
+ const c = document.createElement('canvas');
+ c.width = Math.round(maskRect.width);
+ c.height = Math.round(maskRect.height);
+ const dCtx = c.getContext('2d');
+ if (dCtx) this.frameDraw(dCtx, c.width, c.height);
+ ctx.drawImage(c, maskRect.x, maskRect.y, maskRect.width, maskRect.height);
+ } else {
+ ctx.drawImage(
+ this.frameImage!,
+ maskRect.x,
+ maskRect.y,
+ maskRect.width,
+ maskRect.height,
+ );
+ }
+ return;
+ }
+
+ // Calculate frame dimensions from insets
+ const screenW = maskRect.width;
+ const screenH = maskRect.height;
+ const frameW = screenW / (1 - insets.left - insets.right);
+ const frameH = screenH / (1 - insets.top - insets.bottom);
+ const frameX = maskRect.x - insets.left * frameW;
+ const frameY = maskRect.y - insets.top * frameH;
+
+ if (this.frameDraw) {
+ // Draw at the exact export resolution — no bitmap scaling
+ const c = document.createElement('canvas');
+ c.width = Math.round(frameW);
+ c.height = Math.round(frameH);
+ const dCtx = c.getContext('2d');
+ if (dCtx) this.frameDraw(dCtx, c.width, c.height);
+ ctx.drawImage(c, frameX, frameY, frameW, frameH);
+ } else {
+ ctx.drawImage(this.frameImage!, frameX, frameY, frameW, frameH);
+ }
+ }
+
private drawWebcamOverlay(
ctx: CanvasRenderingContext2D,
width: number,
@@ -1469,5 +1739,8 @@ export class FrameRenderer {
this.webcamBubbleCanvas = null;
this.webcamBubbleCtx = null;
this.lastSyncedWebcamTime = null;
+ this.frameImage = null;
+ this.frameInsets = null;
+ this.frameDraw = null;
}
}
diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts
index 2b2d8d2c..dfd68e2d 100644
--- a/src/lib/exporter/gifExporter.ts
+++ b/src/lib/exporter/gifExporter.ts
@@ -73,6 +73,7 @@ interface GifExporterConfig {
cursorClickBounce?: number;
cursorClickBounceDuration?: number;
cursorSway?: number;
+ frame?: string | null;
previewWidth?: number;
previewHeight?: number;
maxDecodeQueue?: number;
@@ -190,6 +191,7 @@ export class GifExporter {
cursorClickBounce: this.config.cursorClickBounce,
cursorClickBounceDuration: this.config.cursorClickBounceDuration,
cursorSway: this.config.cursorSway,
+ frame: this.config.frame,
});
await this.renderer.initialize();
diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts
index f0897a5e..7fe8fe77 100644
--- a/src/lib/exporter/modernFrameRenderer.ts
+++ b/src/lib/exporter/modernFrameRenderer.ts
@@ -57,8 +57,19 @@ import {
getWebcamOverlaySizePx,
} from "@/components/video-editor/webcamOverlay";
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
+import { extensionHost } from "@/lib/extensions";
+import {
+ mapCursorToCanvasNormalized,
+ mapSmoothedCursorToCanvasNormalized,
+} from "@/lib/extensions/cursorCoordinates";
+import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
import { drawSquircleOnCanvas, drawSquircleOnGraphics } from "@/lib/geometry/squircle";
import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
+import {
+ executeExtensionCursorEffects,
+ executeExtensionRenderHooks,
+ notifyCursorInteraction,
+} from "@/lib/extensions/renderHooks";
import { isVideoWallpaperSource } from "@/lib/wallpapers";
import {
type AnnotationRenderAssets,
@@ -119,6 +130,7 @@ interface FrameRenderConfig {
cursorSway?: number;
zoomSmoothness?: number;
zoomClassicMode?: boolean;
+ frame?: string | null;
}
interface AnimationState {
@@ -362,6 +374,9 @@ export class FrameRenderer {
private webcamTextureUsesStartupStaging = false;
private nativePixelReadbackWarningShown = false;
private nativeReadbackBuffer: Uint8Array | null = null;
+ private compositeCanvas: HTMLCanvasElement | null = null;
+ private compositeCtx: CanvasRenderingContext2D | null = null;
+ private lastEmittedClickTimeMs = -1;
private cleanupWebcamSource: (() => void) | null = null;
constructor(config: FrameRenderConfig) {
@@ -483,6 +498,18 @@ export class FrameRenderer {
await this.setupAnnotationLayer();
this.setupCaptionResources();
+ this.compositeCanvas = document.createElement("canvas");
+ this.compositeCanvas.width = this.config.width;
+ this.compositeCanvas.height = this.config.height;
+ this.compositeCtx = configureHighQuality2DContext(
+ this.compositeCanvas.getContext("2d", {
+ willReadFrequently: false,
+ }),
+ );
+ if (!this.compositeCtx) {
+ throw new Error("Failed to get 2D context for composite canvas");
+ }
+
if (this.shouldUseZoomMotionBlur()) {
this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0);
}
@@ -2165,6 +2192,192 @@ export class FrameRenderer {
this.outputCanvasOverride = null;
this.app.render();
+ this.compositeExtensions(timeMs, cursorTimeMs);
+ }
+
+ private shouldCompositeExtensionFrame(): boolean {
+ return (
+ extensionHost.hasCursorEffects() ||
+ extensionHost.hasRenderHooks("post-zoom") ||
+ extensionHost.hasRenderHooks("post-cursor") ||
+ extensionHost.hasRenderHooks("post-annotations") ||
+ extensionHost.hasRenderHooks("final")
+ );
+ }
+
+ private compositeExtensions(timeMs: number, cursorTimeMs: number): void {
+ if (!this.app || !this.compositeCtx || !this.compositeCanvas) {
+ return;
+ }
+
+ if (!this.shouldCompositeExtensionFrame()) {
+ return;
+ }
+
+ this.compositeCtx.clearRect(0, 0, this.config.width, this.config.height);
+ this.compositeCtx.drawImage(this.app.canvas as HTMLCanvasElement, 0, 0);
+
+ const maskRect = this.layoutCache?.maskRect;
+ const smoothedCursor = mapSmoothedCursorToCanvasNormalized(
+ this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null,
+ {
+ maskRect,
+ canvasWidth: this.config.width,
+ canvasHeight: this.config.height,
+ },
+ );
+ extensionHost.setSmoothedCursor(
+ smoothedCursor
+ ? {
+ timeMs,
+ cx: smoothedCursor.cx,
+ cy: smoothedCursor.cy,
+ trail: smoothedCursor.trail,
+ }
+ : null,
+ );
+ const rawCursor = this.getCursorPosition(cursorTimeMs);
+ const hookParams = {
+ width: this.config.width,
+ height: this.config.height,
+ timeMs,
+ durationMs: 0,
+ cursor: smoothedCursor
+ ? {
+ cx: smoothedCursor.cx,
+ cy: smoothedCursor.cy,
+ interactionType: rawCursor?.interactionType,
+ }
+ : rawCursor,
+ smoothedCursor,
+ videoLayout: maskRect
+ ? {
+ maskRect: {
+ x: maskRect.x,
+ y: maskRect.y,
+ width: maskRect.width,
+ height: maskRect.height,
+ },
+ borderRadius: this.config.borderRadius ?? 0,
+ padding: this.config.padding ?? 0,
+ }
+ : undefined,
+ zoom: {
+ scale: this.animationState.scale,
+ focusX: this.animationState.focusX,
+ focusY: this.animationState.focusY,
+ progress: this.animationState.progress,
+ },
+ shadow: {
+ enabled: this.config.showShadow,
+ intensity: this.config.shadowIntensity,
+ },
+ sceneTransform: {
+ scale: this.animationState.appliedScale,
+ x: this.animationState.x,
+ y: this.animationState.y,
+ },
+ };
+
+ this.compositeCtx.save();
+ applyCanvasSceneTransform(this.compositeCtx, {
+ scale: this.animationState.appliedScale,
+ x: this.animationState.x,
+ y: this.animationState.y,
+ });
+ executeExtensionRenderHooks("post-video", this.compositeCtx, hookParams);
+ executeExtensionRenderHooks("post-zoom", this.compositeCtx, hookParams);
+ executeExtensionRenderHooks("post-cursor", this.compositeCtx, hookParams);
+ this.emitCursorInteractions(cursorTimeMs);
+ executeExtensionCursorEffects(
+ this.compositeCtx,
+ timeMs,
+ this.config.width,
+ this.config.height,
+ {
+ zoom: hookParams.zoom,
+ sceneTransform: hookParams.sceneTransform,
+ videoLayout: hookParams.videoLayout,
+ },
+ );
+ this.compositeCtx.restore();
+
+ executeExtensionRenderHooks("post-webcam", this.compositeCtx, hookParams);
+ executeExtensionRenderHooks("post-annotations", this.compositeCtx, hookParams);
+ executeExtensionRenderHooks("final", this.compositeCtx, hookParams);
+ }
+
+ private getCursorPosition(
+ timeMs: number,
+ ): { cx: number; cy: number; interactionType?: string } | null {
+ const telemetry = this.config.cursorTelemetry;
+ if (!telemetry || telemetry.length === 0) {
+ return null;
+ }
+
+ let closest = telemetry[0];
+ let minDist = Math.abs(telemetry[0].timeMs - timeMs);
+ for (let i = 1; i < telemetry.length; i++) {
+ const dist = Math.abs(telemetry[i].timeMs - timeMs);
+ if (dist < minDist) {
+ minDist = dist;
+ closest = telemetry[i];
+ }
+ if (telemetry[i].timeMs > timeMs) {
+ break;
+ }
+ }
+
+ return mapCursorToCanvasNormalized(
+ { cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType },
+ {
+ maskRect: this.layoutCache?.maskRect,
+ canvasWidth: this.config.width,
+ canvasHeight: this.config.height,
+ },
+ );
+ }
+
+ private emitCursorInteractions(timeMs: number): void {
+ const telemetry = this.config.cursorTelemetry;
+ if (!telemetry || telemetry.length === 0) {
+ return;
+ }
+
+ for (const point of telemetry) {
+ if (point.timeMs > timeMs) {
+ break;
+ }
+ if (point.timeMs < timeMs - 100) {
+ continue;
+ }
+ if (!point.interactionType || point.interactionType === "move") {
+ continue;
+ }
+ if (point.timeMs === this.lastEmittedClickTimeMs) {
+ continue;
+ }
+
+ const mappedCursor = mapCursorToCanvasNormalized(
+ { cx: point.cx, cy: point.cy, interactionType: point.interactionType },
+ {
+ maskRect: this.layoutCache?.maskRect,
+ canvasWidth: this.config.width,
+ canvasHeight: this.config.height,
+ },
+ );
+ if (!mappedCursor) {
+ continue;
+ }
+
+ this.lastEmittedClickTimeMs = point.timeMs;
+ notifyCursorInteraction(
+ point.timeMs,
+ mappedCursor.cx,
+ mappedCursor.cy,
+ point.interactionType,
+ );
+ }
}
private updateLayout(): void {
@@ -2495,7 +2708,7 @@ export class FrameRenderer {
throw new Error("Renderer not initialized");
}
- if (this.outputCanvasOverride) {
+ if (this.outputCanvasOverride || this.shouldCompositeExtensionFrame()) {
return null;
}
@@ -2523,6 +2736,10 @@ export class FrameRenderer {
throw new Error("Renderer not initialized");
}
+ if (this.shouldCompositeExtensionFrame() && this.compositeCanvas) {
+ return this.compositeCanvas;
+ }
+
return this.outputCanvasOverride ?? (this.app.canvas as HTMLCanvasElement);
}
diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts
index 7a70658e..1cfc3dc6 100644
--- a/src/lib/exporter/modernVideoExporter.ts
+++ b/src/lib/exporter/modernVideoExporter.ts
@@ -25,6 +25,7 @@ import {
} from "./mp4Support";
import { VideoMuxer } from "./muxer";
import { captureCanvasFrameForNativeExport } from "./nativeFrameCapture";
+import { extensionHost } from "@/lib/extensions";
import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder";
import type {
ExportConfig,
@@ -81,6 +82,7 @@ interface VideoExporterConfig extends ExportConfig {
cursorSway?: number;
zoomSmoothness?: number;
zoomClassicMode?: boolean;
+ frame?: string | null;
audioRegions?: AudioRegion[];
sourceAudioFallbackPaths?: string[];
previewWidth?: number;
@@ -341,6 +343,7 @@ export class ModernVideoExporter {
cursorSway: this.config.cursorSway,
zoomSmoothness: this.config.zoomSmoothness,
zoomClassicMode: this.config.zoomClassicMode,
+ frame: this.config.frame,
});
await this.renderer.initialize();
this.rendererInitTimeMs = this.getNowMs() - stageStartedAt;
@@ -400,6 +403,7 @@ export class ModernVideoExporter {
frameIndex++;
this.processedFrameCount = frameIndex;
this.reportProgress(frameIndex, totalFrames, "extracting");
+ extensionHost.emitEvent({ type: 'export:frame', data: { frameIndex, totalFrames } });
},
);
this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt;
@@ -455,6 +459,9 @@ export class ModernVideoExporter {
(this.config.sourceAudioFallbackPaths ?? []).length > 0
) {
this.audioProcessor = new AudioProcessor();
+ this.audioProcessor.setOnProgress((progress) => {
+ this.reportFinalizingProgress(totalFrames, 99, progress);
+ });
this.reportFinalizingProgress(totalFrames, 99);
await this.awaitWithFinalizationTimeout(
this.audioProcessor.process(
@@ -857,6 +864,9 @@ export class ModernVideoExporter {
if (audioPlan.audioMode === "edited-track") {
this.audioProcessor = new AudioProcessor();
+ this.audioProcessor.setOnProgress((progress) => {
+ this.reportFinalizingProgress(this.processedFrameCount, 99, progress);
+ });
const audioBlob = await this.awaitWithFinalizationTimeout(
this.audioProcessor.renderEditedAudioTrack(
this.config.videoUrl,
@@ -968,8 +978,8 @@ export class ModernVideoExporter {
}
}
- private reportFinalizingProgress(totalFrames: number, renderProgress: number) {
- this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress);
+ private reportFinalizingProgress(totalFrames: number, renderProgress: number, audioProgress?: number) {
+ this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress);
}
private reportProgress(
@@ -977,6 +987,7 @@ export class ModernVideoExporter {
totalFrames: number,
phase: ExportProgress["phase"] = "extracting",
renderProgress?: number,
+ audioProgress?: number,
) {
const nowMs = this.getNowMs();
const elapsedSeconds = Math.max((nowMs - this.exportStartTimeMs) / 1000, 0.001);
@@ -1047,6 +1058,7 @@ export class ModernVideoExporter {
encoderName: this.encoderName ?? undefined,
phase,
renderProgress: safeRenderProgress,
+ audioProgress,
});
}
}
diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts
index 1943c67c..62825a79 100644
--- a/src/lib/exporter/videoExporter.ts
+++ b/src/lib/exporter/videoExporter.ts
@@ -62,6 +62,7 @@ interface VideoExporterConfig extends ExportConfig {
cursorClickBounceDuration?: number;
cursorSway?: number;
zoomSmoothness?: number;
+ frame?: string | null;
audioRegions?: AudioRegion[];
sourceAudioFallbackPaths?: string[];
previewWidth?: number;
@@ -177,6 +178,7 @@ export class VideoExporter {
cursorClickBounceDuration: this.config.cursorClickBounceDuration,
cursorSway: this.config.cursorSway,
zoomSmoothness: this.config.zoomSmoothness,
+ frame: this.config.frame,
});
await this.renderer.initialize();