Create polished demo videos without editing skills. Mac/Windows/Linux
feat: render captions in the export pipeline
webadderall199e96e07cbe
diff --git a/src/lib/exporter/captionRenderer.ts b/src/lib/exporter/captionRenderer.tsnew file mode 100644index 00000000..8cbf4586--- /dev/null+++ b/src/lib/exporter/captionRenderer.ts@@ -0,0 +1,104 @@+import {+ getDefaultCaptionFontFamily,+ type AutoCaptionSettings,+ type CaptionCue,+} from "@/components/video-editor/types";+import { drawSquircleOnCanvas } from "@/lib/geometry/squircle";+import {+ buildActiveCaptionLayout,+} from "@/components/video-editor/captionLayout";+import {+ CAPTION_FONT_WEIGHT,+ CAPTION_LINE_HEIGHT,+ getCaptionPadding,+ getCaptionScaledRadius,+ getCaptionScaledFontSize,+ getCaptionTextMaxWidth,+ getCaptionWordVisualState,+} from "@/components/video-editor/captionStyle";++export function renderCaptions(+ ctx: CanvasRenderingContext2D,+ cues: CaptionCue[],+ settings: AutoCaptionSettings,+ width: number,+ height: number,+ timeMs: number,+) {+ if (!settings.enabled || cues.length === 0) {+ return;+ }++ ctx.save();++ const fontSize = getCaptionScaledFontSize(settings.fontSize, width, settings.maxWidth);+ ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`;+ const padding = getCaptionPadding(fontSize);++ const activeCaptionLayout = buildActiveCaptionLayout({+ cues,+ timeMs,+ settings,+ maxWidthPx: getCaptionTextMaxWidth(width, settings.maxWidth, fontSize),+ measureText: (text) => ctx.measureText(text).width,+ });+ if (!activeCaptionLayout) {+ ctx.restore();+ return;+ }++ const lineHeight = fontSize * CAPTION_LINE_HEIGHT;+ const paddingX = padding.x;+ const paddingY = padding.y;+ const textBlockHeight = activeCaptionLayout.visibleLines.length * lineHeight;+ const boxHeight = textBlockHeight + paddingY * 2;+ const centerX = width / 2;+ const centerY = height - (height * settings.bottomOffset) / 100 - boxHeight / 2;+ const maxMeasuredWidth = activeCaptionLayout.visibleLines.reduce(+ (largest, line) => Math.max(largest, line.width),+ 0,+ );+ const boxWidth = Math.min(width * (settings.maxWidth / 100) + paddingX * 2, maxMeasuredWidth + paddingX * 2);++ ctx.translate(centerX, centerY + activeCaptionLayout.translateY);+ ctx.scale(activeCaptionLayout.scale, activeCaptionLayout.scale);+ ctx.globalAlpha = activeCaptionLayout.opacity;++ ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`;+ drawSquircleOnCanvas(ctx, {+ x: -boxWidth / 2,+ y: -boxHeight / 2,+ width: boxWidth,+ height: boxHeight,+ radius: getCaptionScaledRadius(settings.boxRadius, fontSize),+ });+ ctx.fill();++ ctx.textAlign = "left";+ ctx.textBaseline = "middle";++ activeCaptionLayout.visibleLines.forEach((line, lineIndex) => {+ let cursorX = -line.width / 2;+ const lineY = -boxHeight / 2 + paddingY + lineHeight * lineIndex + lineHeight / 2;++ line.words.forEach((word) => {+ const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`;+ const segmentWidth = ctx.measureText(segmentText).width;+ const visualState = getCaptionWordVisualState(+ activeCaptionLayout.hasWordTimings,+ word.state,+ );++ ctx.save();+ ctx.translate(cursorX, lineY);+ ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor;+ ctx.globalAlpha = activeCaptionLayout.opacity * visualState.opacity;+ ctx.fillText(segmentText, 0, 0);+ ctx.restore();++ cursorX += segmentWidth;+ });+ });++ ctx.restore();+}\ No newline at end of filediff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.tsindex fb5d049f..be9d8ce3 100644--- a/src/lib/exporter/frameRenderer.ts+++ b/src/lib/exporter/frameRenderer.ts@@ -5,15 +5,20 @@ import {Graphics,BlurFilter,Texture,+ TextureSource,} from "pixi.js";import { MotionBlurFilter } from "pixi-filters/motion-blur";import type {+ AutoCaptionSettings,+ CaptionCue,ZoomRegion,CropRegion,AnnotationRegion,SpeedRegion,+ CursorStyle,CursorTelemetryPoint,WebcamOverlaySettings,+ ZoomTransitionEasing,} from "@/components/video-editor/types";import { ZOOM_DEPTH_SCALES } from "@/components/video-editor/types";import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";@@ -31,6 +36,7 @@ import {ZOOM_TRANSLATION_DEADZONE_PX,} from "@/components/video-editor/videoPlayback/constants";import { renderAnnotations } from "./annotationRenderer";+import { renderCaptions } from "./captionRenderer";import {PixiCursorOverlay,DEFAULT_CURSOR_CONFIG,@@ -52,6 +58,14 @@ interface FrameRenderConfig {backgroundBlur: number;zoomMotionBlur?: number;connectZooms?: boolean;+ zoomInDurationMs?: number;+ zoomInOverlapMs?: number;+ zoomOutDurationMs?: number;+ connectedZoomGapMs?: number;+ connectedZoomDurationMs?: number;+ zoomInEasing?: ZoomTransitionEasing;+ zoomOutEasing?: ZoomTransitionEasing;+ connectedZoomEasing?: ZoomTransitionEasing;borderRadius?: number;padding?: number;cropRegion: CropRegion;@@ -60,11 +74,14 @@ interface FrameRenderConfig {videoWidth: number;videoHeight: number;annotationRegions?: AnnotationRegion[];+ autoCaptions?: CaptionCue[];+ autoCaptionSettings?: AutoCaptionSettings;speedRegions?: SpeedRegion[];previewWidth?: number;previewHeight?: number;cursorTelemetry?: CursorTelemetryPoint[];showCursor?: boolean;+ cursorStyle?: CursorStyle;cursorSize?: number;cursorSmoothing?: number;cursorMotionBlur?: number;@@ -103,6 +120,7 @@ export class FrameRenderer {private videoContainer: Container | null = null;private cursorContainer: Container | null = null;private videoSprite: Sprite | null = null;+ private videoTextureSource: TextureSource<any> | null = null;private backgroundSprite: Sprite | null = null;private maskGraphics: Graphics | null = null;private blurFilter: BlurFilter | null = null;@@ -190,6 +208,7 @@ export class FrameRenderer {this.cursorOverlay = new PixiCursorOverlay({dotRadius:DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4),+ style: this.config.cursorStyle ?? "tahoe",smoothingFactor:this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor,motionBlur: this.config.cursorMotionBlur ?? 0,@@ -693,6 +712,7 @@ export class FrameRenderer {if (!this.videoSprite) {const texture = Texture.from(videoFrame as any);this.videoSprite = new Sprite(texture);+ this.videoTextureSource = texture.source as TextureSource<any>;this.videoContainer.addChild(this.videoSprite);if (this.cursorOverlay && this.cursorContainer) {this.cursorContainer.addChild(this.cursorOverlay.container);@@ -701,11 +721,9 @@ export class FrameRenderer {this.videoContainer.addChild(this.maskGraphics);}} else {- // Destroy old texture to avoid memory leaks, then create new one- const oldTexture = this.videoSprite.texture;- const newTexture = Texture.from(videoFrame as any);- this.videoSprite.texture = newTexture;- oldTexture.destroy(true);+ this.videoTextureSource ??= this.videoSprite.texture.source as TextureSource<any>;+ this.videoTextureSource.resource = videoFrame as any;+ this.videoTextureSource.update();}// Apply layout@@ -783,6 +801,22 @@ export class FrameRenderer {scaleFactor,);}++ if (+ this.config.autoCaptions &&+ this.config.autoCaptions.length > 0 &&+ this.config.autoCaptionSettings &&+ this.compositeCtx+ ) {+ renderCaptions(+ this.compositeCtx,+ this.config.autoCaptions,+ this.config.autoCaptionSettings,+ this.config.width,+ this.config.height,+ timeMs,+ );+ }}private updateLayout(): void {@@ -1220,6 +1254,7 @@ export class FrameRenderer {this.videoSprite.destroy({ texture: false, textureSource: false });videoTexture?.destroy(true);this.videoSprite = null;+ this.videoTextureSource = null;}this.backgroundSprite = null;if (this.app) {diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.tsindex 8b0db134..08dffff3 100644--- a/src/lib/exporter/gifExporter.ts+++ b/src/lib/exporter/gifExporter.ts@@ -9,13 +9,17 @@ import type {import { StreamingVideoDecoder } from "./streamingDecoder";import { FrameRenderer } from "./frameRenderer";import type {+ AutoCaptionSettings,ZoomRegion,CropRegion,TrimRegion,AnnotationRegion,+ CaptionCue,SpeedRegion,+ CursorStyle,CursorTelemetryPoint,WebcamOverlaySettings,+ ZoomTransitionEasing,} from "@/components/video-editor/types";const GIF_WORKER_URL = new URL(@@ -39,6 +43,14 @@ interface GifExporterConfig {backgroundBlur: number;zoomMotionBlur?: number;connectZooms?: boolean;+ zoomInDurationMs?: number;+ zoomInOverlapMs?: number;+ zoomOutDurationMs?: number;+ connectedZoomGapMs?: number;+ connectedZoomDurationMs?: number;+ zoomInEasing?: ZoomTransitionEasing;+ zoomOutEasing?: ZoomTransitionEasing;+ connectedZoomEasing?: ZoomTransitionEasing;borderRadius?: number;padding?: number;videoPadding?: number;@@ -46,8 +58,11 @@ interface GifExporterConfig {webcam?: WebcamOverlaySettings;webcamUrl?: string | null;annotationRegions?: AnnotationRegion[];+ autoCaptions?: CaptionCue[];+ autoCaptionSettings?: AutoCaptionSettings;cursorTelemetry?: CursorTelemetryPoint[];showCursor?: boolean;+ cursorStyle?: CursorStyle;cursorSize?: number;cursorSmoothing?: number;cursorMotionBlur?: number;@@ -126,6 +141,14 @@ export class GifExporter {backgroundBlur: this.config.backgroundBlur,zoomMotionBlur: this.config.zoomMotionBlur,connectZooms: this.config.connectZooms,+ zoomInDurationMs: this.config.zoomInDurationMs,+ zoomInOverlapMs: this.config.zoomInOverlapMs,+ zoomOutDurationMs: this.config.zoomOutDurationMs,+ connectedZoomGapMs: this.config.connectedZoomGapMs,+ connectedZoomDurationMs: this.config.connectedZoomDurationMs,+ zoomInEasing: this.config.zoomInEasing,+ zoomOutEasing: this.config.zoomOutEasing,+ connectedZoomEasing: this.config.connectedZoomEasing,borderRadius: this.config.borderRadius,padding: this.config.padding,cropRegion: this.config.cropRegion,@@ -134,11 +157,14 @@ export class GifExporter {videoWidth: videoInfo.width,videoHeight: videoInfo.height,annotationRegions: this.config.annotationRegions,+ autoCaptions: this.config.autoCaptions,+ autoCaptionSettings: this.config.autoCaptionSettings,speedRegions: this.config.speedRegions,previewWidth: this.config.previewWidth,previewHeight: this.config.previewHeight,cursorTelemetry: this.config.cursorTelemetry,showCursor: this.config.showCursor,+ cursorStyle: this.config.cursorStyle,cursorSize: this.config.cursorSize,cursorSmoothing: this.config.cursorSmoothing,cursorMotionBlur: this.config.cursorMotionBlur,diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.tsindex 5934af04..a052fec0 100644--- a/src/lib/exporter/muxer.ts+++ b/src/lib/exporter/muxer.ts@@ -86,5 +86,12 @@ export class VideoMuxer {return new Blob([buffer], { type: 'video/mp4' });}++ destroy(): void {+ this.output = null;+ this.videoSource = null;+ this.audioSource = null;+ this.target = null;+ }}diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.tsindex 812d850d..a92d28db 100644--- a/src/lib/exporter/types.ts+++ b/src/lib/exporter/types.ts@@ -11,7 +11,7 @@ export interface ExportProgress {totalFrames: number;percentage: number;estimatedTimeRemaining: number; // in seconds- phase?: 'extracting' | 'finalizing'; // Phase of export+ phase?: 'extracting' | 'finalizing' | 'saving'; // Phase of exportrenderProgress?: number; // 0-100, progress of GIF rendering phase}diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.tsindex 12096587..435ac483 100644--- a/src/lib/exporter/videoExporter.ts+++ b/src/lib/exporter/videoExporter.ts@@ -1,11 +1,15 @@import type {+ AutoCaptionSettings,AnnotationRegion,AudioRegion,+ CaptionCue,CropRegion,+ CursorStyle,CursorTelemetryPoint,SpeedRegion,TrimRegion,WebcamOverlaySettings,+ ZoomTransitionEasing,ZoomRegion,} from "@/components/video-editor/types";import { AudioProcessor } from "./audioEncoder";@@ -25,6 +29,14 @@ interface VideoExporterConfig extends ExportConfig {backgroundBlur: number;zoomMotionBlur?: number;connectZooms?: boolean;+ zoomInDurationMs?: number;+ zoomInOverlapMs?: number;+ zoomOutDurationMs?: number;+ connectedZoomGapMs?: number;+ connectedZoomDurationMs?: number;+ zoomInEasing?: ZoomTransitionEasing;+ zoomOutEasing?: ZoomTransitionEasing;+ connectedZoomEasing?: ZoomTransitionEasing;borderRadius?: number;padding?: number;videoPadding?: number;@@ -32,8 +44,11 @@ interface VideoExporterConfig extends ExportConfig {webcam?: WebcamOverlaySettings;webcamUrl?: string | null;annotationRegions?: AnnotationRegion[];+ autoCaptions?: CaptionCue[];+ autoCaptionSettings?: AutoCaptionSettings;cursorTelemetry?: CursorTelemetryPoint[];showCursor?: boolean;+ cursorStyle?: CursorStyle;cursorSize?: number;cursorSmoothing?: number;cursorMotionBlur?: number;@@ -87,6 +102,14 @@ export class VideoExporter {backgroundBlur: this.config.backgroundBlur,zoomMotionBlur: this.config.zoomMotionBlur,connectZooms: this.config.connectZooms,+ zoomInDurationMs: this.config.zoomInDurationMs,+ zoomInOverlapMs: this.config.zoomInOverlapMs,+ zoomOutDurationMs: this.config.zoomOutDurationMs,+ connectedZoomGapMs: this.config.connectedZoomGapMs,+ connectedZoomDurationMs: this.config.connectedZoomDurationMs,+ zoomInEasing: this.config.zoomInEasing,+ zoomOutEasing: this.config.zoomOutEasing,+ connectedZoomEasing: this.config.connectedZoomEasing,borderRadius: this.config.borderRadius,padding: this.config.padding,cropRegion: this.config.cropRegion,@@ -95,11 +118,14 @@ export class VideoExporter {videoWidth: videoInfo.width,videoHeight: videoInfo.height,annotationRegions: this.config.annotationRegions,+ autoCaptions: this.config.autoCaptions,+ autoCaptionSettings: this.config.autoCaptionSettings,speedRegions: this.config.speedRegions,previewWidth: this.config.previewWidth,previewHeight: this.config.previewHeight,cursorTelemetry: this.config.cursorTelemetry,showCursor: this.config.showCursor,+ cursorStyle: this.config.cursorStyle,cursorSize: this.config.cursorSize,cursorSmoothing: this.config.cursorSmoothing,cursorMotionBlur: this.config.cursorMotionBlur,@@ -440,6 +466,14 @@ export class VideoExporter {this.renderer = null;}+ if (this.muxer) {+ try {+ this.muxer.destroy();+ } catch (e) {+ console.warn("Error destroying muxer:", e);+ }+ }+this.muxer = null;this.audioProcessor = null;this.encodeQueue = 0;