Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

feat: render captions in the export pipeline

webadderall199e96e07cbe
diff --git a/src/lib/exporter/captionRenderer.ts b/src/lib/exporter/captionRenderer.ts
new file mode 100644
index 00000000..8cbf4586
--- /dev/null
+++ b/src/lib/exporter/captionRenderer.ts
@@ -0,0 +1,104 @@
+import {
+ getDefaultCaptionFontFamily,
+ type AutoCaptionSettings,
+ type CaptionCue,
+} from "@/components/video-editor/types";
+import { drawSquircleOnCanvas } from "@/lib/geometry/squircle";
+import {
+ buildActiveCaptionLayout,
+} from "@/components/video-editor/captionLayout";
+import {
+ CAPTION_FONT_WEIGHT,
+ CAPTION_LINE_HEIGHT,
+ getCaptionPadding,
+ getCaptionScaledRadius,
+ getCaptionScaledFontSize,
+ getCaptionTextMaxWidth,
+ getCaptionWordVisualState,
+} from "@/components/video-editor/captionStyle";
+
+export function renderCaptions(
+ ctx: CanvasRenderingContext2D,
+ cues: CaptionCue[],
+ settings: AutoCaptionSettings,
+ width: number,
+ height: number,
+ timeMs: number,
+) {
+ if (!settings.enabled || cues.length === 0) {
+ return;
+ }
+
+ ctx.save();
+
+ const fontSize = getCaptionScaledFontSize(settings.fontSize, width, settings.maxWidth);
+ ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`;
+ const padding = getCaptionPadding(fontSize);
+
+ const activeCaptionLayout = buildActiveCaptionLayout({
+ cues,
+ timeMs,
+ settings,
+ maxWidthPx: getCaptionTextMaxWidth(width, settings.maxWidth, fontSize),
+ measureText: (text) => ctx.measureText(text).width,
+ });
+ if (!activeCaptionLayout) {
+ ctx.restore();
+ return;
+ }
+
+ const lineHeight = fontSize * CAPTION_LINE_HEIGHT;
+ const paddingX = padding.x;
+ const paddingY = padding.y;
+ const textBlockHeight = activeCaptionLayout.visibleLines.length * lineHeight;
+ const boxHeight = textBlockHeight + paddingY * 2;
+ const centerX = width / 2;
+ const centerY = height - (height * settings.bottomOffset) / 100 - boxHeight / 2;
+ const maxMeasuredWidth = activeCaptionLayout.visibleLines.reduce(
+ (largest, line) => Math.max(largest, line.width),
+ 0,
+ );
+ const boxWidth = Math.min(width * (settings.maxWidth / 100) + paddingX * 2, maxMeasuredWidth + paddingX * 2);
+
+ ctx.translate(centerX, centerY + activeCaptionLayout.translateY);
+ ctx.scale(activeCaptionLayout.scale, activeCaptionLayout.scale);
+ ctx.globalAlpha = activeCaptionLayout.opacity;
+
+ ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`;
+ drawSquircleOnCanvas(ctx, {
+ x: -boxWidth / 2,
+ y: -boxHeight / 2,
+ width: boxWidth,
+ height: boxHeight,
+ radius: getCaptionScaledRadius(settings.boxRadius, fontSize),
+ });
+ ctx.fill();
+
+ ctx.textAlign = "left";
+ ctx.textBaseline = "middle";
+
+ activeCaptionLayout.visibleLines.forEach((line, lineIndex) => {
+ let cursorX = -line.width / 2;
+ const lineY = -boxHeight / 2 + paddingY + lineHeight * lineIndex + lineHeight / 2;
+
+ line.words.forEach((word) => {
+ const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`;
+ const segmentWidth = ctx.measureText(segmentText).width;
+ const visualState = getCaptionWordVisualState(
+ activeCaptionLayout.hasWordTimings,
+ word.state,
+ );
+
+ ctx.save();
+ ctx.translate(cursorX, lineY);
+ ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor;
+ ctx.globalAlpha = activeCaptionLayout.opacity * visualState.opacity;
+ ctx.fillText(segmentText, 0, 0);
+ ctx.restore();
+
+ cursorX += segmentWidth;
+ });
+ });
+
+ ctx.restore();
+}
\ No newline at end of file
diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts
index fb5d049f..be9d8ce3 100644
--- a/src/lib/exporter/frameRenderer.ts
+++ b/src/lib/exporter/frameRenderer.ts
@@ -5,15 +5,20 @@ import {
Graphics,
BlurFilter,
Texture,
+ TextureSource,
} from "pixi.js";
import { MotionBlurFilter } from "pixi-filters/motion-blur";
import type {
+ AutoCaptionSettings,
+ CaptionCue,
ZoomRegion,
CropRegion,
AnnotationRegion,
SpeedRegion,
+ CursorStyle,
CursorTelemetryPoint,
WebcamOverlaySettings,
+ ZoomTransitionEasing,
} from "@/components/video-editor/types";
import { ZOOM_DEPTH_SCALES } from "@/components/video-editor/types";
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
@@ -31,6 +36,7 @@ import {
ZOOM_TRANSLATION_DEADZONE_PX,
} from "@/components/video-editor/videoPlayback/constants";
import { renderAnnotations } from "./annotationRenderer";
+import { renderCaptions } from "./captionRenderer";
import {
PixiCursorOverlay,
DEFAULT_CURSOR_CONFIG,
@@ -52,6 +58,14 @@ interface FrameRenderConfig {
backgroundBlur: number;
zoomMotionBlur?: number;
connectZooms?: boolean;
+ zoomInDurationMs?: number;
+ zoomInOverlapMs?: number;
+ zoomOutDurationMs?: number;
+ connectedZoomGapMs?: number;
+ connectedZoomDurationMs?: number;
+ zoomInEasing?: ZoomTransitionEasing;
+ zoomOutEasing?: ZoomTransitionEasing;
+ connectedZoomEasing?: ZoomTransitionEasing;
borderRadius?: number;
padding?: number;
cropRegion: CropRegion;
@@ -60,11 +74,14 @@ interface FrameRenderConfig {
videoWidth: number;
videoHeight: number;
annotationRegions?: AnnotationRegion[];
+ autoCaptions?: CaptionCue[];
+ autoCaptionSettings?: AutoCaptionSettings;
speedRegions?: SpeedRegion[];
previewWidth?: number;
previewHeight?: number;
cursorTelemetry?: CursorTelemetryPoint[];
showCursor?: boolean;
+ cursorStyle?: CursorStyle;
cursorSize?: number;
cursorSmoothing?: number;
cursorMotionBlur?: number;
@@ -103,6 +120,7 @@ export class FrameRenderer {
private videoContainer: Container | null = null;
private cursorContainer: Container | null = null;
private videoSprite: Sprite | null = null;
+ private videoTextureSource: TextureSource<any> | null = null;
private backgroundSprite: Sprite | null = null;
private maskGraphics: Graphics | null = null;
private blurFilter: BlurFilter | null = null;
@@ -190,6 +208,7 @@ export class FrameRenderer {
this.cursorOverlay = new PixiCursorOverlay({
dotRadius:
DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4),
+ style: this.config.cursorStyle ?? "tahoe",
smoothingFactor:
this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor,
motionBlur: this.config.cursorMotionBlur ?? 0,
@@ -693,6 +712,7 @@ export class FrameRenderer {
if (!this.videoSprite) {
const texture = Texture.from(videoFrame as any);
this.videoSprite = new Sprite(texture);
+ this.videoTextureSource = texture.source as TextureSource<any>;
this.videoContainer.addChild(this.videoSprite);
if (this.cursorOverlay && this.cursorContainer) {
this.cursorContainer.addChild(this.cursorOverlay.container);
@@ -701,11 +721,9 @@ export class FrameRenderer {
this.videoContainer.addChild(this.maskGraphics);
}
} else {
- // Destroy old texture to avoid memory leaks, then create new one
- const oldTexture = this.videoSprite.texture;
- const newTexture = Texture.from(videoFrame as any);
- this.videoSprite.texture = newTexture;
- oldTexture.destroy(true);
+ this.videoTextureSource ??= this.videoSprite.texture.source as TextureSource<any>;
+ this.videoTextureSource.resource = videoFrame as any;
+ this.videoTextureSource.update();
}
// Apply layout
@@ -783,6 +801,22 @@ export class FrameRenderer {
scaleFactor,
);
}
+
+ if (
+ this.config.autoCaptions &&
+ this.config.autoCaptions.length > 0 &&
+ this.config.autoCaptionSettings &&
+ this.compositeCtx
+ ) {
+ renderCaptions(
+ this.compositeCtx,
+ this.config.autoCaptions,
+ this.config.autoCaptionSettings,
+ this.config.width,
+ this.config.height,
+ timeMs,
+ );
+ }
}
private updateLayout(): void {
@@ -1220,6 +1254,7 @@ export class FrameRenderer {
this.videoSprite.destroy({ texture: false, textureSource: false });
videoTexture?.destroy(true);
this.videoSprite = null;
+ this.videoTextureSource = null;
}
this.backgroundSprite = null;
if (this.app) {
diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts
index 8b0db134..08dffff3 100644
--- a/src/lib/exporter/gifExporter.ts
+++ b/src/lib/exporter/gifExporter.ts
@@ -9,13 +9,17 @@ import type {
import { StreamingVideoDecoder } from "./streamingDecoder";
import { FrameRenderer } from "./frameRenderer";
import type {
+ AutoCaptionSettings,
ZoomRegion,
CropRegion,
TrimRegion,
AnnotationRegion,
+ CaptionCue,
SpeedRegion,
+ CursorStyle,
CursorTelemetryPoint,
WebcamOverlaySettings,
+ ZoomTransitionEasing,
} from "@/components/video-editor/types";
const GIF_WORKER_URL = new URL(
@@ -39,6 +43,14 @@ interface GifExporterConfig {
backgroundBlur: number;
zoomMotionBlur?: number;
connectZooms?: boolean;
+ zoomInDurationMs?: number;
+ zoomInOverlapMs?: number;
+ zoomOutDurationMs?: number;
+ connectedZoomGapMs?: number;
+ connectedZoomDurationMs?: number;
+ zoomInEasing?: ZoomTransitionEasing;
+ zoomOutEasing?: ZoomTransitionEasing;
+ connectedZoomEasing?: ZoomTransitionEasing;
borderRadius?: number;
padding?: number;
videoPadding?: number;
@@ -46,8 +58,11 @@ interface GifExporterConfig {
webcam?: WebcamOverlaySettings;
webcamUrl?: string | null;
annotationRegions?: AnnotationRegion[];
+ autoCaptions?: CaptionCue[];
+ autoCaptionSettings?: AutoCaptionSettings;
cursorTelemetry?: CursorTelemetryPoint[];
showCursor?: boolean;
+ cursorStyle?: CursorStyle;
cursorSize?: number;
cursorSmoothing?: number;
cursorMotionBlur?: number;
@@ -126,6 +141,14 @@ export class GifExporter {
backgroundBlur: this.config.backgroundBlur,
zoomMotionBlur: this.config.zoomMotionBlur,
connectZooms: this.config.connectZooms,
+ zoomInDurationMs: this.config.zoomInDurationMs,
+ zoomInOverlapMs: this.config.zoomInOverlapMs,
+ zoomOutDurationMs: this.config.zoomOutDurationMs,
+ connectedZoomGapMs: this.config.connectedZoomGapMs,
+ connectedZoomDurationMs: this.config.connectedZoomDurationMs,
+ zoomInEasing: this.config.zoomInEasing,
+ zoomOutEasing: this.config.zoomOutEasing,
+ connectedZoomEasing: this.config.connectedZoomEasing,
borderRadius: this.config.borderRadius,
padding: this.config.padding,
cropRegion: this.config.cropRegion,
@@ -134,11 +157,14 @@ export class GifExporter {
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
annotationRegions: this.config.annotationRegions,
+ autoCaptions: this.config.autoCaptions,
+ autoCaptionSettings: this.config.autoCaptionSettings,
speedRegions: this.config.speedRegions,
previewWidth: this.config.previewWidth,
previewHeight: this.config.previewHeight,
cursorTelemetry: this.config.cursorTelemetry,
showCursor: this.config.showCursor,
+ cursorStyle: this.config.cursorStyle,
cursorSize: this.config.cursorSize,
cursorSmoothing: this.config.cursorSmoothing,
cursorMotionBlur: this.config.cursorMotionBlur,
diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts
index 5934af04..a052fec0 100644
--- a/src/lib/exporter/muxer.ts
+++ b/src/lib/exporter/muxer.ts
@@ -86,5 +86,12 @@ export class VideoMuxer {
return new Blob([buffer], { type: 'video/mp4' });
}
+
+ destroy(): void {
+ this.output = null;
+ this.videoSource = null;
+ this.audioSource = null;
+ this.target = null;
+ }
}
diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts
index 812d850d..a92d28db 100644
--- a/src/lib/exporter/types.ts
+++ b/src/lib/exporter/types.ts
@@ -11,7 +11,7 @@ export interface ExportProgress {
totalFrames: number;
percentage: number;
estimatedTimeRemaining: number; // in seconds
- phase?: 'extracting' | 'finalizing'; // Phase of export
+ phase?: 'extracting' | 'finalizing' | 'saving'; // Phase of export
renderProgress?: number; // 0-100, progress of GIF rendering phase
}
diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts
index 12096587..435ac483 100644
--- a/src/lib/exporter/videoExporter.ts
+++ b/src/lib/exporter/videoExporter.ts
@@ -1,11 +1,15 @@
import type {
+ AutoCaptionSettings,
AnnotationRegion,
AudioRegion,
+ CaptionCue,
CropRegion,
+ CursorStyle,
CursorTelemetryPoint,
SpeedRegion,
TrimRegion,
WebcamOverlaySettings,
+ ZoomTransitionEasing,
ZoomRegion,
} from "@/components/video-editor/types";
import { AudioProcessor } from "./audioEncoder";
@@ -25,6 +29,14 @@ interface VideoExporterConfig extends ExportConfig {
backgroundBlur: number;
zoomMotionBlur?: number;
connectZooms?: boolean;
+ zoomInDurationMs?: number;
+ zoomInOverlapMs?: number;
+ zoomOutDurationMs?: number;
+ connectedZoomGapMs?: number;
+ connectedZoomDurationMs?: number;
+ zoomInEasing?: ZoomTransitionEasing;
+ zoomOutEasing?: ZoomTransitionEasing;
+ connectedZoomEasing?: ZoomTransitionEasing;
borderRadius?: number;
padding?: number;
videoPadding?: number;
@@ -32,8 +44,11 @@ interface VideoExporterConfig extends ExportConfig {
webcam?: WebcamOverlaySettings;
webcamUrl?: string | null;
annotationRegions?: AnnotationRegion[];
+ autoCaptions?: CaptionCue[];
+ autoCaptionSettings?: AutoCaptionSettings;
cursorTelemetry?: CursorTelemetryPoint[];
showCursor?: boolean;
+ cursorStyle?: CursorStyle;
cursorSize?: number;
cursorSmoothing?: number;
cursorMotionBlur?: number;
@@ -87,6 +102,14 @@ export class VideoExporter {
backgroundBlur: this.config.backgroundBlur,
zoomMotionBlur: this.config.zoomMotionBlur,
connectZooms: this.config.connectZooms,
+ zoomInDurationMs: this.config.zoomInDurationMs,
+ zoomInOverlapMs: this.config.zoomInOverlapMs,
+ zoomOutDurationMs: this.config.zoomOutDurationMs,
+ connectedZoomGapMs: this.config.connectedZoomGapMs,
+ connectedZoomDurationMs: this.config.connectedZoomDurationMs,
+ zoomInEasing: this.config.zoomInEasing,
+ zoomOutEasing: this.config.zoomOutEasing,
+ connectedZoomEasing: this.config.connectedZoomEasing,
borderRadius: this.config.borderRadius,
padding: this.config.padding,
cropRegion: this.config.cropRegion,
@@ -95,11 +118,14 @@ export class VideoExporter {
videoWidth: videoInfo.width,
videoHeight: videoInfo.height,
annotationRegions: this.config.annotationRegions,
+ autoCaptions: this.config.autoCaptions,
+ autoCaptionSettings: this.config.autoCaptionSettings,
speedRegions: this.config.speedRegions,
previewWidth: this.config.previewWidth,
previewHeight: this.config.previewHeight,
cursorTelemetry: this.config.cursorTelemetry,
showCursor: this.config.showCursor,
+ cursorStyle: this.config.cursorStyle,
cursorSize: this.config.cursorSize,
cursorSmoothing: this.config.cursorSmoothing,
cursorMotionBlur: this.config.cursorMotionBlur,
@@ -440,6 +466,14 @@ export class VideoExporter {
this.renderer = null;
}
+ if (this.muxer) {
+ try {
+ this.muxer.destroy();
+ } catch (e) {
+ console.warn("Error destroying muxer:", e);
+ }
+ }
+
this.muxer = null;
this.audioProcessor = null;
this.encodeQueue = 0;