Create polished demo videos without editing skills. Mac/Windows/Linux
feat: render captions in preview playback
webadderall2e443169c1f7
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsxindex b852489a..0e8fe195 100644--- a/src/components/video-editor/VideoPlayback.tsx+++ b/src/components/video-editor/VideoPlayback.tsx@@ -25,6 +25,8 @@ import {} from "pixi.js";import { MotionBlurFilter } from "pixi-filters/motion-blur";import {+ type AutoCaptionSettings,+ type CaptionCue,ZOOM_DEPTH_SCALES,type ZoomRegion,type ZoomFocus,@@ -33,7 +35,9 @@ import {type SpeedRegion,type AnnotationRegion,type CursorTelemetryPoint,+ type CursorStyle,type WebcamOverlaySettings,+ type ZoomTransitionEasing,} from "./types";import {DEFAULT_FOCUS,@@ -45,6 +49,18 @@ import {PixiCursorOverlay,preloadCursorAssets,} from "./videoPlayback/cursorRenderer";+import {+ buildActiveCaptionLayout,+} from "./captionLayout";+import {+ CAPTION_FONT_WEIGHT,+ CAPTION_LINE_HEIGHT,+ getCaptionPadding,+ getCaptionScaledRadius,+ getCaptionScaledFontSize,+ getCaptionTextMaxWidth,+ getCaptionWordVisualState,+} from "./captionStyle";import { clamp01 } from "./videoPlayback/mathUtils";import { findDominantRegion } from "./videoPlayback/zoomRegionUtils";import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focusUtils";@@ -70,10 +86,19 @@ import {DEFAULT_CURSOR_SIZE,DEFAULT_CURSOR_SMOOTHING,DEFAULT_CURSOR_SWAY,+ DEFAULT_CONNECTED_ZOOM_DURATION_MS,+ DEFAULT_CONNECTED_ZOOM_EASING,+ DEFAULT_CONNECTED_ZOOM_GAP_MS,DEFAULT_WEBCAM_CORNER_RADIUS,DEFAULT_WEBCAM_REACT_TO_ZOOM,DEFAULT_WEBCAM_SHADOW,DEFAULT_WEBCAM_SIZE,+ DEFAULT_ZOOM_IN_DURATION_MS,+ DEFAULT_ZOOM_IN_EASING,+ DEFAULT_ZOOM_IN_OVERLAP_MS,+ DEFAULT_ZOOM_OUT_DURATION_MS,+ DEFAULT_ZOOM_OUT_EASING,+ getDefaultCaptionFontFamily,} from "./types";import { getWebcamOverlayPosition, getWebcamOverlaySizePx } from "./webcamOverlay";import { getSquircleSvgPath } from "@/lib/geometry/squircle";@@ -100,6 +125,26 @@ function createPlaybackAnimationState(): PlaybackAnimationState {};}+function getEffectiveNativeAspectRatio(+ dimensions: { width: number; height: number } | null | undefined,+ cropRegion?: import("./types").CropRegion,+): number {+ if (!dimensions || dimensions.height <= 0 || dimensions.width <= 0) {+ return 16 / 9;+ }++ const cropWidth = cropRegion?.width ?? 1;+ const cropHeight = cropRegion?.height ?? 1;+ const effectiveWidth = dimensions.width * cropWidth;+ const effectiveHeight = dimensions.height * cropHeight;++ if (effectiveWidth <= 0 || effectiveHeight <= 0) {+ return dimensions.width / dimensions.height;+ }++ return effectiveWidth / effectiveHeight;+}+interface VideoPlaybackProps {videoPath: string;onDurationChange: (duration: number) => void;@@ -118,6 +163,14 @@ interface VideoPlaybackProps {backgroundBlur?: number;zoomMotionBlur?: number;connectZooms?: boolean;+ zoomInDurationMs?: number;+ zoomInOverlapMs?: number;+ zoomOutDurationMs?: number;+ connectedZoomGapMs?: number;+ connectedZoomDurationMs?: number;+ zoomInEasing?: ZoomTransitionEasing;+ zoomOutEasing?: ZoomTransitionEasing;+ connectedZoomEasing?: ZoomTransitionEasing;borderRadius?: number;padding?: number;cropRegion?: import("./types").CropRegion;@@ -127,6 +180,8 @@ interface VideoPlaybackProps {speedRegions?: SpeedRegion[];aspectRatio: AspectRatio;annotationRegions?: AnnotationRegion[];+ autoCaptions?: CaptionCue[];+ autoCaptionSettings?: AutoCaptionSettings;selectedAnnotationId?: string | null;onSelectAnnotation?: (id: string | null) => void;onAnnotationPositionChange?: (@@ -139,6 +194,7 @@ interface VideoPlaybackProps {) => void;cursorTelemetry?: CursorTelemetryPoint[];showCursor?: boolean;+ cursorStyle?: CursorStyle;cursorSize?: number;cursorSmoothing?: number;cursorMotionBlur?: number;@@ -179,6 +235,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(backgroundBlur = 0,zoomMotionBlur = 0,connectZooms = true,+ zoomInDurationMs = DEFAULT_ZOOM_IN_DURATION_MS,+ zoomInOverlapMs = DEFAULT_ZOOM_IN_OVERLAP_MS,+ zoomOutDurationMs = DEFAULT_ZOOM_OUT_DURATION_MS,+ connectedZoomGapMs = DEFAULT_CONNECTED_ZOOM_GAP_MS,+ connectedZoomDurationMs = DEFAULT_CONNECTED_ZOOM_DURATION_MS,+ zoomInEasing = DEFAULT_ZOOM_IN_EASING,+ zoomOutEasing = DEFAULT_ZOOM_OUT_EASING,+ connectedZoomEasing = DEFAULT_CONNECTED_ZOOM_EASING,borderRadius = 0,padding = 50,cropRegion,@@ -188,12 +252,15 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(speedRegions = [],aspectRatio,annotationRegions = [],+ autoCaptions = [],+ autoCaptionSettings,selectedAnnotationId,onSelectAnnotation,onAnnotationPositionChange,onAnnotationSizeChange,cursorTelemetry = [],showCursor = false,+ cursorStyle = "tahoe",cursorSize = DEFAULT_CURSOR_SIZE,cursorSmoothing = DEFAULT_CURSOR_SMOOTHING,cursorMotionBlur = DEFAULT_CURSOR_MOTION_BLUR,@@ -219,6 +286,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(const webcamVideoRef = useRef<HTMLVideoElement | null>(null);const webcamBubbleRef = useRef<HTMLDivElement | null>(null);const webcamBubbleInnerRef = useRef<HTMLDivElement | null>(null);+ const captionBoxRef = useRef<HTMLDivElement | null>(null);const currentTimeRef = useRef(0);const zoomRegionsRef = useRef<ZoomRegion[]>([]);const selectedZoomIdRef = useRef<string | null>(null);@@ -259,16 +327,95 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(const lastWebcamSyncTimeRef = useRef<number | null>(null);const zoomMotionBlurRef = useRef(zoomMotionBlur);const connectZoomsRef = useRef(connectZooms);+ const zoomInDurationMsRef = useRef(zoomInDurationMs);+ const zoomInOverlapMsRef = useRef(zoomInOverlapMs);+ const zoomOutDurationMsRef = useRef(zoomOutDurationMs);+ const connectedZoomGapMsRef = useRef(connectedZoomGapMs);+ const connectedZoomDurationMsRef = useRef(connectedZoomDurationMs);+ const zoomInEasingRef = useRef(zoomInEasing);+ const zoomOutEasingRef = useRef(zoomOutEasing);+ const connectedZoomEasingRef = useRef(connectedZoomEasing);const videoReadyRafRef = useRef<number | null>(null);const cursorOverlayRef = useRef<PixiCursorOverlay | null>(null);const cursorTelemetryRef = useRef<CursorTelemetryPoint[]>([]);const showCursorRef = useRef(showCursor);const cursorSizeRef = useRef(cursorSize);+ const cursorStyleRef = useRef(cursorStyle);const cursorSmoothingRef = useRef(cursorSmoothing);const cursorMotionBlurRef = useRef(cursorMotionBlur);const cursorClickBounceRef = useRef(cursorClickBounce);const cursorClickBounceDurationRef = useRef(cursorClickBounceDuration);const cursorSwayRef = useRef(cursorSway);++ const activeCaptionLayout = useMemo(() => {+ if (!autoCaptionSettings?.enabled || autoCaptions.length === 0 || typeof document === "undefined") {+ return null;+ }++ const overlayWidth = overlayRef.current?.clientWidth || 960;+ const fontSize = getCaptionScaledFontSize(+ autoCaptionSettings.fontSize,+ overlayWidth,+ autoCaptionSettings.maxWidth,+ );+ const maxTextWidthPx = getCaptionTextMaxWidth(+ overlayWidth,+ autoCaptionSettings.maxWidth,+ fontSize,+ );+ const measurementCanvas = document.createElement("canvas");+ const measurementContext = measurementCanvas.getContext("2d");+ if (!measurementContext) {+ return null;+ }++ measurementContext.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`;++ return buildActiveCaptionLayout({+ cues: autoCaptions,+ timeMs: Math.round(currentTime * 1000),+ settings: autoCaptionSettings,+ maxWidthPx: maxTextWidthPx,+ measureText: (text) => measurementContext.measureText(text).width,+ });+ }, [autoCaptionSettings, autoCaptions, currentTime]);++ useEffect(() => {+ const captionBox = captionBoxRef.current;+ if (!captionBox || !activeCaptionLayout || !autoCaptionSettings) {+ if (captionBox) {+ captionBox.style.clipPath = "";+ captionBox.style.removeProperty("-webkit-clip-path");+ }+ return;+ }++ const frame = requestAnimationFrame(() => {+ const width = captionBox.offsetWidth;+ const height = captionBox.offsetHeight;+ if (width <= 0 || height <= 0) {+ return;+ }++ const fontSize = getCaptionScaledFontSize(+ autoCaptionSettings.fontSize,+ overlayRef.current?.clientWidth || 960,+ autoCaptionSettings.maxWidth,+ );++ const squirclePath = getSquircleSvgPath({+ x: 0,+ y: 0,+ width,+ height,+ radius: getCaptionScaledRadius(autoCaptionSettings.boxRadius, fontSize),+ });+ captionBox.style.clipPath = `path('${squirclePath}')`;+ captionBox.style.setProperty("-webkit-clip-path", `path('${squirclePath}')`);+ });++ return () => cancelAnimationFrame(frame);+ }, [activeCaptionLayout, autoCaptionSettings]);const motionBlurStateRef = useRef<MotionBlurState>(createMotionBlurState());const applyWebcamBubbleLayout = useCallback((zoomScale: number) => {@@ -620,6 +767,38 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(connectZoomsRef.current = connectZooms;}, [connectZooms]);+ useEffect(() => {+ zoomInDurationMsRef.current = zoomInDurationMs;+ }, [zoomInDurationMs]);++ useEffect(() => {+ zoomInOverlapMsRef.current = zoomInOverlapMs;+ }, [zoomInOverlapMs]);++ useEffect(() => {+ zoomOutDurationMsRef.current = zoomOutDurationMs;+ }, [zoomOutDurationMs]);++ useEffect(() => {+ connectedZoomGapMsRef.current = connectedZoomGapMs;+ }, [connectedZoomGapMs]);++ useEffect(() => {+ connectedZoomDurationMsRef.current = connectedZoomDurationMs;+ }, [connectedZoomDurationMs]);++ useEffect(() => {+ zoomInEasingRef.current = zoomInEasing;+ }, [zoomInEasing]);++ useEffect(() => {+ zoomOutEasingRef.current = zoomOutEasing;+ }, [zoomOutEasing]);++ useEffect(() => {+ connectedZoomEasingRef.current = connectedZoomEasing;+ }, [connectedZoomEasing]);+useEffect(() => {cursorTelemetryRef.current = cursorTelemetry;}, [cursorTelemetry]);@@ -628,6 +807,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(showCursorRef.current = showCursor;}, [showCursor]);+ useEffect(() => {+ cursorStyleRef.current = cursorStyle;+ }, [cursorStyle]);+useEffect(() => {cursorSizeRef.current = cursorSize;}, [cursorSize]);@@ -877,6 +1060,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(if (cursorOverlayEnabled) {const cursorOverlay = new PixiCursorOverlay({dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * cursorSizeRef.current,+ style: cursorStyleRef.current,smoothingFactor: cursorSmoothingRef.current,motionBlur: cursorMotionBlurRef.current,clickBounce: cursorClickBounceRef.current,@@ -1228,6 +1412,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(}overlay.setDotRadius(DEFAULT_CURSOR_CONFIG.dotRadius * cursorSize);+ overlay.setStyle(cursorStyle);overlay.setSmoothingFactor(cursorSmoothing);overlay.setMotionBlur(cursorMotionBlur);overlay.setClickBounce(cursorClickBounce);@@ -1235,6 +1420,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(overlay.setSway(cursorSway);overlay.reset();}, [+ cursorStyle,cursorSize,cursorSmoothing,cursorMotionBlur,@@ -1352,12 +1538,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(const nativeAspectRatio = (() => {const locked = lockedVideoDimensionsRef.current;- if (locked && locked.height > 0) {- return locked.width / locked.height;+ if (locked) {+ return getEffectiveNativeAspectRatio(locked, cropRegion);}const video = videoRef.current;- if (video && video.videoHeight > 0) {- return video.videoWidth / video.videoHeight;+ if (video && video.videoHeight > 0 && video.videoWidth > 0) {+ return getEffectiveNativeAspectRatio(+ {+ width: video.videoWidth,+ height: video.videoHeight,+ },+ cropRegion,+ );}return 16 / 9;})();@@ -1429,6 +1621,97 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(</div></div>) : null}+ {activeCaptionLayout && autoCaptionSettings ? (+ <div+ className="pointer-events-none absolute inset-x-0 flex justify-center"+ style={{+ bottom: `${autoCaptionSettings.bottomOffset}%`,+ }}+ >+ <div+ style={{+ maxWidth: `${autoCaptionSettings.maxWidth}%`,+ opacity: activeCaptionLayout.opacity,+ transform: `translateY(${activeCaptionLayout.translateY}px) scale(${activeCaptionLayout.scale})`,+ transformOrigin: "center bottom",+ filter: "drop-shadow(0 12px 30px rgba(0, 0, 0, 0.28))",+ }}+ >+ <div+ ref={captionBoxRef}+ style={{+ backgroundColor: `rgba(0, 0, 0, ${autoCaptionSettings.backgroundOpacity})`,+ fontFamily: getDefaultCaptionFontFamily(),+ fontSize: `${getCaptionScaledFontSize(+ autoCaptionSettings.fontSize,+ overlayRef.current?.clientWidth || 960,+ autoCaptionSettings.maxWidth,+ )}px`,+ lineHeight: CAPTION_LINE_HEIGHT,+ textAlign: "center",+ fontWeight: CAPTION_FONT_WEIGHT,+ padding: `${getCaptionPadding(+ getCaptionScaledFontSize(+ autoCaptionSettings.fontSize,+ overlayRef.current?.clientWidth || 960,+ autoCaptionSettings.maxWidth,+ ),+ ).y}px ${getCaptionPadding(+ getCaptionScaledFontSize(+ autoCaptionSettings.fontSize,+ overlayRef.current?.clientWidth || 960,+ autoCaptionSettings.maxWidth,+ ),+ ).x}px`,+ borderRadius: `${getCaptionScaledRadius(+ autoCaptionSettings.boxRadius,+ getCaptionScaledFontSize(+ autoCaptionSettings.fontSize,+ overlayRef.current?.clientWidth || 960,+ autoCaptionSettings.maxWidth,+ ),+ )}px`,+ boxSizing: "border-box",+ }}+ >+ {activeCaptionLayout.visibleLines.map((line) => (+ <div+ key={`${activeCaptionLayout.blockKey}-${line.startWordIndex}`}+ style={{+ display: "flex",+ justifyContent: "center",+ flexWrap: "nowrap",+ whiteSpace: "nowrap",+ }}+ >+ {line.words.map((word) => {+ const visualState = getCaptionWordVisualState(+ activeCaptionLayout.hasWordTimings,+ word.state,+ );++ return (+ <span+ key={`${activeCaptionLayout.blockKey}-${word.index}`}+ style={{+ display: "inline-block",+ whiteSpace: "pre",+ color: visualState.isInactive+ ? autoCaptionSettings.inactiveTextColor+ : autoCaptionSettings.textColor,+ opacity: visualState.opacity,+ }}+ >+ {`${word.leadingSpace ? " " : ""}${word.text}`}+ </span>+ );+ })}+ </div>+ ))}+ </div>+ </div>+ </div>+ ) : null}{(() => {const filtered = (annotationRegions || []).filter((annotation) => {diff --git a/src/components/video-editor/videoPlayback/layoutUtils.ts b/src/components/video-editor/videoPlayback/layoutUtils.tsindex 77e6cd7f..ec1dd6af 100644--- a/src/components/video-editor/videoPlayback/layoutUtils.ts+++ b/src/components/video-editor/videoPlayback/layoutUtils.ts@@ -71,7 +71,6 @@ export function layoutVideoContent(params: LayoutParams): LayoutResult | null {const scale = Math.min(maxDisplayWidth / croppedVideoWidth,maxDisplayHeight / croppedVideoHeight,- 1);videoSprite.scale.set(scale);diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.tsindex 0b9080f8..df321bcf 100644--- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts+++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts@@ -1,5 +1,7 @@import type { ZoomFocus, ZoomRegion } from "../types";-import { ZOOM_DEPTH_SCALES } from "../types";+import {+ ZOOM_DEPTH_SCALES,+} from "../types";import { TRANSITION_WINDOW_MS, ZOOM_IN_TRANSITION_WINDOW_MS } from "./constants";import { clampFocusToScale } from "./focusUtils";import { clamp01, cubicBezier, easeOutScreenStudio } from "./mathUtils";@@ -35,7 +37,10 @@ function easeConnectedPan(value: number) {return cubicBezier(0.1, 0.0, 0.2, 1.0, value);}-export function computeRegionStrength(region: ZoomRegion, timeMs: number) {+export function computeRegionStrength(+ region: ZoomRegion,+ timeMs: number,+) {const zoomInEnd = region.startMs + ZOOM_IN_OVERLAP_MS;const leadInStart = zoomInEnd - ZOOM_IN_TRANSITION_WINDOW_MS;const leadOutEnd = region.endMs + TRANSITION_WINDOW_MS;@@ -92,7 +97,11 @@ function getConnectedRegionPairs(regions: ZoomRegion[]) {return pairs;}-function getActiveRegion(regions: ZoomRegion[], timeMs: number, connectedPairs: ConnectedRegionPair[]) {+function getActiveRegion(+ regions: ZoomRegion[],+ timeMs: number,+ connectedPairs: ConnectedRegionPair[],+) {const activeRegions = regions.map((region) => {const outgoingPair = connectedPairs.find((pair) => pair.currentRegion.id === region.id);@@ -198,7 +207,9 @@ export function findDominantRegion(regions: ZoomRegion[], timeMs: number, optionblendedScale: number | null;transition: ConnectedPanTransition | null;} {- const connectedPairs = options.connectZooms ? getConnectedRegionPairs(regions) : [];+ const connectedPairs = options.connectZooms+ ? getConnectedRegionPairs(regions)+ : [];if (options.connectZooms) {const connectedTransition = getConnectedRegionTransition(connectedPairs, timeMs);