Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

Add inline auto-caption editing

wizardAEIbc26584de861
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index 7dc6d812..81b14d31 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -101,6 +101,7 @@ const PhSettings = (props: { className?: string; weight?: "fill" | "regular" })
import { extensionHost } from "@/lib/extensions";
import { resolveAutoCaptionSourcePath } from "./autoCaptionSource";
import { CropControl } from "./CropControl";
+import { updateCaptionCuesForEditedTarget, type CaptionEditTarget } from "./captionEditing";
import { ExportSettingsMenu } from "./ExportSettingsMenu";
import ExtensionManager from "./ExtensionManager";
import { loadEditorPreferences, saveEditorPreferences } from "./editorPreferences";
@@ -2273,6 +2274,14 @@ export default function VideoEditor() {
setAutoCaptionSettings((prev) => ({ ...prev, enabled: false }));
}, []);
+ const handleSaveAutoCaptionEdit = useCallback(
+ (target: CaptionEditTarget, text: string) => {
+ setAutoCaptions((captions) => updateCaptionCuesForEditedTarget(captions, target, text));
+ toast.success(t("settings.captions.editSaved", "Caption updated"));
+ },
+ [t],
+ );
+
const saveProject = useCallback(
async (forceSaveAs: boolean, options?: SaveProjectOptions) => {
clearPendingProjectAutosave();
@@ -5530,6 +5539,7 @@ export default function VideoEditor() {
annotationRegions={annotationRegions}
autoCaptions={autoCaptions}
autoCaptionSettings={autoCaptionSettings}
+ onEditAutoCaption={handleSaveAutoCaptionEdit}
selectedAnnotationId={selectedAnnotationId}
onSelectAnnotation={handleSelectAnnotation}
onAnnotationPositionChange={
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx
index c6c50f0b..bfb8623e 100644
--- a/src/components/video-editor/VideoPlayback.tsx
+++ b/src/components/video-editor/VideoPlayback.tsx
@@ -25,6 +25,7 @@ import {
DEFAULT_WALLPAPER_RELATIVE_PATH,
isVideoWallpaperSource,
} from "@/lib/wallpapers";
+import { type CaptionEditTarget, normalizeCaptionEditText } from "./captionEditing";
import { buildActiveCaptionLayout } from "./captionLayout";
import {
CAPTION_FONT_WEIGHT,
@@ -254,6 +255,7 @@ interface VideoPlaybackProps {
annotationRegions?: AnnotationRegion[];
autoCaptions?: CaptionCue[];
autoCaptionSettings?: AutoCaptionSettings;
+ onEditAutoCaption?: (target: CaptionEditTarget, text: string) => void;
selectedAnnotationId?: string | null;
onSelectAnnotation?: (id: string | null) => void;
onAnnotationPositionChange?: (id: string, position: { x: number; y: number }) => void;
@@ -272,6 +274,11 @@ interface VideoPlaybackProps {
volume?: number;
}
+type CaptionEditSession = {
+ target: CaptionEditTarget;
+ draft: string;
+};
+
export interface VideoPlaybackRef {
video: HTMLVideoElement | null;
app: Application | null;
@@ -324,6 +331,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
annotationRegions = [],
autoCaptions = [],
autoCaptionSettings,
+ onEditAutoCaption,
selectedAnnotationId,
onSelectAnnotation,
onAnnotationPositionChange,
@@ -359,6 +367,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const webcamBubbleRef = useRef<HTMLDivElement | null>(null);
const webcamBubbleInnerRef = useRef<HTMLDivElement | null>(null);
const captionBoxRef = useRef<HTMLDivElement | null>(null);
+ const captionEditInputRef = useRef<HTMLTextAreaElement | null>(null);
+ const captionEditSessionRef = useRef<CaptionEditSession | null>(null);
+ const [captionEditSession, setCaptionEditSession] = useState<CaptionEditSession | null>(
+ null,
+ );
const currentTimeRef = useRef(0);
const zoomRegionsRef = useRef<ZoomRegion[]>([]);
const selectedZoomIdRef = useRef<string | null>(null);
@@ -471,6 +484,127 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
measureText: (text) => measurementContext.measureText(text).width,
});
}, [autoCaptionSettings, autoCaptions, currentTime]);
+ const isCaptionEditing = captionEditSession !== null;
+ const captionEditDraft = captionEditSession?.draft ?? "";
+ const captionEditTargetId = captionEditSession?.target.id ?? null;
+ const captionEditTextMetrics = useMemo(() => {
+ if (!captionEditSession || !autoCaptionSettings || typeof document === "undefined") {
+ return null;
+ }
+
+ const overlayWidth = overlayRef.current?.clientWidth || 960;
+ const fontSize = getCaptionScaledFontSize(
+ autoCaptionSettings.fontSize,
+ overlayWidth,
+ autoCaptionSettings.maxWidth,
+ );
+ const maxTextWidthPx = getCaptionTextMaxWidth(
+ overlayWidth,
+ autoCaptionSettings.maxWidth,
+ fontSize,
+ );
+ const measurementCanvas = document.createElement("canvas");
+ const measurementContext = measurementCanvas.getContext("2d");
+ if (!measurementContext) {
+ return null;
+ }
+
+ measurementContext.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`;
+ const measuredWidth = Math.max(
+ ...captionEditSession.draft
+ .split(/\r?\n/)
+ .map((line) => measurementContext.measureText(line || " ").width),
+ );
+
+ return {
+ fontSize,
+ maxTextWidthPx,
+ widthPx: Math.ceil(
+ Math.min(maxTextWidthPx, Math.max(fontSize * 2, measuredWidth + 2)),
+ ),
+ };
+ }, [autoCaptionSettings, captionEditSession]);
+ const captionEditSizeKey = captionEditSession
+ ? `${captionEditTextMetrics?.widthPx ?? 0}:${captionEditDraft}`
+ : "";
+
+ const beginCaptionEdit = useCallback(() => {
+ if (!activeCaptionLayout?.editTarget || !onEditAutoCaption) {
+ return;
+ }
+
+ videoRef.current?.pause();
+ onPlayStateChange(false);
+ const nextSession = {
+ target: activeCaptionLayout.editTarget,
+ draft: activeCaptionLayout.editTarget.text,
+ };
+ captionEditSessionRef.current = nextSession;
+ setCaptionEditSession(nextSession);
+ }, [activeCaptionLayout, onEditAutoCaption, onPlayStateChange]);
+
+ const commitCaptionEdit = useCallback(() => {
+ const session = captionEditSessionRef.current;
+ if (!session || !onEditAutoCaption) {
+ captionEditSessionRef.current = null;
+ setCaptionEditSession(null);
+ return;
+ }
+
+ const normalizedDraft = normalizeCaptionEditText(session.draft);
+ captionEditSessionRef.current = null;
+ if (!normalizedDraft) {
+ setCaptionEditSession(null);
+ return;
+ }
+
+ if (normalizedDraft !== normalizeCaptionEditText(session.target.text)) {
+ onEditAutoCaption(session.target, session.draft);
+ }
+ setCaptionEditSession(null);
+ }, [onEditAutoCaption]);
+
+ const cancelCaptionEdit = useCallback(() => {
+ captionEditSessionRef.current = null;
+ setCaptionEditSession(null);
+ }, []);
+
+ useEffect(() => {
+ if (!captionEditTargetId) {
+ return;
+ }
+
+ const frame = requestAnimationFrame(() => {
+ const input = captionEditInputRef.current;
+ if (!input) {
+ return;
+ }
+
+ input.focus();
+ const cursorPosition = input.value.length;
+ input.setSelectionRange(cursorPosition, cursorPosition);
+ });
+
+ return () => cancelAnimationFrame(frame);
+ }, [captionEditTargetId]);
+
+ useEffect(() => {
+ if (!captionEditSizeKey) {
+ return;
+ }
+
+ const frame = requestAnimationFrame(() => {
+ const input = captionEditInputRef.current;
+ if (!input) {
+ return;
+ }
+
+ input.style.height = "auto";
+ input.style.height = `${input.scrollHeight}px`;
+ });
+
+ return () => cancelAnimationFrame(frame);
+ }, [captionEditSizeKey]);
useEffect(() => {
const captionBox = captionBoxRef.current;
@@ -483,6 +617,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
}
const frame = requestAnimationFrame(() => {
+ if (isCaptionEditing) {
+ captionBox.dataset.editingCaption = captionEditSizeKey;
+ } else {
+ delete captionBox.dataset.editingCaption;
+ }
+
const width = captionBox.offsetWidth;
const height = captionBox.offsetHeight;
if (width <= 0 || height <= 0) {
@@ -507,7 +647,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
});
return () => cancelAnimationFrame(frame);
- }, [activeCaptionLayout, autoCaptionSettings]);
+ }, [activeCaptionLayout, autoCaptionSettings, captionEditSizeKey, isCaptionEditing]);
const motionBlurStateRef = useRef<MotionBlurState>(createMotionBlurState());
const applyWebcamBubbleLayout = useCallback(
@@ -1020,7 +1160,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
: clampMediaTimeToDuration(clipTimelineTime, videoDuration);
const activeSpeedRegion = speedRegionsRef.current.find(
- (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
+ (region) =>
+ currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
);
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
const syncedPlaybackRate = getMediaSyncPlaybackRate({
@@ -2368,7 +2509,34 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
}}
>
<div
+ role={
+ onEditAutoCaption && !captionEditSession
+ ? "button"
+ : undefined
+ }
+ tabIndex={
+ onEditAutoCaption && !captionEditSession ? 0 : undefined
+ }
+ aria-label={
+ onEditAutoCaption && !captionEditSession
+ ? "Edit current caption"
+ : undefined
+ }
ref={captionBoxRef}
+ onClick={() => {
+ if (!captionEditSession) {
+ beginCaptionEdit();
+ }
+ }}
+ onKeyDown={(event) => {
+ if (!onEditAutoCaption || captionEditSession) {
+ return;
+ }
+ if (event.key === "Enter" || event.key === " ") {
+ event.preventDefault();
+ beginCaptionEdit();
+ }
+ }}
style={{
backgroundColor: `rgba(0, 0, 0, ${autoCaptionSettings.backgroundOpacity})`,
fontFamily: getDefaultCaptionFontFamily(),
@@ -2406,42 +2574,137 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
),
)}px`,
boxSizing: "border-box",
+ cursor:
+ onEditAutoCaption && !captionEditSession
+ ? "text"
+ : undefined,
+ pointerEvents: onEditAutoCaption ? "auto" : undefined,
}}
>
- {activeCaptionLayout.visibleLines.map((line) => (
- <div
- key={`${activeCaptionLayout.blockKey}-${line.startWordIndex}`}
+ {captionEditSession ? (
+ <textarea
+ ref={captionEditInputRef}
+ value={captionEditSession.draft}
+ onChange={(event) => {
+ const draft = event.target.value;
+ setCaptionEditSession((session) => {
+ const nextSession = session
+ ? {
+ ...session,
+ draft,
+ }
+ : session;
+ captionEditSessionRef.current = nextSession;
+ return nextSession;
+ });
+ }}
+ onBlur={commitCaptionEdit}
+ onClick={(event) => event.stopPropagation()}
+ onKeyDown={(event) => {
+ if (event.key === "Escape") {
+ event.preventDefault();
+ cancelCaptionEdit();
+ return;
+ }
+ if (event.key === "Enter" && !event.shiftKey) {
+ event.preventDefault();
+ event.currentTarget.blur();
+ }
+ }}
+ rows={Math.max(
+ 1,
+ activeCaptionLayout.visibleLines.length,
+ )}
+ aria-label="Edit current caption"
style={{
- display: "flex",
- justifyContent: "center",
- flexWrap: "nowrap",
- whiteSpace: "nowrap",
+ display: "block",
+ width: `${
+ captionEditTextMetrics?.widthPx ??
+ Math.max(
+ 48,
+ activeCaptionLayout.visibleLines.reduce(
+ (width, line) =>
+ Math.max(width, line.width),
+ 0,
+ ),
+ )
+ }px`,
+ maxWidth: `${
+ captionEditTextMetrics?.maxTextWidthPx ??
+ getCaptionTextMaxWidth(
+ overlayRef.current?.clientWidth || 960,
+ autoCaptionSettings.maxWidth,
+ getCaptionScaledFontSize(
+ autoCaptionSettings.fontSize,
+ overlayRef.current?.clientWidth ||
+ 960,
+ autoCaptionSettings.maxWidth,
+ ),
+ )
+ }px`,
+ minHeight: `${
+ Math.max(
+ 1,
+ activeCaptionLayout.visibleLines.length,
+ ) *
+ getCaptionScaledFontSize(
+ autoCaptionSettings.fontSize,
+ overlayRef.current?.clientWidth || 960,
+ autoCaptionSettings.maxWidth,
+ ) *
+ CAPTION_LINE_HEIGHT
+ }px`,
+ resize: "none",
+ border: "0",
+ outline: "0",
+ padding: "0",
+ margin: "0",
+ overflow: "hidden",
+ background: "transparent",
+ color: autoCaptionSettings.textColor,
+ font: "inherit",
+ fontWeight: "inherit",
+ lineHeight: "inherit",
+ textAlign: "center",
}}
- >
- {line.words.map((word) => {
- const visualState = getCaptionWordVisualState(
- activeCaptionLayout.hasWordTimings,
- word.state,
- );
-
- return (
- <span
- key={`${activeCaptionLayout.blockKey}-${word.index}`}
- style={{
- display: "inline-block",
- whiteSpace: "pre",
- color: visualState.isInactive
- ? autoCaptionSettings.inactiveTextColor
- : autoCaptionSettings.textColor,
- opacity: visualState.opacity,
- }}
- >
- {`${word.leadingSpace ? " " : ""}${word.text}`}
- </span>
- );
- })}
- </div>
- ))}
+ />
+ ) : (
+ activeCaptionLayout.visibleLines.map((line) => (
+ <div
+ key={`${activeCaptionLayout.blockKey}-${line.startWordIndex}`}
+ style={{
+ display: "flex",
+ justifyContent: "center",
+ flexWrap: "nowrap",
+ whiteSpace: "nowrap",
+ }}
+ >
+ {line.words.map((word) => {
+ const visualState =
+ getCaptionWordVisualState(
+ activeCaptionLayout.hasWordTimings,
+ word.state,
+ );
+
+ return (
+ <span
+ key={`${activeCaptionLayout.blockKey}-${word.index}`}
+ style={{
+ display: "inline-block",
+ whiteSpace: "pre",
+ color: visualState.isInactive
+ ? autoCaptionSettings.inactiveTextColor
+ : autoCaptionSettings.textColor,
+ opacity: visualState.opacity,
+ }}
+ >
+ {`${word.leadingSpace ? " " : ""}${word.text}`}
+ </span>
+ );
+ })}
+ </div>
+ ))
+ )}
</div>
</div>
</div>
diff --git a/src/components/video-editor/captionEditing.test.ts b/src/components/video-editor/captionEditing.test.ts
new file mode 100644
index 00000000..7cfc1831
--- /dev/null
+++ b/src/components/video-editor/captionEditing.test.ts
@@ -0,0 +1,117 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ type CaptionEditTarget,
+ normalizeCaptionEditText,
+ updateCaptionCuesForEditedTarget,
+} from "./captionEditing";
+import { buildActiveCaptionLayout } from "./captionLayout";
+import { type CaptionCue, DEFAULT_AUTO_CAPTION_SETTINGS } from "./types";
+
+const visibleTarget: CaptionEditTarget = {
+ id: "visible-page",
+ startMs: 1_000,
+ endMs: 2_400,
+ text: "Hello Hello 你们好啊",
+ words: [
+ {
+ cueId: "a",
+ cueWordIndex: 0,
+ startMs: 1_000,
+ endMs: 1_500,
+ text: "Hello",
+ leadingSpace: false,
+ },
+ {
+ cueId: "a",
+ cueWordIndex: 1,
+ startMs: 1_500,
+ endMs: 2_000,
+ text: "Hello",
+ leadingSpace: true,
+ },
+ {
+ cueId: "b",
+ cueWordIndex: 0,
+ startMs: 2_000,
+ endMs: 2_400,
+ text: "你们好啊",
+ leadingSpace: true,
+ },
+ ],
+};
+
+describe("captionEditing", () => {
+ it("normalizes edited caption text", () => {
+ expect(normalizeCaptionEditText(" hello \n edited\tcaption ")).toBe(
+ "hello edited caption",
+ );
+ expect(normalizeCaptionEditText(" \n\t ")).toBe("");
+ });
+
+ it("keeps text-only captions text-only after editing", () => {
+ const updated = updateCaptionCuesForEditedTarget(
+ [
+ { id: "a", startMs: 1_000, endMs: 2_000, text: "Hello Hello" },
+ { id: "b", startMs: 2_000, endMs: 3_000, text: "你们好啊 这个是我的屏幕" },
+ ],
+ visibleTarget,
+ "Hi 大家好",
+ );
+
+ expect(updated.map((caption) => caption.text)).toEqual(["Hi", "大家好 这个是我的屏幕"]);
+ expect(updated.every((caption) => caption.words === undefined)).toBe(true);
+
+ const layout = buildActiveCaptionLayout({
+ cues: updated,
+ timeMs: 1_500,
+ settings: DEFAULT_AUTO_CAPTION_SETTINGS,
+ maxWidthPx: 500,
+ measureText: (text) => text.length * 10,
+ });
+ expect(layout?.hasWordTimings).toBe(false);
+ });
+
+ it("preserves cue identity and timing when editing captions with word timings", () => {
+ const cues: CaptionCue[] = [
+ {
+ id: "a",
+ startMs: 1_000,
+ endMs: 2_000,
+ text: "Hello Hello",
+ words: [
+ { text: "Hello", startMs: 1_000, endMs: 1_500 },
+ { text: "Hello", startMs: 1_500, endMs: 2_000, leadingSpace: true },
+ ],
+ },
+ {
+ id: "b",
+ startMs: 2_000,
+ endMs: 3_000,
+ text: "你们好啊 这个是我的屏幕",
+ words: [
+ { text: "你们好啊", startMs: 2_000, endMs: 2_400 },
+ { text: "这个是我的屏幕", startMs: 2_400, endMs: 3_000, leadingSpace: true },
+ ],
+ },
+ ];
+
+ const updated = updateCaptionCuesForEditedTarget(cues, visibleTarget, "Hi 大家好");
+
+ expect(updated.map((caption) => [caption.id, caption.startMs, caption.endMs])).toEqual([
+ ["a", 1_000, 2_000],
+ ["b", 2_000, 3_000],
+ ]);
+ expect(updated[0].words).toEqual([{ text: "Hi", startMs: 1_000, endMs: 2_000 }]);
+ expect(updated[1].words).toEqual([
+ { text: "大家好", startMs: 2_000, endMs: 2_400 },
+ { text: "这个是我的屏幕", startMs: 2_400, endMs: 3_000, leadingSpace: true },
+ ]);
+ });
+
+ it("does not update captions when edited text is blank", () => {
+ const cues: CaptionCue[] = [{ id: "a", startMs: 1_000, endMs: 2_000, text: "Hello Hello" }];
+
+ expect(updateCaptionCuesForEditedTarget(cues, visibleTarget, " \n\t ")).toBe(cues);
+ });
+});
diff --git a/src/components/video-editor/captionEditing.ts b/src/components/video-editor/captionEditing.ts
new file mode 100644
index 00000000..b5084426
--- /dev/null
+++ b/src/components/video-editor/captionEditing.ts
@@ -0,0 +1,234 @@
+import type { CaptionCue, CaptionCueWord } from "./types";
+
+export interface CaptionEditWordRef {
+ cueId: string;
+ cueWordIndex: number;
+ startMs: number;
+ endMs: number;
+ text: string;
+ leadingSpace: boolean;
+}
+
+export interface CaptionEditTarget {
+ id: string;
+ startMs: number;
+ endMs: number;
+ text: string;
+ words: CaptionEditWordRef[];
+}
+
+export function normalizeCaptionEditText(text: string) {
+ return text.trim().replace(/\s+/g, " ");
+}
+
+function buildCaptionWordsForEditedText(
+ text: string,
+ startMs: number,
+ endMs: number,
+): CaptionCueWord[] {
+ const normalizedText = normalizeCaptionEditText(text);
+ const tokens = normalizedText.match(/\S+/g) ?? [];
+ const normalizedStartMs = Math.max(0, Math.round(startMs));
+ const normalizedEndMs = Math.max(normalizedStartMs + 1, Math.round(endMs));
+ const durationMs = normalizedEndMs - normalizedStartMs;
+
+ return tokens.map((token, index) => {
+ const wordStartMs = Math.min(
+ normalizedEndMs - 1,
+ Math.max(
+ normalizedStartMs,
+ Math.round(normalizedStartMs + (durationMs * index) / tokens.length),
+ ),
+ );
+ const nextBoundaryMs =
+ index === tokens.length - 1
+ ? normalizedEndMs
+ : Math.round(normalizedStartMs + (durationMs * (index + 1)) / tokens.length);
+ const wordEndMs = Math.min(normalizedEndMs, Math.max(wordStartMs + 1, nextBoundaryMs));
+
+ return {
+ text: token,
+ startMs: wordStartMs,
+ endMs: wordEndMs,
+ ...(index > 0 ? { leadingSpace: true } : {}),
+ };
+ });
+}
+
+function normalizeCaptionWords(cue: CaptionCue): CaptionCueWord[] {
+ const sourceWords =
+ Array.isArray(cue.words) && cue.words.length > 0
+ ? cue.words
+ : buildCaptionWordsForEditedText(cue.text, cue.startMs, cue.endMs);
+
+ return sourceWords
+ .filter((word): word is CaptionCueWord => Boolean(word && typeof word.text === "string"))
+ .map((word) => {
+ const startMs = Math.max(
+ cue.startMs,
+ Math.min(cue.endMs - 1, Math.round(word.startMs)),
+ );
+ const endMs = Math.max(startMs + 1, Math.min(cue.endMs, Math.round(word.endMs)));
+
+ return {
+ text: normalizeCaptionEditText(word.text),
+ startMs,
+ endMs,
+ ...(word.leadingSpace ? { leadingSpace: true } : {}),
+ };
+ })
+ .filter((word) => word.text.length > 0);
+}
+
+function captionWordsToText(words: CaptionCueWord[]) {
+ return words
+ .map((word, index) => `${index > 0 && word.leadingSpace ? " " : ""}${word.text}`)
+ .join("")
+ .trim();
+}
+
+function normalizeCaptionWordSpacing(words: CaptionCueWord[]): CaptionCueWord[] {
+ return words
+ .slice()
+ .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs)
+ .map((word, index) => ({
+ text: word.text,
+ startMs: word.startMs,
+ endMs: word.endMs,
+ ...(index > 0 ? { leadingSpace: true } : {}),
+ }));
+}
+
+function shouldPreserveCaptionWords(cue: CaptionCue) {
+ return Array.isArray(cue.words) && cue.words.length > 0;
+}
+
+export function updateCaptionCuesForEditedTarget(
+ cues: CaptionCue[],
+ target: CaptionEditTarget,
+ text: string,
+): CaptionCue[] {
+ const normalizedText = normalizeCaptionEditText(text);
+ if (!normalizedText || target.words.length === 0) {
+ return cues;
+ }
+
+ const targetWordsByCue = new Map<string, CaptionEditWordRef[]>();
+ for (const word of target.words) {
+ const words = targetWordsByCue.get(word.cueId) ?? [];
+ words.push(word);
+ targetWordsByCue.set(word.cueId, words);
+ }
+
+ const tokens = normalizedText.match(/\S+/g) ?? [];
+ const targetCueIds = new Set(targetWordsByCue.keys());
+ const cueSegments = cues
+ .filter((cue) => targetCueIds.has(cue.id))
+ .map((cue) => {
+ const words = targetWordsByCue.get(cue.id) ?? [];
+ return {
+ cue,
+ startMs: Math.min(...words.map((word) => word.startMs)),
+ endMs: Math.max(...words.map((word) => word.endMs)),
+ };
+ })
+ .filter((segment) => Number.isFinite(segment.startMs) && Number.isFinite(segment.endMs));
+ const editedWordsByCue = new Map<string, CaptionCueWord[]>();
+ const tokenCountsByCue = distributeEditedTokensAcrossCueSegments(tokens.length, cueSegments);
+ let tokenCursor = 0;
+
+ for (const segment of cueSegments) {
+ const tokenCount = tokenCountsByCue.get(segment.cue.id) ?? 0;
+ if (tokenCount <= 0) {
+ continue;
+ }
+
+ const segmentTokens = tokens.slice(tokenCursor, tokenCursor + tokenCount);
+ tokenCursor += tokenCount;
+ editedWordsByCue.set(
+ segment.cue.id,
+ buildCaptionWordsForEditedText(segmentTokens.join(" "), segment.startMs, segment.endMs),
+ );
+ }
+
+ return cues.map((cue) => {
+ const targetWords = targetWordsByCue.get(cue.id);
+ if (!targetWords) {
+ return cue;
+ }
+
+ const targetIndexes = new Set(targetWords.map((word) => word.cueWordIndex));
+ const existingWords = normalizeCaptionWords(cue);
+ const keptWords = existingWords.filter((_, index) => !targetIndexes.has(index));
+ const nextWords = normalizeCaptionWordSpacing([
+ ...keptWords,
+ ...(editedWordsByCue.get(cue.id) ?? []),
+ ]);
+ const shouldKeepWords = shouldPreserveCaptionWords(cue);
+
+ return {
+ id: cue.id,
+ startMs: cue.startMs,
+ endMs: cue.endMs,
+ text: captionWordsToText(nextWords),
+ ...(shouldKeepWords && nextWords.length > 0 ? { words: nextWords } : {}),
+ };
+ });
+}
+
+function distributeEditedTokensAcrossCueSegments(
+ tokenCount: number,
+ segments: Array<{ cue: CaptionCue; startMs: number; endMs: number }>,
+) {
+ const tokenCountsByCue = new Map<string, number>();
+ if (tokenCount <= 0 || segments.length === 0) {
+ return tokenCountsByCue;
+ }
+
+ if (tokenCount < segments.length) {
+ const largestSegments = [...segments]
+ .sort((a, b) => b.endMs - b.startMs - (a.endMs - a.startMs))
+ .slice(0, tokenCount);
+ const selectedCueIds = new Set(largestSegments.map((segment) => segment.cue.id));
+ for (const segment of segments) {
+ tokenCountsByCue.set(segment.cue.id, selectedCueIds.has(segment.cue.id) ? 1 : 0);
+ }
+ return tokenCountsByCue;
+ }
+
+ const baseTokenCount = 1;
+ const remainingTokens = tokenCount - segments.length;
+ const totalDuration = Math.max(
+ 1,
+ segments.reduce(
+ (total, segment) => total + Math.max(1, segment.endMs - segment.startMs),
+ 0,
+ ),
+ );
+ const weightedCounts = segments.map((segment) => {
+ const exactCount =
+ (Math.max(1, segment.endMs - segment.startMs) / totalDuration) * remainingTokens;
+ const extraCount = Math.floor(exactCount);
+ return {
+ segment,
+ count: baseTokenCount + extraCount,
+ remainder: exactCount - extraCount,
+ };
+ });
+ let assignedTokens = weightedCounts.reduce((total, item) => total + item.count, 0);
+
+ for (const item of [...weightedCounts].sort((a, b) => b.remainder - a.remainder)) {
+ if (assignedTokens >= tokenCount) {
+ break;
+ }
+
+ item.count += 1;
+ assignedTokens += 1;
+ }
+
+ for (const item of weightedCounts) {
+ tokenCountsByCue.set(item.segment.cue.id, item.count);
+ }
+
+ return tokenCountsByCue;
+}
diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.ts
index 076cb5db..6c0d987e 100644
--- a/src/components/video-editor/captionLayout.ts
+++ b/src/components/video-editor/captionLayout.ts
@@ -1,3 +1,4 @@
+import type { CaptionEditTarget } from "./captionEditing";
import type {
AutoCaptionAnimation,
AutoCaptionSettings,
@@ -8,12 +9,15 @@ import type {
export type CaptionWordState = "spoken" | "active" | "upcoming";
export interface CaptionWordLayout {
+ cueId: string;
+ cueWordIndex: number;
text: string;
index: number;
forcedBreakBefore: boolean;
leadingSpace: boolean;
startMs: number;
endMs: number;
+ hasRealTiming: boolean;
state: CaptionWordState;
}
@@ -37,6 +41,7 @@ export interface ActiveCaptionLayout {
hasWordTimings: boolean;
activeWordIndex: number;
activeWordProgress: number;
+ editTarget: CaptionEditTarget;
visiblePageIndex: number;
opacity: number;
translateY: number;
@@ -45,6 +50,7 @@ export interface ActiveCaptionLayout {
type CaptionSourceWord = {
cueId: string;
+ cueWordIndex: number;
text: string;
forcedBreakBefore: boolean;
leadingSpace?: boolean;
@@ -77,6 +83,7 @@ function splitCaptionWordsFromText(text: string) {
.forEach((word, wordIndex) => {
words.push({
cueId: "",
+ cueWordIndex: words.length,
text: word,
forcedBreakBefore: lineIndex > 0 && wordIndex === 0,
});
@@ -92,8 +99,9 @@ function splitCaptionWords(cue: CaptionCue) {
.filter((word): word is CaptionCueWord =>
Boolean(word && typeof word.text === "string"),
)
- .map((word) => ({
+ .map((word, cueWordIndex) => ({
cueId: cue.id,
+ cueWordIndex,
text: word.text.trim(),
forcedBreakBefore: false,
leadingSpace: Boolean(word.leadingSpace),
@@ -103,7 +111,10 @@ function splitCaptionWords(cue: CaptionCue) {
.filter((word) => word.text.length > 0);
}
- return splitCaptionWordsFromText(cue.text);
+ return splitCaptionWordsFromText(cue.text).map((word) => ({
+ ...word,
+ cueId: cue.id,
+ }));
}
function getActiveCaptionCue(cues: CaptionCue[], timeMs: number) {
@@ -119,6 +130,7 @@ function getActiveCaptionCue(cues: CaptionCue[], timeMs: number) {
function flattenCaptionWords(cues: CaptionCue[]) {
const flattened: Array<{
cueId: string;
+ cueWordIndex: number;
text: string;
forcedBreakBefore: boolean;
leadingSpace: boolean;
@@ -160,6 +172,7 @@ function flattenCaptionWords(cues: CaptionCue[]) {
flattened.push({
cueId: cue.id,
+ cueWordIndex: word.cueWordIndex,
text: word.text,
forcedBreakBefore:
word.forcedBreakBefore || (wordIndex === 0 && shouldForceCueBreak),
@@ -380,6 +393,18 @@ function getVisibleCaptionPageIndex(pages: CaptionPageLayout[], timeMs: number)
return -1;
}
+function getVisibleCaptionText(lines: CaptionLineLayout[]) {
+ return lines
+ .map((line) =>
+ line.words
+ .map((word, index) => `${index > 0 && word.leadingSpace ? " " : ""}${word.text}`)
+ .join("")
+ .trim(),
+ )
+ .filter(Boolean)
+ .join(" ");
+}
+
export function buildActiveCaptionLayout(options: {
cues: CaptionCue[];
timeMs: number;
@@ -392,34 +417,32 @@ export function buildActiveCaptionLayout(options: {
return null;
}
- const hasWordTimings = sourceWords.every((word) => word.hasRealTiming);
-
let activeWordIndex = -1;
- if (hasWordTimings) {
- activeWordIndex = sourceWords.findIndex(
- (word) => options.timeMs >= word.startMs && options.timeMs < word.endMs,
- );
- if (activeWordIndex < 0) {
- activeWordIndex = sourceWords.findIndex((word) => options.timeMs < word.startMs);
- activeWordIndex =
- activeWordIndex < 0
- ? sourceWords.length - 1
- : clamp(activeWordIndex - 1, 0, sourceWords.length - 1);
- }
+ activeWordIndex = sourceWords.findIndex(
+ (word) => options.timeMs >= word.startMs && options.timeMs < word.endMs,
+ );
+ if (activeWordIndex < 0) {
+ activeWordIndex = sourceWords.findIndex((word) => options.timeMs < word.startMs);
+ activeWordIndex =
+ activeWordIndex < 0
+ ? sourceWords.length - 1
+ : clamp(activeWordIndex - 1, 0, sourceWords.length - 1);
}
const maxRows = clamp(Math.round(options.settings.maxRows || 1), 1, 4);
const words: CaptionWordLayout[] = sourceWords.map((word, index) => {
return {
+ cueId: word.cueId,
+ cueWordIndex: word.cueWordIndex,
text: word.text,
index,
forcedBreakBefore: word.forcedBreakBefore,
leadingSpace: word.leadingSpace,
startMs: word.startMs,
endMs: word.endMs,
- state: !hasWordTimings
- ? "spoken"
- : index < activeWordIndex
+ hasRealTiming: word.hasRealTiming,
+ state:
+ index < activeWordIndex
? "spoken"
: index === activeWordIndex
? "active"
@@ -461,6 +484,8 @@ export function buildActiveCaptionLayout(options: {
const animationStartMs = visiblePage?.startMs ?? sourceWords[0].startMs;
const animationEndMs = visiblePage?.endMs ?? sourceWords[sourceWords.length - 1].endMs;
const pageStartWordIndex = visibleLines[0]?.startWordIndex ?? 0;
+ const visibleWords = visibleLines.flatMap((line) => line.words);
+ const visibleHasWordTimings = visibleWords.every((word) => word.hasRealTiming);
const pageCueId = sourceWords[pageStartWordIndex]?.cueId ?? sourceWords[0].cueId;
const activeCue =
getActiveCaptionCue(options.cues, options.timeMs) ??
@@ -478,9 +503,25 @@ export function buildActiveCaptionLayout(options: {
cue: activeCue,
blockKey: `${Math.round(animationStartMs)}-${Math.round(animationEndMs)}`,
visibleLines,
- hasWordTimings,
+ hasWordTimings: visibleHasWordTimings,
activeWordIndex,
activeWordProgress,
+ editTarget: {
+ id: `${Math.round(animationStartMs)}-${Math.round(animationEndMs)}:${visibleWords
+ .map((word) => `${word.cueId}:${word.cueWordIndex}`)
+ .join("|")}`,
+ startMs: animationStartMs,
+ endMs: animationEndMs,
+ text: getVisibleCaptionText(visibleLines),
+ words: visibleWords.map((word) => ({
+ cueId: word.cueId,
+ cueWordIndex: word.cueWordIndex,
+ startMs: word.startMs,
+ endMs: word.endMs,
+ text: word.text,
+ leadingSpace: word.leadingSpace,
+ })),
+ },
visiblePageIndex,
opacity: animation.opacity,
translateY: animation.translateY,
diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json
index c788b132..9a214fef 100644
--- a/src/i18n/locales/en/settings.json
+++ b/src/i18n/locales/en/settings.json
@@ -120,6 +120,7 @@
"generateFull": "Generate Captions",
"regenerateFull": "Regenerate Captions",
"clearFull": "Clear Captions",
+ "editSaved": "Caption updated",
"fontSettings": "Font Settings",
"defaultFont": "Default",
"fontFamily": "Font",
diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json
index ab686b30..f1b88cbb 100644
--- a/src/i18n/locales/es/settings.json
+++ b/src/i18n/locales/es/settings.json
@@ -120,6 +120,7 @@
"generateFull": "Generar subtítulos",
"regenerateFull": "Regenerar subtítulos",
"clearFull": "Borrar subtítulos",
+ "editSaved": "Caption updated",
"fontSettings": "Tipografía",
"defaultFont": "Predeterminado",
"fontFamily": "Fuente",
diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json
index c4748e3d..22feb6fb 100644
--- a/src/i18n/locales/fr/settings.json
+++ b/src/i18n/locales/fr/settings.json
@@ -120,6 +120,7 @@
"generateFull": "Générer les sous-titres",
"regenerateFull": "Régénérer les sous-titres",
"clearFull": "Effacer les sous-titres",
+ "editSaved": "Caption updated",
"fontSettings": "Paramètres de police",
"defaultFont": "Par défaut",
"fontFamily": "Police",
diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json
index 62f6f8fd..ce8db65a 100644
--- a/src/i18n/locales/ko/settings.json
+++ b/src/i18n/locales/ko/settings.json
@@ -120,6 +120,7 @@
"generateFull": "자막 생성",
"regenerateFull": "자막 다시 생성",
"clearFull": "자막 지우기",
+ "editSaved": "Caption updated",
"fontSettings": "글꼴 설정",
"defaultFont": "기본값",
"fontFamily": "글꼴",
diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json
index 2aaa43e9..8b8bd169 100644
--- a/src/i18n/locales/nl/settings.json
+++ b/src/i18n/locales/nl/settings.json
@@ -120,6 +120,7 @@
"generateFull": "Ondertiteling genereren",
"regenerateFull": "Ondertiteling opnieuw genereren",
"clearFull": "Ondertiteling wissen",
+ "editSaved": "Caption updated",
"fontSettings": "Lettertype-instellingen",
"defaultFont": "Standaard",
"fontFamily": "Lettertype",
diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json
index 86e969d5..dbb751e6 100644
--- a/src/i18n/locales/pt-BR/settings.json
+++ b/src/i18n/locales/pt-BR/settings.json
@@ -120,6 +120,7 @@
"generateFull": "Gerar legendas",
"regenerateFull": "Gerar legendas novamente",
"clearFull": "Limpar legendas",
+ "editSaved": "Caption updated",
"fontSettings": "Configurações da fonte",
"defaultFont": "Padrão",
"fontFamily": "Fonte",
diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json
index dae55a79..92674a8b 100644
--- a/src/i18n/locales/zh-CN/settings.json
+++ b/src/i18n/locales/zh-CN/settings.json
@@ -120,6 +120,7 @@
"generateFull": "生成字幕",
"regenerateFull": "重新生成字幕",
"clearFull": "清除字幕",
+ "editSaved": "字幕已更新",
"fontSettings": "字体设置",
"defaultFont": "默认",
"fontFamily": "字体",
diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json
index b2185b3c..8edf24d3 100644
--- a/src/i18n/locales/zh-TW/settings.json
+++ b/src/i18n/locales/zh-TW/settings.json
@@ -29,15 +29,16 @@
"loopCursor": "游標循環",
"cursorStyle": "游標樣式",
"cursorStyleOptions": {
- "tahoe": "Tahoe",
- "dot": "圓點",
- "figma": "極簡",
- "mono": "反相",
- "lavender": "Lavender",
- "parched": "Parched",
- "chooper": "Chooper",
- "amongus": "Among Us",
- "turtle": "Turtle"
+ "macos": "macOS",
+ "tahoe": "Tahoe",
+ "tahoe-inverted": "Tahoe Inverted",
+ "dot": "圓點",
+ "figma": "極簡",
+ "lavender": "Lavender",
+ "parched": "Parched",
+ "chooper": "Chooper",
+ "amongus": "Among Us",
+ "turtle": "Turtle"
},
"backgroundBlur": "背景模糊",
"zoomMotionBlur": "縮放動態模糊",
@@ -62,11 +63,11 @@
"connectedZoomDuration": "連接縮放時間",
"connectedZoomEasing": "連接平移曲線",
"zoomEasingOptions": {
- "recordly": "Recordly",
- "glide": "滑行",
- "smooth": "平滑",
- "snappy": "俐落",
- "linear": "線性"
+ "recordly": "Recordly",
+ "glide": "滑行",
+ "smooth": "平滑",
+ "snappy": "俐落",
+ "linear": "線性"
},
"cursorSize": "游標大小",
"cursorSmoothing": "游標平滑",
@@ -91,9 +92,6 @@
"radius": "半徑",
"roundness": "圓角",
"padding": "內距",
- "paddingAdvanced": "進階",
- "paddingAdvancedShow": "顯示進階內距控制",
- "paddingAdvancedHide": "隱藏進階內距控制",
"paddingLinked": "連動(等距)",
"paddingUnlinked": "不連動(不對稱)",
"paddingTop": "上",
@@ -122,6 +120,7 @@
"generateFull": "產生字幕",
"regenerateFull": "重新產生字幕",
"clearFull": "清除字幕",
+ "editSaved": "Caption updated",
"fontSettings": "字型設定",
"defaultFont": "預設",
"fontFamily": "字型",
@@ -163,10 +162,10 @@
"mp4": "MP4",
"gif": "GIF",
"quality": {
- "low": "低",
- "medium": "中",
- "high": "高",
- "original": "原始"
+ "low": "低",
+ "medium": "中",
+ "high": "高",
+ "original": "原始"
},
"fpsTitle": "FPS",
"loop": "循環",
@@ -180,4 +179,4 @@
"reportBug": "回報錯誤",
"starOnGithub": "在 GitHub 按讚"
}
-}
\ No newline at end of file
+}