The open-source CapCut alternative
Add support for basic SRT subtitle import to Captions panel (#756)
kcfanchere3d4f3fa92a0
diff --git a/apps/web/src/components/editor/panels/assets/views/captions.tsx b/apps/web/src/components/editor/panels/assets/views/captions.tsxindex c89e587b..488875a8 100644--- a/apps/web/src/components/editor/panels/assets/views/captions.tsx+++ b/apps/web/src/components/editor/panels/assets/views/captions.tsx@@ -1,179 +1,282 @@-import { Button } from "@/components/ui/button";-import { PanelView } from "@/components/editor/panels/assets/views/base-panel";-import {- Select,- SelectContent,- SelectItem,- SelectTrigger,- SelectValue,-} from "@/components/ui/select";-import { useState, useRef } from "react";-import { extractTimelineAudio } from "@/lib/media/mediabunny";-import { useEditor } from "@/hooks/use-editor";-import {- DEFAULT_TRANSCRIPTION_SAMPLE_RATE,- TRANSCRIPTION_LANGUAGES,-} from "@/constants/transcription-constants";-import type {- TranscriptionLanguage,- TranscriptionProgress,-} from "@/lib/transcription/types";-import { transcriptionService } from "@/services/transcription/service";-import { decodeAudioToFloat32 } from "@/lib/media/audio";-import { buildCaptionChunks } from "@/lib/transcription/caption";-import { Spinner } from "@/components/ui/spinner";-import {- Section,- SectionContent,- SectionField,- SectionFields,-} from "@/components/section";-import { DEFAULTS } from "@/lib/timeline/defaults";-import {- AddTrackCommand,- BatchCommand,- InsertElementCommand,-} from "@/lib/commands";--export function Captions() {- const [selectedLanguage, setSelectedLanguage] =- useState<TranscriptionLanguage>("auto");- const [isProcessing, setIsProcessing] = useState(false);- const [processingStep, setProcessingStep] = useState("");- const [error, setError] = useState<string | null>(null);- const containerRef = useRef<HTMLDivElement>(null);- const editor = useEditor();-- const handleProgress = (progress: TranscriptionProgress) => {- if (progress.status === "loading-model") {- setProcessingStep(`Loading model ${Math.round(progress.progress)}%`);- } else if (progress.status === "transcribing") {- setProcessingStep("Transcribing...");- }- };-- const handleGenerateTranscript = async () => {- try {- setIsProcessing(true);- setError(null);- setProcessingStep("Extracting audio...");-- const audioBlob = await extractTimelineAudio({- tracks: editor.scenes.getActiveScene().tracks,- mediaAssets: editor.media.getAssets(),- totalDuration: editor.timeline.getTotalDuration(),- });-- setProcessingStep("Preparing audio...");- const { samples } = await decodeAudioToFloat32({- audioBlob,- sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE,- });-- const result = await transcriptionService.transcribe({- audioData: samples,- language: selectedLanguage === "auto" ? undefined : selectedLanguage,- onProgress: handleProgress,- });-- setProcessingStep("Generating captions...");- const captionChunks = buildCaptionChunks({ segments: result.segments });-- const addTrackCommand = new AddTrackCommand("text", 0);- const insertCommands = captionChunks.map(- (caption, i) =>- new InsertElementCommand({- placement: {- mode: "explicit",- trackId: addTrackCommand.getTrackId(),- },- element: {- ...DEFAULTS.text.element,- name: `Caption ${i + 1}`,- content: caption.text,- duration: caption.duration,- startTime: caption.startTime,- fontSize: 65,- fontWeight: "bold",- },- }),- );-- editor.command.execute({- command: new BatchCommand([addTrackCommand, ...insertCommands]),- });- } catch (error) {- console.error("Transcription failed:", error);- setError(- error instanceof Error ? error.message : "An unexpected error occurred",- );- } finally {- setIsProcessing(false);- setProcessingStep("");- }- };-- const handleLanguageChange = ({ value }: { value: string }) => {- if (value === "auto") {- setSelectedLanguage("auto");- return;- }-- const matchedLanguage = TRANSCRIPTION_LANGUAGES.find(- (language) => language.code === value,- );- if (!matchedLanguage) return;- setSelectedLanguage(matchedLanguage.code);- };-- return (- <PanelView- title="Captions"- contentClassName="px-0 flex flex-col h-full"- ref={containerRef}- >- <Section showTopBorder={false} showBottomBorder={false} className="flex-1">- <SectionContent className="flex flex-col gap-4 h-full pt-1">- <SectionFields>- <SectionField label="Language">- <Select- value={selectedLanguage}- onValueChange={(value) => handleLanguageChange({ value })}- >- <SelectTrigger>- <SelectValue placeholder="Select a language" />- </SelectTrigger>- <SelectContent>- <SelectItem value="auto">Auto detect</SelectItem>- {TRANSCRIPTION_LANGUAGES.map((language) => (- <SelectItem key={language.code} value={language.code}>- {language.name}- </SelectItem>- ))}- </SelectContent>- </Select>- </SectionField>- </SectionFields>-- {error && (- <div className="bg-destructive/10 border-destructive/20 rounded-md border p-3">- <p className="text-destructive text-sm">{error}</p>- </div>- )}- </SectionContent>- </Section>- <Section showBottomBorder={false} showTopBorder={false}>- <SectionContent>- <Button- className="w-full"- onClick={handleGenerateTranscript}- disabled={isProcessing}- >- {isProcessing && <Spinner className="mr-1" />}- {isProcessing ? processingStep : "Generate transcript"}- </Button>- </SectionContent>- </Section>- </PanelView>- );-}+import { Button } from "@/components/ui/button";+import { PanelView } from "@/components/editor/panels/assets/views/base-panel";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";+import { useReducer, useRef, useState } from "react";+import { extractTimelineAudio } from "@/lib/media/mediabunny";+import { useEditor } from "@/hooks/use-editor";+import {+ DEFAULT_TRANSCRIPTION_SAMPLE_RATE,+ TRANSCRIPTION_LANGUAGES,+} from "@/constants/transcription-constants";+import type {+ CaptionChunk,+ TranscriptionLanguage,+ TranscriptionProgress,+} from "@/lib/transcription/types";+import { transcriptionService } from "@/services/transcription/service";+import { decodeAudioToFloat32 } from "@/lib/media/audio";+import { buildCaptionChunks } from "@/lib/transcription/caption";+import { insertCaptionChunksAsTextTrack } from "@/lib/subtitles/insert";+import { parseSubtitleFile } from "@/lib/subtitles/parse";+import { Spinner } from "@/components/ui/spinner";+import {+ Section,+ SectionContent,+ SectionField,+ SectionFields,+} from "@/components/section";+import { CloudUploadIcon } from "@hugeicons/core-free-icons";+import { HugeiconsIcon } from "@hugeicons/react";++type ProcessingState =+ | { status: "idle"; error: string | null; warnings: string[] }+ | { status: "processing"; step: string };++type ProcessingAction =+ | { type: "start"; step: string }+ | { type: "update_step"; step: string }+ | { type: "succeed"; warnings: string[] }+ | { type: "fail"; error: string };++const IDLE_STATE: ProcessingState = { status: "idle", error: null, warnings: [] };++function processingReducer(+ state: ProcessingState,+ action: ProcessingAction,+): ProcessingState {+ switch (action.type) {+ case "start":+ return { status: "processing", step: action.step };+ case "update_step":+ if (state.status !== "processing") return state;+ return { status: "processing", step: action.step };+ case "succeed":+ return { status: "idle", error: null, warnings: action.warnings };+ case "fail":+ return { status: "idle", error: action.error, warnings: [] };+ }+}++export function Captions() {+ const [selectedLanguage, setSelectedLanguage] =+ useState<TranscriptionLanguage>("auto");+ const [processing, dispatch] = useReducer(processingReducer, IDLE_STATE);+ const containerRef = useRef<HTMLDivElement>(null);+ const fileInputRef = useRef<HTMLInputElement>(null);+ const editor = useEditor();++ const isProcessing = processing.status === "processing";++ const handleProgress = (progress: TranscriptionProgress) => {+ if (progress.status === "loading-model") {+ dispatch({+ type: "update_step",+ step: `Loading model ${Math.round(progress.progress)}%`,+ });+ } else if (progress.status === "transcribing") {+ dispatch({ type: "update_step", step: "Transcribing..." });+ }+ };++ const insertCaptions = ({ captions }: { captions: CaptionChunk[] }): boolean => {+ const trackId = insertCaptionChunksAsTextTrack({ editor, captions });+ return trackId !== null;+ };++ const handleGenerateTranscript = async () => {+ dispatch({ type: "start", step: "Extracting audio..." });+ try {+ const audioBlob = await extractTimelineAudio({+ tracks: editor.scenes.getActiveScene().tracks,+ mediaAssets: editor.media.getAssets(),+ totalDuration: editor.timeline.getTotalDuration(),+ });++ dispatch({ type: "update_step", step: "Preparing audio..." });+ const { samples } = await decodeAudioToFloat32({+ audioBlob,+ sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE,+ });++ const result = await transcriptionService.transcribe({+ audioData: samples,+ language: selectedLanguage === "auto" ? undefined : selectedLanguage,+ onProgress: handleProgress,+ });++ dispatch({ type: "update_step", step: "Generating captions..." });+ const captionChunks = buildCaptionChunks({ segments: result.segments });++ if (!insertCaptions({ captions: captionChunks })) {+ dispatch({ type: "fail", error: "No captions were generated" });+ return;+ }++ dispatch({ type: "succeed", warnings: [] });+ } catch (error) {+ console.error("Transcription failed:", error);+ dispatch({+ type: "fail",+ error: error instanceof Error ? error.message : "An unexpected error occurred",+ });+ }+ };++ const handleImportClick = () => {+ fileInputRef.current?.click();+ };++ const handleImportFile = async ({ file }: { file: File }) => {+ dispatch({ type: "start", step: "Reading subtitle file..." });+ try {+ const input = await file.text();+ const result = parseSubtitleFile({+ fileName: file.name,+ input,+ });++ if (result.captions.length === 0) {+ dispatch({+ type: "fail",+ error: "No valid subtitle cues were found in the subtitle file",+ });+ return;+ }++ dispatch({ type: "update_step", step: "Importing subtitles..." });++ if (!insertCaptions({ captions: result.captions })) {+ dispatch({ type: "fail", error: "No captions were generated" });+ return;+ }++ const nextWarnings = [...result.warnings];+ if (result.skippedCueCount > 0) {+ nextWarnings.unshift(+ `Imported ${result.captions.length} subtitle cue(s) and skipped ${result.skippedCueCount} malformed cue(s).`,+ );+ }++ dispatch({ type: "succeed", warnings: nextWarnings });+ } catch (error) {+ console.error("Subtitle import failed:", error);+ dispatch({+ type: "fail",+ error: error instanceof Error ? error.message : "An unexpected error occurred",+ });+ }+ };++ const handleFileChange = async ({+ event,+ }: {+ event: React.ChangeEvent<HTMLInputElement>;+ }) => {+ const file = event.target.files?.[0];+ if (event.target) {+ event.target.value = "";+ }+ if (!file) return;++ await handleImportFile({ file });+ };++ const handleLanguageChange = ({ value }: { value: string }) => {+ if (value === "auto") {+ setSelectedLanguage("auto");+ return;+ }++ const matchedLanguage = TRANSCRIPTION_LANGUAGES.find(+ (language) => language.code === value,+ );+ if (!matchedLanguage) return;+ setSelectedLanguage(matchedLanguage.code);+ };++ const error = processing.status === "idle" ? processing.error : null;+ const warnings = processing.status === "idle" ? processing.warnings : [];++ return (+ <PanelView+ title="Captions"+ contentClassName="px-0 flex flex-col h-full"+ actions={+ <Button+ type="button"+ variant="outline"+ size="sm"+ onClick={handleImportClick}+ disabled={isProcessing}+ className="items-center justify-center gap-1.5"+ >+ <HugeiconsIcon icon={CloudUploadIcon} />+ Import+ </Button>+ }+ ref={containerRef}+ >+ <input+ ref={fileInputRef}+ type="file"+ accept=".srt,.ass"+ className="hidden"+ onChange={(event) => void handleFileChange({ event })}+ />+ <Section showTopBorder={false} showBottomBorder={false} className="flex-1">+ <SectionContent className="flex flex-col gap-4 h-full pt-1">+ <SectionFields>+ <SectionField label="Language">+ <Select+ value={selectedLanguage}+ onValueChange={(value) => handleLanguageChange({ value })}+ >+ <SelectTrigger>+ <SelectValue placeholder="Select a language" />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="auto">Auto detect</SelectItem>+ {TRANSCRIPTION_LANGUAGES.map((language) => (+ <SelectItem key={language.code} value={language.code}>+ {language.name}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </SectionField>+ </SectionFields>++ <Button+ type="button"+ className="mt-auto w-full"+ onClick={handleGenerateTranscript}+ disabled={isProcessing}+ >+ {isProcessing && <Spinner className="mr-1" />}+ {isProcessing ? processing.step : "Generate transcript"}+ </Button>+ {error && (+ <div className="bg-destructive/10 border-destructive/20 rounded-md border p-3">+ <p className="text-destructive text-sm">{error}</p>+ </div>+ )}+ {warnings.length > 0 && (+ <div className="rounded-md border border-amber-500/20 bg-amber-500/10 p-3">+ <ul className="space-y-1 text-sm text-amber-700">+ {warnings.map((warning) => (+ <li key={warning}>{warning}</li>+ ))}+ </ul>+ </div>+ )}+ </SectionContent>+ </Section>+ </PanelView>+ );+}diff --git a/apps/web/src/lib/subtitles/ass.ts b/apps/web/src/lib/subtitles/ass.tsnew file mode 100644index 00000000..030c5b36--- /dev/null+++ b/apps/web/src/lib/subtitles/ass.ts@@ -0,0 +1,514 @@+import type {+ ParseSubtitleResult,+ SubtitleCue,+ SubtitleStyleOverrides,+} from "./types";++const ASS_DEFAULT_PLAY_RES_X = 384;+const ASS_DEFAULT_PLAY_RES_Y = 288;+const STYLE_SECTION_NAMES = new Set(["v4 styles", "v4+ styles"]);+const ALIGNMENT_MAP: Record<+ number,+ {+ textAlign: NonNullable<SubtitleStyleOverrides["textAlign"]>;+ verticalAlign: NonNullable<+ NonNullable<SubtitleStyleOverrides["placement"]>["verticalAlign"]+ >;+ }+> = {+ 1: { textAlign: "left", verticalAlign: "bottom" },+ 2: { textAlign: "center", verticalAlign: "bottom" },+ 3: { textAlign: "right", verticalAlign: "bottom" },+ 4: { textAlign: "left", verticalAlign: "middle" },+ 5: { textAlign: "center", verticalAlign: "middle" },+ 6: { textAlign: "right", verticalAlign: "middle" },+ 7: { textAlign: "left", verticalAlign: "top" },+ 8: { textAlign: "center", verticalAlign: "top" },+ 9: { textAlign: "right", verticalAlign: "top" },+};++interface AssScriptInfo {+ playResX: number;+ playResY: number;+}++interface AssStyleRecord {+ name: string;+ fontname?: string;+ fontsize?: string;+ primarycolour?: string;+ secondarycolour?: string;+ outlinecolour?: string;+ backcolour?: string;+ bold?: string;+ italic?: string;+ underline?: string;+ strikeout?: string;+ scalex?: string;+ scaley?: string;+ spacing?: string;+ angle?: string;+ borderstyle?: string;+ outline?: string;+ shadow?: string;+ alignment?: string;+ marginl?: string;+ marginr?: string;+ marginv?: string;+}++export function parseAss({ input }: { input: string }): ParseSubtitleResult {+ const normalized = input.replace(/\r\n?/g, "\n").trim();+ if (!normalized) {+ return {+ captions: [],+ skippedCueCount: 0,+ warnings: [],+ };+ }++ const scriptInfo: AssScriptInfo = {+ playResX: ASS_DEFAULT_PLAY_RES_X,+ playResY: ASS_DEFAULT_PLAY_RES_Y,+ };+ const warnings = new Set<string>();+ const styles = new Map<string, SubtitleStyleOverrides>();+ const captions: SubtitleCue[] = [];++ let currentSection = "";+ let styleFormat: string[] | null = null;+ let eventFormat: string[] | null = null;+ let skippedCueCount = 0;+ let strippedInlineTagCueCount = 0;+ let skippedNonDialogueEventCount = 0;+ let ignoredEffectCount = 0;+ let missingStyleCueCount = 0;+ let usesHeavilyUnsupportedStyles = false;++ for (const rawLine of normalized.split("\n")) {+ const line = rawLine.trim();+ if (!line || line.startsWith(";")) {+ continue;+ }++ const sectionMatch = line.match(/^\[(.+)\]$/);+ if (sectionMatch) {+ currentSection = sectionMatch[1].trim().toLowerCase();+ continue;+ }++ if (currentSection === "script info") {+ const [rawKey, ...rest] = line.split(":");+ if (!rawKey || rest.length === 0) continue;++ const key = rawKey.trim().toLowerCase();+ const value = rest.join(":").trim();+ const parsedValue = parseFloat(value);++ if (!Number.isFinite(parsedValue) || parsedValue <= 0) {+ continue;+ }++ if (key === "playresx") {+ scriptInfo.playResX = parsedValue;+ }+ if (key === "playresy") {+ scriptInfo.playResY = parsedValue;+ }+ continue;+ }++ if (STYLE_SECTION_NAMES.has(currentSection)) {+ if (line.startsWith("Format:")) {+ styleFormat = parseAssFormat({ line });+ continue;+ }++ if (line.startsWith("Style:") && styleFormat) {+ const values = splitAssFields({+ value: line.slice("Style:".length).trim(),+ expectedFieldCount: styleFormat.length,+ });+ if (values.length !== styleFormat.length) {+ continue;+ }++ const record = mapFieldsToRecord<AssStyleRecord>({+ fields: styleFormat,+ values,+ });+ if (!record.name) {+ continue;+ }++ const mappedStyle = mapAssStyleToSubtitleStyle({+ style: record,+ scriptInfo,+ });+ usesHeavilyUnsupportedStyles ||= mappedStyle.hasUnsupportedFeatures;+ styles.set(record.name.toLowerCase(), mappedStyle.style);+ }+ continue;+ }++ if (currentSection !== "events") {+ continue;+ }++ if (line.startsWith("Format:")) {+ eventFormat = parseAssFormat({ line });+ continue;+ }++ if (!eventFormat) {+ continue;+ }++ if (line.startsWith("Dialogue:")) {+ const values = splitAssFields({+ value: line.slice("Dialogue:".length).trim(),+ expectedFieldCount: eventFormat.length,+ });+ if (values.length !== eventFormat.length) {+ skippedCueCount += 1;+ continue;+ }++ const record = mapFieldsToRecord<Record<string, string>>({+ fields: eventFormat,+ values,+ });+ const startTime = parseAssTimestamp({ input: record.start });+ const endTime = parseAssTimestamp({ input: record.end });+ const duration = endTime - startTime;+ if (+ !Number.isFinite(startTime) ||+ !Number.isFinite(endTime) ||+ duration <= 0+ ) {+ skippedCueCount += 1;+ continue;+ }++ const strippedText = stripAssText({+ input: record.text ?? "",+ });+ strippedInlineTagCueCount += strippedText.hadInlineTags ? 1 : 0;+ if (!strippedText.text) {+ skippedCueCount += 1;+ continue;+ }++ if (record.effect?.trim()) {+ ignoredEffectCount += 1;+ }++ const referencedStyleName = record.style?.trim().toLowerCase();+ const referencedStyle = referencedStyleName+ ? styles.get(referencedStyleName)+ : undefined;+ if (referencedStyleName && !referencedStyle) {+ missingStyleCueCount += 1;+ }+ const style = referencedStyle ?? styles.get("default");++ captions.push({+ text: strippedText.text,+ startTime,+ duration,+ style,+ });+ continue;+ }++ if (line.includes(":")) {+ skippedNonDialogueEventCount += 1;+ }+ }++ if (strippedInlineTagCueCount > 0) {+ warnings.add(+ `Stripped unsupported ASS inline override tags from ${strippedInlineTagCueCount} subtitle cue(s).`,+ );+ }++ if (ignoredEffectCount > 0) {+ warnings.add(+ `Ignored ASS event effects in ${ignoredEffectCount} subtitle cue(s).`,+ );+ }++ if (missingStyleCueCount > 0) {+ warnings.add(+ `Fell back to default subtitle styling for ${missingStyleCueCount} cue(s) that referenced missing ASS styles.`,+ );+ }++ if (skippedNonDialogueEventCount > 0) {+ warnings.add(+ `Ignored ${skippedNonDialogueEventCount} non-dialogue ASS event(s).`,+ );+ }++ if (usesHeavilyUnsupportedStyles) {+ warnings.add(+ "Ignored unsupported ASS style features such as outline, shadow, rotation, or scaling.",+ );+ }++ return {+ captions,+ skippedCueCount,+ warnings: [...warnings],+ };+}++function parseAssFormat({ line }: { line: string }): string[] {+ return line+ .slice(line.indexOf(":") + 1)+ .split(",")+ .map((field) => field.trim().toLowerCase());+}++function splitAssFields({+ value,+ expectedFieldCount,+}: {+ value: string;+ expectedFieldCount: number;+}): string[] {+ if (expectedFieldCount <= 1) {+ return [value];+ }++ const result: string[] = [];+ let current = "";++ for (const character of value) {+ if (character === "," && result.length < expectedFieldCount - 1) {+ result.push(current.trim());+ current = "";+ continue;+ }++ current += character;+ }++ result.push(current.trim());+ return result;+}++function mapFieldsToRecord<T extends object>({+ fields,+ values,+}: {+ fields: string[];+ values: string[];+}): T {+ const record = {} as Record<string, string | undefined>;++ for (let index = 0; index < fields.length; index++) {+ record[fields[index]] = values[index];+ }++ return record as T;+}++function parseAssTimestamp({ input }: { input: string | undefined }): number {+ if (!input) {+ return Number.NaN;+ }++ const match = input.trim().match(/^(\d+):(\d{2}):(\d{2})[.](\d{1,2}|\d{3})$/);+ if (!match) {+ return Number.NaN;+ }++ const [, hours, minutes, seconds, fraction] = match;+ const parsedHours = Number.parseInt(hours, 10);+ const parsedMinutes = Number.parseInt(minutes, 10);+ const parsedSeconds = Number.parseInt(seconds, 10);+ const parsedMilliseconds = Number.parseInt(fraction.padEnd(3, "0"), 10);++ return (+ parsedHours * 3600 ++ parsedMinutes * 60 ++ parsedSeconds ++ parsedMilliseconds / 1000+ );+}++function stripAssText({ input }: { input: string }): {+ text: string;+ hadInlineTags: boolean;+} {+ const hadInlineTags = /\{[^}]*\}/.test(input);+ const text = input+ .replace(/\{[^}]*\}/g, "")+ .replace(/\\N/gi, "\n")+ .replace(/\\h/g, " ")+ .replace(/\\n/g, "\n")+ .trim();++ return {+ text,+ hadInlineTags,+ };+}++function mapAssStyleToSubtitleStyle({+ style,+ scriptInfo,+}: {+ style: AssStyleRecord;+ scriptInfo: AssScriptInfo;+}): {+ style: SubtitleStyleOverrides;+ hasUnsupportedFeatures: boolean;+} {+ const fontSize = parseFloat(style.fontsize ?? "");+ const primaryColor = parseAssColor({ input: style.primarycolour });+ const backColor = parseAssColor({ input: style.backcolour });+ const bold = parseAssBoolean({ input: style.bold });+ const italic = parseAssBoolean({ input: style.italic });+ const underline = parseAssBoolean({ input: style.underline });+ const strikeOut = parseAssBoolean({ input: style.strikeout });+ const borderStyle = parseFloat(style.borderstyle ?? "");+ const spacing = parseFloat(style.spacing ?? "");+ const alignment = parseFloat(style.alignment ?? "");+ const marginLeft = parseFloat(style.marginl ?? "");+ const marginRight = parseFloat(style.marginr ?? "");+ const marginVertical = parseFloat(style.marginv ?? "");+ // Store as a ratio of playResY so the builder can convert to app units+ // without the parser needing to know the app's coordinate system.+ const fontSizeRatioOfPlayHeight = Number.isFinite(fontSize)+ ? Math.round((fontSize / scriptInfo.playResY) * 1000) / 1000+ : null;++ const mappedAlignment =+ ALIGNMENT_MAP[Math.round(alignment)] ?? ALIGNMENT_MAP[2];+ const placement =+ Number.isFinite(marginLeft) ||+ Number.isFinite(marginRight) ||+ Number.isFinite(marginVertical) ||+ mappedAlignment.verticalAlign !== "bottom"+ ? {+ verticalAlign: mappedAlignment.verticalAlign,+ marginLeftRatio: Number.isFinite(marginLeft)+ ? marginLeft / scriptInfo.playResX+ : undefined,+ marginRightRatio: Number.isFinite(marginRight)+ ? marginRight / scriptInfo.playResX+ : undefined,+ marginVerticalRatio: Number.isFinite(marginVertical)+ ? marginVertical / scriptInfo.playResY+ : undefined,+ }+ : undefined;++ const styleOverrides: SubtitleStyleOverrides = {+ ...(style.fontname ? { fontFamily: style.fontname.trim() } : {}),+ ...(fontSizeRatioOfPlayHeight !== null && fontSizeRatioOfPlayHeight > 0+ ? { fontSizeRatioOfPlayHeight }+ : {}),+ ...(primaryColor?.cssColor ? { color: primaryColor.cssColor } : {}),+ ...(bold !== null ? { fontWeight: bold ? "bold" : "normal" } : {}),+ ...(italic !== null ? { fontStyle: italic ? "italic" : "normal" } : {}),+ ...(underline || strikeOut+ ? {+ textDecoration: underline+ ? "underline"+ : strikeOut+ ? "line-through"+ : "none",+ }+ : {}),+ ...(Number.isFinite(spacing) ? { letterSpacing: spacing } : {}),+ textAlign: mappedAlignment.textAlign,+ ...(placement ? { placement } : {}),+ ...(backColor?.cssColor && Math.round(borderStyle) === 3+ ? {+ background: {+ enabled: backColor.alpha > 0,+ color: backColor.alpha > 0 ? backColor.cssColor : "transparent",+ },+ }+ : {}),+ };++ const hasUnsupportedFeatures =+ Math.round(borderStyle) !== 1 && Math.round(borderStyle) !== 3+ ? true+ : (parseFloat(style.outline ?? "") || 0) > 0 ||+ (parseFloat(style.shadow ?? "") || 0) > 0 ||+ (parseFloat(style.angle ?? "") || 0) !== 0 ||+ (Number.isFinite(parseFloat(style.scalex ?? "")) &&+ parseFloat(style.scalex ?? "") !== 100) ||+ (Number.isFinite(parseFloat(style.scaley ?? "")) &&+ parseFloat(style.scaley ?? "") !== 100) ||+ Boolean(underline && strikeOut);++ return {+ style: styleOverrides,+ hasUnsupportedFeatures,+ };+}++function parseAssBoolean({+ input,+}: {+ input: string | undefined;+}): boolean | null {+ if (input === undefined) {+ return null;+ }++ const trimmed = input.trim();+ if (!trimmed) {+ return null;+ }++ const parsed = Number.parseInt(trimmed, 10);+ if (!Number.isFinite(parsed)) {+ return null;+ }++ return parsed !== 0;+}++function parseAssColor({+ input,+}: {+ input: string | undefined;+}): { cssColor: string; alpha: number } | null {+ if (!input) {+ return null;+ }++ const normalized = input.trim().replace(/^&?H/i, "").padStart(8, "0");+ if (!/^[0-9a-fA-F]{8}$/.test(normalized)) {+ return null;+ }++ const alphaHex = normalized.slice(0, 2);+ const blueHex = normalized.slice(2, 4);+ const greenHex = normalized.slice(4, 6);+ const redHex = normalized.slice(6, 8);++ const red = Number.parseInt(redHex, 16);+ const green = Number.parseInt(greenHex, 16);+ const blue = Number.parseInt(blueHex, 16);+ const alpha = 1 - Number.parseInt(alphaHex, 16) / 255;++ if (alpha >= 1) {+ return {+ cssColor: `#${redHex}${greenHex}${blueHex}`.toLowerCase(),+ alpha,+ };+ }++ return {+ cssColor: `rgba(${red}, ${green}, ${blue}, ${Math.round(alpha * 1000) / 1000})`,+ alpha,+ };+}diff --git a/apps/web/src/lib/subtitles/build-subtitle-text-element.ts b/apps/web/src/lib/subtitles/build-subtitle-text-element.tsnew file mode 100644index 00000000..8be93e9d--- /dev/null+++ b/apps/web/src/lib/subtitles/build-subtitle-text-element.ts@@ -0,0 +1,333 @@+import { FONT_SIZE_SCALE_REFERENCE } from "@/constants/text-constants";+import {+ getTextVisualRect,+ measureTextBlock,+ setCanvasLetterSpacing,+} from "@/lib/text/layout";+import { DEFAULTS } from "@/lib/timeline/defaults";+import { TICKS_PER_SECOND } from "@/lib/wasm";+import type { CreateTextElement } from "@/lib/timeline";+import type { SubtitleCue, SubtitleStyleOverrides } from "./types";++const SUBTITLE_MAX_WIDTH_RATIO = 0.8;+const SUBTITLE_BOTTOM_MARGIN_RATIO = 0.05;+const SUBTITLE_FONT_SIZE = 5;+const MEASUREMENT_CANVAS_SIZE = 4096;++function quoteFontFamily({ fontFamily }: { fontFamily: string }): string {+ return `"${fontFamily.replace(/"/g, '\\"')}"`;+}++function createMeasurementContext(): CanvasRenderingContext2D | null {+ const canvas = document.createElement("canvas");+ canvas.width = MEASUREMENT_CANVAS_SIZE;+ canvas.height = MEASUREMENT_CANVAS_SIZE;+ return canvas.getContext("2d");+}++function measureLineWidth({+ ctx,+ text,+}: {+ ctx: CanvasRenderingContext2D;+ text: string;+}): number {+ return ctx.measureText(text).width;+}++function wrapSubtitleText({+ ctx,+ text,+ maxWidth,+}: {+ ctx: CanvasRenderingContext2D;+ text: string;+ maxWidth: number;+}): string {+ const normalized = text.trim().replace(/\r\n/g, "\n");+ const paragraphs = normalized.split("\n");+ const wrappedParagraphs: string[] = [];++ for (const paragraph of paragraphs) {+ const trimmedParagraph = paragraph.trim();+ if (!trimmedParagraph) {+ wrappedParagraphs.push("");+ continue;+ }++ const words = trimmedParagraph.split(/\s+/);+ let currentLine = words[0] ?? "";+ const lines: string[] = [];++ for (let i = 1; i < words.length; i++) {+ const nextLine = `${currentLine} ${words[i]}`;+ if (measureLineWidth({ ctx, text: nextLine }) <= maxWidth) {+ currentLine = nextLine;+ continue;+ }++ lines.push(currentLine);+ currentLine = words[i];+ }++ lines.push(currentLine);+ wrappedParagraphs.push(lines.join("\n"));+ }++ return wrappedParagraphs.join("\n");+}++function measureWrappedTextBlock({+ ctx,+ content,+ canvasHeight,+ textAlign,+ background,+ fontSize,+ lineHeight,+}: {+ ctx: CanvasRenderingContext2D;+ content: string;+ canvasHeight: number;+ textAlign: CreateTextElement["textAlign"];+ background: CreateTextElement["background"];+ fontSize: number;+ lineHeight: number;+}) {+ const scaledFontSize = fontSize * (canvasHeight / FONT_SIZE_SCALE_REFERENCE);+ const lineHeightPx = lineHeight * scaledFontSize;+ const lines = content.split("\n");+ const lineMetrics = lines.map((line) => ctx.measureText(line));++ const block = measureTextBlock({+ lineMetrics,+ lineHeightPx,+ });+ const visualRect = getTextVisualRect({+ textAlign,+ block,+ background,+ fontSizeRatio: fontSize / DEFAULTS.text.element.fontSize,+ });++ return {+ block,+ visualRect,+ };+}++function resolveSubtitleStyle({+ style,+}: {+ style: SubtitleStyleOverrides | undefined;+}): {+ fontFamily: string;+ fontSize: number;+ color: string;+ textAlign: CreateTextElement["textAlign"];+ fontWeight: CreateTextElement["fontWeight"];+ fontStyle: CreateTextElement["fontStyle"];+ textDecoration: CreateTextElement["textDecoration"];+ letterSpacing: number;+ lineHeight: number;+ background: CreateTextElement["background"];+ placement: NonNullable<SubtitleStyleOverrides["placement"]>;+} {+ const fontSize =+ style?.fontSizeRatioOfPlayHeight != null+ ? style.fontSizeRatioOfPlayHeight * FONT_SIZE_SCALE_REFERENCE+ : (style?.fontSize ?? SUBTITLE_FONT_SIZE);++ return {+ fontFamily: style?.fontFamily ?? DEFAULTS.text.element.fontFamily,+ fontSize,+ color: style?.color ?? DEFAULTS.text.element.color,+ textAlign: style?.textAlign ?? "center",+ fontWeight: style?.fontWeight ?? "bold",+ fontStyle: style?.fontStyle ?? DEFAULTS.text.element.fontStyle,+ textDecoration:+ style?.textDecoration ?? DEFAULTS.text.element.textDecoration,+ letterSpacing: style?.letterSpacing ?? DEFAULTS.text.letterSpacing,+ lineHeight: style?.lineHeight ?? DEFAULTS.text.lineHeight,+ background: {+ ...DEFAULTS.text.element.background,+ enabled: false,+ ...(style?.background ?? {}),+ },+ placement: {+ verticalAlign: style?.placement?.verticalAlign ?? "bottom",+ marginLeftRatio: style?.placement?.marginLeftRatio,+ marginRightRatio: style?.placement?.marginRightRatio,+ marginVerticalRatio: style?.placement?.marginVerticalRatio,+ },+ };+}++function resolveTargetWidth({+ canvasWidth,+ placement,+}: {+ canvasWidth: number;+ placement: ReturnType<typeof resolveSubtitleStyle>["placement"];+}): number {+ const leftRatio = placement.marginLeftRatio ?? 0;+ const rightRatio = placement.marginRightRatio ?? 0;+ const hasExplicitMargins = leftRatio > 0 || rightRatio > 0;+ if (!hasExplicitMargins) {+ return canvasWidth * SUBTITLE_MAX_WIDTH_RATIO;+ }++ const availableWidth = canvasWidth * (1 - leftRatio - rightRatio);+ return Math.max(0, availableWidth);+}++function resolvePositionX({+ canvasWidth,+ textAlign,+ placement,+ visualRect,+}: {+ canvasWidth: number;+ textAlign: CreateTextElement["textAlign"];+ placement: ReturnType<typeof resolveSubtitleStyle>["placement"];+ visualRect: { left: number; width: number };+}): number {+ const leftMargin = canvasWidth * (placement.marginLeftRatio ?? 0);+ const rightMargin = canvasWidth * (placement.marginRightRatio ?? 0);+ const canvasCenterX = canvasWidth / 2;++ if (textAlign === "left") {+ return leftMargin - visualRect.left - canvasCenterX;+ }++ if (textAlign === "right") {+ return (+ canvasWidth -+ rightMargin -+ (visualRect.left + visualRect.width) -+ canvasCenterX+ );+ }++ const availableWidth = canvasWidth - leftMargin - rightMargin;+ const targetCenterX = leftMargin + availableWidth / 2;+ return (+ targetCenterX - (visualRect.left + visualRect.width / 2) - canvasCenterX+ );+}++function resolvePositionY({+ canvasHeight,+ placement,+ visualRect,+}: {+ canvasHeight: number;+ placement: ReturnType<typeof resolveSubtitleStyle>["placement"];+ visualRect: { top: number; height: number };+}): number {+ const margin =+ canvasHeight *+ (placement.marginVerticalRatio ?? SUBTITLE_BOTTOM_MARGIN_RATIO);+ const canvasCenterY = canvasHeight / 2;++ if (placement.verticalAlign === "top") {+ return margin - visualRect.top - canvasCenterY;+ }++ if (placement.verticalAlign === "middle") {+ const targetCenterY = canvasHeight / 2;+ return (+ targetCenterY - (visualRect.top + visualRect.height / 2) - canvasCenterY+ );+ }++ return (+ canvasHeight - margin - (visualRect.top + visualRect.height) - canvasCenterY+ );+}++export function buildSubtitleTextElement({+ index,+ caption,+ canvasSize,+}: {+ index: number;+ caption: SubtitleCue;+ canvasSize: { width: number; height: number };+}): CreateTextElement {+ const ctx = createMeasurementContext();+ const style = resolveSubtitleStyle({+ style: caption.style,+ });+ const fontFamily = quoteFontFamily({+ fontFamily: style.fontFamily,+ });+ const fontWeight = style.fontWeight;+ const fontStyle = style.fontStyle === "italic" ? "italic" : "normal";+ const scaledFontSize =+ style.fontSize * (canvasSize.height / FONT_SIZE_SCALE_REFERENCE);+ const fontString = `${fontStyle} ${fontWeight} ${scaledFontSize}px ${fontFamily}, sans-serif`;+ const maxWidth = resolveTargetWidth({+ canvasWidth: canvasSize.width,+ placement: style.placement,+ });++ let content = caption.text;+ let positionX = 0;+ let positionY = 0;++ if (ctx) {+ ctx.font = fontString;+ setCanvasLetterSpacing({ ctx, letterSpacingPx: style.letterSpacing });+ content = wrapSubtitleText({+ ctx,+ text: caption.text,+ maxWidth,+ });+ const measurement = measureWrappedTextBlock({+ ctx,+ content,+ canvasHeight: canvasSize.height,+ textAlign: style.textAlign,+ background: style.background,+ fontSize: style.fontSize,+ lineHeight: style.lineHeight,+ });+ positionX = resolvePositionX({+ canvasWidth: canvasSize.width,+ textAlign: style.textAlign,+ placement: style.placement,+ visualRect: measurement.visualRect,+ });+ positionY = resolvePositionY({+ canvasHeight: canvasSize.height,+ placement: style.placement,+ visualRect: measurement.visualRect,+ });+ }++ return {+ ...DEFAULTS.text.element,+ name: `Caption ${index + 1}`,+ content,+ duration: Math.round(caption.duration * TICKS_PER_SECOND),+ startTime: Math.round(caption.startTime * TICKS_PER_SECOND),+ fontSize: style.fontSize,+ fontFamily: style.fontFamily,+ color: style.color,+ textAlign: style.textAlign,+ fontWeight: style.fontWeight,+ fontStyle: style.fontStyle,+ textDecoration: style.textDecoration,+ letterSpacing: style.letterSpacing,+ lineHeight: style.lineHeight,+ background: style.background,+ transform: {+ ...DEFAULTS.text.element.transform,+ position: {+ x: positionX,+ y: positionY,+ },+ },+ };+}diff --git a/apps/web/src/lib/subtitles/insert.ts b/apps/web/src/lib/subtitles/insert.tsnew file mode 100644index 00000000..1bb04fc8--- /dev/null+++ b/apps/web/src/lib/subtitles/insert.ts@@ -0,0 +1,40 @@+import type { EditorCore } from "@/core";+import {+ AddTrackCommand,+ BatchCommand,+ InsertElementCommand,+} from "@/lib/commands";+import { buildSubtitleTextElement } from "./build-subtitle-text-element";+import type { SubtitleCue } from "./types";++export function insertCaptionChunksAsTextTrack({+ editor,+ captions,+}: {+ editor: EditorCore;+ captions: SubtitleCue[];+}): string | null {+ if (captions.length === 0) {+ return null;+ }++ const addTrackCommand = new AddTrackCommand("text", 0);+ const trackId = addTrackCommand.getTrackId();+ const canvasSize = editor.project.getActive().settings.canvasSize;+ const insertCommands = captions.map(+ (caption, index) =>+ new InsertElementCommand({+ placement: { mode: "explicit", trackId },+ element: buildSubtitleTextElement({+ index,+ caption,+ canvasSize,+ }),+ }),+ );+ editor.command.execute({+ command: new BatchCommand([addTrackCommand, ...insertCommands]),+ });++ return trackId;+}diff --git a/apps/web/src/lib/subtitles/parse.ts b/apps/web/src/lib/subtitles/parse.tsnew file mode 100644index 00000000..7a2d2342--- /dev/null+++ b/apps/web/src/lib/subtitles/parse.ts@@ -0,0 +1,28 @@+import { parseAss } from "./ass";+import { parseSrt } from "./srt";+import type { ParseSubtitleResult } from "./types";+export type { ParseSubtitleResult, SubtitleCue } from "./types";++export function parseSubtitleFile({+ fileName,+ input,+}: {+ fileName: string;+ input: string;+}): ParseSubtitleResult {+ const extension = getFileExtension({ fileName });++ switch (extension) {+ case "srt":+ return parseSrt({ input });+ case "ass":+ return parseAss({ input });+ default:+ throw new Error("Unsupported subtitle format");+ }+}++function getFileExtension({ fileName }: { fileName: string }): string {+ const extension = fileName.split(".").pop();+ return extension?.toLowerCase() ?? "";+}diff --git a/apps/web/src/lib/subtitles/srt.ts b/apps/web/src/lib/subtitles/srt.tsnew file mode 100644index 00000000..197e96f9--- /dev/null+++ b/apps/web/src/lib/subtitles/srt.ts@@ -0,0 +1,98 @@+import type { ParseSubtitleResult, SubtitleCue } from "./types";++const TIMESTAMP_SEPARATOR = /\s*-->\s*/;+const TIMESTAMP_PATTERN =+ /^(\d{2}:\d{2}:\d{2}[,.]\d{1,3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,.]\d{1,3})/;++export function parseSrt({ input }: { input: string }): ParseSubtitleResult {+ const normalized = input.replace(/\r\n?/g, "\n").trim();+ if (!normalized) {+ return {+ captions: [],+ skippedCueCount: 0,+ warnings: [],+ };+ }++ const blocks = normalized.split(/\n{2,}/);+ const cues: SubtitleCue[] = [];+ let skippedCueCount = 0;++ for (const block of blocks) {+ const lines = block+ .split("\n")+ .map((line) => line.trim())+ .filter((line) => line.length > 0);++ if (lines.length < 2) {+ skippedCueCount += 1;+ continue;+ }++ const timestampIndex = TIMESTAMP_SEPARATOR.test(lines[0]) ? 0 : 1;+ const timestampLine = lines[timestampIndex];+ if (!timestampLine || !TIMESTAMP_PATTERN.test(timestampLine)) {+ skippedCueCount += 1;+ continue;+ }++ const textLines = lines.slice(timestampIndex + 1);+ const text = textLines.join("\n").trim();+ if (!text) {+ skippedCueCount += 1;+ continue;+ }++ const [rawStart, rawEnd] = timestampLine.split(TIMESTAMP_SEPARATOR);+ if (!rawStart || !rawEnd) {+ skippedCueCount += 1;+ continue;+ }++ const startTime = parseSrtTimestamp({ input: rawStart });+ const endTime = parseSrtTimestamp({ input: rawEnd });+ const duration = endTime - startTime;++ if (+ !Number.isFinite(startTime) ||+ !Number.isFinite(endTime) ||+ duration <= 0+ ) {+ skippedCueCount += 1;+ continue;+ }++ cues.push({+ text,+ startTime,+ duration,+ });+ }++ return {+ captions: cues,+ skippedCueCount,+ warnings: [],+ };+}++function parseSrtTimestamp({ input }: { input: string }): number {+ const normalized = input.trim().replace(",", ".");+ const match = normalized.match(/^(\d{2}):(\d{2}):(\d{2})\.(\d{1,3})$/);+ if (!match) {+ return Number.NaN;+ }++ const [, hours, minutes, seconds, milliseconds] = match;+ const parsedHours = Number.parseInt(hours, 10);+ const parsedMinutes = Number.parseInt(minutes, 10);+ const parsedSeconds = Number.parseInt(seconds, 10);+ const parsedMilliseconds = Number.parseInt(milliseconds.padEnd(3, "0"), 10);++ return (+ parsedHours * 3600 ++ parsedMinutes * 60 ++ parsedSeconds ++ parsedMilliseconds / 1000+ );+}diff --git a/apps/web/src/lib/subtitles/types.ts b/apps/web/src/lib/subtitles/types.tsnew file mode 100644index 00000000..86f6525d--- /dev/null+++ b/apps/web/src/lib/subtitles/types.ts@@ -0,0 +1,46 @@+import type { TextBackground, TextElement } from "@/lib/timeline";+import type { CaptionChunk } from "@/lib/transcription/types";++export interface SubtitlePlacementStyle {+ verticalAlign?: "top" | "middle" | "bottom";+ marginLeftRatio?: number;+ marginRightRatio?: number;+ marginVerticalRatio?: number;+}++export interface SubtitleStyleOverrides {+ /**+ * Font size in app units (same coordinate space as TextElement.fontSize).+ * Use fontSizeRatioOfPlayHeight when the source coordinate space is unknown+ * (e.g. ASS files, where font size is relative to the script's play resolution).+ */+ fontSize?: number;+ /**+ * Font size expressed as a fraction of the reference canvas height.+ * Set by the ASS parser so the builder can convert to app units without+ * the parser needing to know about the app's coordinate system.+ * Takes precedence over fontSize when both are present.+ */+ fontSizeRatioOfPlayHeight?: number;+ fontFamily?: string;+ color?: string;+ background?: Pick<TextBackground, "enabled" | "color"> &+ Partial<Omit<TextBackground, "enabled" | "color">>;+ textAlign?: TextElement["textAlign"];+ fontWeight?: TextElement["fontWeight"];+ fontStyle?: TextElement["fontStyle"];+ textDecoration?: TextElement["textDecoration"];+ letterSpacing?: number;+ lineHeight?: number;+ placement?: SubtitlePlacementStyle;+}++export interface SubtitleCue extends CaptionChunk {+ style?: SubtitleStyleOverrides;+}++export interface ParseSubtitleResult {+ captions: SubtitleCue[];+ skippedCueCount: number;+ warnings: string[];+}