π₯ Make videos programmatically with React
`@remotion/studio`: Import SRT and provider transcripts as captions (#11665)
Igor Samokhovets11e14f0c1817
diff --git a/bun.lock b/bun.lockindex 61dddd8c5d..b9cd369e7e 100644--- a/bun.lock+++ b/bun.lock@@ -735,6 +735,7 @@"version": "4.0.529","dependencies": {"@remotion/captions": "workspace:*",+ "zod": "catalog:",},"devDependencies": {"@remotion/eslint-config-internal": "workspace:*",@@ -1431,6 +1432,7 @@"version": "4.0.529","dependencies": {"@remotion/captions": "workspace:*",+ "zod": "catalog:",},"devDependencies": {"@remotion/eslint-config-internal": "workspace:*",@@ -1863,8 +1865,10 @@"@remotion/canvas": "workspace:*","@remotion/captions": "workspace:*","@remotion/codemods": "workspace:*",+ "@remotion/elevenlabs": "workspace:*","@remotion/media": "workspace:*","@remotion/media-utils": "workspace:*",+ "@remotion/openai-whisper": "workspace:*","@remotion/paths": "workspace:*","@remotion/player": "workspace:*","@remotion/renderer": "workspace:*",diff --git a/packages/captions/src/parse-srt.ts b/packages/captions/src/parse-srt.tsindex bf6dce3281..8be48c513a 100644--- a/packages/captions/src/parse-srt.ts+++ b/packages/captions/src/parse-srt.ts@@ -33,13 +33,17 @@ function toMilliseconds(time: string) {export type ParseSrtInput = {input: string;+ strict?: boolean;};export type ParseSrtOutput = {captions: Caption[];};-export const parseSrt = ({input}: ParseSrtInput): ParseSrtOutput => {+export const parseSrt = ({+ input,+ strict = false,+}: ParseSrtInput): ParseSrtOutput => {const inputLines = input.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n')@@ -47,6 +51,101 @@ export const parseSrt = ({input}: ParseSrtInput): ParseSrtOutput => {.split('\n');const captions: Caption[] = [];+ if (strict) {+ const timestamp = (token: string, cue: number, line: number): number => {+ const match = /^(\d+):(\d{2}):(\d{2})[,.](\d{3})$/.exec(token);+ if (!match || Number(match[2]) > 59 || Number(match[3]) > 59) {+ throw new Error(+ `SRT cue ${cue}, line ${line}: invalid timestamp "${token}". Use HH:MM:SS,mmm.`,+ );+ }++ const value =+ Number(match[1]) * 3600000 ++ Number(match[2]) * 60000 ++ Number(match[3]) * 1000 ++ Number(match[4]);+ if (!Number.isFinite(value)) {+ throw new Error(+ `SRT cue ${cue}, line ${line}: timestamp is too large.`,+ );+ }++ return value;+ };++ let i = 0;+ while (i < inputLines.length) {+ if (inputLines[i].trim() === '') {+ i++;+ continue;+ }++ const cue = captions.length + 1;+ const headerLine = i + 1;+ if (!/^\d+$/.test(inputLines[i].trim())) {+ throw new Error(+ `SRT cue ${cue}, line ${headerLine}: expected a numeric cue index.`,+ );+ }++ i++;+ const timingLine = inputLines[i];+ const timing = timingLine?.match(/^\s*(\S+)\s+-->\s+(\S+)(.*)$/);+ const settings = timing?.[3].trim();+ if (+ !timing ||+ (settings &&+ !settings+ .split(/\s+/)+ .every((setting) =>+ /^(?:(?:X[12]|Y[12]):\d+|(?:align|position|line|size|vertical):\S+)$/.test(+ setting,+ ),+ ))+ ) {+ throw new Error(+ `SRT cue ${cue}, line ${i + 1}: expected start --> end timestamps.`,+ );+ }++ const startMs = timestamp(timing[1], cue, i + 1);+ const endMs = timestamp(timing[2], cue, i + 1);+ if (endMs < startMs) {+ throw new Error(+ `SRT cue ${cue}, line ${i + 1}: end must not be earlier than start.`,+ );+ }++ i++;+ const text: string[] = [];+ while (i < inputLines.length && inputLines[i].trim() !== '') {+ text.push(inputLines[i]);+ i++;+ }++ if (text.length === 0) {+ throw new Error(`SRT cue ${cue}, line ${i + 1}: expected cue text.`);+ }++ captions.push({+ text: text.join('\n').trimEnd(),+ startMs,+ endMs,+ timestampMs: (startMs + endMs) / 2,+ confidence: 1,+ });+ }++ if (captions.length === 0) {+ throw new Error(+ 'SRT: No timed captions found. Provide numbered cues with timestamps and text.',+ );+ }++ return {captions};+ }+for (let i = 0; i < inputLines.length; i++) {const line = inputLines[i];const nextLine = inputLines[i + 1];diff --git a/packages/docs/docs/captions/parse-srt.mdx b/packages/docs/docs/captions/parse-srt.mdxindex 8d5bf8f890..bf95321e36 100644--- a/packages/docs/docs/captions/parse-srt.mdx+++ b/packages/docs/docs/captions/parse-srt.mdx@@ -59,6 +59,10 @@ const {captions} = parseSrt({input});The contents of a `.srt` file as a `string`.+### `strict`?<AvailableFrom v="4.0.530" />++Defaults to `false` for compatibility with tolerant parsing. Set to `true` to reject malformed or empty files instead of ignoring unrecognized cues. Strict parsing validates every cue's timestamps (including complete tokens and minute/second ranges), end-before-start errors, and trailing content. Recognized timing-line settings such as `align:start position:0%` or `X1:100 X2:200 Y1:300 Y2:400` are ignored. Errors include cue and line context. Studio imports use strict parsing.+## Return valueAn object with the following properties:diff --git a/packages/docs/docs/elevenlabs/TableOfContents.tsx b/packages/docs/docs/elevenlabs/TableOfContents.tsxindex 069651aee2..a2e0b1a516 100644--- a/packages/docs/docs/elevenlabs/TableOfContents.tsx+++ b/packages/docs/docs/elevenlabs/TableOfContents.tsx@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {<code>Caption</code></div></TOCItem>+ <TOCItem link="/docs/elevenlabs/detect-elevenlabs-transcript-format">+ <strong>{'detectElevenLabsTranscriptFormat()'}</strong>+ <div>Recognize ElevenLabs transcript format signatures</div>+ </TOCItem></Grid></div>);diff --git a/packages/docs/docs/elevenlabs/detect-elevenlabs-transcript-format.mdx b/packages/docs/docs/elevenlabs/detect-elevenlabs-transcript-format.mdxnew file mode 100644index 0000000000..19943dc2c6--- /dev/null+++ b/packages/docs/docs/elevenlabs/detect-elevenlabs-transcript-format.mdx@@ -0,0 +1,34 @@+---+image: /generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.png+title: detectElevenLabsTranscriptFormat()+crumb: '@remotion/elevenlabs'+---++# detectElevenLabsTranscriptFormat()<AvailableFrom v="4.0.530" />++Recognizes the structure of an ElevenLabs Speech-to-Text response or segmented JSON export. This function does not validate a transcript: identifiable malformed data may be recognized. Use [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions) to validate and convert it.++```ts title="recognize.ts"+import {detectElevenLabsTranscriptFormat} from '@remotion/elevenlabs';++const format = detectElevenLabsTranscriptFormat(input);+// 'speech-to-text' | 'segmented' | null+```++## API++### `input`++An unknown value, such as parsed JSON. Recognition does not depend on the file name and does not throw for malformed values.++## Return value++`'speech-to-text'`, `'segmented'`, or `null` for unrecognized values. A non-null result does not imply valid timestamps or a valid transcript.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />++## See also++- [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions)diff --git a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdxindex 7c91de0729..f5727471dc 100644--- a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx+++ b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx@@ -43,8 +43,23 @@ An object with the following property:#### `transcript`-The response from the [ElevenLabs Speech to Text API](https://elevenlabs.io/docs/api-reference/speech-to-text/convert).-Must include a `words` array with word-level timing β ensure the API is called with `timestamps_granularity` set to `"word"`.+A Speech-to-Text response with a `words` array (request `timestamps_granularity: "word"`), or a segmented JSON export with a top-level `language_code` field (which can be `null`) and `segments` containing `text`, `start_time`, `end_time`, and optionally timed `words`. <AvailableFrom v="4.0.530" inline /> Segmented exports use absolute seconds; when timed words are available they are preserved, otherwise each segment becomes one caption.++A segmented export uses this structure (example text and times are illustrative):++```json title="Segmented JSON export"+{+ "language_code": null,+ "segments": [+ {+ "text": "Hello",+ "start_time": 0,+ "end_time": 0.5,+ "words": [{"text": "Hello", "start_time": 0, "end_time": 0.5}]+ }+ ]+}+```The `words` array should contain objects with the following fields:@@ -53,6 +68,8 @@ The `words` array should contain objects with the following fields:- `start`: Start time in seconds- `end`: End time in seconds+Invalid consumed fields, negative or reversed timings, and unknown entry types throw an error naming the field. Audio events are intentionally omitted; spacing entries inform the timing of subsequent words. The converter runs locally and does not call the ElevenLabs API.+### Return valueAn object with the following property:diff --git a/packages/docs/docs/openai-whisper/TableOfContents.tsx b/packages/docs/docs/openai-whisper/TableOfContents.tsxindex 0488a2a185..03b484a2bb 100644--- a/packages/docs/docs/openai-whisper/TableOfContents.tsx+++ b/packages/docs/docs/openai-whisper/TableOfContents.tsx@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {<code>Caption</code></div></TOCItem>+ <TOCItem link="/docs/openai-whisper/is-openai-whisper-transcript">+ <strong>{'isOpenAiWhisperTranscript()'}</strong>+ <div>Recognize OpenAI Whisper transcript signatures</div>+ </TOCItem></Grid></div>);diff --git a/packages/docs/docs/openai-whisper/is-openai-whisper-transcript.mdx b/packages/docs/docs/openai-whisper/is-openai-whisper-transcript.mdxnew file mode 100644index 0000000000..d12868965f--- /dev/null+++ b/packages/docs/docs/openai-whisper/is-openai-whisper-transcript.mdx@@ -0,0 +1,34 @@+---+image: /generated/articles-docs-openai-whisper-is-openai-whisper-transcript.png+title: isOpenAiWhisperTranscript()+crumb: '@remotion/openai-whisper'+---++# isOpenAiWhisperTranscript()<AvailableFrom v="4.0.530" />++Recognizes structural signatures of an OpenAI Whisper verbose JSON transcription. This function does not validate a transcript: recognizable malformed input may return `true`. Use [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions) to validate and convert it.++```ts title="recognize.ts"+import {isOpenAiWhisperTranscript} from '@remotion/openai-whisper';++const recognized = isOpenAiWhisperTranscript(input);+// boolean+```++## API++### `input`++An unknown value, such as parsed JSON. Recognition does not depend on the file name and does not throw for malformed values.++## Return value++A boolean indicating whether a Whisper verbose transcription signature was recognized. `true` does not establish that its fields or timings are valid.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />++## See also++- [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions)diff --git a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdxindex 6a14303ad4..892a155fe1 100644--- a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx+++ b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx@@ -30,6 +30,10 @@ const transcription = await openai.audio.transcriptions.create({const {captions} = openAiWhisperApiToCaptions({transcription});```+## Input<AvailableFrom v="4.0.530" />++Accepts a verbose JSON transcription with `text` and timed `words` (`word`, `start`, `end` in seconds). Word-level conversion reconstructs punctuation from the full text. If words are absent or empty, timed `segments` (`text`, `start`, `end` in seconds) produce one caption per segment, without invented word timings. Invalid supplied word timings throw an error rather than falling back to segments. A transcription without timed entries throws an actionable error. Conversion is local and never calls OpenAI.+## Compatibility<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />diff --git a/packages/docs/elements/captions/index.mdx b/packages/docs/elements/captions/index.mdxindex 44075c61de..80c9a54d33 100644--- a/packages/docs/elements/captions/index.mdx+++ b/packages/docs/elements/captions/index.mdx@@ -16,7 +16,9 @@ import {ElementLibrary} from '@site/src/components/Elements/ElementLibrary';<Step>2</Step> Select its caption layer in Studio.</div><div>-<Step>3</Step> In the Captions inspector, click <kbd>Import</kbd> and choose a JSON file containing [`Caption`](/docs/captions/caption) elements.+<Step>3</Step> In the Captions inspector, click <kbd>Import</kbd> and choose a Remotion [`Caption[]`](/docs/captions/caption) JSON, SRT, ElevenLabs Speech-to-Text or segmented JSON, or OpenAI Whisper verbose JSON file.</div>+<AvailableFrom v="4.0.530" /> Studio detects the format and converts it locally. Timed words retain word-level timing; exports with only timed segments produce segment-level captions. Invalid or unsupported files show an error and leave existing captions unchanged. You can choose a local file or a public asset.+<ElementLibrary category="captions" />diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex 5b8972e503..3863a195d4 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -356,7 +356,10 @@ const sidebars: SidebarsConfig = {type: 'doc',id: 'elevenlabs/index',},- items: ['elevenlabs/elevenlabs-transcript-to-captions'],+ items: [+ 'elevenlabs/elevenlabs-transcript-to-captions',+ 'elevenlabs/detect-elevenlabs-transcript-format',+ ],},{type: 'category',@@ -585,7 +588,10 @@ const sidebars: SidebarsConfig = {type: 'doc',id: 'openai-whisper/index',},- items: ['openai-whisper/openai-whisper-api-to-captions'],+ items: [+ 'openai-whisper/openai-whisper-api-to-captions',+ 'openai-whisper/is-openai-whisper-transcript',+ ],},{type: 'category',diff --git a/packages/docs/src/components/Elements/ElementPage.tsx b/packages/docs/src/components/Elements/ElementPage.tsxindex 9b55d3b00c..74d8ea58b4 100644--- a/packages/docs/src/components/Elements/ElementPage.tsx+++ b/packages/docs/src/components/Elements/ElementPage.tsx@@ -299,7 +299,7 @@ export const ElementPage: React.FC<ElementPageProps> = ({<p className={styles.description} style={{marginTop: 8}}>Have captions?{' '}<a href="/elements/captions/#importing-captions-into-studio">- Import Remotion Caption[] JSON from Studio.+ Import captions from JSON or SRT in Studio.</a></p>) : null}diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex 26596a6975..1de1992464 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -3114,6 +3114,15 @@ export const articles = [noAi: false,slug: 'electron',},+ {+ id: 'elevenlabs/detect-elevenlabs-transcript-format',+ title: 'detectElevenLabsTranscriptFormat()',+ relativePath: 'docs/elevenlabs/detect-elevenlabs-transcript-format.mdx',+ compId: 'articles-docs-elevenlabs-detect-elevenlabs-transcript-format',+ crumb: '@remotion/elevenlabs',+ noAi: false,+ slug: 'elevenlabs/detect-elevenlabs-transcript-format',+ },{id: 'elevenlabs/elevenlabs-transcript-to-captions',title: 'elevenLabsTranscriptToCaptions()',@@ -5733,6 +5742,15 @@ export const articles = [noAi: false,slug: 'openai-whisper/index',},+ {+ id: 'openai-whisper/is-openai-whisper-transcript',+ title: 'isOpenAiWhisperTranscript()',+ relativePath: 'docs/openai-whisper/is-openai-whisper-transcript.mdx',+ compId: 'articles-docs-openai-whisper-is-openai-whisper-transcript',+ crumb: '@remotion/openai-whisper',+ noAi: false,+ slug: 'openai-whisper/is-openai-whisper-transcript',+ },{id: 'openai-whisper/openai-whisper-api-to-captions',title: 'openAiWhisperApiToCaptions()',diff --git a/packages/docs/static/generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.png b/packages/docs/static/generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.pngnew file mode 100644index 0000000000..ec63e0dec2Binary files /dev/null and b/packages/docs/static/generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.png differdiff --git a/packages/docs/static/generated/articles-docs-openai-whisper-is-openai-whisper-transcript.png b/packages/docs/static/generated/articles-docs-openai-whisper-is-openai-whisper-transcript.pngnew file mode 100644index 0000000000..2d1b3fbd0cBinary files /dev/null and b/packages/docs/static/generated/articles-docs-openai-whisper-is-openai-whisper-transcript.png differdiff --git a/packages/elevenlabs/.npmignore b/packages/elevenlabs/.npmignoreindex d037cfc2fd..c8337bff57 100644--- a/packages/elevenlabs/.npmignore+++ b/packages/elevenlabs/.npmignore@@ -5,3 +5,4 @@ tsconfig.tsbuildinfo.turbosrceslint.config.mjs+bundle.tsdiff --git a/packages/elevenlabs/bundle.ts b/packages/elevenlabs/bundle.tsnew file mode 100644index 0000000000..94ba1370ec--- /dev/null+++ b/packages/elevenlabs/bundle.ts@@ -0,0 +1,7 @@+import {buildPackage} from '../.monorepo/builder';++await buildPackage({+ formats: {esm: 'build', cjs: 'use-tsc'},+ external: 'dependencies',+ entrypoints: [{path: 'src/index.ts', target: 'browser'}],+});diff --git a/packages/elevenlabs/package.json b/packages/elevenlabs/package.jsonindex 79eebc231a..a14aaf72f5 100644--- a/packages/elevenlabs/package.json+++ b/packages/elevenlabs/package.json@@ -6,6 +6,7 @@"version": "4.0.529","description": "Work with the output of the ElevenLabs API","main": "dist/index.js",+ "module": "dist/esm/index.mjs","bugs": {"url": "https://github.com/remotion-dev/remotion/issues"},@@ -14,12 +15,13 @@"format": "oxfmt src","lint": "eslint src","test": "bun test src",- "make": "tsgo -d"+ "make": "tsgo -d && bun --env-file=../.env.bundle bundle.ts"},"author": "Jonny Burger <[email protected]>","license": "MIT","dependencies": {- "@remotion/captions": "workspace:*"+ "@remotion/captions": "workspace:*",+ "zod": "catalog:"},"peerDependencies": {},"devDependencies": {@@ -34,5 +36,14 @@"publishConfig": {"access": "public"},- "homepage": "https://www.remotion.dev/docs/elevenlabs"+ "homepage": "https://www.remotion.dev/docs/elevenlabs",+ "exports": {+ "./package.json": "./package.json",+ ".": {+ "types": "./dist/index.d.ts",+ "require": "./dist/index.js",+ "module": "./dist/esm/index.mjs",+ "import": "./dist/esm/index.mjs"+ }+ }}diff --git a/packages/elevenlabs/src/detect-elevenlabs-transcript-format.ts b/packages/elevenlabs/src/detect-elevenlabs-transcript-format.tsnew file mode 100644index 0000000000..d5f92dd350--- /dev/null+++ b/packages/elevenlabs/src/detect-elevenlabs-transcript-format.ts@@ -0,0 +1,26 @@+import * as z from 'zod/mini';++// Recognition deliberately uses permissive schemas; conversion validates the fields.+const stt = z.object({language_code: z.unknown(), words: z.unknown()});+const sttWithId = z.object({transcription_id: z.unknown(), words: z.unknown()});+const segmented = z.object({+ language_code: z.unknown(),+ segments: z.unknown(),+});++export const detectElevenLabsTranscriptFormat = (+ input: unknown,+): 'speech-to-text' | 'segmented' | null => {+ if (+ z.safeParse(stt, input).success ||+ z.safeParse(sttWithId, input).success+ ) {+ return 'speech-to-text';+ }++ if (z.safeParse(segmented, input).success) {+ return 'segmented';+ }++ return null;+};diff --git a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.tsindex 671644ba95..60d676aa86 100644--- a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts+++ b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts@@ -1,31 +1,115 @@import type {Caption} from '@remotion/captions';-import type {ElevenLabsTranscript} from './elevenlabs-transcript';+import * as z from 'zod/mini';+import {detectElevenLabsTranscriptFormat} from './detect-elevenlabs-transcript-format';export type ElevenLabsTranscriptToCaptionsInput = {- transcript: ElevenLabsTranscript;+ transcript: unknown;};export type ElevenLabsTranscriptToCaptionsOutput = {captions: Caption[];};+const seconds = z.number().check(z.nonnegative());+const sttWord = z.object({+ text: z.string(),+ start: seconds,+ end: seconds,+ type: z.enum(['word', 'spacing', 'audio_event']),+});+const stt = z.object({words: z.array(sttWord)});+const segmentWord = z.object({+ text: z.string(),+ start_time: seconds,+ end_time: seconds,+});+const segment = z.object({+ text: z.string(),+ start_time: seconds,+ end_time: seconds,+ words: z.optional(z.array(segmentWord)),+});+const segmented = z.object({segments: z.array(segment)});+export const elevenLabsTranscriptToCaptions = ({transcript,}: ElevenLabsTranscriptToCaptionsInput): ElevenLabsTranscriptToCaptionsOutput => {- const captions: Caption[] = [];+ const format = detectElevenLabsTranscriptFormat(transcript);+ if (format === null) {+ throw new Error(+ 'Invalid ElevenLabs transcript: expected Speech-to-Text words or a segmented JSON export with timed segments.',+ );+ }- if (!transcript || !transcript.words || !Array.isArray(transcript.words)) {+ const schema = format === 'speech-to-text' ? stt : segmented;+ const result = z.safeParse(schema, transcript);+ if (!result.success) {+ const issue = result.error.issues[0];+ const path = issue.path.reduce<string>(+ (acc, key) =>+ typeof key === 'number'+ ? `${acc}[${key}]`+ : `${acc}${acc ? '.' : ''}${String(key)}`,+ '',+ );throw new Error(- 'Invalid ElevenLabs transcript. The transcript must be generated with `timestamps_granularity` set to `"word"`. See https://www.remotion.dev/docs/elevenlabs/elevenlabs-transcript-to-captions',+ `ElevenLabs ${path || 'transcript'} ${/(^|\.)(start|end|start_time|end_time)$/.test(path) ? 'must be a finite, non-negative number (seconds)' : `is invalid: ${issue.message}`}.`,);}- const {words} = transcript;+ const captions: Caption[] = [];+ if (format === 'segmented') {+ const parsed = z.parse(segmented, transcript);+ for (const [index, entry] of parsed.segments.entries()) {+ if (entry.end_time < entry.start_time) {+ throw new Error(+ `ElevenLabs segments[${index}].end_time must not be earlier than start_time.`,+ );+ }- let isFirst = true;+ if (entry.words && entry.words.length > 0) {+ for (const [wordIndex, word] of entry.words.entries()) {+ if (word.end_time < word.start_time) {+ throw new Error(+ `ElevenLabs segments[${index}].words[${wordIndex}].end_time must not be earlier than start_time.`,+ );+ }+ const startMs = word.start_time * 1000;+ const endMs = word.end_time * 1000;+ captions.push({+ text: word.text,+ startMs,+ endMs,+ timestampMs: (startMs + endMs) / 2,+ confidence: null,+ });+ }+ } else {+ const startMs = entry.start_time * 1000;+ const endMs = entry.end_time * 1000;+ captions.push({+ text: entry.text,+ startMs,+ endMs,+ timestampMs: (startMs + endMs) / 2,+ confidence: null,+ });+ }+ }++ return {captions};+ }++ const {words} = z.parse(stt, transcript);+ let isFirst = true;for (let i = 0; i < words.length; i++) {const entry = words[i];+ if (entry.end < entry.start) {+ throw new Error(+ `ElevenLabs words[${i}].end must not be earlier than start.`,+ );+ }if (entry.type !== 'word') {continue;@@ -33,12 +117,10 @@ export const elevenLabsTranscriptToCaptions = ({const prevEntry = i > 0 ? words[i - 1] : null;const hasSpacing = prevEntry !== null && prevEntry.type === 'spacing';-const startMs =!isFirst && hasSpacing ? prevEntry.start * 1000 : entry.start * 1000;const endMs = entry.end * 1000;const text = isFirst ? entry.text : ` ${entry.text}`;-captions.push({confidence: null,startMs,@@ -46,7 +128,6 @@ export const elevenLabsTranscriptToCaptions = ({text,timestampMs: (startMs + endMs) / 2,});-isFirst = false;}diff --git a/packages/elevenlabs/src/index.ts b/packages/elevenlabs/src/index.tsindex 50006f266f..43451cb4cf 100644--- a/packages/elevenlabs/src/index.ts+++ b/packages/elevenlabs/src/index.ts@@ -1,3 +1,5 @@+export {detectElevenLabsTranscriptFormat} from './detect-elevenlabs-transcript-format';+export {ElevenLabsTranscriptToCaptionsInput,ElevenLabsTranscriptToCaptionsOutput,diff --git a/packages/openai-whisper/.npmignore b/packages/openai-whisper/.npmignoreindex 5510046421..c8337bff57 100644--- a/packages/openai-whisper/.npmignore+++ b/packages/openai-whisper/.npmignore@@ -4,4 +4,5 @@ tsconfig.tsbuildinfo.env.turbosrc-eslint.config.mjs\ No newline at end of file+eslint.config.mjs+bundle.tsdiff --git a/packages/openai-whisper/bundle.ts b/packages/openai-whisper/bundle.tsnew file mode 100644index 0000000000..94ba1370ec--- /dev/null+++ b/packages/openai-whisper/bundle.ts@@ -0,0 +1,7 @@+import {buildPackage} from '../.monorepo/builder';++await buildPackage({+ formats: {esm: 'build', cjs: 'use-tsc'},+ external: 'dependencies',+ entrypoints: [{path: 'src/index.ts', target: 'browser'}],+});diff --git a/packages/openai-whisper/package.json b/packages/openai-whisper/package.jsonindex b03d38191e..0690481532 100644--- a/packages/openai-whisper/package.json+++ b/packages/openai-whisper/package.json@@ -6,6 +6,7 @@"version": "4.0.529","description": "Work with the output of the OpenAI Whisper API","main": "dist/index.js",+ "module": "dist/esm/index.mjs","bugs": {"url": "https://github.com/remotion-dev/remotion/issues"},@@ -14,12 +15,13 @@"format": "oxfmt src","lint": "eslint src","test": "bun test src",- "make": "tsgo -d"+ "make": "tsgo -d && bun --env-file=../.env.bundle bundle.ts"},"author": "Jonny Burger <[email protected]>","license": "MIT","dependencies": {- "@remotion/captions": "workspace:*"+ "@remotion/captions": "workspace:*",+ "zod": "catalog:"},"peerDependencies": {},"devDependencies": {@@ -34,5 +36,14 @@"publishConfig": {"access": "public"},- "homepage": "https://www.remotion.dev/docs/openai-whisper"+ "homepage": "https://www.remotion.dev/docs/openai-whisper",+ "exports": {+ "./package.json": "./package.json",+ ".": {+ "types": "./dist/index.d.ts",+ "require": "./dist/index.js",+ "module": "./dist/esm/index.mjs",+ "import": "./dist/esm/index.mjs"+ }+ }}diff --git a/packages/openai-whisper/src/index.ts b/packages/openai-whisper/src/index.tsindex 11916012c8..a6cf1b6f3f 100644--- a/packages/openai-whisper/src/index.ts+++ b/packages/openai-whisper/src/index.ts@@ -5,3 +5,4 @@ export {} from './openai-whisper-api-to-captions';export {OpenAiVerboseTranscription} from './openai-format';+export {isOpenAiWhisperTranscript} from './is-openai-whisper-transcript';diff --git a/packages/openai-whisper/src/is-openai-whisper-transcript.ts b/packages/openai-whisper/src/is-openai-whisper-transcript.tsnew file mode 100644index 0000000000..5af73e28b6--- /dev/null+++ b/packages/openai-whisper/src/is-openai-whisper-transcript.ts@@ -0,0 +1,28 @@+import * as z from 'zod/mini';++// Permissive signatures, not proof that the transcript is valid.+const withTask = z.object({task: z.unknown()});+const withWords = z.object({+ text: z.unknown(),+ language: z.unknown(),+ words: z.unknown(),+});+const withDuration = z.object({+ text: z.unknown(),+ duration: z.unknown(),+ segments: z.unknown(),+});+const withSegments = z.object({+ text: z.unknown(),+ language: z.unknown(),+ segments: z.unknown(),+});++export const isOpenAiWhisperTranscript = (input: unknown): boolean => {+ return (+ z.safeParse(withTask, input).success ||+ z.safeParse(withWords, input).success ||+ z.safeParse(withDuration, input).success ||+ z.safeParse(withSegments, input).success+ );+};diff --git a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts b/packages/openai-whisper/src/openai-whisper-api-to-captions.tsindex 60809ce5aa..68baf24844 100644--- a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts+++ b/packages/openai-whisper/src/openai-whisper-api-to-captions.ts@@ -1,8 +1,8 @@import type {Caption} from '@remotion/captions';-import type {OpenAiVerboseTranscription} from './openai-format';+import * as z from 'zod/mini';export type OpenAiToCaptionsInput = {- transcription: OpenAiVerboseTranscription;+ transcription: unknown;};export type OpenAiToCaptionsOutput = {@@ -26,27 +26,94 @@ const escapeWordForRegex = (text: string) => {.join('');};+const seconds = z.number().check(z.nonnegative());+const wordSchema = z.object({word: z.string(), start: seconds, end: seconds});+const segmentSchema = z.object({+ text: z.string(),+ start: seconds,+ end: seconds,+});+const transcriptionSchema = z.object({+ text: z.string(),+ words: z.optional(z.array(wordSchema)),+ task: z.optional(z.string()),+});+const segmentsSchema = z.object({segments: z.optional(z.array(segmentSchema))});++const invalidTranscript = (issue: z.core.$ZodIssue): Error => {+ const path = issue.path.reduce<string>(+ (acc, key) =>+ typeof key === 'number'+ ? `${acc}[${key}]`+ : `${acc}${acc ? '.' : ''}${String(key)}`,+ '',+ );+ return new Error(+ `OpenAI Whisper ${path || 'transcription'} ${/(^|\.)(start|end)$/.test(path) ? 'must be a finite, non-negative number (seconds)' : `is invalid: ${issue.message}`}.`,+ );+};+export const openAiWhisperApiToCaptions = ({transcription,}: OpenAiToCaptionsInput): OpenAiToCaptionsOutput => {const captions: Caption[] = [];+ const result = z.safeParse(transcriptionSchema, transcription);+ if (!result.success) {+ throw invalidTranscript(result.error.issues[0]);+ }++ const parsed = result.data;+ if (parsed.task && parsed.task !== 'transcribe' && !parsed.words?.length) {+ throw new Error(+ `The transcription does need to be a "transcribe" task. The input you gave is "task": "${parsed.task}"`,+ );+ }++ if (!parsed.words?.length) {+ const segmentResult = z.safeParse(segmentsSchema, transcription);+ if (!segmentResult.success) {+ throw invalidTranscript(segmentResult.error.issues[0]);+ }- if (!transcription.words) {- if (transcription.task && transcription.task !== 'transcribe') {+ if (!segmentResult.data.segments?.length) {throw new Error(- `The transcription does need to be a "transcribe" task. The input you gave is "task": "${transcription.task}"`,+ 'OpenAI Whisper: No timed captions found. Export verbose_json with timestamp_granularities: ["word"] or timed segments.',);}- throw new Error(- 'The transcription does need to be been generated with `timestamp_granularities: ["word"]`',- );+ for (const [index, segment] of segmentResult.data.segments.entries()) {+ if (segment.end < segment.start) {+ throw new Error(+ `OpenAI Whisper segments[${index}].end must not be earlier than start.`,+ );+ }++ const startMs = segment.start * 1000;+ const endMs = segment.end * 1000;+ captions.push({+ text: segment.text,+ startMs,+ endMs,+ timestampMs: (startMs + endMs) / 2,+ confidence: null,+ });+ }++ return {captions};+ }++ for (const [index, word] of parsed.words.entries()) {+ if (word.end < word.start) {+ throw new Error(+ `OpenAI Whisper words[${index}].end must not be earlier than start.`,+ );+ }}- let remainingText = transcription.text;+ let remainingText = parsed.text;- for (let i = 0; i < transcription.words.length; i++) {- const word = transcription.words[i];+ for (let i = 0; i < parsed.words.length; i++) {+ const word = parsed.words[i];const firstWord = i === 0;// https://github.com/remotion-dev/remotion/issues/5031if (firstWord) {diff --git a/packages/studio/package.json b/packages/studio/package.jsonindex be297c15c5..61540272f2 100644--- a/packages/studio/package.json+++ b/packages/studio/package.json@@ -37,6 +37,8 @@"@jridgewell/trace-mapping": "0.3.31","@remotion/canvas": "workspace:*","@remotion/captions": "workspace:*",+ "@remotion/elevenlabs": "workspace:*",+ "@remotion/openai-whisper": "workspace:*","@remotion/media": "workspace:*","@remotion/media-utils": "workspace:*","@remotion/paths": "workspace:*",diff --git a/packages/studio/src/components/CaptionInspector.tsx b/packages/studio/src/components/CaptionInspector.tsxindex 1504eee212..7e635f5821 100644--- a/packages/studio/src/components/CaptionInspector.tsx+++ b/packages/studio/src/components/CaptionInspector.tsx@@ -14,7 +14,7 @@ import {parseCaptionFile} from './parse-caption-file';const importTooltip = `Import captions-Supports Remotion Caption[] JSON. Files are processed locally.`;+Supports Remotion Caption[] JSON, SRT, ElevenLabs Speech-to-Text and segmented JSON, and OpenAI Whisper verbose JSON. Files are processed locally.`;const readOnlyStatus: React.CSSProperties = {color: LIGHT_TEXT,@@ -107,12 +107,20 @@ export const CaptionInspector: React.FC<{mode: 'assets',invocationTimestamp: Date.now(),assetSelection: {- initialQuery: 'type:json',+ initialQuery: '',onSelectFile: () => fileInput.current?.click(),onSelected: (asset) => {return importCaptions({fileName: asset.name,- contents: fetch(asset.src).then((response) => response.text()),+ contents: fetch(asset.src).then((response) => {+ if (!response.ok) {+ throw new Error(+ `Could not read asset (HTTP ${response.status}). Check that the file is available.`,+ );+ }++ return response.text();+ }),}).catch(() => undefined);},},@@ -130,7 +138,7 @@ export const CaptionInspector: React.FC<{<><inputref={fileInput}- accept=".json"+ accept=".json,.srt"aria-label="Import captions file"hiddenonChange={onFileSelected}diff --git a/packages/studio/src/components/parse-caption-file.ts b/packages/studio/src/components/parse-caption-file.tsindex 5aacaf660b..26f4b48160 100644--- a/packages/studio/src/components/parse-caption-file.ts+++ b/packages/studio/src/components/parse-caption-file.ts@@ -1,4 +1,12 @@-import type {Caption} from '@remotion/captions';+import {parseSrt, type Caption} from '@remotion/captions';+import {+ detectElevenLabsTranscriptFormat,+ elevenLabsTranscriptToCaptions,+} from '@remotion/elevenlabs';+import {+ isOpenAiWhisperTranscript,+ openAiWhisperApiToCaptions,+} from '@remotion/openai-whisper';const isObject = (value: unknown): value is Record<string, unknown> => {return typeof value === 'object' && value !== null && !Array.isArray(value);@@ -15,23 +23,57 @@ export const parseCaptionFile = ({fileName: string;contents: string;}): Caption[] => {- if (!fileName.toLowerCase().endsWith('.json')) {- throw new Error('Unsupported caption file. Choose a .json file.');+ const name = fileName.toLowerCase();+ if (!name.endsWith('.json') && !name.endsWith('.srt')) {+ throw new Error('Unsupported caption file. Choose a .json or .srt file.');}let parsed: unknown;- try {- parsed = JSON.parse(contents);- } catch (error) {- throw new Error(- `Invalid JSON: ${error instanceof Error ? error.message : String(error)}`,- );+ let canonical = false;+ if (name.endsWith('.srt')) {+ parsed = parseSrt({input: contents, strict: true}).captions;+ } else {+ try {+ parsed = JSON.parse(contents.replace(/^\uFEFF/, ''));+ } catch (error) {+ throw new Error(+ `Invalid JSON: ${error instanceof Error ? error.message : String(error)}`,+ );+ }++ if (Array.isArray(parsed)) {+ canonical = true;+ } else {+ const elevenLabs = detectElevenLabsTranscriptFormat(parsed);+ const whisper = isOpenAiWhisperTranscript(parsed);+ if (elevenLabs && whisper) {+ throw new Error(+ 'Ambiguous transcript: matches both ElevenLabs and OpenAI Whisper signatures. Remove conflicting provider fields.',+ );+ }++ if (elevenLabs) {+ parsed = elevenLabsTranscriptToCaptions({transcript: parsed}).captions;+ } else if (whisper) {+ parsed = openAiWhisperApiToCaptions({transcription: parsed}).captions;+ } else {+ throw new Error(+ 'Unsupported JSON shape. Expected Remotion Caption[], ElevenLabs Speech-to-Text or segmented JSON, or OpenAI Whisper verbose JSON with timed words or segments.',+ );+ }+ }}if (!Array.isArray(parsed)) {throw new Error('Expected a Remotion Caption[] JSON array.');}+ if (!canonical && parsed.length === 0) {+ throw new Error(+ 'No timed captions found. Export word or segment timestamps, or choose a caption file with timed cues.',+ );+ }+let previousStart: number | null = null;for (const [index, caption] of parsed.entries()) {const path = `captions[${index}]`;diff --git a/packages/studio/src/test/caption-inspector-import.test.tsx b/packages/studio/src/test/caption-inspector-import.test.tsxindex 2912f76edc..fd9d95a328 100644--- a/packages/studio/src/test/caption-inspector-import.test.tsx+++ b/packages/studio/src/test/caption-inspector-import.test.tsx@@ -61,7 +61,7 @@ test('imports captions from a JSON file in the public folder', async () => {const {assetSelection} = modal;- expect(assetSelection.initialQuery).toBe('type:json');+ expect(assetSelection.initialQuery).toBe('');await act(async () => {await (assetSelection.onSelected({lastModified: 0,diff --git a/packages/studio/src/test/parse-caption-file.test.ts b/packages/studio/src/test/parse-caption-file.test.tsindex 5265fdf004..525708727e 100644--- a/packages/studio/src/test/parse-caption-file.test.ts+++ b/packages/studio/src/test/parse-caption-file.test.ts@@ -41,7 +41,7 @@ test.each([{name: 'non-array JSON',value: '{}',- error: 'Expected a Remotion Caption[] JSON array',+ error: 'Unsupported JSON shape',},{name: 'missing field',