Skip to content

πŸŽ₯ Make videos programmatically with React

`@remotion/studio`: Import SRT and provider transcripts as captions (#11665)

Igor Samokhovets11e14f0c1817
diff --git a/bun.lock b/bun.lock
index 61dddd8c5d..b9cd369e7e 100644
--- a/bun.lock
+++ b/bun.lock
@@ -735,6 +735,7 @@
"version": "4.0.529",
"dependencies": {
"@remotion/captions": "workspace:*",
+ "zod": "catalog:",
},
"devDependencies": {
"@remotion/eslint-config-internal": "workspace:*",
@@ -1431,6 +1432,7 @@
"version": "4.0.529",
"dependencies": {
"@remotion/captions": "workspace:*",
+ "zod": "catalog:",
},
"devDependencies": {
"@remotion/eslint-config-internal": "workspace:*",
@@ -1863,8 +1865,10 @@
"@remotion/canvas": "workspace:*",
"@remotion/captions": "workspace:*",
"@remotion/codemods": "workspace:*",
+ "@remotion/elevenlabs": "workspace:*",
"@remotion/media": "workspace:*",
"@remotion/media-utils": "workspace:*",
+ "@remotion/openai-whisper": "workspace:*",
"@remotion/paths": "workspace:*",
"@remotion/player": "workspace:*",
"@remotion/renderer": "workspace:*",
diff --git a/packages/captions/src/parse-srt.ts b/packages/captions/src/parse-srt.ts
index bf6dce3281..8be48c513a 100644
--- a/packages/captions/src/parse-srt.ts
+++ b/packages/captions/src/parse-srt.ts
@@ -33,13 +33,17 @@ function toMilliseconds(time: string) {
export type ParseSrtInput = {
input: string;
+ strict?: boolean;
};
export type ParseSrtOutput = {
captions: Caption[];
};
-export const parseSrt = ({input}: ParseSrtInput): ParseSrtOutput => {
+export const parseSrt = ({
+ input,
+ strict = false,
+}: ParseSrtInput): ParseSrtOutput => {
const inputLines = input
.replace(/^\uFEFF/, '')
.replace(/\r\n/g, '\n')
@@ -47,6 +51,101 @@ export const parseSrt = ({input}: ParseSrtInput): ParseSrtOutput => {
.split('\n');
const captions: Caption[] = [];
+ if (strict) {
+ const timestamp = (token: string, cue: number, line: number): number => {
+ const match = /^(\d+):(\d{2}):(\d{2})[,.](\d{3})$/.exec(token);
+ if (!match || Number(match[2]) > 59 || Number(match[3]) > 59) {
+ throw new Error(
+ `SRT cue ${cue}, line ${line}: invalid timestamp "${token}". Use HH:MM:SS,mmm.`,
+ );
+ }
+
+ const value =
+ Number(match[1]) * 3600000 +
+ Number(match[2]) * 60000 +
+ Number(match[3]) * 1000 +
+ Number(match[4]);
+ if (!Number.isFinite(value)) {
+ throw new Error(
+ `SRT cue ${cue}, line ${line}: timestamp is too large.`,
+ );
+ }
+
+ return value;
+ };
+
+ let i = 0;
+ while (i < inputLines.length) {
+ if (inputLines[i].trim() === '') {
+ i++;
+ continue;
+ }
+
+ const cue = captions.length + 1;
+ const headerLine = i + 1;
+ if (!/^\d+$/.test(inputLines[i].trim())) {
+ throw new Error(
+ `SRT cue ${cue}, line ${headerLine}: expected a numeric cue index.`,
+ );
+ }
+
+ i++;
+ const timingLine = inputLines[i];
+ const timing = timingLine?.match(/^\s*(\S+)\s+-->\s+(\S+)(.*)$/);
+ const settings = timing?.[3].trim();
+ if (
+ !timing ||
+ (settings &&
+ !settings
+ .split(/\s+/)
+ .every((setting) =>
+ /^(?:(?:X[12]|Y[12]):\d+|(?:align|position|line|size|vertical):\S+)$/.test(
+ setting,
+ ),
+ ))
+ ) {
+ throw new Error(
+ `SRT cue ${cue}, line ${i + 1}: expected start --> end timestamps.`,
+ );
+ }
+
+ const startMs = timestamp(timing[1], cue, i + 1);
+ const endMs = timestamp(timing[2], cue, i + 1);
+ if (endMs < startMs) {
+ throw new Error(
+ `SRT cue ${cue}, line ${i + 1}: end must not be earlier than start.`,
+ );
+ }
+
+ i++;
+ const text: string[] = [];
+ while (i < inputLines.length && inputLines[i].trim() !== '') {
+ text.push(inputLines[i]);
+ i++;
+ }
+
+ if (text.length === 0) {
+ throw new Error(`SRT cue ${cue}, line ${i + 1}: expected cue text.`);
+ }
+
+ captions.push({
+ text: text.join('\n').trimEnd(),
+ startMs,
+ endMs,
+ timestampMs: (startMs + endMs) / 2,
+ confidence: 1,
+ });
+ }
+
+ if (captions.length === 0) {
+ throw new Error(
+ 'SRT: No timed captions found. Provide numbered cues with timestamps and text.',
+ );
+ }
+
+ return {captions};
+ }
+
for (let i = 0; i < inputLines.length; i++) {
const line = inputLines[i];
const nextLine = inputLines[i + 1];
diff --git a/packages/docs/docs/captions/parse-srt.mdx b/packages/docs/docs/captions/parse-srt.mdx
index 8d5bf8f890..bf95321e36 100644
--- a/packages/docs/docs/captions/parse-srt.mdx
+++ b/packages/docs/docs/captions/parse-srt.mdx
@@ -59,6 +59,10 @@ const {captions} = parseSrt({input});
The contents of a `.srt` file as a `string`.
+### `strict`?<AvailableFrom v="4.0.530" />
+
+Defaults to `false` for compatibility with tolerant parsing. Set to `true` to reject malformed or empty files instead of ignoring unrecognized cues. Strict parsing validates every cue's timestamps (including complete tokens and minute/second ranges), end-before-start errors, and trailing content. Recognized timing-line settings such as `align:start position:0%` or `X1:100 X2:200 Y1:300 Y2:400` are ignored. Errors include cue and line context. Studio imports use strict parsing.
+
## Return value
An object with the following properties:
diff --git a/packages/docs/docs/elevenlabs/TableOfContents.tsx b/packages/docs/docs/elevenlabs/TableOfContents.tsx
index 069651aee2..a2e0b1a516 100644
--- a/packages/docs/docs/elevenlabs/TableOfContents.tsx
+++ b/packages/docs/docs/elevenlabs/TableOfContents.tsx
@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {
<code>Caption</code>
</div>
</TOCItem>
+ <TOCItem link="/docs/elevenlabs/detect-elevenlabs-transcript-format">
+ <strong>{'detectElevenLabsTranscriptFormat()'}</strong>
+ <div>Recognize ElevenLabs transcript format signatures</div>
+ </TOCItem>
</Grid>
</div>
);
diff --git a/packages/docs/docs/elevenlabs/detect-elevenlabs-transcript-format.mdx b/packages/docs/docs/elevenlabs/detect-elevenlabs-transcript-format.mdx
new file mode 100644
index 0000000000..19943dc2c6
--- /dev/null
+++ b/packages/docs/docs/elevenlabs/detect-elevenlabs-transcript-format.mdx
@@ -0,0 +1,34 @@
+---
+image: /generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.png
+title: detectElevenLabsTranscriptFormat()
+crumb: '@remotion/elevenlabs'
+---
+
+# detectElevenLabsTranscriptFormat()<AvailableFrom v="4.0.530" />
+
+Recognizes the structure of an ElevenLabs Speech-to-Text response or segmented JSON export. This function does not validate a transcript: identifiable malformed data may be recognized. Use [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions) to validate and convert it.
+
+```ts title="recognize.ts"
+import {detectElevenLabsTranscriptFormat} from '@remotion/elevenlabs';
+
+const format = detectElevenLabsTranscriptFormat(input);
+// 'speech-to-text' | 'segmented' | null
+```
+
+## API
+
+### `input`
+
+An unknown value, such as parsed JSON. Recognition does not depend on the file name and does not throw for malformed values.
+
+## Return value
+
+`'speech-to-text'`, `'segmented'`, or `null` for unrecognized values. A non-null result does not imply valid timestamps or a valid transcript.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />
+
+## See also
+
+- [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions)
diff --git a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx
index 7c91de0729..f5727471dc 100644
--- a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx
+++ b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx
@@ -43,8 +43,23 @@ An object with the following property:
#### `transcript`
-The response from the [ElevenLabs Speech to Text API](https://elevenlabs.io/docs/api-reference/speech-to-text/convert).
-Must include a `words` array with word-level timing β€” ensure the API is called with `timestamps_granularity` set to `"word"`.
+A Speech-to-Text response with a `words` array (request `timestamps_granularity: "word"`), or a segmented JSON export with a top-level `language_code` field (which can be `null`) and `segments` containing `text`, `start_time`, `end_time`, and optionally timed `words`. <AvailableFrom v="4.0.530" inline /> Segmented exports use absolute seconds; when timed words are available they are preserved, otherwise each segment becomes one caption.
+
+A segmented export uses this structure (example text and times are illustrative):
+
+```json title="Segmented JSON export"
+{
+ "language_code": null,
+ "segments": [
+ {
+ "text": "Hello",
+ "start_time": 0,
+ "end_time": 0.5,
+ "words": [{"text": "Hello", "start_time": 0, "end_time": 0.5}]
+ }
+ ]
+}
+```
The `words` array should contain objects with the following fields:
@@ -53,6 +68,8 @@ The `words` array should contain objects with the following fields:
- `start`: Start time in seconds
- `end`: End time in seconds
+Invalid consumed fields, negative or reversed timings, and unknown entry types throw an error naming the field. Audio events are intentionally omitted; spacing entries inform the timing of subsequent words. The converter runs locally and does not call the ElevenLabs API.
+
### Return value
An object with the following property:
diff --git a/packages/docs/docs/openai-whisper/TableOfContents.tsx b/packages/docs/docs/openai-whisper/TableOfContents.tsx
index 0488a2a185..03b484a2bb 100644
--- a/packages/docs/docs/openai-whisper/TableOfContents.tsx
+++ b/packages/docs/docs/openai-whisper/TableOfContents.tsx
@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {
<code>Caption</code>
</div>
</TOCItem>
+ <TOCItem link="/docs/openai-whisper/is-openai-whisper-transcript">
+ <strong>{'isOpenAiWhisperTranscript()'}</strong>
+ <div>Recognize OpenAI Whisper transcript signatures</div>
+ </TOCItem>
</Grid>
</div>
);
diff --git a/packages/docs/docs/openai-whisper/is-openai-whisper-transcript.mdx b/packages/docs/docs/openai-whisper/is-openai-whisper-transcript.mdx
new file mode 100644
index 0000000000..d12868965f
--- /dev/null
+++ b/packages/docs/docs/openai-whisper/is-openai-whisper-transcript.mdx
@@ -0,0 +1,34 @@
+---
+image: /generated/articles-docs-openai-whisper-is-openai-whisper-transcript.png
+title: isOpenAiWhisperTranscript()
+crumb: '@remotion/openai-whisper'
+---
+
+# isOpenAiWhisperTranscript()<AvailableFrom v="4.0.530" />
+
+Recognizes structural signatures of an OpenAI Whisper verbose JSON transcription. This function does not validate a transcript: recognizable malformed input may return `true`. Use [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions) to validate and convert it.
+
+```ts title="recognize.ts"
+import {isOpenAiWhisperTranscript} from '@remotion/openai-whisper';
+
+const recognized = isOpenAiWhisperTranscript(input);
+// boolean
+```
+
+## API
+
+### `input`
+
+An unknown value, such as parsed JSON. Recognition does not depend on the file name and does not throw for malformed values.
+
+## Return value
+
+A boolean indicating whether a Whisper verbose transcription signature was recognized. `true` does not establish that its fields or timings are valid.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />
+
+## See also
+
+- [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions)
diff --git a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx
index 6a14303ad4..892a155fe1 100644
--- a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx
+++ b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx
@@ -30,6 +30,10 @@ const transcription = await openai.audio.transcriptions.create({
const {captions} = openAiWhisperApiToCaptions({transcription});
```
+## Input<AvailableFrom v="4.0.530" />
+
+Accepts a verbose JSON transcription with `text` and timed `words` (`word`, `start`, `end` in seconds). Word-level conversion reconstructs punctuation from the full text. If words are absent or empty, timed `segments` (`text`, `start`, `end` in seconds) produce one caption per segment, without invented word timings. Invalid supplied word timings throw an error rather than falling back to segments. A transcription without timed entries throws an actionable error. Conversion is local and never calls OpenAI.
+
## Compatibility
<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />
diff --git a/packages/docs/elements/captions/index.mdx b/packages/docs/elements/captions/index.mdx
index 44075c61de..80c9a54d33 100644
--- a/packages/docs/elements/captions/index.mdx
+++ b/packages/docs/elements/captions/index.mdx
@@ -16,7 +16,9 @@ import {ElementLibrary} from '@site/src/components/Elements/ElementLibrary';
<Step>2</Step> Select its caption layer in Studio.
</div>
<div>
-<Step>3</Step> In the Captions inspector, click <kbd>Import</kbd> and choose a JSON file containing [`Caption`](/docs/captions/caption) elements.
+<Step>3</Step> In the Captions inspector, click <kbd>Import</kbd> and choose a Remotion [`Caption[]`](/docs/captions/caption) JSON, SRT, ElevenLabs Speech-to-Text or segmented JSON, or OpenAI Whisper verbose JSON file.
</div>
+<AvailableFrom v="4.0.530" /> Studio detects the format and converts it locally. Timed words retain word-level timing; exports with only timed segments produce segment-level captions. Invalid or unsupported files show an error and leave existing captions unchanged. You can choose a local file or a public asset.
+
<ElementLibrary category="captions" />
diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.ts
index 5b8972e503..3863a195d4 100644
--- a/packages/docs/sidebars.ts
+++ b/packages/docs/sidebars.ts
@@ -356,7 +356,10 @@ const sidebars: SidebarsConfig = {
type: 'doc',
id: 'elevenlabs/index',
},
- items: ['elevenlabs/elevenlabs-transcript-to-captions'],
+ items: [
+ 'elevenlabs/elevenlabs-transcript-to-captions',
+ 'elevenlabs/detect-elevenlabs-transcript-format',
+ ],
},
{
type: 'category',
@@ -585,7 +588,10 @@ const sidebars: SidebarsConfig = {
type: 'doc',
id: 'openai-whisper/index',
},
- items: ['openai-whisper/openai-whisper-api-to-captions'],
+ items: [
+ 'openai-whisper/openai-whisper-api-to-captions',
+ 'openai-whisper/is-openai-whisper-transcript',
+ ],
},
{
type: 'category',
diff --git a/packages/docs/src/components/Elements/ElementPage.tsx b/packages/docs/src/components/Elements/ElementPage.tsx
index 9b55d3b00c..74d8ea58b4 100644
--- a/packages/docs/src/components/Elements/ElementPage.tsx
+++ b/packages/docs/src/components/Elements/ElementPage.tsx
@@ -299,7 +299,7 @@ export const ElementPage: React.FC<ElementPageProps> = ({
<p className={styles.description} style={{marginTop: 8}}>
Have captions?{' '}
<a href="/elements/captions/#importing-captions-into-studio">
- Import Remotion Caption[] JSON from Studio.
+ Import captions from JSON or SRT in Studio.
</a>
</p>
) : null}
diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts
index 26596a6975..1de1992464 100644
--- a/packages/docs/src/data/articles.ts
+++ b/packages/docs/src/data/articles.ts
@@ -3114,6 +3114,15 @@ export const articles = [
noAi: false,
slug: 'electron',
},
+ {
+ id: 'elevenlabs/detect-elevenlabs-transcript-format',
+ title: 'detectElevenLabsTranscriptFormat()',
+ relativePath: 'docs/elevenlabs/detect-elevenlabs-transcript-format.mdx',
+ compId: 'articles-docs-elevenlabs-detect-elevenlabs-transcript-format',
+ crumb: '@remotion/elevenlabs',
+ noAi: false,
+ slug: 'elevenlabs/detect-elevenlabs-transcript-format',
+ },
{
id: 'elevenlabs/elevenlabs-transcript-to-captions',
title: 'elevenLabsTranscriptToCaptions()',
@@ -5733,6 +5742,15 @@ export const articles = [
noAi: false,
slug: 'openai-whisper/index',
},
+ {
+ id: 'openai-whisper/is-openai-whisper-transcript',
+ title: 'isOpenAiWhisperTranscript()',
+ relativePath: 'docs/openai-whisper/is-openai-whisper-transcript.mdx',
+ compId: 'articles-docs-openai-whisper-is-openai-whisper-transcript',
+ crumb: '@remotion/openai-whisper',
+ noAi: false,
+ slug: 'openai-whisper/is-openai-whisper-transcript',
+ },
{
id: 'openai-whisper/openai-whisper-api-to-captions',
title: 'openAiWhisperApiToCaptions()',
diff --git a/packages/docs/static/generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.png b/packages/docs/static/generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.png
new file mode 100644
index 0000000000..ec63e0dec2
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-elevenlabs-detect-elevenlabs-transcript-format.png differ
diff --git a/packages/docs/static/generated/articles-docs-openai-whisper-is-openai-whisper-transcript.png b/packages/docs/static/generated/articles-docs-openai-whisper-is-openai-whisper-transcript.png
new file mode 100644
index 0000000000..2d1b3fbd0c
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-openai-whisper-is-openai-whisper-transcript.png differ
diff --git a/packages/elevenlabs/.npmignore b/packages/elevenlabs/.npmignore
index d037cfc2fd..c8337bff57 100644
--- a/packages/elevenlabs/.npmignore
+++ b/packages/elevenlabs/.npmignore
@@ -5,3 +5,4 @@ tsconfig.tsbuildinfo
.turbo
src
eslint.config.mjs
+bundle.ts
diff --git a/packages/elevenlabs/bundle.ts b/packages/elevenlabs/bundle.ts
new file mode 100644
index 0000000000..94ba1370ec
--- /dev/null
+++ b/packages/elevenlabs/bundle.ts
@@ -0,0 +1,7 @@
+import {buildPackage} from '../.monorepo/builder';
+
+await buildPackage({
+ formats: {esm: 'build', cjs: 'use-tsc'},
+ external: 'dependencies',
+ entrypoints: [{path: 'src/index.ts', target: 'browser'}],
+});
diff --git a/packages/elevenlabs/package.json b/packages/elevenlabs/package.json
index 79eebc231a..a14aaf72f5 100644
--- a/packages/elevenlabs/package.json
+++ b/packages/elevenlabs/package.json
@@ -6,6 +6,7 @@
"version": "4.0.529",
"description": "Work with the output of the ElevenLabs API",
"main": "dist/index.js",
+ "module": "dist/esm/index.mjs",
"bugs": {
"url": "https://github.com/remotion-dev/remotion/issues"
},
@@ -14,12 +15,13 @@
"format": "oxfmt src",
"lint": "eslint src",
"test": "bun test src",
- "make": "tsgo -d"
+ "make": "tsgo -d && bun --env-file=../.env.bundle bundle.ts"
},
"author": "Jonny Burger <[email protected]>",
"license": "MIT",
"dependencies": {
- "@remotion/captions": "workspace:*"
+ "@remotion/captions": "workspace:*",
+ "zod": "catalog:"
},
"peerDependencies": {},
"devDependencies": {
@@ -34,5 +36,14 @@
"publishConfig": {
"access": "public"
},
- "homepage": "https://www.remotion.dev/docs/elevenlabs"
+ "homepage": "https://www.remotion.dev/docs/elevenlabs",
+ "exports": {
+ "./package.json": "./package.json",
+ ".": {
+ "types": "./dist/index.d.ts",
+ "require": "./dist/index.js",
+ "module": "./dist/esm/index.mjs",
+ "import": "./dist/esm/index.mjs"
+ }
+ }
}
diff --git a/packages/elevenlabs/src/detect-elevenlabs-transcript-format.ts b/packages/elevenlabs/src/detect-elevenlabs-transcript-format.ts
new file mode 100644
index 0000000000..d5f92dd350
--- /dev/null
+++ b/packages/elevenlabs/src/detect-elevenlabs-transcript-format.ts
@@ -0,0 +1,26 @@
+import * as z from 'zod/mini';
+
+// Recognition deliberately uses permissive schemas; conversion validates the fields.
+const stt = z.object({language_code: z.unknown(), words: z.unknown()});
+const sttWithId = z.object({transcription_id: z.unknown(), words: z.unknown()});
+const segmented = z.object({
+ language_code: z.unknown(),
+ segments: z.unknown(),
+});
+
+export const detectElevenLabsTranscriptFormat = (
+ input: unknown,
+): 'speech-to-text' | 'segmented' | null => {
+ if (
+ z.safeParse(stt, input).success ||
+ z.safeParse(sttWithId, input).success
+ ) {
+ return 'speech-to-text';
+ }
+
+ if (z.safeParse(segmented, input).success) {
+ return 'segmented';
+ }
+
+ return null;
+};
diff --git a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts
index 671644ba95..60d676aa86 100644
--- a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts
+++ b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts
@@ -1,31 +1,115 @@
import type {Caption} from '@remotion/captions';
-import type {ElevenLabsTranscript} from './elevenlabs-transcript';
+import * as z from 'zod/mini';
+import {detectElevenLabsTranscriptFormat} from './detect-elevenlabs-transcript-format';
export type ElevenLabsTranscriptToCaptionsInput = {
- transcript: ElevenLabsTranscript;
+ transcript: unknown;
};
export type ElevenLabsTranscriptToCaptionsOutput = {
captions: Caption[];
};
+const seconds = z.number().check(z.nonnegative());
+const sttWord = z.object({
+ text: z.string(),
+ start: seconds,
+ end: seconds,
+ type: z.enum(['word', 'spacing', 'audio_event']),
+});
+const stt = z.object({words: z.array(sttWord)});
+const segmentWord = z.object({
+ text: z.string(),
+ start_time: seconds,
+ end_time: seconds,
+});
+const segment = z.object({
+ text: z.string(),
+ start_time: seconds,
+ end_time: seconds,
+ words: z.optional(z.array(segmentWord)),
+});
+const segmented = z.object({segments: z.array(segment)});
+
export const elevenLabsTranscriptToCaptions = ({
transcript,
}: ElevenLabsTranscriptToCaptionsInput): ElevenLabsTranscriptToCaptionsOutput => {
- const captions: Caption[] = [];
+ const format = detectElevenLabsTranscriptFormat(transcript);
+ if (format === null) {
+ throw new Error(
+ 'Invalid ElevenLabs transcript: expected Speech-to-Text words or a segmented JSON export with timed segments.',
+ );
+ }
- if (!transcript || !transcript.words || !Array.isArray(transcript.words)) {
+ const schema = format === 'speech-to-text' ? stt : segmented;
+ const result = z.safeParse(schema, transcript);
+ if (!result.success) {
+ const issue = result.error.issues[0];
+ const path = issue.path.reduce<string>(
+ (acc, key) =>
+ typeof key === 'number'
+ ? `${acc}[${key}]`
+ : `${acc}${acc ? '.' : ''}${String(key)}`,
+ '',
+ );
throw new Error(
- 'Invalid ElevenLabs transcript. The transcript must be generated with `timestamps_granularity` set to `"word"`. See https://www.remotion.dev/docs/elevenlabs/elevenlabs-transcript-to-captions',
+ `ElevenLabs ${path || 'transcript'} ${/(^|\.)(start|end|start_time|end_time)$/.test(path) ? 'must be a finite, non-negative number (seconds)' : `is invalid: ${issue.message}`}.`,
);
}
- const {words} = transcript;
+ const captions: Caption[] = [];
+ if (format === 'segmented') {
+ const parsed = z.parse(segmented, transcript);
+ for (const [index, entry] of parsed.segments.entries()) {
+ if (entry.end_time < entry.start_time) {
+ throw new Error(
+ `ElevenLabs segments[${index}].end_time must not be earlier than start_time.`,
+ );
+ }
- let isFirst = true;
+ if (entry.words && entry.words.length > 0) {
+ for (const [wordIndex, word] of entry.words.entries()) {
+ if (word.end_time < word.start_time) {
+ throw new Error(
+ `ElevenLabs segments[${index}].words[${wordIndex}].end_time must not be earlier than start_time.`,
+ );
+ }
+ const startMs = word.start_time * 1000;
+ const endMs = word.end_time * 1000;
+ captions.push({
+ text: word.text,
+ startMs,
+ endMs,
+ timestampMs: (startMs + endMs) / 2,
+ confidence: null,
+ });
+ }
+ } else {
+ const startMs = entry.start_time * 1000;
+ const endMs = entry.end_time * 1000;
+ captions.push({
+ text: entry.text,
+ startMs,
+ endMs,
+ timestampMs: (startMs + endMs) / 2,
+ confidence: null,
+ });
+ }
+ }
+
+ return {captions};
+ }
+
+ const {words} = z.parse(stt, transcript);
+ let isFirst = true;
for (let i = 0; i < words.length; i++) {
const entry = words[i];
+ if (entry.end < entry.start) {
+ throw new Error(
+ `ElevenLabs words[${i}].end must not be earlier than start.`,
+ );
+ }
if (entry.type !== 'word') {
continue;
@@ -33,12 +117,10 @@ export const elevenLabsTranscriptToCaptions = ({
const prevEntry = i > 0 ? words[i - 1] : null;
const hasSpacing = prevEntry !== null && prevEntry.type === 'spacing';
-
const startMs =
!isFirst && hasSpacing ? prevEntry.start * 1000 : entry.start * 1000;
const endMs = entry.end * 1000;
const text = isFirst ? entry.text : ` ${entry.text}`;
-
captions.push({
confidence: null,
startMs,
@@ -46,7 +128,6 @@ export const elevenLabsTranscriptToCaptions = ({
text,
timestampMs: (startMs + endMs) / 2,
});
-
isFirst = false;
}
diff --git a/packages/elevenlabs/src/index.ts b/packages/elevenlabs/src/index.ts
index 50006f266f..43451cb4cf 100644
--- a/packages/elevenlabs/src/index.ts
+++ b/packages/elevenlabs/src/index.ts
@@ -1,3 +1,5 @@
+export {detectElevenLabsTranscriptFormat} from './detect-elevenlabs-transcript-format';
+
export {
ElevenLabsTranscriptToCaptionsInput,
ElevenLabsTranscriptToCaptionsOutput,
diff --git a/packages/openai-whisper/.npmignore b/packages/openai-whisper/.npmignore
index 5510046421..c8337bff57 100644
--- a/packages/openai-whisper/.npmignore
+++ b/packages/openai-whisper/.npmignore
@@ -4,4 +4,5 @@ tsconfig.tsbuildinfo
.env
.turbo
src
-eslint.config.mjs
\ No newline at end of file
+eslint.config.mjs
+bundle.ts
diff --git a/packages/openai-whisper/bundle.ts b/packages/openai-whisper/bundle.ts
new file mode 100644
index 0000000000..94ba1370ec
--- /dev/null
+++ b/packages/openai-whisper/bundle.ts
@@ -0,0 +1,7 @@
+import {buildPackage} from '../.monorepo/builder';
+
+await buildPackage({
+ formats: {esm: 'build', cjs: 'use-tsc'},
+ external: 'dependencies',
+ entrypoints: [{path: 'src/index.ts', target: 'browser'}],
+});
diff --git a/packages/openai-whisper/package.json b/packages/openai-whisper/package.json
index b03d38191e..0690481532 100644
--- a/packages/openai-whisper/package.json
+++ b/packages/openai-whisper/package.json
@@ -6,6 +6,7 @@
"version": "4.0.529",
"description": "Work with the output of the OpenAI Whisper API",
"main": "dist/index.js",
+ "module": "dist/esm/index.mjs",
"bugs": {
"url": "https://github.com/remotion-dev/remotion/issues"
},
@@ -14,12 +15,13 @@
"format": "oxfmt src",
"lint": "eslint src",
"test": "bun test src",
- "make": "tsgo -d"
+ "make": "tsgo -d && bun --env-file=../.env.bundle bundle.ts"
},
"author": "Jonny Burger <[email protected]>",
"license": "MIT",
"dependencies": {
- "@remotion/captions": "workspace:*"
+ "@remotion/captions": "workspace:*",
+ "zod": "catalog:"
},
"peerDependencies": {},
"devDependencies": {
@@ -34,5 +36,14 @@
"publishConfig": {
"access": "public"
},
- "homepage": "https://www.remotion.dev/docs/openai-whisper"
+ "homepage": "https://www.remotion.dev/docs/openai-whisper",
+ "exports": {
+ "./package.json": "./package.json",
+ ".": {
+ "types": "./dist/index.d.ts",
+ "require": "./dist/index.js",
+ "module": "./dist/esm/index.mjs",
+ "import": "./dist/esm/index.mjs"
+ }
+ }
}
diff --git a/packages/openai-whisper/src/index.ts b/packages/openai-whisper/src/index.ts
index 11916012c8..a6cf1b6f3f 100644
--- a/packages/openai-whisper/src/index.ts
+++ b/packages/openai-whisper/src/index.ts
@@ -5,3 +5,4 @@ export {
} from './openai-whisper-api-to-captions';
export {OpenAiVerboseTranscription} from './openai-format';
+export {isOpenAiWhisperTranscript} from './is-openai-whisper-transcript';
diff --git a/packages/openai-whisper/src/is-openai-whisper-transcript.ts b/packages/openai-whisper/src/is-openai-whisper-transcript.ts
new file mode 100644
index 0000000000..5af73e28b6
--- /dev/null
+++ b/packages/openai-whisper/src/is-openai-whisper-transcript.ts
@@ -0,0 +1,28 @@
+import * as z from 'zod/mini';
+
+// Permissive signatures, not proof that the transcript is valid.
+const withTask = z.object({task: z.unknown()});
+const withWords = z.object({
+ text: z.unknown(),
+ language: z.unknown(),
+ words: z.unknown(),
+});
+const withDuration = z.object({
+ text: z.unknown(),
+ duration: z.unknown(),
+ segments: z.unknown(),
+});
+const withSegments = z.object({
+ text: z.unknown(),
+ language: z.unknown(),
+ segments: z.unknown(),
+});
+
+export const isOpenAiWhisperTranscript = (input: unknown): boolean => {
+ return (
+ z.safeParse(withTask, input).success ||
+ z.safeParse(withWords, input).success ||
+ z.safeParse(withDuration, input).success ||
+ z.safeParse(withSegments, input).success
+ );
+};
diff --git a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts b/packages/openai-whisper/src/openai-whisper-api-to-captions.ts
index 60809ce5aa..68baf24844 100644
--- a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts
+++ b/packages/openai-whisper/src/openai-whisper-api-to-captions.ts
@@ -1,8 +1,8 @@
import type {Caption} from '@remotion/captions';
-import type {OpenAiVerboseTranscription} from './openai-format';
+import * as z from 'zod/mini';
export type OpenAiToCaptionsInput = {
- transcription: OpenAiVerboseTranscription;
+ transcription: unknown;
};
export type OpenAiToCaptionsOutput = {
@@ -26,27 +26,94 @@ const escapeWordForRegex = (text: string) => {
.join('');
};
+const seconds = z.number().check(z.nonnegative());
+const wordSchema = z.object({word: z.string(), start: seconds, end: seconds});
+const segmentSchema = z.object({
+ text: z.string(),
+ start: seconds,
+ end: seconds,
+});
+const transcriptionSchema = z.object({
+ text: z.string(),
+ words: z.optional(z.array(wordSchema)),
+ task: z.optional(z.string()),
+});
+const segmentsSchema = z.object({segments: z.optional(z.array(segmentSchema))});
+
+const invalidTranscript = (issue: z.core.$ZodIssue): Error => {
+ const path = issue.path.reduce<string>(
+ (acc, key) =>
+ typeof key === 'number'
+ ? `${acc}[${key}]`
+ : `${acc}${acc ? '.' : ''}${String(key)}`,
+ '',
+ );
+ return new Error(
+ `OpenAI Whisper ${path || 'transcription'} ${/(^|\.)(start|end)$/.test(path) ? 'must be a finite, non-negative number (seconds)' : `is invalid: ${issue.message}`}.`,
+ );
+};
+
export const openAiWhisperApiToCaptions = ({
transcription,
}: OpenAiToCaptionsInput): OpenAiToCaptionsOutput => {
const captions: Caption[] = [];
+ const result = z.safeParse(transcriptionSchema, transcription);
+ if (!result.success) {
+ throw invalidTranscript(result.error.issues[0]);
+ }
+
+ const parsed = result.data;
+ if (parsed.task && parsed.task !== 'transcribe' && !parsed.words?.length) {
+ throw new Error(
+ `The transcription does need to be a "transcribe" task. The input you gave is "task": "${parsed.task}"`,
+ );
+ }
+
+ if (!parsed.words?.length) {
+ const segmentResult = z.safeParse(segmentsSchema, transcription);
+ if (!segmentResult.success) {
+ throw invalidTranscript(segmentResult.error.issues[0]);
+ }
- if (!transcription.words) {
- if (transcription.task && transcription.task !== 'transcribe') {
+ if (!segmentResult.data.segments?.length) {
throw new Error(
- `The transcription does need to be a "transcribe" task. The input you gave is "task": "${transcription.task}"`,
+ 'OpenAI Whisper: No timed captions found. Export verbose_json with timestamp_granularities: ["word"] or timed segments.',
);
}
- throw new Error(
- 'The transcription does need to be been generated with `timestamp_granularities: ["word"]`',
- );
+ for (const [index, segment] of segmentResult.data.segments.entries()) {
+ if (segment.end < segment.start) {
+ throw new Error(
+ `OpenAI Whisper segments[${index}].end must not be earlier than start.`,
+ );
+ }
+
+ const startMs = segment.start * 1000;
+ const endMs = segment.end * 1000;
+ captions.push({
+ text: segment.text,
+ startMs,
+ endMs,
+ timestampMs: (startMs + endMs) / 2,
+ confidence: null,
+ });
+ }
+
+ return {captions};
+ }
+
+ for (const [index, word] of parsed.words.entries()) {
+ if (word.end < word.start) {
+ throw new Error(
+ `OpenAI Whisper words[${index}].end must not be earlier than start.`,
+ );
+ }
}
- let remainingText = transcription.text;
+ let remainingText = parsed.text;
- for (let i = 0; i < transcription.words.length; i++) {
- const word = transcription.words[i];
+ for (let i = 0; i < parsed.words.length; i++) {
+ const word = parsed.words[i];
const firstWord = i === 0;
// https://github.com/remotion-dev/remotion/issues/5031
if (firstWord) {
diff --git a/packages/studio/package.json b/packages/studio/package.json
index be297c15c5..61540272f2 100644
--- a/packages/studio/package.json
+++ b/packages/studio/package.json
@@ -37,6 +37,8 @@
"@jridgewell/trace-mapping": "0.3.31",
"@remotion/canvas": "workspace:*",
"@remotion/captions": "workspace:*",
+ "@remotion/elevenlabs": "workspace:*",
+ "@remotion/openai-whisper": "workspace:*",
"@remotion/media": "workspace:*",
"@remotion/media-utils": "workspace:*",
"@remotion/paths": "workspace:*",
diff --git a/packages/studio/src/components/CaptionInspector.tsx b/packages/studio/src/components/CaptionInspector.tsx
index 1504eee212..7e635f5821 100644
--- a/packages/studio/src/components/CaptionInspector.tsx
+++ b/packages/studio/src/components/CaptionInspector.tsx
@@ -14,7 +14,7 @@ import {parseCaptionFile} from './parse-caption-file';
const importTooltip = `Import captions
-Supports Remotion Caption[] JSON. Files are processed locally.`;
+Supports Remotion Caption[] JSON, SRT, ElevenLabs Speech-to-Text and segmented JSON, and OpenAI Whisper verbose JSON. Files are processed locally.`;
const readOnlyStatus: React.CSSProperties = {
color: LIGHT_TEXT,
@@ -107,12 +107,20 @@ export const CaptionInspector: React.FC<{
mode: 'assets',
invocationTimestamp: Date.now(),
assetSelection: {
- initialQuery: 'type:json',
+ initialQuery: '',
onSelectFile: () => fileInput.current?.click(),
onSelected: (asset) => {
return importCaptions({
fileName: asset.name,
- contents: fetch(asset.src).then((response) => response.text()),
+ contents: fetch(asset.src).then((response) => {
+ if (!response.ok) {
+ throw new Error(
+ `Could not read asset (HTTP ${response.status}). Check that the file is available.`,
+ );
+ }
+
+ return response.text();
+ }),
}).catch(() => undefined);
},
},
@@ -130,7 +138,7 @@ export const CaptionInspector: React.FC<{
<>
<input
ref={fileInput}
- accept=".json"
+ accept=".json,.srt"
aria-label="Import captions file"
hidden
onChange={onFileSelected}
diff --git a/packages/studio/src/components/parse-caption-file.ts b/packages/studio/src/components/parse-caption-file.ts
index 5aacaf660b..26f4b48160 100644
--- a/packages/studio/src/components/parse-caption-file.ts
+++ b/packages/studio/src/components/parse-caption-file.ts
@@ -1,4 +1,12 @@
-import type {Caption} from '@remotion/captions';
+import {parseSrt, type Caption} from '@remotion/captions';
+import {
+ detectElevenLabsTranscriptFormat,
+ elevenLabsTranscriptToCaptions,
+} from '@remotion/elevenlabs';
+import {
+ isOpenAiWhisperTranscript,
+ openAiWhisperApiToCaptions,
+} from '@remotion/openai-whisper';
const isObject = (value: unknown): value is Record<string, unknown> => {
return typeof value === 'object' && value !== null && !Array.isArray(value);
@@ -15,23 +23,57 @@ export const parseCaptionFile = ({
fileName: string;
contents: string;
}): Caption[] => {
- if (!fileName.toLowerCase().endsWith('.json')) {
- throw new Error('Unsupported caption file. Choose a .json file.');
+ const name = fileName.toLowerCase();
+ if (!name.endsWith('.json') && !name.endsWith('.srt')) {
+ throw new Error('Unsupported caption file. Choose a .json or .srt file.');
}
let parsed: unknown;
- try {
- parsed = JSON.parse(contents);
- } catch (error) {
- throw new Error(
- `Invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
- );
+ let canonical = false;
+ if (name.endsWith('.srt')) {
+ parsed = parseSrt({input: contents, strict: true}).captions;
+ } else {
+ try {
+ parsed = JSON.parse(contents.replace(/^\uFEFF/, ''));
+ } catch (error) {
+ throw new Error(
+ `Invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+
+ if (Array.isArray(parsed)) {
+ canonical = true;
+ } else {
+ const elevenLabs = detectElevenLabsTranscriptFormat(parsed);
+ const whisper = isOpenAiWhisperTranscript(parsed);
+ if (elevenLabs && whisper) {
+ throw new Error(
+ 'Ambiguous transcript: matches both ElevenLabs and OpenAI Whisper signatures. Remove conflicting provider fields.',
+ );
+ }
+
+ if (elevenLabs) {
+ parsed = elevenLabsTranscriptToCaptions({transcript: parsed}).captions;
+ } else if (whisper) {
+ parsed = openAiWhisperApiToCaptions({transcription: parsed}).captions;
+ } else {
+ throw new Error(
+ 'Unsupported JSON shape. Expected Remotion Caption[], ElevenLabs Speech-to-Text or segmented JSON, or OpenAI Whisper verbose JSON with timed words or segments.',
+ );
+ }
+ }
}
if (!Array.isArray(parsed)) {
throw new Error('Expected a Remotion Caption[] JSON array.');
}
+ if (!canonical && parsed.length === 0) {
+ throw new Error(
+ 'No timed captions found. Export word or segment timestamps, or choose a caption file with timed cues.',
+ );
+ }
+
let previousStart: number | null = null;
for (const [index, caption] of parsed.entries()) {
const path = `captions[${index}]`;
diff --git a/packages/studio/src/test/caption-inspector-import.test.tsx b/packages/studio/src/test/caption-inspector-import.test.tsx
index 2912f76edc..fd9d95a328 100644
--- a/packages/studio/src/test/caption-inspector-import.test.tsx
+++ b/packages/studio/src/test/caption-inspector-import.test.tsx
@@ -61,7 +61,7 @@ test('imports captions from a JSON file in the public folder', async () => {
const {assetSelection} = modal;
- expect(assetSelection.initialQuery).toBe('type:json');
+ expect(assetSelection.initialQuery).toBe('');
await act(async () => {
await (assetSelection.onSelected({
lastModified: 0,
diff --git a/packages/studio/src/test/parse-caption-file.test.ts b/packages/studio/src/test/parse-caption-file.test.ts
index 5265fdf004..525708727e 100644
--- a/packages/studio/src/test/parse-caption-file.test.ts
+++ b/packages/studio/src/test/parse-caption-file.test.ts
@@ -41,7 +41,7 @@ test.each([
{
name: 'non-array JSON',
value: '{}',
- error: 'Expected a Remotion Caption[] JSON array',
+ error: 'Unsupported JSON shape',
},
{
name: 'missing field',