Skip to content

πŸŽ₯ Make videos programmatically with React

@remotion/studio: Add JSON and SRT caption importing

Igor Samokhovets91287f7f1566
diff --git a/bun.lock b/bun.lock
index 6582230b33..f5ece55fd3 100644
--- a/bun.lock
+++ b/bun.lock
@@ -1805,8 +1805,10 @@
"@jridgewell/trace-mapping": "0.3.31",
"@remotion/canvas": "workspace:*",
"@remotion/captions": "workspace:*",
+ "@remotion/elevenlabs": "workspace:*",
"@remotion/media": "workspace:*",
"@remotion/media-utils": "workspace:*",
+ "@remotion/openai-whisper": "workspace:*",
"@remotion/player": "workspace:*",
"@remotion/renderer": "workspace:*",
"@remotion/studio-codemods": "workspace:*",
diff --git a/packages/docs/docs/elevenlabs/TableOfContents.tsx b/packages/docs/docs/elevenlabs/TableOfContents.tsx
index 069651aee2..251b106f10 100644
--- a/packages/docs/docs/elevenlabs/TableOfContents.tsx
+++ b/packages/docs/docs/elevenlabs/TableOfContents.tsx
@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {
<code>Caption</code>
</div>
</TOCItem>
+ <TOCItem link="/docs/elevenlabs/is-elevenlabs-transcript-json">
+ <strong>{'isElevenLabsTranscriptJson()'}</strong>
+ <div>Detect ElevenLabs Speech to Text JSON</div>
+ </TOCItem>
</Grid>
</div>
);
diff --git a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx
index 2a3e06608c..f968d501fe 100644
--- a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx
+++ b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx
@@ -63,6 +63,7 @@ An array of [`Caption`](/docs/captions/caption) objects.
## See also
+- [`isElevenLabsTranscriptJson()`](/docs/elevenlabs/is-elevenlabs-transcript-json)
- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts)
- [`@remotion/elevenlabs`](/docs/elevenlabs)
- [`@remotion/openai-whisper`](/docs/openai-whisper)
diff --git a/packages/docs/docs/elevenlabs/index.mdx b/packages/docs/docs/elevenlabs/index.mdx
index d665ed253a..94649d6148 100644
--- a/packages/docs/docs/elevenlabs/index.mdx
+++ b/packages/docs/docs/elevenlabs/index.mdx
@@ -6,7 +6,7 @@ crumb: 'Subtitle videos'
# @remotion/elevenlabs<AvailableFrom v="4.0.443" />
-The `@remotion/elevenlabs` package converts the output of the [ElevenLabs Speech to Text API](https://elevenlabs.io/docs/api-reference/speech-to-text/convert) into an array of [`Caption`](/docs/captions/caption) objects, making them compatible with other caption-related Remotion APIs.
+The `@remotion/elevenlabs` package detects and converts the output of the [ElevenLabs Speech to Text API](https://elevenlabs.io/docs/api-reference/speech-to-text/convert) into an array of [`Caption`](/docs/captions/caption) objects, making them compatible with other caption-related Remotion APIs.
import {TableOfContents} from './TableOfContents';
diff --git a/packages/docs/docs/elevenlabs/is-elevenlabs-transcript-json.mdx b/packages/docs/docs/elevenlabs/is-elevenlabs-transcript-json.mdx
new file mode 100644
index 0000000000..c5a8653e50
--- /dev/null
+++ b/packages/docs/docs/elevenlabs/is-elevenlabs-transcript-json.mdx
@@ -0,0 +1,52 @@
+---
+image: /generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.png
+title: isElevenLabsTranscriptJson()
+crumb: '@remotion/elevenlabs'
+---
+
+# isElevenLabsTranscriptJson()<AvailableFrom v="4.0.521" />
+
+Returns whether a value has the top-level fields of an ElevenLabs Speech to Text JSON response.
+
+Use it before [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions) when accepting JSON whose source is unknown.
+
+This function detects the format but does not validate every word. The converter validates the fields it uses and throws an actionable error if the recognized input is malformed.
+
+## Example
+
+```ts twoslash title="convert-upload.ts"
+import {
+ elevenLabsTranscriptToCaptions,
+ isElevenLabsTranscriptJson,
+} from '@remotion/elevenlabs';
+
+const contents = '{"language_code":"eng","text":"Hello","words":[]}';
+const json = JSON.parse(contents);
+
+if (isElevenLabsTranscriptJson(json)) {
+ const {captions} = elevenLabsTranscriptToCaptions({transcript: json});
+ console.log(captions);
+}
+```
+
+## API
+
+### `value`
+
+The value to inspect. It may have any JavaScript type.
+
+### Return value
+
+A boolean. It is `true` when the value is an object with `language_code` and either `words` or `text` as top-level fields.
+
+Additional fields do not affect detection.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />
+
+## See also
+
+- [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions)
+- [`isOpenAiWhisperJson()`](/docs/openai-whisper/is-open-ai-whisper-json)
+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts)
diff --git a/packages/docs/docs/openai-whisper/TableOfContents.tsx b/packages/docs/docs/openai-whisper/TableOfContents.tsx
index 0488a2a185..a82c9a7932 100644
--- a/packages/docs/docs/openai-whisper/TableOfContents.tsx
+++ b/packages/docs/docs/openai-whisper/TableOfContents.tsx
@@ -6,6 +6,10 @@ export const TableOfContents: React.FC = () => {
return (
<div>
<Grid>
+ <TOCItem link="/docs/openai-whisper/is-open-ai-whisper-json">
+ <strong>{'isOpenAiWhisperJson()'}</strong>
+ <div>Detect OpenAI Whisper verbose JSON</div>
+ </TOCItem>
<TOCItem link="/docs/openai-whisper/openai-whisper-api-to-captions">
<strong>{'openAiWhisperApiToCaptions()'}</strong>
<div>
diff --git a/packages/docs/docs/openai-whisper/is-open-ai-whisper-json.mdx b/packages/docs/docs/openai-whisper/is-open-ai-whisper-json.mdx
new file mode 100644
index 0000000000..8829011cb2
--- /dev/null
+++ b/packages/docs/docs/openai-whisper/is-open-ai-whisper-json.mdx
@@ -0,0 +1,52 @@
+---
+image: /generated/articles-docs-openai-whisper-is-open-ai-whisper-json.png
+title: isOpenAiWhisperJson()
+crumb: '@remotion/openai-whisper'
+---
+
+# isOpenAiWhisperJson()<AvailableFrom v="4.0.521" />
+
+Returns whether a value has the top-level fields of an OpenAI Whisper verbose JSON response.
+
+Use it before [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions) when accepting JSON whose source is unknown.
+
+This function detects the format but does not validate every word. The converter validates the fields it uses and throws an actionable error if the recognized input is malformed.
+
+## Example
+
+```ts twoslash title="convert-upload.ts"
+import {
+ isOpenAiWhisperJson,
+ openAiWhisperApiToCaptions,
+} from '@remotion/openai-whisper';
+
+const contents = '{"language":"english","text":"Hello","words":[]}';
+const json = JSON.parse(contents);
+
+if (isOpenAiWhisperJson(json)) {
+ const {captions} = openAiWhisperApiToCaptions({transcription: json});
+ console.log(captions);
+}
+```
+
+## API
+
+### `value`
+
+The value to inspect. It may have any JavaScript type.
+
+### Return value
+
+A boolean. It is `true` when the value is an object without `language_code`, with `text`, and with at least one of `task`, `language`, `duration`, `segments`, or `words` as top-level fields.
+
+Other additional fields do not affect detection.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />
+
+## See also
+
+- [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions)
+- [`isElevenLabsTranscriptJson()`](/docs/elevenlabs/is-elevenlabs-transcript-json)
+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/openai-whisper/src/openai-whisper-api-to-captions.ts)
diff --git a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx
index a690eb6c30..762cde20ee 100644
--- a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx
+++ b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx
@@ -32,6 +32,7 @@ const {captions} = openAiWhisperApiToCaptions({transcription});
## See also
+- [`isOpenAiWhisperJson()`](/docs/openai-whisper/is-open-ai-whisper-json)
- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/openai-whisper/src/openai-whisper-api-to-captions.ts)
- [`@remotion/openai-whisper`](/docs/openai-whisper)
- [`@remotion/install-whisper-cpp`](/docs/install-whisper-cpp)
diff --git a/packages/docs/elements/captions/index.mdx b/packages/docs/elements/captions/index.mdx
index 1280cdd7c3..25ef2cd6c5 100644
--- a/packages/docs/elements/captions/index.mdx
+++ b/packages/docs/elements/captions/index.mdx
@@ -7,4 +7,14 @@ import {ElementLibrary} from '@site/src/components/Elements/ElementLibrary';
# Captions
+## Importing captions into Studio<AvailableFrom v="4.0.521" />
+
+<Step>1</Step> Install an Element from the [Captions category](/elements/captions), or add [`Interactive.captionsSchema`](/docs/interactivity-schema#interactivecaptionsschema) to an interactive component that accepts [`Caption[]`](/docs/captions/caption).<br />
+<Step>2</Step> Select its caption layer in Studio.<br />
+<Step>3</Step> In the Captions inspector, click **Import captions** and choose the JSON or SRT file exported by ElevenLabs or your Whisper transcription tool.
+
+Studio detects and converts Remotion [`Caption[]`](/docs/captions/caption) JSON, ElevenLabs Speech-to-Text JSON, ElevenLabs segmented export JSON, OpenAI Whisper verbose JSON, and [SRT subtitles](/docs/captions/parse-srt). JSON transcripts must include word-level timestamps. SRT cue timestamps work, but cannot provide true word-by-word animation.
+
+Files are processed locally.
+
<ElementLibrary category="captions" />
diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.ts
index 03e4a8385f..a0d4cd4b3d 100644
--- a/packages/docs/sidebars.ts
+++ b/packages/docs/sidebars.ts
@@ -354,7 +354,10 @@ const sidebars: SidebarsConfig = {
type: 'doc',
id: 'elevenlabs/index',
},
- items: ['elevenlabs/elevenlabs-transcript-to-captions'],
+ items: [
+ 'elevenlabs/elevenlabs-transcript-to-captions',
+ 'elevenlabs/is-elevenlabs-transcript-json',
+ ],
},
{
type: 'category',
@@ -578,7 +581,10 @@ const sidebars: SidebarsConfig = {
type: 'doc',
id: 'openai-whisper/index',
},
- items: ['openai-whisper/openai-whisper-api-to-captions'],
+ items: [
+ 'openai-whisper/is-open-ai-whisper-json',
+ 'openai-whisper/openai-whisper-api-to-captions',
+ ],
},
{
type: 'category',
diff --git a/packages/docs/src/components/Elements/ElementPage.tsx b/packages/docs/src/components/Elements/ElementPage.tsx
index a8bd4ae5e4..378d11a9fd 100644
--- a/packages/docs/src/components/Elements/ElementPage.tsx
+++ b/packages/docs/src/components/Elements/ElementPage.tsx
@@ -343,6 +343,14 @@ export const ElementPage: React.FC<ElementPageProps> = ({
<div className={styles.details}>
<p className={styles.description}>{description}</p>
+ {definition.category === 'captions' ? (
+ <p className={styles.description} style={{marginTop: 8}}>
+ Have a transcript?{' '}
+ <a href="/elements/captions/#importing-captions-into-studio">
+ Import JSON or SRT from the Captions inspector in Studio.
+ </a>
+ </p>
+ ) : null}
<dl className={styles.metadata}>
<div>
<dt>Dimensions</dt>
diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts
index a492a185f7..ddb63e5b6e 100644
--- a/packages/docs/src/data/articles.ts
+++ b/packages/docs/src/data/articles.ts
@@ -2528,6 +2528,15 @@ export const articles = [
noAi: false,
slug: 'elevenlabs/index',
},
+ {
+ id: 'elevenlabs/is-elevenlabs-transcript-json',
+ title: 'isElevenLabsTranscriptJson()',
+ relativePath: 'docs/elevenlabs/is-elevenlabs-transcript-json.mdx',
+ compId: 'articles-docs-elevenlabs-is-elevenlabs-transcript-json',
+ crumb: '@remotion/elevenlabs',
+ noAi: false,
+ slug: 'elevenlabs/is-elevenlabs-transcript-json',
+ },
{
id: 'enable-scss/enable-scss',
title: 'enableScss()',
@@ -5066,6 +5075,15 @@ export const articles = [
noAi: false,
slug: 'openai-whisper/index',
},
+ {
+ id: 'openai-whisper/is-open-ai-whisper-json',
+ title: 'isOpenAiWhisperJson()',
+ relativePath: 'docs/openai-whisper/is-open-ai-whisper-json.mdx',
+ compId: 'articles-docs-openai-whisper-is-open-ai-whisper-json',
+ crumb: '@remotion/openai-whisper',
+ noAi: false,
+ slug: 'openai-whisper/is-open-ai-whisper-json',
+ },
{
id: 'openai-whisper/openai-whisper-api-to-captions',
title: 'openAiWhisperApiToCaptions()',
diff --git a/packages/docs/static/generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.png b/packages/docs/static/generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.png
new file mode 100644
index 0000000000..e4abd628f3
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.png differ
diff --git a/packages/docs/static/generated/articles-docs-openai-whisper-is-open-ai-whisper-json.png b/packages/docs/static/generated/articles-docs-openai-whisper-is-open-ai-whisper-json.png
new file mode 100644
index 0000000000..d0c1aab130
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-openai-whisper-is-open-ai-whisper-json.png differ
diff --git a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts
index 671644ba95..9df34fdea1 100644
--- a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts
+++ b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts
@@ -9,34 +9,126 @@ export type ElevenLabsTranscriptToCaptionsOutput = {
captions: Caption[];
};
+export const isElevenLabsTranscriptJson = (value: unknown): boolean => {
+ return (
+ typeof value === 'object' &&
+ value !== null &&
+ !Array.isArray(value) &&
+ 'language_code' in value &&
+ ('words' in value || 'text' in value)
+ );
+};
+
export const elevenLabsTranscriptToCaptions = ({
transcript,
}: ElevenLabsTranscriptToCaptionsInput): ElevenLabsTranscriptToCaptionsOutput => {
const captions: Caption[] = [];
- if (!transcript || !transcript.words || !Array.isArray(transcript.words)) {
+ if (
+ typeof transcript !== 'object' ||
+ transcript === null ||
+ Array.isArray(transcript)
+ ) {
throw new Error(
- 'Invalid ElevenLabs transcript. The transcript must be generated with `timestamps_granularity` set to `"word"`. See https://www.remotion.dev/docs/elevenlabs/elevenlabs-transcript-to-captions',
+ 'Invalid ElevenLabs transcript: transcript must be an object.',
);
}
- const {words} = transcript;
+ const {words}: {words: unknown} = transcript;
+ if (!Array.isArray(words)) {
+ throw new Error(
+ 'Invalid ElevenLabs transcript: words must be an array generated with `timestamps_granularity` set to `"word"`. See https://www.remotion.dev/docs/elevenlabs/elevenlabs-transcript-to-captions',
+ );
+ }
let isFirst = true;
for (let i = 0; i < words.length; i++) {
- const entry = words[i];
+ const value: unknown = words[i];
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i}] must be an object.`,
+ );
+ }
+ const entry = value as Record<string, unknown>;
if (entry.type !== 'word') {
continue;
}
- const prevEntry = i > 0 ? words[i - 1] : null;
- const hasSpacing = prevEntry !== null && prevEntry.type === 'spacing';
+ if (typeof entry.text !== 'string') {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i}].text must be a string.`,
+ );
+ }
+
+ if (
+ typeof entry.start !== 'number' ||
+ !Number.isFinite(entry.start) ||
+ entry.start < 0
+ ) {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i}].start must be a finite, non-negative number.`,
+ );
+ }
+
+ if (
+ typeof entry.end !== 'number' ||
+ !Number.isFinite(entry.end) ||
+ entry.end < 0
+ ) {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i}].end must be a finite, non-negative number.`,
+ );
+ }
+
+ if (entry.end < entry.start) {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i}].end must not be earlier than words[${i}].start.`,
+ );
+ }
+
+ let spacingStart: number | null = null;
+ const previousValue: unknown = i > 0 ? words[i - 1] : null;
+ if (
+ !isFirst &&
+ typeof previousValue === 'object' &&
+ previousValue !== null &&
+ !Array.isArray(previousValue)
+ ) {
+ const previousEntry = previousValue as Record<string, unknown>;
+ if (previousEntry.type === 'spacing') {
+ if (
+ typeof previousEntry.start !== 'number' ||
+ !Number.isFinite(previousEntry.start) ||
+ previousEntry.start < 0
+ ) {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i - 1}].start must be a finite, non-negative number.`,
+ );
+ }
+
+ spacingStart = previousEntry.start;
+ }
+ }
- const startMs =
- !isFirst && hasSpacing ? prevEntry.start * 1000 : entry.start * 1000;
+ const startMs = (spacingStart ?? entry.start) * 1000;
const endMs = entry.end * 1000;
+ if (endMs < startMs) {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i}].end must not be earlier than the caption start.`,
+ );
+ }
+
+ if (
+ captions.length > 0 &&
+ startMs < captions[captions.length - 1].startMs
+ ) {
+ throw new Error(
+ `Invalid ElevenLabs transcript: words[${i}].start is out of timestamp order.`,
+ );
+ }
+
const text = isFirst ? entry.text : ` ${entry.text}`;
captions.push({
diff --git a/packages/elevenlabs/src/index.ts b/packages/elevenlabs/src/index.ts
index 50006f266f..75f90603b2 100644
--- a/packages/elevenlabs/src/index.ts
+++ b/packages/elevenlabs/src/index.ts
@@ -2,6 +2,7 @@ export {
ElevenLabsTranscriptToCaptionsInput,
ElevenLabsTranscriptToCaptionsOutput,
elevenLabsTranscriptToCaptions,
+ isElevenLabsTranscriptJson,
} from './elevenlabs-transcript-to-captions';
export {
diff --git a/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.ts b/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.ts
index 4152504afb..0af8d82445 100644
--- a/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.ts
+++ b/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.ts
@@ -1,9 +1,21 @@
import {expect, test} from 'bun:test';
import type {ElevenLabsTranscript} from '../elevenlabs-transcript';
-import {elevenLabsTranscriptToCaptions} from '../elevenlabs-transcript-to-captions';
+import {
+ elevenLabsTranscriptToCaptions,
+ isElevenLabsTranscriptJson,
+} from '../elevenlabs-transcript-to-captions';
import expectedCaptions from './gimme-gimme-captions-snapshot.json';
import transcript from './gimme-gimme-transcript.json';
+test('recognizes ElevenLabs transcript JSON', () => {
+ expect(
+ isElevenLabsTranscriptJson({language_code: 'eng', text: 'Hello'}),
+ ).toBe(true);
+ expect(isElevenLabsTranscriptJson({language: 'english', text: 'Hello'})).toBe(
+ false,
+ );
+});
+
test('converts elevenlabs transcript to captions', () => {
const {captions} = elevenLabsTranscriptToCaptions({
transcript: transcript as ElevenLabsTranscript,
@@ -33,3 +45,76 @@ test('handles empty words array', () => {
expect(captions).toEqual([]);
});
+
+test.each([
+ {
+ name: 'non-object entry',
+ words: [null],
+ error: 'words[0] must be an object',
+ },
+ {
+ name: 'invalid timestamp',
+ words: [{type: 'word', text: 'Hello', start: Number.NaN, end: 1}],
+ error: 'words[0].start must be a finite, non-negative number',
+ },
+ {
+ name: 'invalid consumed spacing timestamp',
+ words: [
+ {type: 'word', text: 'Hello', start: 0, end: 1},
+ {type: 'spacing', text: ' ', start: 'later', end: 2},
+ {type: 'word', text: 'world', start: 2, end: 3},
+ ],
+ error: 'words[1].start must be a finite, non-negative number',
+ },
+ {
+ name: 'reversed timestamps',
+ words: [{type: 'word', text: 'Hello', start: 2, end: 1}],
+ error: 'words[0].end must not be earlier than words[0].start',
+ },
+ {
+ name: 'out-of-order words',
+ words: [
+ {type: 'word', text: 'Later', start: 2, end: 3},
+ {type: 'word', text: 'Earlier', start: 1, end: 2},
+ ],
+ error: 'words[1].start is out of timestamp order',
+ },
+])('rejects $name', ({words, error}) => {
+ expect(() =>
+ elevenLabsTranscriptToCaptions({
+ transcript: {words} as unknown as ElevenLabsTranscript,
+ }),
+ ).toThrow(`Invalid ElevenLabs transcript: ${error}`);
+});
+
+test('ignores unknown fields and entry types', () => {
+ const {captions} = elevenLabsTranscriptToCaptions({
+ transcript: {
+ language_code: 'eng',
+ language_probability: 1,
+ text: 'Hello',
+ transcription_id: 'test',
+ provider_added_field: true,
+ words: [
+ {type: 'provider_event', provider_added_field: true},
+ {
+ type: 'word',
+ text: 'Hello',
+ start: 0,
+ end: 1,
+ provider_added_field: true,
+ },
+ ],
+ } as unknown as ElevenLabsTranscript,
+ });
+
+ expect(captions).toEqual([
+ {
+ confidence: null,
+ startMs: 0,
+ endMs: 1000,
+ text: 'Hello',
+ timestampMs: 500,
+ },
+ ]);
+});
diff --git a/packages/example/e2e/captions-inspector.test.mts b/packages/example/e2e/captions-inspector.test.mts
index 1aae3ad836..a5d9ca3523 100644
--- a/packages/example/e2e/captions-inspector.test.mts
+++ b/packages/example/e2e/captions-inspector.test.mts
@@ -100,17 +100,92 @@ test.describe('captions inspector', () => {
}).toPass({timeout: 30_000});
await expect(defaultCaption).toBeEnabled();
- await defaultCaption.fill('Editable captions');
+ const importCaptionsButton = page.getByRole('button', {
+ name: 'Import captions',
+ exact: true,
+ });
+ const importCaptionsInput = page.getByLabel('Import captions file');
+ await expect(importCaptionsButton).toBeVisible();
+
+ const sourceBeforeFailedImport = fs.readFileSync(
+ elementCaptionsFile,
+ 'utf-8',
+ );
+ await importCaptionsInput.setInputFiles({
+ name: 'broken.json',
+ mimeType: 'application/json',
+ buffer: Buffer.from(
+ JSON.stringify({
+ language_code: 'eng',
+ segments: [
+ {
+ text: 'Broken',
+ start_time: 0,
+ end_time: 1,
+ words: [{text: 'Broken', end_time: 1}],
+ },
+ ],
+ }),
+ ),
+ });
+ await expect(
+ page.getByText(
+ /broken\.json:.*segments\[0\]\.words\[0\]\.start_time must be a finite number/,
+ ),
+ ).toBeVisible();
+ expect(fs.readFileSync(elementCaptionsFile, 'utf-8')).toBe(
+ sourceBeforeFailedImport,
+ );
+
+ await importCaptionsInput.setInputFiles({
+ name: 'captions.json',
+ mimeType: 'application/json',
+ buffer: Buffer.from(
+ JSON.stringify({
+ language_code: 'eng',
+ segments: [
+ {
+ text: 'Imported captions',
+ start_time: 0.1,
+ end_time: 0.9,
+ words: [
+ {text: 'Imported', start_time: 0.1, end_time: 0.4},
+ {text: ' ', start_time: 0.4, end_time: 0.5},
+ {text: 'captions', start_time: 0.5, end_time: 0.9},
+ ],
+ },
+ ],
+ }),
+ ),
+ });
+ await expect(
+ page.getByText('Replace captions?', {exact: true}),
+ ).toBeVisible();
+ expect(fs.readFileSync(elementCaptionsFile, 'utf-8')).toBe(
+ sourceBeforeFailedImport,
+ );
+ await page.getByRole('button', {name: 'Replace captions'}).click();
+ await expect(
+ page.getByText(
+ 'Imported 2 captions from ElevenLabs segmented JSON. Undo is available.',
+ ),
+ ).toBeVisible();
+ await expect(defaultCaption).toHaveValue('Imported');
+ await expect
+ .poll(() => fs.readFileSync(elementCaptionsFile, 'utf-8'))
+ .toMatch(/startMs:\s*100[\s\S]*text:\s*['"]Imported['"]/);
+ expect(fs.readFileSync(elementCallSiteFile, 'utf-8')).toBe(
+ elementCallSiteSourceBefore,
+ );
+
+ await defaultCaption.fill('Edited imported caption');
await defaultCaption.blur();
await expect
.poll(() => {
- return /text:\s*['"]Editable captions['"]/.test(
+ return /text:\s*['"]Edited imported caption['"]/.test(
fs.readFileSync(elementCaptionsFile, 'utf-8'),
);
})
.toBe(true);
- expect(fs.readFileSync(elementCallSiteFile, 'utf-8')).toBe(
- elementCallSiteSourceBefore,
- );
});
});
diff --git a/packages/openai-whisper/src/index.ts b/packages/openai-whisper/src/index.ts
index 11916012c8..1a88934b0e 100644
--- a/packages/openai-whisper/src/index.ts
+++ b/packages/openai-whisper/src/index.ts
@@ -1,6 +1,7 @@
export {
OpenAiToCaptionsInput,
OpenAiToCaptionsOutput,
+ isOpenAiWhisperJson,
openAiWhisperApiToCaptions,
} from './openai-whisper-api-to-captions';
diff --git a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts b/packages/openai-whisper/src/openai-whisper-api-to-captions.ts
index 60809ce5aa..920a20cba3 100644
--- a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts
+++ b/packages/openai-whisper/src/openai-whisper-api-to-captions.ts
@@ -9,6 +9,21 @@ export type OpenAiToCaptionsOutput = {
captions: Caption[];
};
+export const isOpenAiWhisperJson = (value: unknown): boolean => {
+ return (
+ typeof value === 'object' &&
+ value !== null &&
+ !Array.isArray(value) &&
+ !('language_code' in value) &&
+ 'text' in value &&
+ ('task' in value ||
+ 'language' in value ||
+ 'duration' in value ||
+ 'segments' in value ||
+ 'words' in value)
+ );
+};
+
const escapeRegex = (text: string) => {
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
};
@@ -31,36 +46,107 @@ export const openAiWhisperApiToCaptions = ({
}: OpenAiToCaptionsInput): OpenAiToCaptionsOutput => {
const captions: Caption[] = [];
+ if (
+ typeof transcription !== 'object' ||
+ transcription === null ||
+ Array.isArray(transcription)
+ ) {
+ throw new Error(
+ 'Invalid OpenAI Whisper transcription: transcription must be an object.',
+ );
+ }
+
+ if (transcription.task && transcription.task !== 'transcribe') {
+ throw new Error(
+ `Invalid OpenAI Whisper transcription: The transcription does need to be a "transcribe" task. The input you gave is "task": "${transcription.task}"`,
+ );
+ }
+
if (!transcription.words) {
- if (transcription.task && transcription.task !== 'transcribe') {
- throw new Error(
- `The transcription does need to be a "transcribe" task. The input you gave is "task": "${transcription.task}"`,
- );
- }
+ throw new Error(
+ 'Invalid OpenAI Whisper transcription: The transcription does need to be been generated with `timestamp_granularities: ["word"]`',
+ );
+ }
+ const {words} = transcription as unknown as {words: unknown};
+ if (!Array.isArray(words)) {
throw new Error(
- 'The transcription does need to be been generated with `timestamp_granularities: ["word"]`',
+ 'Invalid OpenAI Whisper transcription: words must be an array.',
+ );
+ }
+
+ if (typeof transcription.text !== 'string') {
+ throw new Error(
+ 'Invalid OpenAI Whisper transcription: text must be a string.',
);
}
let remainingText = transcription.text;
+ let previousStart: number | null = null;
+
+ for (let i = 0; i < words.length; i++) {
+ const value: unknown = words[i];
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new Error(
+ `Invalid OpenAI Whisper transcription: words[${i}] must be an object.`,
+ );
+ }
+
+ const word = value as Record<string, unknown>;
+ if (typeof word.word !== 'string') {
+ throw new Error(
+ `Invalid OpenAI Whisper transcription: words[${i}].word must be a string.`,
+ );
+ }
+
+ if (
+ typeof word.start !== 'number' ||
+ !Number.isFinite(word.start) ||
+ word.start < 0
+ ) {
+ throw new Error(
+ `Invalid OpenAI Whisper transcription: words[${i}].start must be a finite, non-negative number.`,
+ );
+ }
+
+ if (
+ typeof word.end !== 'number' ||
+ !Number.isFinite(word.end) ||
+ word.end < 0
+ ) {
+ throw new Error(
+ `Invalid OpenAI Whisper transcription: words[${i}].end must be a finite, non-negative number.`,
+ );
+ }
+
+ if (word.end < word.start) {
+ throw new Error(
+ `Invalid OpenAI Whisper transcription: words[${i}].end must not be earlier than words[${i}].start.`,
+ );
+ }
+
+ if (previousStart !== null && word.start < previousStart) {
+ throw new Error(
+ `Invalid OpenAI Whisper transcription: words[${i}].start is out of timestamp order.`,
+ );
+ }
- for (let i = 0; i < transcription.words.length; i++) {
- const word = transcription.words[i];
+ previousStart = word.start;
const firstWord = i === 0;
// https://github.com/remotion-dev/remotion/issues/5031
+ const wordText = firstWord ? word.word.trimStart() : word.word;
if (firstWord) {
- word.word = word.word.trimStart();
+ word.word = wordText;
}
const punctuation = `\\?,\\.\\%\\–\\!\\;\\:\\'\\"\\-\\_\\(\\)\\[\\]\\{\\}\\@\\#\\$\\^\\&\\*\\+\\=\\/\\|\\<\\>\\~\`\\u2018\\u2019\\u02bc\\uff07`;
- const wordToMatch = word.word.replace(new RegExp(`^[${punctuation}]+`), '');
+ const wordToMatch = wordText.replace(new RegExp(`^[${punctuation}]+`), '');
const match = new RegExp(
`^([\\s?${punctuation}]{0,4})${escapeWordForRegex(wordToMatch)}([${punctuation}]{0,3})?`,
).exec(remainingText);
if (!match) {
throw new Error(
- `Unable to parse punctuation from OpenAI Whisper output. Could not find word "${word.word}" in text "${remainingText.slice(0, 100)}". File an issue under https://remotion.dev/issue and post the input for openAiWhisperApiToCaptions() to ask for a fix.`,
+ `Unable to parse punctuation from OpenAI Whisper output. Could not find word "${wordText}" in text "${remainingText.slice(0, 100)}". File an issue under https://remotion.dev/issue and post the input for openAiWhisperApiToCaptions() to ask for a fix.`,
);
}
diff --git a/packages/openai-whisper/src/test/regressions.test.ts b/packages/openai-whisper/src/test/regressions.test.ts
index 2863b69365..07d3584040 100644
--- a/packages/openai-whisper/src/test/regressions.test.ts
+++ b/packages/openai-whisper/src/test/regressions.test.ts
@@ -1,6 +1,9 @@
import {expect, test} from 'bun:test';
import type {OpenAiVerboseTranscription} from '../openai-format';
-import {openAiWhisperApiToCaptions} from '../openai-whisper-api-to-captions';
+import {
+ isOpenAiWhisperJson,
+ openAiWhisperApiToCaptions,
+} from '../openai-whisper-api-to-captions';
const transcript1: OpenAiVerboseTranscription = {
task: 'transcribe',
@@ -31,6 +34,15 @@ const transcript1: OpenAiVerboseTranscription = {
language: 'english',
};
+test('recognizes OpenAI Whisper JSON', () => {
+ expect(
+ isOpenAiWhisperJson({language: 'english', text: 'Hello', words: []}),
+ ).toBe(true);
+ expect(
+ isOpenAiWhisperJson({language_code: 'eng', text: 'Hello', words: []}),
+ ).toBe(false);
+});
+
const transcript2: OpenAiVerboseTranscription = {
task: 'transcribe',
text: 'in real-time functions.',
@@ -168,3 +180,69 @@ test('Issue 7298 - apostrophe variants', () => {
},
]);
});
+
+test.each([
+ {
+ name: 'malformed word text',
+ words: [{word: 42, start: 0, end: 1}],
+ error: 'words[0].word must be a string',
+ },
+ {
+ name: 'malformed word timestamp',
+ words: [{word: 'Hello', start: Number.POSITIVE_INFINITY, end: 1}],
+ error: 'words[0].start must be a finite, non-negative number',
+ },
+ {
+ name: 'reversed timestamps',
+ words: [{word: 'Hello', start: 2, end: 1}],
+ error: 'words[0].end must not be earlier than words[0].start',
+ },
+ {
+ name: 'out-of-order words',
+ words: [
+ {word: 'Hello', start: 2, end: 3},
+ {word: 'world', start: 1, end: 2},
+ ],
+ error: 'words[1].start is out of timestamp order',
+ },
+])('rejects $name', ({words, error}) => {
+ expect(() =>
+ openAiWhisperApiToCaptions({
+ transcription: {
+ text: 'Hello world',
+ words,
+ } as unknown as OpenAiVerboseTranscription,
+ }),
+ ).toThrow(`Invalid OpenAI Whisper transcription: ${error}`);
+});
+
+test('accepts extra provider fields', () => {
+ expect(
+ openAiWhisperApiToCaptions({
+ transcription: {
+ duration: 1,
+ language: 'english',
+ text: 'Hello',
+ words: [
+ {
+ word: 'Hello',
+ start: 0,
+ end: 1,
+ provider_added_field: true,
+ },
+ ],
+ provider_added_field: true,
+ } as unknown as OpenAiVerboseTranscription,
+ }),
+ ).toEqual({
+ captions: [
+ {
+ confidence: null,
+ endMs: 1000,
+ startMs: 0,
+ text: 'Hello',
+ timestampMs: 500,
+ },
+ ],
+ });
+});
diff --git a/packages/studio/package.json b/packages/studio/package.json
index 341f8ec78d..275ef86508 100644
--- a/packages/studio/package.json
+++ b/packages/studio/package.json
@@ -31,9 +31,11 @@
"semver": "7.5.3",
"remotion": "workspace:*",
"@remotion/captions": "workspace:*",
+ "@remotion/elevenlabs": "workspace:*",
"@remotion/media": "workspace:*",
"@remotion/player": "workspace:*",
"@remotion/media-utils": "workspace:*",
+ "@remotion/openai-whisper": "workspace:*",
"@remotion/renderer": "workspace:*",
"@remotion/web-renderer": "workspace:*",
"@remotion/studio-shared": "workspace:*",
diff --git a/packages/studio/src/components/CaptionInspector.tsx b/packages/studio/src/components/CaptionInspector.tsx
index f312e7111b..a3a1edb600 100644
--- a/packages/studio/src/components/CaptionInspector.tsx
+++ b/packages/studio/src/components/CaptionInspector.tsx
@@ -1,10 +1,28 @@
import type {Caption} from '@remotion/captions';
-import React from 'react';
+import React, {useCallback, useRef, useState} from 'react';
import {LIGHT_TEXT} from '../helpers/colors';
+import {FOCUS_VISIBLE_ONLY_CLASS_NAME} from '../helpers/hoverable';
+import {UploadIcon} from '../icons/upload';
import {CaptionTextEditor} from './CaptionTextEditor';
+import {useConfirmationDialog} from './ConfirmationDialog';
+import {InlineAction} from './InlineAction';
import {CollapsibleInspectorSectionHeader} from './InspectorPanel/CollapsibleInspectorSectionHeader';
import {InspectorSectionHeader} from './InspectorPanel/common';
import {sectionHeaderEnd} from './InspectorPanel/styles';
+import {showNotification} from './Notifications/NotificationCenter';
+import {parseCaptionFile, type ParsedCaptionFile} from './parse-caption-file';
+
+const importTooltip = `Import captions
+
+Supports Remotion Caption[] JSON, ElevenLabs Speech-to-Text JSON, ElevenLabs segmented export JSON, OpenAI Whisper verbose JSON, and SRT subtitles. JSON transcripts need word-level timestamps. Files are processed locally.`;
+
+const formatLabels: Record<ParsedCaptionFile['format'], string> = {
+ remotion: 'Remotion JSON',
+ elevenlabs: 'ElevenLabs JSON',
+ 'elevenlabs-segments': 'ElevenLabs segmented JSON',
+ 'openai-whisper': 'OpenAI Whisper JSON',
+ srt: 'SRT',
+};
const readOnlyStatus: React.CSSProperties = {
color: LIGHT_TEXT,
@@ -21,6 +39,7 @@ export const CaptionInspector: React.FC<{
readonly onTextChange: (captions: Caption[]) => void;
readonly onTextSave: ((captions: Caption[]) => void) | null;
readonly onTextCancel: (() => void) | null;
+ readonly onReplaceCaptions: ((captions: Caption[]) => Promise<void>) | null;
readonly onToggle: () => void;
readonly readOnly: boolean;
readonly readOnlyTitle: string | null;
@@ -30,16 +49,94 @@ export const CaptionInspector: React.FC<{
onTextChange,
onTextSave,
onTextCancel,
+ onReplaceCaptions,
onToggle,
readOnly,
readOnlyTitle,
}) => {
+ const fileInput = useRef<HTMLInputElement>(null);
+ const [isImporting, setIsImporting] = useState(false);
+ const confirm = useConfirmationDialog();
+
+ const importCaptions = useCallback(
+ async (event: React.ChangeEvent<HTMLInputElement>) => {
+ const file = event.currentTarget.files?.[0];
+ event.currentTarget.value = '';
+ if (!file || onReplaceCaptions === null) {
+ return;
+ }
+
+ setIsImporting(true);
+ try {
+ const parsed = parseCaptionFile({
+ fileName: file.name,
+ contents: await file.text(),
+ });
+ if (
+ captions.length > 0 &&
+ !(await confirm({
+ title: 'Replace captions?',
+ message: `This will replace ${captions.length} existing ${captions.length === 1 ? 'caption' : 'captions'} with ${parsed.captions.length} imported ${parsed.captions.length === 1 ? 'caption' : 'captions'}. You can undo this action.`,
+ confirmLabel: 'Replace captions',
+ cancelLabel: 'Cancel',
+ }))
+ ) {
+ return;
+ }
+
+ await onReplaceCaptions(parsed.captions);
+ const captionLabel =
+ parsed.captions.length === 1 ? 'caption' : 'captions';
+ showNotification(
+ `Imported ${parsed.captions.length} ${captionLabel} from ${formatLabels[parsed.format]}. Undo is available.`,
+ 4000,
+ );
+ } catch (error) {
+ showNotification(
+ `Could not import ${file.name}: ${error instanceof Error ? error.message : String(error)}`,
+ 5000,
+ );
+ } finally {
+ setIsImporting(false);
+ }
+ },
+ [captions.length, confirm, onReplaceCaptions],
+ );
+
return (
<>
<InspectorSectionHeader>
<CollapsibleInspectorSectionHeader
action={
<div style={sectionHeaderEnd}>
+ {onReplaceCaptions === null ? null : (
+ <>
+ <input
+ ref={fileInput}
+ accept=".json,.srt"
+ aria-label="Import captions file"
+ hidden
+ onChange={importCaptions}
+ type="file"
+ />
+ <InlineAction
+ aria-label="Import captions"
+ className={FOCUS_VISIBLE_ONLY_CLASS_NAME}
+ disabled={isImporting}
+ onClick={() => fileInput.current?.click()}
+ renderAction={(color) => (
+ <UploadIcon
+ aria-hidden="true"
+ color={color}
+ focusable="false"
+ style={{height: 16, width: 16}}
+ />
+ )}
+ title={importTooltip}
+ variant={null}
+ />
+ </>
+ )}
{readOnly ? (
<div style={readOnlyStatus} title={readOnlyTitle ?? undefined}>
Read only
diff --git a/packages/studio/src/components/InlineCaptionInspector.tsx b/packages/studio/src/components/InlineCaptionInspector.tsx
index 2591d1fc1e..d43935b008 100644
--- a/packages/studio/src/components/InlineCaptionInspector.tsx
+++ b/packages/studio/src/components/InlineCaptionInspector.tsx
@@ -15,7 +15,10 @@ import {Internals} from 'remotion';
import type {CodePosition} from '../error-overlay/react-overlay/utils/get-source-map';
import {StudioServerConnectionCtx} from '../helpers/client-id';
import {CaptionInspector} from './CaptionInspector';
-import {saveInlineCaptionPatches} from './Timeline/save-sequence-prop';
+import {
+ saveInlineCaptionPatches,
+ saveSequencePropsOrThrow,
+} from './Timeline/save-sequence-prop';
const serializeCaptions = (captions: Caption[]): string => {
return JSON.stringify(captions);
@@ -171,6 +174,58 @@ export const InlineCaptionInspector: React.FC<{
],
);
+ const replaceCaptions = useCallback(
+ async (nextCaptions: Caption[]) => {
+ if (!canSave || clientId === null) {
+ throw new Error('Caption importing is unavailable');
+ }
+
+ setDraftCaptions(nextCaptions);
+ setDragOverrides(
+ nodePath,
+ 'captions',
+ Internals.makeStaticDragOverride(nextCaptions),
+ );
+
+ try {
+ await saveSequencePropsOrThrow({
+ changes: [
+ {
+ fileName: validatedLocation.source,
+ nodePath,
+ fieldKey: 'captions',
+ value: nextCaptions,
+ defaultValue: null,
+ schema: controls.schema,
+ },
+ ],
+ addedKeyframes: null,
+ movedKeyframes: null,
+ setPropStatuses,
+ clientId,
+ undoLabel: 'Import captions',
+ redoLabel: 'Import captions again',
+ });
+ savedCaptions.current = nextCaptions;
+ } catch (error) {
+ setDraftCaptions(savedCaptions.current);
+ throw error;
+ } finally {
+ clearDragOverrides(nodePath);
+ }
+ },
+ [
+ canSave,
+ clearDragOverrides,
+ clientId,
+ controls.schema,
+ nodePath,
+ setDragOverrides,
+ setPropStatuses,
+ validatedLocation.source,
+ ],
+ );
+
const readOnlyTitle = readOnlyStudio
? 'Caption editing is unavailable in read-only Studio'
: clientId === null
@@ -189,6 +244,7 @@ export const InlineCaptionInspector: React.FC<{
onToggle={onToggle}
readOnly={!canSave}
readOnlyTitle={canSave ? null : readOnlyTitle}
+ onReplaceCaptions={canSave ? replaceCaptions : null}
/>
);
};
diff --git a/packages/studio/src/components/Timeline/save-prop-queue.ts b/packages/studio/src/components/Timeline/save-prop-queue.ts
index ccc8ba7c6c..4f093e36f9 100644
--- a/packages/studio/src/components/Timeline/save-prop-queue.ts
+++ b/packages/studio/src/components/Timeline/save-prop-queue.ts
@@ -54,14 +54,17 @@ export type EnqueueSaveOptions<TResponse> = {
errorLabel: string;
};
-export const enqueueSavePropChange = <TResponse>({
- nodePath,
- setPropStatuses,
- applyOptimistic,
- applyServerResponse,
- apiCall,
- errorLabel,
-}: EnqueueSaveOptions<TResponse>): Promise<void> => {
+const enqueueSavePropChangeInternal = <TResponse>(
+ {
+ nodePath,
+ setPropStatuses,
+ applyOptimistic,
+ applyServerResponse,
+ apiCall,
+ errorLabel,
+ }: EnqueueSaveOptions<TResponse>,
+ rejectOnError: boolean,
+): Promise<void> => {
const q = getQueue(nodePath);
if (q.cancelled) {
@@ -98,6 +101,10 @@ export const enqueueSavePropChange = <TResponse>({
myQueue.cancelled = true;
dropQueue(nodePath, myQueue);
+ if (rejectOnError) {
+ throw err;
+ }
+
showNotification(
`${errorLabel}: ${err instanceof Error ? err.message : String(err)}`,
4000,
@@ -108,3 +115,15 @@ export const enqueueSavePropChange = <TResponse>({
myQueue.chain = next;
return next;
};
+
+export const enqueueSavePropChange = <TResponse>(
+ options: EnqueueSaveOptions<TResponse>,
+): Promise<void> => {
+ return enqueueSavePropChangeInternal(options, false);
+};
+
+export const enqueueSavePropChangeOrThrow = <TResponse>(
+ options: EnqueueSaveOptions<TResponse>,
+): Promise<void> => {
+ return enqueueSavePropChangeInternal(options, true);
+};
diff --git a/packages/studio/src/components/Timeline/save-sequence-prop.ts b/packages/studio/src/components/Timeline/save-sequence-prop.ts
index 8cd844e3ce..e7bef9caf5 100644
--- a/packages/studio/src/components/Timeline/save-sequence-prop.ts
+++ b/packages/studio/src/components/Timeline/save-sequence-prop.ts
@@ -16,7 +16,10 @@ import {
type MoveEffectKeyframeChange,
type MoveSequenceKeyframeChange,
} from './call-move-keyframe';
-import {enqueueSavePropChange} from './save-prop-queue';
+import {
+ enqueueSavePropChange,
+ enqueueSavePropChangeOrThrow,
+} from './save-prop-queue';
export type SetPropStatuses = (
nodePath: SequencePropsSubscriptionKey,
@@ -102,15 +105,18 @@ export const saveInlineCaptionPatches = ({
});
};
-export const saveSequenceProps = ({
- changes,
- addedKeyframes,
- movedKeyframes,
- setPropStatuses,
- clientId,
- undoLabel,
- redoLabel,
-}: SaveSequencePropsOptions): Promise<void> => {
+const saveSequencePropsInternal = (
+ {
+ changes,
+ addedKeyframes,
+ movedKeyframes,
+ setPropStatuses,
+ clientId,
+ undoLabel,
+ redoLabel,
+ }: SaveSequencePropsOptions,
+ rejectOnError: boolean,
+): Promise<void> => {
const keyframesToAdd = addedKeyframes === null ? [] : addedKeyframes;
const sequenceKeyframes =
movedKeyframes === null ? [] : movedKeyframes.sequenceKeyframes;
@@ -136,7 +142,10 @@ export const saveSequenceProps = ({
throw new Error('Expected a sequence prop change');
}
- return enqueueSavePropChange({
+ const enqueue = rejectOnError
+ ? enqueueSavePropChangeOrThrow
+ : enqueueSavePropChange;
+ return enqueue({
nodePath: change.nodePath,
setPropStatuses,
applyOptimistic: (prev) =>
@@ -256,3 +265,15 @@ export const saveSequenceProps = ({
redoLabel,
}).then(() => undefined);
};
+
+export const saveSequenceProps = (
+ options: SaveSequencePropsOptions,
+): Promise<void> => {
+ return saveSequencePropsInternal(options, false);
+};
+
+export const saveSequencePropsOrThrow = (
+ options: SaveSequencePropsOptions,
+): Promise<void> => {
+ return saveSequencePropsInternal(options, true);
+};
diff --git a/packages/studio/src/components/parse-caption-file.ts b/packages/studio/src/components/parse-caption-file.ts
new file mode 100644
index 0000000000..d5cdcde565
--- /dev/null
+++ b/packages/studio/src/components/parse-caption-file.ts
@@ -0,0 +1,284 @@
+import type {Caption} from '@remotion/captions';
+import {parseSrt} from '@remotion/captions';
+import {
+ elevenLabsTranscriptToCaptions,
+ isElevenLabsTranscriptJson,
+ type ElevenLabsTranscript,
+} from '@remotion/elevenlabs';
+import {
+ isOpenAiWhisperJson,
+ openAiWhisperApiToCaptions,
+ type OpenAiVerboseTranscription,
+} from '@remotion/openai-whisper';
+
+export type ParsedCaptionFile = {
+ readonly captions: Caption[];
+ readonly format:
+ | 'remotion'
+ | 'elevenlabs'
+ | 'elevenlabs-segments'
+ | 'openai-whisper'
+ | 'srt';
+};
+
+const isObject = (value: unknown): value is Record<string, unknown> => {
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
+};
+
+const isElevenLabsSegmentedJson = (value: unknown): boolean => {
+ return isObject(value) && 'language_code' in value && 'segments' in value;
+};
+
+const isFiniteNumber = (value: unknown): value is number => {
+ return typeof value === 'number' && Number.isFinite(value);
+};
+
+const errorMessage = (error: unknown) => {
+ return error instanceof Error ? error.message : String(error);
+};
+
+const validateCaptions = (value: unknown): Caption[] => {
+ if (!Array.isArray(value)) {
+ throw new Error('captions must be an array of Remotion captions.');
+ }
+
+ let previousStart: number | null = null;
+ for (const [index, caption] of value.entries()) {
+ const path = `captions[${index}]`;
+ if (!isObject(caption)) {
+ throw new Error(`${path} must be an object.`);
+ }
+
+ if (typeof caption.text !== 'string') {
+ throw new Error(`${path}.text must be a string.`);
+ }
+
+ if (!isFiniteNumber(caption.startMs)) {
+ throw new Error(`${path}.startMs must be a finite number.`);
+ }
+
+ if (caption.startMs < 0) {
+ throw new Error(`${path}.startMs must be a non-negative number.`);
+ }
+
+ if (!isFiniteNumber(caption.endMs)) {
+ throw new Error(`${path}.endMs must be a finite number.`);
+ }
+
+ if (caption.endMs < 0) {
+ throw new Error(`${path}.endMs must be a non-negative number.`);
+ }
+
+ if (caption.endMs < caption.startMs) {
+ throw new Error(`${path}.endMs must not be earlier than startMs.`);
+ }
+
+ if (caption.timestampMs !== null && !isFiniteNumber(caption.timestampMs)) {
+ throw new Error(`${path}.timestampMs must be a finite number or null.`);
+ }
+
+ if (caption.confidence !== null && !isFiniteNumber(caption.confidence)) {
+ throw new Error(`${path}.confidence must be a finite number or null.`);
+ }
+
+ if (
+ typeof caption.confidence === 'number' &&
+ (caption.confidence < 0 || caption.confidence > 1)
+ ) {
+ throw new Error(`${path}.confidence must be between 0 and 1.`);
+ }
+
+ if (
+ caption.pageBreakAfter !== undefined &&
+ typeof caption.pageBreakAfter !== 'boolean'
+ ) {
+ throw new Error(
+ `${path}.pageBreakAfter must be a boolean when provided.`,
+ );
+ }
+
+ if (previousStart !== null && caption.startMs < previousStart) {
+ throw new Error(`${path}.startMs is out of timestamp order.`);
+ }
+
+ previousStart = caption.startMs;
+ }
+
+ return value as Caption[];
+};
+
+const convertSegmentedElevenLabsTranscript = (
+ transcript: unknown,
+): Caption[] => {
+ if (!isObject(transcript) || !Array.isArray(transcript.segments)) {
+ throw new Error('transcript.segments must be an array.');
+ }
+
+ const captions: Caption[] = [];
+ let pendingSpacing = '';
+ let pendingSpacingStart: number | null = null;
+ let previousStart: number | null = null;
+
+ for (const [segmentIndex, segment] of transcript.segments.entries()) {
+ const segmentPath = `transcript.segments[${segmentIndex}]`;
+ if (!isObject(segment)) {
+ throw new Error(`${segmentPath} must be an object.`);
+ }
+
+ if (!Array.isArray(segment.words)) {
+ throw new Error(`${segmentPath}.words must be an array.`);
+ }
+
+ for (const [wordIndex, word] of segment.words.entries()) {
+ const path = `${segmentPath}.words[${wordIndex}]`;
+ if (!isObject(word)) {
+ throw new Error(`${path} must be an object.`);
+ }
+
+ if (typeof word.text !== 'string') {
+ throw new Error(`${path}.text must be a string.`);
+ }
+
+ if (!isFiniteNumber(word.start_time)) {
+ throw new Error(`${path}.start_time must be a finite number.`);
+ }
+
+ if (word.start_time < 0) {
+ throw new Error(`${path}.start_time must be a non-negative number.`);
+ }
+
+ if (!isFiniteNumber(word.end_time)) {
+ throw new Error(`${path}.end_time must be a finite number.`);
+ }
+
+ if (word.end_time < 0) {
+ throw new Error(`${path}.end_time must be a non-negative number.`);
+ }
+
+ if (word.end_time < word.start_time) {
+ throw new Error(
+ `${path}.end_time must not be earlier than start_time.`,
+ );
+ }
+
+ if (previousStart !== null && word.start_time < previousStart) {
+ throw new Error(`${path}.start_time is out of timestamp order.`);
+ }
+
+ previousStart = word.start_time;
+ if (word.text.trim() === '') {
+ pendingSpacingStart ??= word.start_time;
+ pendingSpacing += word.text;
+ continue;
+ }
+
+ const startMs =
+ captions.length > 0 && pendingSpacingStart !== null
+ ? pendingSpacingStart * 1000
+ : word.start_time * 1000;
+ const endMs = word.end_time * 1000;
+ captions.push({
+ confidence: null,
+ startMs,
+ endMs,
+ text: `${pendingSpacing}${word.text}`,
+ timestampMs: (startMs + endMs) / 2,
+ });
+ pendingSpacing = '';
+ pendingSpacingStart = null;
+ }
+ }
+
+ return captions;
+};
+
+export const parseCaptionFile = ({
+ fileName,
+ contents,
+}: {
+ fileName: string;
+ contents: string;
+}): ParsedCaptionFile => {
+ const lowerCaseFileName = fileName.toLowerCase();
+ if (lowerCaseFileName.endsWith('.srt')) {
+ try {
+ return {
+ captions: validateCaptions(parseSrt({input: contents}).captions),
+ format: 'srt',
+ };
+ } catch (error) {
+ throw new Error(`Invalid SRT file: ${errorMessage(error)}`);
+ }
+ }
+
+ if (!lowerCaseFileName.endsWith('.json')) {
+ throw new Error('Unsupported caption file. Choose a .json or .srt file.');
+ }
+
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(contents);
+ } catch (error) {
+ throw new Error(`Invalid JSON: ${errorMessage(error)}`);
+ }
+
+ if (Array.isArray(parsed)) {
+ return {captions: validateCaptions(parsed), format: 'remotion'};
+ }
+
+ if (!isObject(parsed)) {
+ throw new Error(
+ 'Unsupported transcription JSON. Expected an object or Caption[] array.',
+ );
+ }
+
+ if (isElevenLabsSegmentedJson(parsed)) {
+ try {
+ return {
+ captions: validateCaptions(
+ convertSegmentedElevenLabsTranscript(parsed),
+ ),
+ format: 'elevenlabs-segments',
+ };
+ } catch (error) {
+ throw new Error(
+ `Detected ElevenLabs segmented JSON, but ${errorMessage(error)}`,
+ );
+ }
+ }
+
+ if (isElevenLabsTranscriptJson(parsed)) {
+ try {
+ const {captions} = elevenLabsTranscriptToCaptions({
+ transcript: parsed as ElevenLabsTranscript,
+ });
+ return {
+ captions: validateCaptions(captions),
+ format: 'elevenlabs',
+ };
+ } catch (error) {
+ throw new Error(`Detected ElevenLabs JSON, but ${errorMessage(error)}`);
+ }
+ }
+
+ if (isOpenAiWhisperJson(parsed)) {
+ try {
+ const {captions} = openAiWhisperApiToCaptions({
+ transcription: parsed as unknown as OpenAiVerboseTranscription,
+ });
+ return {
+ captions: validateCaptions(captions),
+ format: 'openai-whisper',
+ };
+ } catch (error) {
+ throw new Error(
+ `Detected OpenAI Whisper JSON, but ${errorMessage(error)}`,
+ );
+ }
+ }
+
+ const keys = Object.keys(parsed);
+ throw new Error(
+ `Unsupported transcription JSON. Found top-level keys: ${keys.length === 0 ? '(none)' : keys.join(', ')}.`,
+ );
+};
diff --git a/packages/studio/src/test/parse-caption-file.test.ts b/packages/studio/src/test/parse-caption-file.test.ts
new file mode 100644
index 0000000000..d3c8acb9a1
--- /dev/null
+++ b/packages/studio/src/test/parse-caption-file.test.ts
@@ -0,0 +1,225 @@
+import {expect, test} from 'bun:test';
+import {parseCaptionFile} from '../components/parse-caption-file';
+
+const parseJson = (value: unknown) => {
+ return parseCaptionFile({
+ fileName: 'captions.json',
+ contents: JSON.stringify(value),
+ });
+};
+
+test('imports supported caption and transcription formats', () => {
+ const remotion = parseJson([
+ {
+ text: 'Hello',
+ startMs: 0,
+ endMs: 500,
+ timestampMs: 250,
+ confidence: null,
+ },
+ ]);
+ expect(remotion).toEqual({
+ format: 'remotion',
+ captions: [
+ {
+ text: 'Hello',
+ startMs: 0,
+ endMs: 500,
+ timestampMs: 250,
+ confidence: null,
+ },
+ ],
+ });
+ expect(parseJson([])).toEqual({format: 'remotion', captions: []});
+
+ const elevenLabs = parseJson({
+ language_code: 'eng',
+ text: 'Hello world',
+ words: [
+ {text: 'Hello', type: 'word', start: 0, end: 0.4},
+ {text: ' ', type: 'spacing', start: 0.4, end: 0.5},
+ {text: 'world', type: 'word', start: 0.5, end: 0.9},
+ ],
+ });
+ expect(elevenLabs.format).toBe('elevenlabs');
+ expect(elevenLabs.captions.map(({text}) => text)).toEqual([
+ 'Hello',
+ ' world',
+ ]);
+
+ const segmented = parseJson({
+ language_code: 'eng',
+ segments: [
+ {
+ text: 'Hello world',
+ start_time: 0,
+ end_time: 0.9,
+ words: [
+ {text: 'Hello', start_time: 0, end_time: 0.4},
+ {text: ' ', start_time: 0.4, end_time: 0.5},
+ {text: 'world', start_time: 0.5, end_time: 0.9},
+ ],
+ },
+ ],
+ });
+ expect(segmented).toEqual({
+ format: 'elevenlabs-segments',
+ captions: [
+ {
+ confidence: null,
+ endMs: 400,
+ startMs: 0,
+ text: 'Hello',
+ timestampMs: 200,
+ },
+ {
+ confidence: null,
+ endMs: 900,
+ startMs: 400,
+ text: ' world',
+ timestampMs: 650,
+ },
+ ],
+ });
+
+ const openAi = parseJson({
+ duration: 0.9,
+ language: 'english',
+ task: 'transcribe',
+ text: 'Hello, world!',
+ words: [
+ {word: 'Hello', start: 0, end: 0.4},
+ {word: 'world', start: 0.5, end: 0.9},
+ ],
+ });
+ expect(openAi.format).toBe('openai-whisper');
+ expect(openAi.captions.map(({text}) => text)).toEqual(['Hello,', ' world!']);
+
+ const srt = parseCaptionFile({
+ fileName: 'CAPTIONS.SRT',
+ contents: '1\n00:00:00,000 --> 00:00:01,000\nHello world\n',
+ });
+ expect(srt.format).toBe('srt');
+ expect(srt.captions[0]).toEqual({
+ confidence: 1,
+ endMs: 1000,
+ startMs: 0,
+ text: 'Hello world',
+ timestampMs: 500,
+ });
+});
+
+test.each([
+ {
+ name: 'invalid JSON',
+ fileName: 'captions.json',
+ contents: '{',
+ error: 'Invalid JSON:',
+ },
+ {
+ name: 'unknown JSON',
+ fileName: 'captions.json',
+ contents: JSON.stringify({language: 'english', utterances: []}),
+ error: 'Found top-level keys: language, utterances',
+ },
+ {
+ name: 'OpenAI response without words',
+ fileName: 'captions.json',
+ contents: JSON.stringify({
+ duration: 1,
+ language: 'english',
+ text: 'Hello',
+ }),
+ error: 'timestamp_granularities: ["word"]',
+ },
+ {
+ name: 'malformed ElevenLabs segmented word',
+ fileName: 'captions.json',
+ contents: JSON.stringify({
+ language_code: 'eng',
+ segments: [
+ {
+ text: 'Hello',
+ start_time: 0,
+ end_time: 1,
+ words: [{text: 'Hello', end_time: 1}],
+ },
+ ],
+ }),
+ error: 'segments[0].words[0].start_time must be a finite number',
+ },
+ {
+ name: 'malformed ElevenLabs API word',
+ fileName: 'captions.json',
+ contents: JSON.stringify({
+ language_code: 'eng',
+ text: 'Hello',
+ words: [{type: 'word', text: 'Hello', start: 'now', end: 1}],
+ }),
+ error:
+ 'Detected ElevenLabs JSON, but Invalid ElevenLabs transcript: words[0].start',
+ },
+ {
+ name: 'malformed OpenAI Whisper word',
+ fileName: 'captions.json',
+ contents: JSON.stringify({
+ duration: 1,
+ language: 'english',
+ text: 'Hello',
+ words: [{word: 'Hello', start: 0, end: 'later'}],
+ }),
+ error:
+ 'Detected OpenAI Whisper JSON, but Invalid OpenAI Whisper transcription: words[0].end',
+ },
+ {
+ name: 'invalid canonical confidence',
+ fileName: 'captions.json',
+ contents: JSON.stringify([
+ {
+ text: 'Hello',
+ startMs: 0,
+ endMs: 1,
+ timestampMs: 0.5,
+ confidence: 2,
+ },
+ ]),
+ error: 'captions[0].confidence must be between 0 and 1',
+ },
+ {
+ name: 'reversed canonical timing',
+ fileName: 'captions.json',
+ contents: JSON.stringify([
+ {
+ text: 'Hello',
+ startMs: 2,
+ endMs: 1,
+ timestampMs: 1.5,
+ confidence: null,
+ },
+ ]),
+ error: 'captions[0].endMs must not be earlier than startMs',
+ },
+ {
+ name: 'out-of-order canonical captions',
+ fileName: 'captions.json',
+ contents: JSON.stringify([
+ {
+ text: 'Later',
+ startMs: 2,
+ endMs: 3,
+ timestampMs: 2.5,
+ confidence: null,
+ },
+ {
+ text: 'Earlier',
+ startMs: 1,
+ endMs: 2,
+ timestampMs: 1.5,
+ confidence: null,
+ },
+ ]),
+ error: 'captions[1].startMs is out of timestamp order',
+ },
+])('rejects $name', ({fileName, contents, error}) => {
+ expect(() => parseCaptionFile({fileName, contents})).toThrow(error);
+});