Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add subtitle support and enhance transcript handling

konstantin-pauluse3bbc1e77b80
diff --git a/apps/web/src/components/engine/api/assets.ts b/apps/web/src/components/engine/api/assets.ts
index db454380..edcf43c7 100644
--- a/apps/web/src/components/engine/api/assets.ts
+++ b/apps/web/src/components/engine/api/assets.ts
@@ -342,7 +342,7 @@ async function probeMediaMetadata(file: Blob, mimeType: string) {
: await getRasterDimensions(file);
return { type: 'IMAGE', width, height } as const;
- } else if (mimeType === 'application/json') {
+ } else if (mimeType === 'application/json' || mimeType === 'application/x-subrip' || mimeType === 'text/vtt') {
return { type: 'TRANSCRIPT' } as const;
} else if (mimeType.match(/^(audio\/|video\/)/)) {
const input = new Input({
@@ -858,6 +858,16 @@ export async function detectMimeType(handle: MediaInput): Promise<string | null>
mimeType = handle.type;
}
+ // Subtitle files: the extension is authoritative — OS registries rarely map .srt.
+ const fileName = typeof handle === 'string'
+ ? handle.split(/[?#]/)[0]
+ : (handle as File).name ?? '';
+ if (/\.srt$/i.test(fileName) || mimeType === 'application/x-subrip') {
+ return 'application/x-subrip';
+ } else if (/\.vtt$/i.test(fileName) || mimeType === 'text/vtt') {
+ return 'text/vtt';
+ }
+
if (mimeType?.startsWith('image/')) {
return mimeType;
} else if (mimeType?.startsWith('text/html')) {
diff --git a/apps/web/src/components/engine/api/utils.ts b/apps/web/src/components/engine/api/utils.ts
index ea19d714..2ee8a060 100644
--- a/apps/web/src/components/engine/api/utils.ts
+++ b/apps/web/src/components/engine/api/utils.ts
@@ -173,6 +173,10 @@ export function reactToAssetChange(world: EngineWorld, eid: number) {
reactToPaintChange(world, eid);
} else if (hasComponent(world, eid, c.Audio)) {
reactToGeometryDurationChange(world, eid);
+ } else if (hasComponent(world, eid, c.Caption)) {
+ // Re-pointed transcript: drop the decoder so it re-resolves with the new asset.
+ c.CaptionDecoder[eid]?.dispose();
+ c.CaptionDecoder[eid] = null;
}
}
diff --git a/apps/web/src/components/engine/decoders/caption/subtitles.ts b/apps/web/src/components/engine/decoders/caption/subtitles.ts
new file mode 100644
index 00000000..7e5e9a8a
--- /dev/null
+++ b/apps/web/src/components/engine/decoders/caption/subtitles.ts
@@ -0,0 +1,60 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { Transcript } from '@diffusionstudio/api-contract';
+
+const TIMESTAMP_RE = /(?:\d{1,2}:)?\d{1,2}:\d{2}[.,]\d{1,3}/g;
+
+function parseTimestamp(raw: string): number {
+ return raw
+ .replace(',', '.')
+ .split(':')
+ .reduce((acc, part) => acc * 60 + Number(part), 0);
+}
+
+/**
+ * Parses SRT or WebVTT text into a `Transcript`. Each cue becomes one segment
+ * (a sentence boundary for grouping). Cues carry no word timings, so each
+ * word's window is synthesized proportional to its length within the cue.
+ */
+export function parseSubtitles(text: string): Transcript {
+ const transcript: Transcript = [];
+ const blocks = text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n').split(/\n{2,}/);
+
+ for (const block of blocks) {
+ const lines = block.split('\n');
+ const timingIndex = lines.findIndex((line) => line.includes('-->'));
+ if (timingIndex === -1) continue; // WEBVTT header, NOTE/STYLE/REGION blocks, stray indices
+
+ const timestamps = lines[timingIndex].match(TIMESTAMP_RE);
+ if (!timestamps || timestamps.length < 2) continue;
+ const start = parseTimestamp(timestamps[0]);
+ const end = parseTimestamp(timestamps[1]);
+ if (!(end > start)) continue;
+
+ // Strip inline markup: HTML/VTT tags (<i>, <c.class>, <00:01.000>) and
+ // SSA-style overrides ({\an8}).
+ const content = lines
+ .slice(timingIndex + 1)
+ .join(' ')
+ .replace(/<[^>]*>/g, '')
+ .replace(/\{[^}]*\}/g, '');
+
+ const tokens = content.split(/\s+/).filter(Boolean);
+ if (tokens.length === 0) continue;
+
+ const total = tokens.reduce((acc, token) => acc + token.length, 0);
+ const duration = end - start;
+ let elapsed = 0;
+ const words = tokens.map((token) => {
+ const wordStart = start + (elapsed / total) * duration;
+ elapsed += token.length;
+ return { text: token, start: wordStart, end: start + (elapsed / total) * duration };
+ });
+
+ transcript.push({ text: tokens.join(' '), words });
+ }
+
+ return transcript.sort((a, b) => a.words[0].start - b.words[0].start);
+}
diff --git a/apps/web/src/components/engine/decoders/caption/utils.ts b/apps/web/src/components/engine/decoders/caption/utils.ts
index de7cb6bd..bc121a05 100644
--- a/apps/web/src/components/engine/decoders/caption/utils.ts
+++ b/apps/web/src/components/engine/decoders/caption/utils.ts
@@ -6,6 +6,7 @@ import { query } from 'bitecs';
import { ChildOf } from '../../components/relations';
import { deleteEntity } from '../../api/entities';
import { getAssetFile } from '../../api/assets';
+import { parseSubtitles } from './subtitles';
import type { Transcript, WordGroup } from '@diffusionstudio/api-contract';
import type { EngineWorld } from '../../api/world';
@@ -110,6 +111,9 @@ export async function resolveTranscript(asset: Asset): Promise<Transcript> {
if (asset.type === 'TRANSCRIPT') {
const file = await getAssetFile(asset);
const text = await file.text();
+ if (asset.mimeType === 'application/x-subrip' || asset.mimeType === 'text/vtt') {
+ return parseSubtitles(text);
+ }
return JSON.parse(text);
}
diff --git a/apps/web/src/components/engine/systems/playback.ts b/apps/web/src/components/engine/systems/playback.ts
index bdc4b0da..2bd05986 100644
--- a/apps/web/src/components/engine/systems/playback.ts
+++ b/apps/web/src/components/engine/systems/playback.ts
@@ -229,28 +229,30 @@ function forwardDecoders(world: EngineWorld, sid: number, eid: number): void {
const hidden =
hasComponent(world, eid, c.Hidden)
- || hasComponent(world, eid, c.Culled)
- || world.mode === 'offline-audio';
+ || hasComponent(world, eid, c.Culled);
+ const visualsEnabled = !hidden && world.mode !== 'offline-audio';
let paintAudioEid: number | undefined;
if (!hidden) {
for (const fid of query(world, [ChildOf(eid), c.Paint, Not(c.Deleted), Not(c.Hidden)])) {
if (c.Paint[fid] === PaintType.VIDEO) {
- forwardVideoDecoder(world, sid, eid, fid);
+ if (visualsEnabled) {
+ forwardVideoDecoder(world, sid, eid, fid);
+ }
paintAudioEid = fid;
}
- if (c.Paint[fid] === PaintType.SEQUENCE) {
+ if (c.Paint[fid] === PaintType.SEQUENCE && visualsEnabled) {
forwardSequenceDecoder(world, sid, eid, fid);
}
- if (c.Paint[fid] === PaintType.IMAGE) {
+ if (c.Paint[fid] === PaintType.IMAGE && visualsEnabled) {
forwardImageDecoder(world, sid, eid, fid);
}
}
}
- if (hasComponent(world, eid, c.Caption) && !hidden) {
+ if (hasComponent(world, eid, c.Caption) && visualsEnabled) {
forwardCaptionDecoder(world, sid, eid);
}
diff --git a/apps/web/src/components/engine/utils/captions.ts b/apps/web/src/components/engine/utils/captions.ts
index db3d1554..8e6c30b6 100644
--- a/apps/web/src/components/engine/utils/captions.ts
+++ b/apps/web/src/components/engine/utils/captions.ts
@@ -81,7 +81,7 @@ function fnv1a(input: string): string {
return (hash >>> 0).toString(16);
}
-function transcriptTrim(transcript: Transcript): { start: number; end: number } {
+export function transcriptTrim(transcript: Transcript): { start: number; end: number } {
const firstWord = transcript.at(0)?.words.at(0);
const lastWord = transcript.at(-1)?.words.at(-1);
return { start: secondsToFrames(firstWord?.start), end: secondsToFrames(lastWord?.end) };
diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.ts
index c80c3abf..0a85cfc4 100644
--- a/apps/web/src/utils/jsx.ts
+++ b/apps/web/src/utils/jsx.ts
@@ -33,7 +33,9 @@ import {
setComponent,
setKeyframeTrack,
transcribeScene,
+ transcriptTrim,
} from "@/components/engine";
+import { resolveTranscript } from "@/components/engine/decoders/caption/utils";
import { ChildOf } from "@/components/engine/components";
import { ASPECT_RATIO_DIMENSIONS, findEmptyPlacement } from "@/utils/genai";
import { resolveAsset, resolveGeneratedAsset } from "@/utils/jsx-generation";
@@ -903,7 +905,12 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {
const c = world.components;
const eid = node.eid;
- if (asset.type === 'AUDIO') {
+ if (asset.type === 'TRANSCRIPT') {
+ const transcript = await resolveTranscript(asset);
+ // AssetId set here makes commit's caption phase skip transcription.
+ setComponent(world, eid, c.AssetId, asset.id);
+ setComponent(world, eid, c.Trim, transcriptTrim(transcript));
+ } else if (asset.type === 'AUDIO') {
setComponent(world, eid, c.AssetId, asset.id);
resizeEntity(world, eid, { width: 500, height: 150 });
} else if (asset.type === 'IMAGE' || asset.type === 'VIDEO') {
diff --git a/apps/web/src/utils/media.ts b/apps/web/src/utils/media.ts
index b57ee7a0..bfd20f62 100644
--- a/apps/web/src/utils/media.ts
+++ b/apps/web/src/utils/media.ts
@@ -58,6 +58,8 @@ export function mimeTypeToExtension(mimeType: string): string {
// Web files
'text/html': '.html',
'application/json': '.json',
+ 'application/x-subrip': '.srt',
+ 'text/vtt': '.vtt',
'text/xml': '.xml',
'application/xml': '.xml',
};
diff --git a/docs/composition.md b/docs/composition.md
index c4a74223..76b8f679 100644
--- a/docs/composition.md
+++ b/docs/composition.md
@@ -37,7 +37,7 @@ The first rendered root becomes the active scene. New roots are auto-placed on t
| `<video>` / `<image>` | Media; `src` is required. |
| `<audio>` | Sound only, no spatial props. |
| `<sequence>` | Track container for back-to-back clips (positions are explicit); hosts transitions. |
-| `<captions>` | Transcribes the scene's audio into styled captions. |
+| `<captions>` | Styled captions: transcribes the scene's audio, or mounts a transcript given via `src`. |
| `<solidPaint>`, `<linearGradientPaint>`, `<radialGradientPaint>`, `<colorStop>` | Fills, declared as children; see [Paints](#paints). |
User-defined components are ordinary Solid components; only intrinsic elements produce nodes.
@@ -213,6 +213,16 @@ Declarations are pure: nothing generates until the mount commits, and refs never
<captions preset="spotlight" colors={["#FF0055"]} />
```
+### Manual captions
+
+`src` supplies the transcript instead of transcribing: a path, URL, or asset id of an `.srt` or `.vtt` subtitle file, or a transcript `.json` (the `dapi transcribe` output format: `[{ text, words: [{ text, start, end }] }]`, times in seconds). Subtitle cues carry no word timings, so word-level presets highlight on timings synthesized within each cue.
+
+```tsx
+<captions src="./subtitles.srt" preset="classic" />
+```
+
+Times are scene-relative; set `start` to shift the whole caption track when the transcript was written against a clip that begins later in the scene.
+
| Preset | Style |
| --- | --- |
| `"classic"` (default) | Centered lowercase text, soft drop shadow, a few words at a time. |
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index bf5a6325..a1159526 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -122,7 +122,11 @@ export type PatchProps = {
transition?: TransitionSpec | null;
/** Any CSS color, applied to the node's solid fill (created if absent); alpha is ignored — use `opacity`. */
fill?: string;
- /** Path, URL, asset id, or a `generate.*` declaration. */
+ /**
+ * Path, URL, asset id, or a `generate.*` declaration. On `<captions>` a
+ * transcript source (.srt, .vtt, or transcript .json) mounted instead of
+ * transcribing the scene; `generate.*` is not accepted there.
+ */
src?: string | AssetRef;
/** How the source maps into the box. Default "cover" on `<video>`, "contain" on `<image>`. */
objectFit?: Fit;
@@ -257,4 +261,4 @@ export type SequenceProps = Pick<PatchProps, "name"> & {
children?: SolidJSX.Element;
};
-export type CaptionsProps = Pick<PatchProps, "preset" | "colors">;
+export type CaptionsProps = Pick<PatchProps, "preset" | "colors" | "src" | "start">;