The VS Code of video editing, built for agents.
TypeScript
Add subtitle support and enhance transcript handling
konstantin-pauluse3bbc1e77b80
diff --git a/apps/web/src/components/engine/api/assets.ts b/apps/web/src/components/engine/api/assets.tsindex db454380..edcf43c7 100644--- a/apps/web/src/components/engine/api/assets.ts+++ b/apps/web/src/components/engine/api/assets.ts@@ -342,7 +342,7 @@ async function probeMediaMetadata(file: Blob, mimeType: string) {: await getRasterDimensions(file);return { type: 'IMAGE', width, height } as const;- } else if (mimeType === 'application/json') {+ } else if (mimeType === 'application/json' || mimeType === 'application/x-subrip' || mimeType === 'text/vtt') {return { type: 'TRANSCRIPT' } as const;} else if (mimeType.match(/^(audio\/|video\/)/)) {const input = new Input({@@ -858,6 +858,16 @@ export async function detectMimeType(handle: MediaInput): Promise<string | null>mimeType = handle.type;}+ // Subtitle files: the extension is authoritative — OS registries rarely map .srt.+ const fileName = typeof handle === 'string'+ ? handle.split(/[?#]/)[0]+ : (handle as File).name ?? '';+ if (/\.srt$/i.test(fileName) || mimeType === 'application/x-subrip') {+ return 'application/x-subrip';+ } else if (/\.vtt$/i.test(fileName) || mimeType === 'text/vtt') {+ return 'text/vtt';+ }+if (mimeType?.startsWith('image/')) {return mimeType;} else if (mimeType?.startsWith('text/html')) {diff --git a/apps/web/src/components/engine/api/utils.ts b/apps/web/src/components/engine/api/utils.tsindex ea19d714..2ee8a060 100644--- a/apps/web/src/components/engine/api/utils.ts+++ b/apps/web/src/components/engine/api/utils.ts@@ -173,6 +173,10 @@ export function reactToAssetChange(world: EngineWorld, eid: number) {reactToPaintChange(world, eid);} else if (hasComponent(world, eid, c.Audio)) {reactToGeometryDurationChange(world, eid);+ } else if (hasComponent(world, eid, c.Caption)) {+ // Re-pointed transcript: drop the decoder so it re-resolves with the new asset.+ c.CaptionDecoder[eid]?.dispose();+ c.CaptionDecoder[eid] = null;}}diff --git a/apps/web/src/components/engine/decoders/caption/subtitles.ts b/apps/web/src/components/engine/decoders/caption/subtitles.tsnew file mode 100644index 00000000..7e5e9a8a--- /dev/null+++ b/apps/web/src/components/engine/decoders/caption/subtitles.ts@@ -0,0 +1,60 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import type { Transcript } from '@diffusionstudio/api-contract';++const TIMESTAMP_RE = /(?:\d{1,2}:)?\d{1,2}:\d{2}[.,]\d{1,3}/g;++function parseTimestamp(raw: string): number {+ return raw+ .replace(',', '.')+ .split(':')+ .reduce((acc, part) => acc * 60 + Number(part), 0);+}++/**+ * Parses SRT or WebVTT text into a `Transcript`. Each cue becomes one segment+ * (a sentence boundary for grouping). Cues carry no word timings, so each+ * word's window is synthesized proportional to its length within the cue.+ */+export function parseSubtitles(text: string): Transcript {+ const transcript: Transcript = [];+ const blocks = text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n').split(/\n{2,}/);++ for (const block of blocks) {+ const lines = block.split('\n');+ const timingIndex = lines.findIndex((line) => line.includes('-->'));+ if (timingIndex === -1) continue; // WEBVTT header, NOTE/STYLE/REGION blocks, stray indices++ const timestamps = lines[timingIndex].match(TIMESTAMP_RE);+ if (!timestamps || timestamps.length < 2) continue;+ const start = parseTimestamp(timestamps[0]);+ const end = parseTimestamp(timestamps[1]);+ if (!(end > start)) continue;++ // Strip inline markup: HTML/VTT tags (<i>, <c.class>, <00:01.000>) and+ // SSA-style overrides ({\an8}).+ const content = lines+ .slice(timingIndex + 1)+ .join(' ')+ .replace(/<[^>]*>/g, '')+ .replace(/\{[^}]*\}/g, '');++ const tokens = content.split(/\s+/).filter(Boolean);+ if (tokens.length === 0) continue;++ const total = tokens.reduce((acc, token) => acc + token.length, 0);+ const duration = end - start;+ let elapsed = 0;+ const words = tokens.map((token) => {+ const wordStart = start + (elapsed / total) * duration;+ elapsed += token.length;+ return { text: token, start: wordStart, end: start + (elapsed / total) * duration };+ });++ transcript.push({ text: tokens.join(' '), words });+ }++ return transcript.sort((a, b) => a.words[0].start - b.words[0].start);+}diff --git a/apps/web/src/components/engine/decoders/caption/utils.ts b/apps/web/src/components/engine/decoders/caption/utils.tsindex de7cb6bd..bc121a05 100644--- a/apps/web/src/components/engine/decoders/caption/utils.ts+++ b/apps/web/src/components/engine/decoders/caption/utils.ts@@ -6,6 +6,7 @@ import { query } from 'bitecs';import { ChildOf } from '../../components/relations';import { deleteEntity } from '../../api/entities';import { getAssetFile } from '../../api/assets';+import { parseSubtitles } from './subtitles';import type { Transcript, WordGroup } from '@diffusionstudio/api-contract';import type { EngineWorld } from '../../api/world';@@ -110,6 +111,9 @@ export async function resolveTranscript(asset: Asset): Promise<Transcript> {if (asset.type === 'TRANSCRIPT') {const file = await getAssetFile(asset);const text = await file.text();+ if (asset.mimeType === 'application/x-subrip' || asset.mimeType === 'text/vtt') {+ return parseSubtitles(text);+ }return JSON.parse(text);}diff --git a/apps/web/src/components/engine/systems/playback.ts b/apps/web/src/components/engine/systems/playback.tsindex bdc4b0da..2bd05986 100644--- a/apps/web/src/components/engine/systems/playback.ts+++ b/apps/web/src/components/engine/systems/playback.ts@@ -229,28 +229,30 @@ function forwardDecoders(world: EngineWorld, sid: number, eid: number): void {const hidden =hasComponent(world, eid, c.Hidden)- || hasComponent(world, eid, c.Culled)- || world.mode === 'offline-audio';+ || hasComponent(world, eid, c.Culled);+ const visualsEnabled = !hidden && world.mode !== 'offline-audio';let paintAudioEid: number | undefined;if (!hidden) {for (const fid of query(world, [ChildOf(eid), c.Paint, Not(c.Deleted), Not(c.Hidden)])) {if (c.Paint[fid] === PaintType.VIDEO) {- forwardVideoDecoder(world, sid, eid, fid);+ if (visualsEnabled) {+ forwardVideoDecoder(world, sid, eid, fid);+ }paintAudioEid = fid;}- if (c.Paint[fid] === PaintType.SEQUENCE) {+ if (c.Paint[fid] === PaintType.SEQUENCE && visualsEnabled) {forwardSequenceDecoder(world, sid, eid, fid);}- if (c.Paint[fid] === PaintType.IMAGE) {+ if (c.Paint[fid] === PaintType.IMAGE && visualsEnabled) {forwardImageDecoder(world, sid, eid, fid);}}}- if (hasComponent(world, eid, c.Caption) && !hidden) {+ if (hasComponent(world, eid, c.Caption) && visualsEnabled) {forwardCaptionDecoder(world, sid, eid);}diff --git a/apps/web/src/components/engine/utils/captions.ts b/apps/web/src/components/engine/utils/captions.tsindex db3d1554..8e6c30b6 100644--- a/apps/web/src/components/engine/utils/captions.ts+++ b/apps/web/src/components/engine/utils/captions.ts@@ -81,7 +81,7 @@ function fnv1a(input: string): string {return (hash >>> 0).toString(16);}-function transcriptTrim(transcript: Transcript): { start: number; end: number } {+export function transcriptTrim(transcript: Transcript): { start: number; end: number } {const firstWord = transcript.at(0)?.words.at(0);const lastWord = transcript.at(-1)?.words.at(-1);return { start: secondsToFrames(firstWord?.start), end: secondsToFrames(lastWord?.end) };diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.tsindex c80c3abf..0a85cfc4 100644--- a/apps/web/src/utils/jsx.ts+++ b/apps/web/src/utils/jsx.ts@@ -33,7 +33,9 @@ import {setComponent,setKeyframeTrack,transcribeScene,+ transcriptTrim,} from "@/components/engine";+import { resolveTranscript } from "@/components/engine/decoders/caption/utils";import { ChildOf } from "@/components/engine/components";import { ASPECT_RATIO_DIMENSIONS, findEmptyPlacement } from "@/utils/genai";import { resolveAsset, resolveGeneratedAsset } from "@/utils/jsx-generation";@@ -903,7 +905,12 @@ export class WorldDocument implements ProjectDocument<DocumentNode> {const c = world.components;const eid = node.eid;- if (asset.type === 'AUDIO') {+ if (asset.type === 'TRANSCRIPT') {+ const transcript = await resolveTranscript(asset);+ // AssetId set here makes commit's caption phase skip transcription.+ setComponent(world, eid, c.AssetId, asset.id);+ setComponent(world, eid, c.Trim, transcriptTrim(transcript));+ } else if (asset.type === 'AUDIO') {setComponent(world, eid, c.AssetId, asset.id);resizeEntity(world, eid, { width: 500, height: 150 });} else if (asset.type === 'IMAGE' || asset.type === 'VIDEO') {diff --git a/apps/web/src/utils/media.ts b/apps/web/src/utils/media.tsindex b57ee7a0..bfd20f62 100644--- a/apps/web/src/utils/media.ts+++ b/apps/web/src/utils/media.ts@@ -58,6 +58,8 @@ export function mimeTypeToExtension(mimeType: string): string {// Web files'text/html': '.html','application/json': '.json',+ 'application/x-subrip': '.srt',+ 'text/vtt': '.vtt','text/xml': '.xml','application/xml': '.xml',};diff --git a/docs/composition.md b/docs/composition.mdindex c4a74223..76b8f679 100644--- a/docs/composition.md+++ b/docs/composition.md@@ -37,7 +37,7 @@ The first rendered root becomes the active scene. New roots are auto-placed on t| `<video>` / `<image>` | Media; `src` is required. || `<audio>` | Sound only, no spatial props. || `<sequence>` | Track container for back-to-back clips (positions are explicit); hosts transitions. |-| `<captions>` | Transcribes the scene's audio into styled captions. |+| `<captions>` | Styled captions: transcribes the scene's audio, or mounts a transcript given via `src`. || `<solidPaint>`, `<linearGradientPaint>`, `<radialGradientPaint>`, `<colorStop>` | Fills, declared as children; see [Paints](#paints). |User-defined components are ordinary Solid components; only intrinsic elements produce nodes.@@ -213,6 +213,16 @@ Declarations are pure: nothing generates until the mount commits, and refs never<captions preset="spotlight" colors={["#FF0055"]} />```+### Manual captions++`src` supplies the transcript instead of transcribing: a path, URL, or asset id of an `.srt` or `.vtt` subtitle file, or a transcript `.json` (the `dapi transcribe` output format: `[{ text, words: [{ text, start, end }] }]`, times in seconds). Subtitle cues carry no word timings, so word-level presets highlight on timings synthesized within each cue.++```tsx+<captions src="./subtitles.srt" preset="classic" />+```++Times are scene-relative; set `start` to shift the whole caption track when the transcript was written against a clip that begins later in the scene.+| Preset | Style || --- | --- || `"classic"` (default) | Centered lowercase text, soft drop shadow, a few words at a time. |diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex bf5a6325..a1159526 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -122,7 +122,11 @@ export type PatchProps = {transition?: TransitionSpec | null;/** Any CSS color, applied to the node's solid fill (created if absent); alpha is ignored — use `opacity`. */fill?: string;- /** Path, URL, asset id, or a `generate.*` declaration. */+ /**+ * Path, URL, asset id, or a `generate.*` declaration. On `<captions>` a+ * transcript source (.srt, .vtt, or transcript .json) mounted instead of+ * transcribing the scene; `generate.*` is not accepted there.+ */src?: string | AssetRef;/** How the source maps into the box. Default "cover" on `<video>`, "contain" on `<image>`. */objectFit?: Fit;@@ -257,4 +261,4 @@ export type SequenceProps = Pick<PatchProps, "name"> & {children?: SolidJSX.Element;};-export type CaptionsProps = Pick<PatchProps, "preset" | "colors">;+export type CaptionsProps = Pick<PatchProps, "preset" | "colors" | "src" | "start">;