Skip to content

The VS Code of video editing, built for agents.

TypeScript
3.5 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Auto-captions: the scene's audible mix, encoded and transcribed, mounted as
 * the scene's `<captions>`. A scene has no geometry to stand in with — it is
 * the frame everything else sits in — so the wait is shown as the blue
 * shimmer around it instead of a pulse; only a failure is told in a toast.
 */

import { Captions } from '@diffusionstudio/reconciler';
import { Caption, getEntityTree, Library, Shimmering } from '@diffusionstudio/runtime';
import { assetName } from '@diffusionstudio/assets';
import { toast } from 'somoto';

import { jobFailure, runJob } from '@/lib/jobs';
import { uploadFile } from '@/lib/uploads';

import { getDocumentEditor } from '../editor';
import { encodeSceneAudio, hasAudio } from '../scene-audio';
import { shimmer } from './shimmer';

import type { AssetLibrary } from '@diffusionstudio/assets';
import type { Entity, World } from 'koota';

const TITLE = 'Caption generation failed';

/** The next free `Captions N.json`. */
function captionsName(library: AssetLibrary): string {
	let max = 0;
	for (const asset of library.list()) {
		const match = assetName(asset).match(/^Captions (\d+)\.json$/);
		if (match) max = Math.max(max, Number(match[1]));
	}
	return `Captions ${max + 1}.json`;
}

/**
 * Transcribes `scene` and mounts the transcript as its captions: the scene's
 * `<captions>` is pointed at the new transcript, or one is added. A scene
 * already being captioned is left to the run in flight.
 */
export async function generateCaptions(world: World, scene: Entity, dir?: string): Promise<void> {
	const library = world.get(Library);
	if (!library || scene.has(Shimmering)) return;
	if (!hasAudio(world, scene)) {
		toast(TITLE, { description: 'No audio found. Add an audio or video clip to the scene to generate captions.' });
		return;
	}

	const release = shimmer(world, scene);
	getDocumentEditor(world).deselect(scene);

	try {
		const audio = await uploadFile(await encodeSceneAudio(world, scene, dir));
		const job = await runJob({ model: 'universal-3.5-pro', audio }, () => {});
		if (job.status !== 'succeeded') throw new Error(jobFailure(job));

		const [file] = job.assets;
		if (!file) throw new Error('The transcription returned nothing.');
		const response = await fetch(file.url);
		if (!response.ok) throw new Error(`Could not download the transcript (${response.status})`);
		const transcript = await response.json() as { words: unknown[] }[];
		if (!transcript.some((segment) => segment.words.length > 0)) {
			throw new Error('No speech detected. The audio does not appear to contain recognizable speech.');
		}

		const blob = new Blob([JSON.stringify(transcript)], { type: 'application/json' });
		const asset = library.update(await library.store(blob, { name: captionsName(library) }), { job: job.id });

		const editor = getDocumentEditor(world);
		const existing = getEntityTree(world, scene).find((entity) => entity !== scene && entity.has(Caption));
		if (existing) {
			editor.editProperty(existing, 'src', asset.path);
			editor.select(existing);
		} else {
			const [captions] = editor.insertElement(scene, () => <Captions src={asset.path} />);
			if (captions) editor.select(captions);
		}
	} catch (error) {
		console.error(`[generate] ${TITLE}:`, error);
		toast.error(TITLE, { description: error instanceof Error ? error.message : String(error) });
	} finally {
		release?.();
	}
}