Skip to content

The VS Code of video editing, built for agents.

TypeScript
11.4 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Generations on the canvas, from the request to the file in the library.
 *
 * - A prompt (`generate`) gets one empty geometry per output, placed beside
 *   the work (see ./placement) and brought into view. Each stands in for its
 *   output — the pulse, headed by the job's progress (see ./placeholder) — until
 *   the result lands as the geometry's paint.
 * - A transform (`transform`) is run over the media a node shows on top (see
 *   ./media); the node pulses meanwhile, and the result is laid over the paint
 *   it was made from, which is hidden rather than removed.
 *
 * What a model returns is downloaded into the project library, which only
 * ever holds files that exist: nothing is reserved there for a job in flight,
 * and a job that fails leaves no trace but the toast saying why.
 */

import { Audio, ImagePaint, Rect, VideoPaint, authoredElement, renderAuthored } from '@diffusionstudio/reconciler';
import { AssetId, Cache, getCameraMatrix, getNextName, getParentEntity, Library, Paint, revealRect, Root, Selected, Size, Source } from '@diffusionstudio/runtime';
import { assetName } from '@diffusionstudio/assets';
import { TRPCClientError } from '@trpc/client';
import { toast } from 'somoto';

import { jobFailure, runJob } from '@/lib/jobs';
import { uploadFile } from '@/lib/uploads';

import { getDocumentEditor } from '../editor';
import { findPlacement, PLACEMENT_GAP } from './placement';
import { trackPlaceholder } from './placeholder';
import { topMedia } from './media';

import type { Asset, AssetLibrary } from '@diffusionstudio/assets';
import type { AssetRef, GenerateRequestInput, Job } from '@diffusionstudio/api-contract';
import type { Entity, World } from 'koota';
import type { NodeMedia } from './media';
import type { GenerationState, Placeholder } from './placeholder';

/** What a prompt asks for, as far as the canvas is concerned. */
export interface GenerationOutput {
	/** What each output lands as: a paint of its placeholder, or an `<audio>` in its place. */
	kind: 'image' | 'video' | 'audio';
	/** How many outputs the request makes, each with a placeholder of its own. */
	count: number;
	/** The box of each placeholder; a picture's is corrected to the result's proportions. */
	size: { width: number; height: number };
	/** What a failure is reported under, e.g. "Image generation failed". */
	title: string;
}

/**
 * The progress of one job, as its placeholders show it: "Queued" until the
 * job runs, then "Generating" with the share of its estimate that has passed
 * — counted on to the end, so a job that lands in time goes from a progress
 * straight to its name. Once the estimate is spent there is no progress left
 * to show: it reads "Taking a bit longer" while the model works, and "Almost
 * done" once the model is done and the result is on its way into the project.
 */
class Progress {
	private job: Job | undefined;
	/** When the job was first seen running, on this clock. */
	private since = 0;

	public readonly update = (job: Job): void => {
		if (job.status !== 'queued' && !this.since) this.since = Date.now();
		this.job = job;
	};

	public readonly state = (): GenerationState => {
		const job = this.job;
		if (!job || job.status === 'queued') return { label: 'Queued' };
		const progress = (Date.now() - this.since) / (job.etaSeconds * 1000);
		if (progress < 1) return { label: 'Generating', progress };
		return { label: job.status === 'succeeded' || job.phase === 'finalizing' ? 'Almost done' : 'Taking a bit longer' };
	};
}

/**
 * Generates what `request` asks for into empty geometries on the canvas (see
 * the module comment). `request` may take making — inputs to upload — so it
 * is made once the placeholders are there, from the moment of the prompt.
 */
export async function generate(
	world: World,
	output: GenerationOutput,
	request: GenerateRequestInput | (() => Promise<GenerateRequestInput>),
): Promise<void> {
	const library = world.get(Library);
	const root = world.get(Root);
	if (!library || !root?.get(Source)?.value) {
		toast('Nothing to generate into', { description: 'Open a project first.' });
		return;
	}

	const editor = getDocumentEditor(world);
	const { width, height } = output.size;
	const block = findPlacement(world, output.count * width + (output.count - 1) * PLACEMENT_GAP, height);

	revealRect(world, block);
	editor.reportEdit(root, 'camera', getCameraMatrix(world));

	const progress = new Progress();
	const placeholders: Placeholder[] = [];
	const inserted: Entity[] = [];

	// A picture or a clip keeps its proportions; a sound's box is only where it sits.
	const proportions = output.kind === 'audio' ? {} : { keepAspectRatio: true };

	for (let index = 0; index < output.count; index++) {
		const x = Math.round(block.x + index * (width + PLACEMENT_GAP));
		const y = Math.round(block.y);
		const [entity] = editor.insertElement(root, () => (
			<Rect {...proportions} x={x} y={y} width={width} height={height} />
		));
		const placeholder = entity && trackPlaceholder(world, entity, progress.state);
		if (!placeholder) continue;
		inserted.push(entity);
		placeholders.push(placeholder);
	}
	if (inserted.length) editor.select(inserted);

	try {
		const job = await runJob(typeof request === 'function' ? await request() : request, progress.update);
		if (job.status !== 'succeeded') throw new Error(jobFailure(job));

		const results = await Promise.all(job.assets.map((file) => storeResult(library, job, file)));
		let orphaned = 0;
		placeholders.forEach((placeholder, index) => {
			const entity = placeholder.entity();
			const result = results[index];
			if (!entity) orphaned += result ? 1 : 0;
			else if (result) land(world, entity, result);
			else editor.remove(entity);
		});
		if (orphaned) {
			toast('Generation finished', { description: 'Its placeholder was removed, so the result is in the library only.' });
		}
	} catch (error) {
		editor.remove(placeholders.map((placeholder) => placeholder.entity()).filter((entity) => entity !== undefined));
		report(output.title, error);
	} finally {
		for (const placeholder of placeholders) placeholder.release();
	}
}

/**
 * Runs `request` over the media `media` shows (see the module comment);
 * `request` is handed the uploaded input.
 */
export async function transform(
	world: World,
	media: NodeMedia,
	title: string,
	request: (input: AssetRef) => GenerateRequestInput,
): Promise<void> {
	const library = world.get(Library);
	const input = library?.get(media.paint.get(AssetId)?.value ?? '');
	if (!library || !input) {
		toast(title, { description: 'The media has not loaded yet.' });
		return;
	}

	const progress = new Progress();
	const placeholder = trackPlaceholder(world, media.node, progress.state);
	const paint = media.paint === media.node ? undefined : media.paint.get(Source)?.value;

	try {
		const job = await runJob(request(await uploadAsset(library, input)), progress.update);
		if (job.status !== 'succeeded') throw new Error(jobFailure(job));

		const [file] = job.assets;
		if (!file) throw new Error('The model returned nothing.');
		const result = await storeResult(library, job, file);

		const node = placeholder?.entity();
		if (node) overlay(world, node, paint, result);
		else toast('Generation finished', { description: 'Its element was removed, so the result is in the library only.' });
	} catch (error) {
		report(title, error);
	} finally {
		placeholder?.release();
	}
}

/** Uploads made this session, by library asset id: an input is sent once. */
const uploads = new Map<string, Promise<AssetRef>>();

/** The ref a request names a library asset by, uploading its bytes when the API lacks them. */
export function uploadAsset(library: AssetLibrary, asset: Asset): Promise<AssetRef> {
	let ref = uploads.get(asset.id);
	if (!ref) {
		ref = library.file(asset).then(uploadFile);
		ref.catch(() => uploads.delete(asset.id));
		uploads.set(asset.id, ref);
	}
	return ref;
}

/** Downloads one output of `job` into the library, noting the job it came from. */
async function storeResult(library: AssetLibrary, job: Job, file: Job['assets'][number]): Promise<Asset> {
	const response = await fetch(file.url);
	if (!response.ok) throw new Error(`Could not download the result (${response.status})`);
	const asset = await library.store(await response.blob(), { name: file.filename });
	return library.update(asset, { job: job.id });
}

const stem = (asset: Asset): string => assetName(asset).replace(/\.[^.]+$/, '');

/** Fills a prompt's placeholder with what it stood in for, named after it. */
function land(world: World, placeholder: Entity, asset: Asset): void {
	const editor = getDocumentEditor(world);
	const name = getNextName(world, stem(asset));

	if (asset.type === 'AUDIO') {
		// A sound is an element of its own, not a paint: it takes the
		// placeholder's place, and its selection.
		const parent = getParentEntity(placeholder);
		const props = authoredElement(placeholder)?.props ?? {};
		const selected = placeholder.has(Selected);
		if (!parent) return;
		const [audio] = editor.insertElement(parent, () => (
			<Audio name={name} src={asset.path} x={props.x as number} y={props.y as number} width={props.width as number} height={props.height as number} />
		), placeholder);
		editor.remove(placeholder);
		if (audio && selected) editor.select(audio, { extend: true });
		return;
	}

	editor.insertElement(placeholder, () => (
		asset.type === 'VIDEO' ? <VideoPaint src={asset.path} /> : <ImagePaint src={asset.path} />
	));
	editor.editProperty(placeholder, 'name', name);

	// The box was a guess at the result's proportions (a model may not take
	// the aspect ratio asked for); the width stays, the height follows.
	const size = placeholder.get(Size);
	if ('width' in asset && asset.width > 0 && size) {
		const height = Math.round((size.width * asset.height) / asset.width);
		if (Math.abs(height - size.height) > 1) editor.editProperty(placeholder, 'height', height);
	}
}

/**
 * Lays a transform's result over the paint it was made from, which is hidden
 * — not removed — so the original is a toggle away. Media that is the node's
 * own (`<image src>`) has no paint to hide; its `src` is pointed at the
 * result instead.
 */
function overlay(world: World, node: Entity, paintSource: string | undefined, asset: Asset): void {
	const editor = getDocumentEditor(world);
	const fills = node.get(Cache)?.fills ?? [];
	const paint = (paintSource && fills.find((fill) => fill.get(Source)?.value === paintSource)) || topMedia(node)?.paint;

	if (!paint || paint === node || !paint.has(Paint)) {
		editor.editProperty(node, 'src', asset.path);
		return;
	}

	// The new paint is drawn the way the old one was: fit, opacity, blending.
	const props = { ...authoredElement(paint)?.props };
	delete props.src;
	delete props.hidden;
	editor.editProperty(paint, 'hidden', true);
	editor.insertElement(node, () => renderAuthored({
		tag: asset.type === 'VIDEO' ? 'videoPaint' : 'imagePaint',
		props: { ...props, src: asset.path },
		children: [],
	}));
}

/** Tells the user why a generation came to nothing — unless the upgrade dialog already does. */
function report(title: string, error: unknown): void {
	if (error instanceof TRPCClientError && error.data?.code === 'PAYMENT_REQUIRED') return;
	console.error(`[generate] ${title}:`, error);
	toast.error(title, { description: error instanceof Error ? error.message : String(error) });
}