The VS Code of video editing, built for agents.
TypeScript
Add shimmer effect for ongoing caption generation
konstantin-pauluse836ffce14d5
diff --git a/apps/web/src/engine/generate/captions.tsx b/apps/web/src/engine/generate/captions.tsxindex 2c81ebe4..7fab09f9 100644--- a/apps/web/src/engine/generate/captions.tsx+++ b/apps/web/src/engine/generate/captions.tsx@@ -5,12 +5,12 @@/*** Auto-captions: the scene's audible mix, encoded and transcribed, mounted as* the scene's `<captions>`. A scene has no geometry to stand in with — it is- * the frame everything else sits in — so the wait is told in a toast that- * follows the job instead of a pulse.+ * the frame everything else sits in — so the wait is shown as the blue+ * shimmer around it instead of a pulse; only a failure is told in a toast.*/import { Captions } from '@diffusionstudio/reconciler';-import { Caption, getEntityTree, Library } from '@diffusionstudio/runtime';+import { Caption, getEntityTree, Library, Shimmering } from '@diffusionstudio/runtime';import { assetName } from '@diffusionstudio/assets';import { toast } from 'somoto';@@ -19,9 +19,9 @@ import { uploadFile } from '@/lib/uploads';import { getDocumentEditor } from '../editor';import { encodeSceneAudio, hasAudio } from '../scene-audio';+import { shimmer } from './shimmer';import type { AssetLibrary } from '@diffusionstudio/assets';-import type { Job } from '@diffusionstudio/api-contract';import type { Entity, World } from 'koota';const TITLE = 'Caption generation failed';@@ -36,34 +36,25 @@ function captionsName(library: AssetLibrary): string {return `Captions ${max + 1}.json`;}-/** The toast line for a transcription in flight. */-function status(job: Job | undefined): string {- if (!job) return 'Preparing the audio…';- if (job.status === 'queued') return 'Queued';- if (job.phase === 'finalizing' || job.status === 'succeeded') return 'Almost done…';- return 'Transcribing…';-}-/*** Transcribes `scene` and mounts the transcript as its captions: the scene's- * `<captions>` is pointed at the new transcript, or one is added.+ * `<captions>` is pointed at the new transcript, or one is added. A scene+ * already being captioned is left to the run in flight.*/export async function generateCaptions(world: World, scene: Entity, dir?: string): Promise<void> {const library = world.get(Library);- if (!library) return;+ if (!library || scene.has(Shimmering)) return;if (!hasAudio(world, scene)) {toast(TITLE, { description: 'No audio found. Add an audio or video clip to the scene to generate captions.' });return;}- const id = `captions:${scene.id()}`;- toast.loading('Generating captions', { id, description: status(undefined) });+ const release = shimmer(world, scene);+ getDocumentEditor(world).deselect(scene);try {const audio = await uploadFile(await encodeSceneAudio(world, scene, dir));- const job = await runJob({ model: 'universal-3.5-pro', audio }, (update) => {- toast.loading('Generating captions', { id, description: status(update) });- });+ const job = await runJob({ model: 'universal-3.5-pro', audio }, () => {});if (job.status !== 'succeeded') throw new Error(jobFailure(job));const [file] = job.assets;@@ -87,9 +78,10 @@ export async function generateCaptions(world: World, scene: Entity, dir?: stringconst [captions] = editor.insertElement(scene, () => <Captions src={asset.path} />);if (captions) editor.select(captions);}- toast.dismiss(id);} catch (error) {console.error(`[generate] ${TITLE}:`, error);- toast.error(TITLE, { id, description: error instanceof Error ? error.message : String(error) });+ toast.error(TITLE, { description: error instanceof Error ? error.message : String(error) });+ } finally {+ release?.();}}diff --git a/apps/web/src/engine/generate/shimmer.ts b/apps/web/src/engine/generate/shimmer.tsnew file mode 100644index 00000000..6bd36c62--- /dev/null+++ b/apps/web/src/engine/generate/shimmer.ts@@ -0,0 +1,53 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * The blue shimmer around a node that work is running on, for work that has+ * nothing to stand in with on the node itself: a scene is the frame+ * everything else sits in, so captioning it outlines it instead of pulsing.+ *+ * Like a placeholder, the node is held by its `Source` stamp, never by+ * entity: a remount or an undo/redo makes new entities, a stamp survives (and+ * a pending stamp is followed to the real one the write answers with). The+ * trait is put on again every `REFRESH`, so an entity that replaced the old+ * one picks it up.+ */++import { Shimmering, Source, Time } from '@diffusionstudio/runtime';++import { getDocumentEditor } from '../editor';++import type { Entity, World } from 'koota';++/** How often the shimmer is put onto the node's entity. */+const REFRESH = 250; // ms++/**+ * Outlines `entity` with the shimmer until the returned release is called.+ * Returns undefined for an entity the file cannot name.+ */+export function shimmer(world: World, entity: Entity): (() => void) | undefined {+ let source = entity.get(Source)?.value;+ if (!source) return undefined;++ const start = world.get(Time)?.now ?? 0;+ const find = () => world.query(Source).find((candidate) => candidate.isAlive() && candidate.get(Source)!.value === source);++ const apply = () => {+ const target = find();+ if (target && !target.has(Shimmering)) target.add(Shimmering({ start }));+ };++ const unsubscribe = getDocumentEditor(world).onRename((ids) => {+ source = ids[source!] ?? source;+ });+ const timer = setInterval(apply, REFRESH);+ apply();++ return () => {+ clearInterval(timer);+ unsubscribe();+ find()?.remove(Shimmering);+ };+}diff --git a/package-lock.json b/package-lock.jsonindex 80be6d6e..f622b4bb 100644--- a/package-lock.json+++ b/package-lock.json@@ -92,6 +92,7 @@"license": "MPL-2.0","dependencies": {"@diffusionstudio/agent-chat": "*",+ "@diffusionstudio/api-contract": "^0.4.0","@diffusionstudio/assets": "*","@diffusionstudio/dapi": "*","@diffusionstudio/encoder": "*",@@ -772,6 +773,16 @@"resolved": "packages/agent-chat","link": true},+ "node_modules/@diffusionstudio/api-contract": {+ "version": "0.4.0",+ "resolved": "https://registry.npmjs.org/@diffusionstudio/api-contract/-/api-contract-0.4.0.tgz",+ "integrity": "sha512-btw8Opi862XPYNxiAzx43TGj/E6SbmLnzHyuunz3LQIoaJDGXV7b31hsXhfGjeU3gnrg7bz/IFCMUdhDY1gNkA==",+ "license": "MIT",+ "dependencies": {+ "@trpc/server": "^11.10.0",+ "zod": "^4.3.6"+ }+ },"node_modules/@diffusionstudio/assets": {"resolved": "packages/assets","link": true@@ -14172,6 +14183,7 @@"version": "0.1.0","license": "MPL-2.0","dependencies": {+ "@diffusionstudio/api-contract": "^0.4.0","@diffusionstudio/jsx": "*","zod": "^4.4.3"},@@ -14198,7 +14210,7 @@},"packages/jsx": {"name": "@diffusionstudio/jsx",- "version": "0.207.0",+ "version": "0.210.0","license": "MPL-2.0","dependencies": {"koota": "^0.6.6"diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex b34d028b..0865cfc3 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -21,10 +21,10 @@ import {ClipsContent, Geometry, Group, Paint, Color, Caption, ScaleMode, Shader,BlendMode, Effect, Mask, Transition, MixedCornerRadius,LocalTransform, WorldTransform, Computed, Cache,- Host,+ Host, Scene, Shimmering,Mode, FrameRate, Camera, Background, RenderSurface,HitRegions,- Root,+ Root, Time,} from '../traits';import { getParentNode } from '../queries/hierarchy';import { getViewMatrix } from '../queries/camera';@@ -610,6 +610,103 @@ function renderPendingFill(world: World, entity: Entity): void {ctx.fill();}+const SHIMMER_PERIOD = 2400; // ms per turn of the highlights+const SHIMMER_SWEEP = 1400; // ms the light takes to cross the scene after each turn+const SHIMMER_BAND = 1; // width of the light, in diagonals of the scene+const SHIMMER_PRIMARY = [0, 140, 255] as const; // #008CFF+const SHIMMER_DEPTH = 0.7; // how far the highlights darken toward the stage background+const SHIMMER_SWEEP_OPACITY = 0.2;++/**+ * The outline of a scene that work is running on (see `Shimmering`), over+ * everything the scene draws: a primary blue stroke along its edge with two+ * highlights travelling around it that darken toward the stage background,+ * and after each turn a band of translucent primary blue sweeping across the+ * scene, corner to corner. Counted on the world's clock from when+ * the work began. The editor's only, never drawn into an export. The stroke+ * keeps its width on screen whatever the zoom, so it is drawn in device+ * pixels from the scene's corners.+ */+function renderShimmer(world: World, entity: Entity): void {+ if (!entity.has(Shimmering) || !entity.has(Scene)) return;++ const ctx = getCtx(world);+ const computed = store(world, Computed);+ const w = computed.width[entity.id()]!;+ const h = computed.height[entity.id()]!;+ const resolution = world.get(RenderSurface)?.resolution ?? 1;++ const m = ctx.getTransform();+ const quad = [[0, 0], [w, 0], [w, h], [0, h]].map(([x, y]) => m.transformPoint({ x, y }));+ const cx = (quad[0]!.x + quad[1]!.x + quad[2]!.x + quad[3]!.x) / 4;+ const cy = (quad[0]!.y + quad[1]!.y + quad[2]!.y + quad[3]!.y) / 4;+ const trace = () => {+ ctx.beginPath();+ ctx.moveTo(quad[0]!.x, quad[0]!.y);+ ctx.lineTo(quad[1]!.x, quad[1]!.y);+ ctx.lineTo(quad[2]!.x, quad[2]!.y);+ ctx.lineTo(quad[3]!.x, quad[3]!.y);+ ctx.closePath();+ };++ const elapsed = Math.max(0, (world.get(Time)?.now ?? 0) - entity.get(Shimmering)!.start);+ const turn = elapsed % SHIMMER_PERIOD;++ ctx.save();+ ctx.resetTransform();+ ctx.globalAlpha = 1;+ ctx.globalCompositeOperation = 'source-over';+ ctx.filter = 'none';++ if (elapsed >= SHIMMER_PERIOD && turn < SHIMMER_SWEEP) {+ // The band's centre goes from half a band before the first corner to+ // half a band past the opposite one: it enters and leaves the scene+ // exactly, and is on it the whole sweep.+ const t = turn / SHIMMER_SWEEP;+ const eased = (1 - Math.cos(Math.PI * t)) / 2;+ const centre = -SHIMMER_BAND / 2 + eased * (1 + SHIMMER_BAND);+ const dx = quad[2]!.x - quad[0]!.x;+ const dy = quad[2]!.y - quad[0]!.y;+ const from = centre - SHIMMER_BAND / 2;+ const to = centre + SHIMMER_BAND / 2;+ const light = ctx.createLinearGradient(+ quad[0]!.x + dx * from, quad[0]!.y + dy * from,+ quad[0]!.x + dx * to, quad[0]!.y + dy * to,+ );+ const [r, g, b] = SHIMMER_PRIMARY;+ // Full strength across the middle of the band, fading at its edges.+ light.addColorStop(0, `rgba(${r}, ${g}, ${b}, 0)`);+ light.addColorStop(0.3, `rgba(${r}, ${g}, ${b}, ${SHIMMER_SWEEP_OPACITY})`);+ light.addColorStop(0.7, `rgba(${r}, ${g}, ${b}, ${SHIMMER_SWEEP_OPACITY})`);+ light.addColorStop(1, `rgba(${r}, ${g}, ${b}, 0)`);++ trace();+ ctx.fillStyle = light;+ ctx.fill();+ }++ const background = world.get(Root)!.get(Background)?.value ?? 0;+ const backdrop = [(background >> 16) & 0xff, (background >> 8) & 0xff, background & 0xff];+ const primary = `rgb(${SHIMMER_PRIMARY.join(', ')})`;+ const dark = `rgb(${SHIMMER_PRIMARY.map((channel, i) => Math.round(channel + (backdrop[i]! - channel) * SHIMMER_DEPTH)).join(', ')})`;++ const ring = ctx.createConicGradient(turn / SHIMMER_PERIOD * Math.PI * 2, cx, cy);+ for (const start of [0, 0.5]) {+ ring.addColorStop(start, primary);+ ring.addColorStop(start + 0.12, dark);+ ring.addColorStop(start + 0.24, primary);+ }+ ring.addColorStop(1, primary);++ trace();+ ctx.strokeStyle = ring;+ ctx.lineWidth = Math.round(2 * resolution);+ ctx.shadowColor = primary;+ ctx.shadowBlur = 8 * resolution;+ ctx.stroke();+ ctx.restore();+}+// ── WAVEFORM paint ─────────────────────────────────────//// Renders an audio asset's pre-computed peaks as a bar chart inside the@@ -829,6 +926,8 @@ export function renderNode(world: World, entity: Entity): void {renderContent(world, entity, buildEffects(world, entity));}+ renderShimmer(world, entity);+ctx.restore();}diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex f199f8e1..037c0b32 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -49,6 +49,13 @@ export const ClipsContent = trait();export const Generating = trait({ label: '', progress: undefined as number | undefined });+// On a scene that work is running on (being captioned): the stage outlines it+// with the blue shimmer. A scene is the frame everything else sits in, so it+// has nothing to stand in with the way a generating node pulses. `start` is+// when the work began on the world's clock, which the shimmer's turns count+// from.+export const Shimmering = trait({ start: 0 });+export const Name = trait({ value: '' });// Stable identity for entities.