Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add shimmer effect for ongoing caption generation

konstantin-pauluse836ffce14d5
diff --git a/apps/web/src/engine/generate/captions.tsx b/apps/web/src/engine/generate/captions.tsx
index 2c81ebe4..7fab09f9 100644
--- a/apps/web/src/engine/generate/captions.tsx
+++ b/apps/web/src/engine/generate/captions.tsx
@@ -5,12 +5,12 @@
/**
* Auto-captions: the scene's audible mix, encoded and transcribed, mounted as
* the scene's `<captions>`. A scene has no geometry to stand in with — it is
- * the frame everything else sits in — so the wait is told in a toast that
- * follows the job instead of a pulse.
+ * the frame everything else sits in — so the wait is shown as the blue
+ * shimmer around it instead of a pulse; only a failure is told in a toast.
*/
import { Captions } from '@diffusionstudio/reconciler';
-import { Caption, getEntityTree, Library } from '@diffusionstudio/runtime';
+import { Caption, getEntityTree, Library, Shimmering } from '@diffusionstudio/runtime';
import { assetName } from '@diffusionstudio/assets';
import { toast } from 'somoto';
@@ -19,9 +19,9 @@ import { uploadFile } from '@/lib/uploads';
import { getDocumentEditor } from '../editor';
import { encodeSceneAudio, hasAudio } from '../scene-audio';
+import { shimmer } from './shimmer';
import type { AssetLibrary } from '@diffusionstudio/assets';
-import type { Job } from '@diffusionstudio/api-contract';
import type { Entity, World } from 'koota';
const TITLE = 'Caption generation failed';
@@ -36,34 +36,25 @@ function captionsName(library: AssetLibrary): string {
return `Captions ${max + 1}.json`;
}
-/** The toast line for a transcription in flight. */
-function status(job: Job | undefined): string {
- if (!job) return 'Preparing the audio…';
- if (job.status === 'queued') return 'Queued';
- if (job.phase === 'finalizing' || job.status === 'succeeded') return 'Almost done…';
- return 'Transcribing…';
-}
-
/**
* Transcribes `scene` and mounts the transcript as its captions: the scene's
- * `<captions>` is pointed at the new transcript, or one is added.
+ * `<captions>` is pointed at the new transcript, or one is added. A scene
+ * already being captioned is left to the run in flight.
*/
export async function generateCaptions(world: World, scene: Entity, dir?: string): Promise<void> {
const library = world.get(Library);
- if (!library) return;
+ if (!library || scene.has(Shimmering)) return;
if (!hasAudio(world, scene)) {
toast(TITLE, { description: 'No audio found. Add an audio or video clip to the scene to generate captions.' });
return;
}
- const id = `captions:${scene.id()}`;
- toast.loading('Generating captions', { id, description: status(undefined) });
+ const release = shimmer(world, scene);
+ getDocumentEditor(world).deselect(scene);
try {
const audio = await uploadFile(await encodeSceneAudio(world, scene, dir));
- const job = await runJob({ model: 'universal-3.5-pro', audio }, (update) => {
- toast.loading('Generating captions', { id, description: status(update) });
- });
+ const job = await runJob({ model: 'universal-3.5-pro', audio }, () => {});
if (job.status !== 'succeeded') throw new Error(jobFailure(job));
const [file] = job.assets;
@@ -87,9 +78,10 @@ export async function generateCaptions(world: World, scene: Entity, dir?: string
const [captions] = editor.insertElement(scene, () => <Captions src={asset.path} />);
if (captions) editor.select(captions);
}
- toast.dismiss(id);
} catch (error) {
console.error(`[generate] ${TITLE}:`, error);
- toast.error(TITLE, { id, description: error instanceof Error ? error.message : String(error) });
+ toast.error(TITLE, { description: error instanceof Error ? error.message : String(error) });
+ } finally {
+ release?.();
}
}
diff --git a/apps/web/src/engine/generate/shimmer.ts b/apps/web/src/engine/generate/shimmer.ts
new file mode 100644
index 00000000..6bd36c62
--- /dev/null
+++ b/apps/web/src/engine/generate/shimmer.ts
@@ -0,0 +1,53 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * The blue shimmer around a node that work is running on, for work that has
+ * nothing to stand in with on the node itself: a scene is the frame
+ * everything else sits in, so captioning it outlines it instead of pulsing.
+ *
+ * Like a placeholder, the node is held by its `Source` stamp, never by
+ * entity: a remount or an undo/redo makes new entities, a stamp survives (and
+ * a pending stamp is followed to the real one the write answers with). The
+ * trait is put on again every `REFRESH`, so an entity that replaced the old
+ * one picks it up.
+ */
+
+import { Shimmering, Source, Time } from '@diffusionstudio/runtime';
+
+import { getDocumentEditor } from '../editor';
+
+import type { Entity, World } from 'koota';
+
+/** How often the shimmer is put onto the node's entity. */
+const REFRESH = 250; // ms
+
+/**
+ * Outlines `entity` with the shimmer until the returned release is called.
+ * Returns undefined for an entity the file cannot name.
+ */
+export function shimmer(world: World, entity: Entity): (() => void) | undefined {
+ let source = entity.get(Source)?.value;
+ if (!source) return undefined;
+
+ const start = world.get(Time)?.now ?? 0;
+ const find = () => world.query(Source).find((candidate) => candidate.isAlive() && candidate.get(Source)!.value === source);
+
+ const apply = () => {
+ const target = find();
+ if (target && !target.has(Shimmering)) target.add(Shimmering({ start }));
+ };
+
+ const unsubscribe = getDocumentEditor(world).onRename((ids) => {
+ source = ids[source!] ?? source;
+ });
+ const timer = setInterval(apply, REFRESH);
+ apply();
+
+ return () => {
+ clearInterval(timer);
+ unsubscribe();
+ find()?.remove(Shimmering);
+ };
+}
diff --git a/package-lock.json b/package-lock.json
index 80be6d6e..f622b4bb 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -92,6 +92,7 @@
"license": "MPL-2.0",
"dependencies": {
"@diffusionstudio/agent-chat": "*",
+ "@diffusionstudio/api-contract": "^0.4.0",
"@diffusionstudio/assets": "*",
"@diffusionstudio/dapi": "*",
"@diffusionstudio/encoder": "*",
@@ -772,6 +773,16 @@
"resolved": "packages/agent-chat",
"link": true
},
+ "node_modules/@diffusionstudio/api-contract": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/@diffusionstudio/api-contract/-/api-contract-0.4.0.tgz",
+ "integrity": "sha512-btw8Opi862XPYNxiAzx43TGj/E6SbmLnzHyuunz3LQIoaJDGXV7b31hsXhfGjeU3gnrg7bz/IFCMUdhDY1gNkA==",
+ "license": "MIT",
+ "dependencies": {
+ "@trpc/server": "^11.10.0",
+ "zod": "^4.3.6"
+ }
+ },
"node_modules/@diffusionstudio/assets": {
"resolved": "packages/assets",
"link": true
@@ -14172,6 +14183,7 @@
"version": "0.1.0",
"license": "MPL-2.0",
"dependencies": {
+ "@diffusionstudio/api-contract": "^0.4.0",
"@diffusionstudio/jsx": "*",
"zod": "^4.4.3"
},
@@ -14198,7 +14210,7 @@
},
"packages/jsx": {
"name": "@diffusionstudio/jsx",
- "version": "0.207.0",
+ "version": "0.210.0",
"license": "MPL-2.0",
"dependencies": {
"koota": "^0.6.6"
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
index b34d028b..0865cfc3 100644
--- a/packages/runtime/src/systems/render.ts
+++ b/packages/runtime/src/systems/render.ts
@@ -21,10 +21,10 @@ import {
ClipsContent, Geometry, Group, Paint, Color, Caption, ScaleMode, Shader,
BlendMode, Effect, Mask, Transition, MixedCornerRadius,
LocalTransform, WorldTransform, Computed, Cache,
- Host,
+ Host, Scene, Shimmering,
Mode, FrameRate, Camera, Background, RenderSurface,
HitRegions,
- Root,
+ Root, Time,
} from '../traits';
import { getParentNode } from '../queries/hierarchy';
import { getViewMatrix } from '../queries/camera';
@@ -610,6 +610,103 @@ function renderPendingFill(world: World, entity: Entity): void {
ctx.fill();
}
+const SHIMMER_PERIOD = 2400; // ms per turn of the highlights
+const SHIMMER_SWEEP = 1400; // ms the light takes to cross the scene after each turn
+const SHIMMER_BAND = 1; // width of the light, in diagonals of the scene
+const SHIMMER_PRIMARY = [0, 140, 255] as const; // #008CFF
+const SHIMMER_DEPTH = 0.7; // how far the highlights darken toward the stage background
+const SHIMMER_SWEEP_OPACITY = 0.2;
+
+/**
+ * The outline of a scene that work is running on (see `Shimmering`), over
+ * everything the scene draws: a primary blue stroke along its edge with two
+ * highlights travelling around it that darken toward the stage background,
+ * and after each turn a band of translucent primary blue sweeping across the
+ * scene, corner to corner. Counted on the world's clock from when
+ * the work began. The editor's only, never drawn into an export. The stroke
+ * keeps its width on screen whatever the zoom, so it is drawn in device
+ * pixels from the scene's corners.
+ */
+function renderShimmer(world: World, entity: Entity): void {
+ if (!entity.has(Shimmering) || !entity.has(Scene)) return;
+
+ const ctx = getCtx(world);
+ const computed = store(world, Computed);
+ const w = computed.width[entity.id()]!;
+ const h = computed.height[entity.id()]!;
+ const resolution = world.get(RenderSurface)?.resolution ?? 1;
+
+ const m = ctx.getTransform();
+ const quad = [[0, 0], [w, 0], [w, h], [0, h]].map(([x, y]) => m.transformPoint({ x, y }));
+ const cx = (quad[0]!.x + quad[1]!.x + quad[2]!.x + quad[3]!.x) / 4;
+ const cy = (quad[0]!.y + quad[1]!.y + quad[2]!.y + quad[3]!.y) / 4;
+ const trace = () => {
+ ctx.beginPath();
+ ctx.moveTo(quad[0]!.x, quad[0]!.y);
+ ctx.lineTo(quad[1]!.x, quad[1]!.y);
+ ctx.lineTo(quad[2]!.x, quad[2]!.y);
+ ctx.lineTo(quad[3]!.x, quad[3]!.y);
+ ctx.closePath();
+ };
+
+ const elapsed = Math.max(0, (world.get(Time)?.now ?? 0) - entity.get(Shimmering)!.start);
+ const turn = elapsed % SHIMMER_PERIOD;
+
+ ctx.save();
+ ctx.resetTransform();
+ ctx.globalAlpha = 1;
+ ctx.globalCompositeOperation = 'source-over';
+ ctx.filter = 'none';
+
+ if (elapsed >= SHIMMER_PERIOD && turn < SHIMMER_SWEEP) {
+ // The band's centre goes from half a band before the first corner to
+ // half a band past the opposite one: it enters and leaves the scene
+ // exactly, and is on it the whole sweep.
+ const t = turn / SHIMMER_SWEEP;
+ const eased = (1 - Math.cos(Math.PI * t)) / 2;
+ const centre = -SHIMMER_BAND / 2 + eased * (1 + SHIMMER_BAND);
+ const dx = quad[2]!.x - quad[0]!.x;
+ const dy = quad[2]!.y - quad[0]!.y;
+ const from = centre - SHIMMER_BAND / 2;
+ const to = centre + SHIMMER_BAND / 2;
+ const light = ctx.createLinearGradient(
+ quad[0]!.x + dx * from, quad[0]!.y + dy * from,
+ quad[0]!.x + dx * to, quad[0]!.y + dy * to,
+ );
+ const [r, g, b] = SHIMMER_PRIMARY;
+ // Full strength across the middle of the band, fading at its edges.
+ light.addColorStop(0, `rgba(${r}, ${g}, ${b}, 0)`);
+ light.addColorStop(0.3, `rgba(${r}, ${g}, ${b}, ${SHIMMER_SWEEP_OPACITY})`);
+ light.addColorStop(0.7, `rgba(${r}, ${g}, ${b}, ${SHIMMER_SWEEP_OPACITY})`);
+ light.addColorStop(1, `rgba(${r}, ${g}, ${b}, 0)`);
+
+ trace();
+ ctx.fillStyle = light;
+ ctx.fill();
+ }
+
+ const background = world.get(Root)!.get(Background)?.value ?? 0;
+ const backdrop = [(background >> 16) & 0xff, (background >> 8) & 0xff, background & 0xff];
+ const primary = `rgb(${SHIMMER_PRIMARY.join(', ')})`;
+ const dark = `rgb(${SHIMMER_PRIMARY.map((channel, i) => Math.round(channel + (backdrop[i]! - channel) * SHIMMER_DEPTH)).join(', ')})`;
+
+ const ring = ctx.createConicGradient(turn / SHIMMER_PERIOD * Math.PI * 2, cx, cy);
+ for (const start of [0, 0.5]) {
+ ring.addColorStop(start, primary);
+ ring.addColorStop(start + 0.12, dark);
+ ring.addColorStop(start + 0.24, primary);
+ }
+ ring.addColorStop(1, primary);
+
+ trace();
+ ctx.strokeStyle = ring;
+ ctx.lineWidth = Math.round(2 * resolution);
+ ctx.shadowColor = primary;
+ ctx.shadowBlur = 8 * resolution;
+ ctx.stroke();
+ ctx.restore();
+}
+
// ── WAVEFORM paint ─────────────────────────────────────
//
// Renders an audio asset's pre-computed peaks as a bar chart inside the
@@ -829,6 +926,8 @@ export function renderNode(world: World, entity: Entity): void {
renderContent(world, entity, buildEffects(world, entity));
}
+ renderShimmer(world, entity);
+
ctx.restore();
}
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index f199f8e1..037c0b32 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -49,6 +49,13 @@ export const ClipsContent = trait();
export const Generating = trait({ label: '', progress: undefined as number | undefined });
+// On a scene that work is running on (being captioned): the stage outlines it
+// with the blue shimmer. A scene is the frame everything else sits in, so it
+// has nothing to stand in with the way a generating node pulses. `start` is
+// when the work began on the world's clock, which the shimmer's turns count
+// from.
+export const Shimmering = trait({ start: 0 });
+
export const Name = trait({ value: '' });
// Stable identity for entities.