Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(examples): re-cut Relay with flowing motion, Gemini voice and Lyria score

okanaslab1cc065b43f
diff --git a/.changeset/focus-zoom-drift.md b/.changeset/focus-zoom-drift.md
new file mode 100644
index 00000000..77b12846
--- /dev/null
+++ b/.changeset/focus-zoom-drift.md
@@ -0,0 +1,5 @@
+---
+"framefields": minor
+---
+
+FocusZoom takes `drift`: a slow push while each shot holds (zoom gained per second, e.g. `0.012`), so the picture never stops between camera moves. Defaults to 0, which holds still as before.
diff --git a/examples/32_relay/assets/audio.json b/examples/32_relay/assets/audio.json
index fee00bdd..6a88c495 100644
--- a/examples/32_relay/assets/audio.json
+++ b/examples/32_relay/assets/audio.json
@@ -2,49 +2,78 @@
"voice": {
"ask": {
"text": "A client asks a simple question.",
- "prompt": "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product. Read the following: 'A client asks a simple question.'",
+ "input": {
+ "prompt": "A client asks a simple question.",
+ "voice": "Sulafat",
+ "style_instructions": "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent."
+ },
"heard": "A client asks a simple question",
- "seconds": 2.179
+ "seconds": 2.2
},
"inbox": {
"text": "In Relay, it becomes a task in one click.",
- "prompt": "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product. Read the following: 'In Relay, it becomes a task in one click.'",
+ "input": {
+ "prompt": "In Relay, it becomes a task in one click.",
+ "voice": "Sulafat",
+ "style_instructions": "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent."
+ },
"heard": "In Relay, it becomes a task in one click",
- "seconds": 3.001542
+ "seconds": 3.32
},
"owner": {
"text": "An owner.",
- "prompt": "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product. Read the following: 'An owner.'",
+ "input": {
+ "prompt": "An owner.",
+ "voice": "Sulafat",
+ "style_instructions": "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent."
+ },
"heard": "An owner",
- "seconds": 0.76
+ "seconds": 0.9
},
"deadline": {
"text": "A deadline.",
- "prompt": "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product. Read the following: 'A deadline.'",
+ "input": {
+ "prompt": "A deadline.",
+ "voice": "Sulafat",
+ "style_instructions": "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent."
+ },
"heard": "A deadline",
- "seconds": 1
+ "seconds": 1.04
},
"step": {
"text": "And a clear next step.",
- "prompt": "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product. Read the following: 'And a clear next step.'",
- "heard": "And a clear next step",
- "seconds": 1.495
+ "input": {
+ "prompt": "And a clear next step.",
+ "voice": "Sulafat",
+ "style_instructions": "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent."
+ },
+ "heard": "and a clear next step",
+ "seconds": 1.96
},
"board": {
"text": "Maya sees it on the board, ready to start.",
- "prompt": "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product. Read the following: 'Maya sees it on the board, ready to start.'",
- "heard": "Maya sees it on the board ready to start",
- "seconds": 3.03275
+ "input": {
+ "prompt": "Maya sees it on the board, ready to start.",
+ "voice": "Sulafat",
+ "style_instructions": "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent."
+ },
+ "heard": "Maya sees it on the board, ready to start",
+ "seconds": 2.6
},
"close": {
"text": "Relay. Keep client work moving.",
- "prompt": "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product. Read the following: 'Relay. Keep client work moving.'",
- "heard": "Relay. Keep client work moving.",
- "seconds": 3.14
+ "input": {
+ "prompt": "Relay. Keep client work moving.",
+ "voice": "Sulafat",
+ "style_instructions": "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent."
+ },
+ "heard": "Relay. Keep client work moving",
+ "seconds": 2.52
}
},
"music": {
- "seconds": 26.81
+ "prompt": "An instrumental underscore for a 30-second software product film. No vocals, no singing, no spoken words.\nCalm, optimistic and modern at 100 BPM in D major: warm felt piano, a soft plucked synth arpeggio, round sub bass, airy pads and light brushed percussion with a gentle shaker.\n[0:00-0:03] The piano motif alone, intimate.\n[0:03-0:08] A soft pulse enters with the plucked arpeggio.\n[0:08-0:14] Brushed percussion and shaker join; light, focused momentum.\n[0:14-0:20] It lifts: sub bass and a warm pad open up the chorus.\n[0:20-0:26] It resolves on one sustained, warm final chord that rings out.\n[0:26-0:30] Silence.",
+ "seconds": 26
},
"click": {
"seconds": 0.12,
diff --git a/examples/32_relay/src/audio.ts b/examples/32_relay/src/audio.ts
index 9acad2c1..df372cbb 100644
--- a/examples/32_relay/src/audio.ts
+++ b/examples/32_relay/src/audio.ts
@@ -1,22 +1,23 @@
/**
- * The film's generated sound, all from the framefields AI gateway with Seed
- * Audio 1.0: a narrator, a music bed and the mouse click. `pnpm audio` makes
- * the files (media stays out of git) and records what it measured in
- * assets/audio.json, which is committed so the film can be timed without them.
+ * The film's generated sound, all through the framefields AI gateway: Gemini
+ * 3.8 Flash TTS reads the narration, Lyria 3.5 writes the music, and Seed
+ * Audio 1.0 makes the mouse click. `pnpm audio` makes the files (media stays
+ * out of git) and records what it measured in assets/audio.json, which is
+ * committed so the film can be timed without them.
*/
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import { ASSETS } from "./theme.js";
-export const AUDIO_MODEL = "seed-audio/1.0";
+export const VOICE_MODEL = "google/gemini-3.8-flash-tts";
+export const MUSIC_MODEL = "google/lyria-3.5";
+export const CLICK_MODEL = "seed-audio/1.0";
export const TRANSCRIBER = "elevenlabs/speech-to-text/scribe-v2";
-/** Read most naturally of the presets tried: accurate, about 2.9 words a second. */
-export const VOICE = "cedric_en_zh";
-export const DIRECTION =
- "Warm, clear and conversational, at an easy natural pace with no long pauses, like a founder showing a friend their product.";
-export const voicePrompt = (text: string) =>
- `${DIRECTION} Read the following: '${text}'`;
+/** Gemini's warm female voice. */
+export const VOICE = "Sulafat";
+export const STYLE =
+ "A warm, confident American woman narrating a calm product film. Natural and conversational, smiling slightly, at an easy pace with no long pauses. Standard American English accent.";
/** The narration, in the order it is heard. */
export const LINES = {
@@ -32,22 +33,26 @@ export const LINES = {
export type LineId = keyof typeof LINES;
export const LINE_IDS = Object.keys(LINES) as LineId[];
+/** What the voice model is asked for one line: its text and the shared delivery. */
+export const voiceInput = (id: LineId) => ({
+ prompt: LINES[id],
+ voice: VOICE,
+ style_instructions: STYLE,
+});
+
/**
- * Music, laid out on the film's sections. The gateway turns away Seed Audio
- * prompts with nothing to say (every music-only prompt fails its content
- * check), so the music is a scene that ends on one spoken word, `MUSIC_CUE`,
- * and is cut off just before it.
+ * Music laid out on the film's sections, instrumental throughout, a little
+ * longer than the film so it can be cut and faded on the last frame.
*/
export const MUSIC_SECONDS = 26;
-export const MUSIC_CUE = "Relay.";
-export const MUSIC_PROMPT = `A bright, quiet design studio in the morning.
-Music: a calm, optimistic, minimal product film underscore at 96 BPM in D major. Warm felt piano, a soft plucked synth arpeggio, round sub bass and light brushed percussion.
-Total duration: 0:30.
-[0:00-0:03] Music only: the piano motif alone.
-[0:03-0:14] Music only: a gentle pulse enters with the plucked arpeggio and brushed percussion.
-[0:14-0:20] Music only: it lifts as the sub bass and a warm pad join.
-[0:20-0:${MUSIC_SECONDS}] Music only: it resolves on one sustained final chord that rings out softly.
-[0:27-0:30] A calm male narrator says: "${MUSIC_CUE}"`;
+export const MUSIC_PROMPT = `An instrumental underscore for a 30-second software product film. No vocals, no singing, no spoken words.
+Calm, optimistic and modern at 100 BPM in D major: warm felt piano, a soft plucked synth arpeggio, round sub bass, airy pads and light brushed percussion with a gentle shaker.
+[0:00-0:03] The piano motif alone, intimate.
+[0:03-0:08] A soft pulse enters with the plucked arpeggio.
+[0:08-0:14] Brushed percussion and shaker join; light, focused momentum.
+[0:14-0:20] It lifts: sub bass and a warm pad open up the chorus.
+[0:20-0:26] It resolves on one sustained, warm final chord that rings out.
+[0:26-0:30] Silence.`;
export const CLICK_PROMPT = `Generate sound effects only. No voice, no music.
One single crisp click of a computer mouse button, recorded close and dry in a quiet room, then silence.`;
@@ -57,16 +62,16 @@ export interface AudioManifest {
LineId,
{
text: string;
- /** What Seed Audio was asked, so a changed direction is read again. */
- prompt: string;
+ /** What the voice model was asked, so a changed line or delivery is read again. */
+ input: ReturnType<typeof voiceInput>;
/** What the transcriber heard, which must match `text` word for word. */
heard: string;
/** Trimmed to the speech: 60 ms before the first word, 300 ms after the last. */
seconds: number;
}
>;
- /** Everything before the spoken cue. */
- music: { seconds: number };
+ /** The prompt it was made from, and its length once cut to the film. */
+ music: { prompt: string; seconds: number };
click: {
seconds: number;
/** Share of the click's energy above 1.5 kHz: a click is bright, a thump is not. */
diff --git a/examples/32_relay/src/backdrop.ts b/examples/32_relay/src/backdrop.ts
new file mode 100644
index 00000000..e0ec83b6
--- /dev/null
+++ b/examples/32_relay/src/backdrop.ts
@@ -0,0 +1,51 @@
+/**
+ * The wall behind everything: the warm gradient, with two soft glows (one
+ * daylight, one a faint tint of the accent) drifting slowly across it, so
+ * even a held frame breathes.
+ */
+import { Color, Layer, LayerAnimation, type LayoutNode } from "framefields";
+import { ACCENT, H, W, WALL, WALL_LIGHT } from "./theme.js";
+
+function glow(
+ color: string,
+ o: { x: number; y: number; size: number; seed: number },
+): LayoutNode {
+ return Layer.shape("circle", {
+ position: "absolute",
+ x: o.x - o.size / 2,
+ y: o.y - o.size / 2,
+ width: o.size,
+ height: o.size,
+ fill: `radial-gradient(circle, ${color} 0%, transparent 70%)`,
+ }).animate(
+ LayerAnimation.create()
+ .wiggle("x", { frequency: 0.12, amplitude: 160, seed: o.seed })
+ .wiggle("y", { frequency: 0.1, amplitude: 90, seed: o.seed + 1 }),
+ );
+}
+
+export function backdrop(): LayoutNode {
+ return Layer.box({
+ position: "absolute",
+ x: 0,
+ y: 0,
+ width: W,
+ height: H,
+ overflow: "hidden",
+ background: `radial-gradient(ellipse at 50% 30%, ${WALL_LIGHT} 0%, ${WALL} 75%)`,
+ children: [
+ glow(Color.alpha("#ffffff", 0.75), {
+ x: W * 0.28,
+ y: H * 0.2,
+ size: 1500,
+ seed: 11,
+ }),
+ glow(Color.alpha(ACCENT, 0.09), {
+ x: W * 0.78,
+ y: H * 0.85,
+ size: 1400,
+ seed: 29,
+ }),
+ ],
+ });
+}
diff --git a/examples/32_relay/src/components/focus-zoom/index.ts b/examples/32_relay/src/components/focus-zoom/index.ts
index a805394c..622ed82f 100644
--- a/examples/32_relay/src/components/focus-zoom/index.ts
+++ b/examples/32_relay/src/components/focus-zoom/index.ts
@@ -100,6 +100,8 @@ export const focusZoom = /* @__PURE__ */ defineComponent({
smoothing: z.number().min(0).max(1).default(0.5),
/** Space kept around click points / targets. */
padding: z.number().min(0).default(160),
+ /** Slow push while a shot holds: zoom gained per second (0.02 = 2 %). */
+ drift: z.number().min(0).max(0.2).default(0),
/** Never show past the content's edges. */
clamp: z.boolean().default(true),
/** When zoomed, keep the view inside this region (e.g. the app window). */
@@ -204,6 +206,7 @@ export const focusZoom = /* @__PURE__ */ defineComponent({
fps,
shots,
smoothing: props.smoothing,
+ drift: props.drift,
clamp: props.clamp,
bounds,
});
diff --git a/examples/32_relay/src/components/focus-zoom/plan.ts b/examples/32_relay/src/components/focus-zoom/plan.ts
index aa2863e6..eaad49a6 100644
--- a/examples/32_relay/src/components/focus-zoom/plan.ts
+++ b/examples/32_relay/src/components/focus-zoom/plan.ts
@@ -36,6 +36,11 @@ export interface CameraInput {
/** 0 = quick moves, 1 = slow, soft moves. */
smoothing: number;
clamp: boolean;
+ /**
+ * Slow push while a shot holds: zoom gained per second (0.02 = 2 %), so
+ * the picture keeps moving between moves. 0 holds perfectly still.
+ */
+ drift?: number;
/** The region the view must stay inside when zoomed (default: the whole view). */
bounds?: { x: number; y: number; width: number; height: number };
}
@@ -125,7 +130,8 @@ export function solveCamera(input: CameraInput): CameraPlan {
const s = input.shots[index];
if (!s) return full;
const c = typeof s.center === "function" ? s.center(frame) : s.center;
- return clampState({ cx: c[0], cy: c[1], zoom: s.zoom });
+ const push = 1 + (input.drift ?? 0) * Math.max(0, (frame - s.start) / fps);
+ return clampState({ cx: c[0], cy: c[1], zoom: s.zoom * push });
};
// Move length: a base set by `smoothing`, longer for bigger zoom changes.
diff --git a/examples/32_relay/src/film.test.ts b/examples/32_relay/src/film.test.ts
index 6399d797..52e94017 100644
--- a/examples/32_relay/src/film.test.ts
+++ b/examples/32_relay/src/film.test.ts
@@ -15,21 +15,18 @@ describe("relay", () => {
expect(film.durationMs).toBe(25_000);
}, 60_000);
- it("holds the camera still through every click", async () => {
+ it("never moves the camera through a click, past its slow drift", async () => {
const { camera, clicks } = await buildRelay();
expect(clicks).toHaveLength(8);
+ const moves = camera.cues.shots.map((s) => [s.frame, s.settled] as const);
for (const frame of clicks) {
// From just before the press until the button is back up.
- const before = camera.cameraAt(frame - 7);
- const after = camera.cameraAt(frame + 7);
- expect(after.zoom, `zoom at click ${frame}`).toBeCloseTo(before.zoom, 3);
- expect(
- Math.hypot(
- after.center[0] - before.center[0],
- after.center[1] - before.center[1],
- ),
- `pan at click ${frame}`,
- ).toBeLessThan(0.5);
+ const [from, to] = [frame - 7, frame + 7];
+ const moving = moves.filter(([start, end]) => start < to && end > from);
+ expect(moving, `camera move over click ${frame}`).toEqual([]);
+ // What is left is the drift: under 1 % of zoom across a press.
+ const ratio = camera.cameraAt(to).zoom / camera.cameraAt(from).zoom;
+ expect(ratio, `zoom at click ${frame}`).toBeLessThan(1.01);
}
}, 60_000);
diff --git a/examples/32_relay/src/film.ts b/examples/32_relay/src/film.ts
index 4d663a0b..38367a87 100644
--- a/examples/32_relay/src/film.ts
+++ b/examples/32_relay/src/film.ts
@@ -23,6 +23,7 @@ import {
readManifest,
voicePath,
} from "./audio.js";
+import { backdrop } from "./backdrop.js";
import { AppFrame } from "./components/app-frame/index.js";
import { FocusZoom } from "./components/focus-zoom/index.js";
import {
@@ -42,7 +43,6 @@ import {
sec,
W,
WALL,
- WALL_LIGHT,
} from "./theme.js";
import { captions, endCard, handover, opening } from "./titles.js";
import { relayApp } from "./ui/app.js";
@@ -324,6 +324,11 @@ export async function buildRelay(): Promise<Relay> {
bounds: FRAME,
length: END_CARD,
fps: FPS,
+ // Long, soft moves, and a slow push while each shot holds: the picture
+ // never stops moving.
+ smoothing: 0.8,
+ drift: 0.012,
+ smear: "standard",
shots: [
{
at: settled(t.select),
@@ -365,8 +370,19 @@ export async function buildRelay(): Promise<Relay> {
y: 0,
width: W,
height: H,
+ perspective: 2200,
children: [camera],
}).animate(
+ LayerAnimation.keys("rotateX", [
+ [0, 22],
+ [WINDOW_IN, 22, "hold"],
+ [LANDED - 2, 0, "expo.out"],
+ ]),
+ LayerAnimation.keys("rotateY", [
+ [0, -18],
+ [WINDOW_IN, -18, "hold"],
+ [LANDED - 2, 0, "expo.out"],
+ ]),
LayerAnimation.keys("opacity", [
[0, 1],
[SCENE_OUT, 1, "hold"],
@@ -397,20 +413,12 @@ export async function buildRelay(): Promise<Relay> {
fonts: [`${FONTS}Inter.ttf`],
});
comp.add([
- Layer.box({
- position: "absolute",
- x: 0,
- y: 0,
- width: W,
- height: H,
- background: `radial-gradient(ellipse at 50% 30%, ${WALL_LIGHT} 0%, ${WALL} 75%)`,
- }),
+ backdrop(),
scene,
...opening(FLY, LANDED, targets.request),
captions(
["An owner.", "A deadline.", "A clear next step."],
[t.maya.at + 3, t.friday.at + 3, checklistItemAt(t, 0)],
- modalIn(t) + sec(1),
t.save.down,
),
...handover(onScreen, CARD_RADIUS * zoomed, SCENE_OUT, END_CARD),
diff --git a/examples/32_relay/src/generate-audio.ts b/examples/32_relay/src/generate-audio.ts
index e727c653..d6532793 100644
--- a/examples/32_relay/src/generate-audio.ts
+++ b/examples/32_relay/src/generate-audio.ts
@@ -3,11 +3,12 @@
* and what was measured in assets/audio.json
* pnpm audio cost → print the estimate, no calls
*
- * Seed Audio 1.0 reads each line of the narration, plays the music bed and
- * makes the click. Scribe v2 checks that every line was read as written and
- * finds its first and last word, so each line is trimmed to its speech and
- * the film can start it exactly where it belongs. Idempotent: delete a file
- * to make it again. Raw takes are kept in assets/takes/.
+ * Gemini 3.8 Flash TTS reads each line of the narration, Lyria 3.5 plays the
+ * music bed and Seed Audio 1.0 makes the click. Scribe v2 checks that every
+ * line was read as written and finds its first and last word, so each line is
+ * trimmed to its speech and the film can start it exactly where it belongs;
+ * it also checks the music has no voice in it. Idempotent: delete a file to
+ * make it again. Raw takes are kept in assets/takes/.
*/
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
@@ -16,28 +17,33 @@ import fs from "node:fs/promises";
import path from "node:path";
import { ai } from "framefields";
import {
- AUDIO_MODEL,
type AudioManifest,
+ CLICK_MODEL,
CLICK_PATH,
CLICK_PROMPT,
LINE_IDS,
LINES,
type LineId,
MANIFEST,
- MUSIC_CUE,
+ MUSIC_MODEL,
MUSIC_PATH,
MUSIC_PROMPT,
MUSIC_SECONDS,
TRANSCRIBER,
- VOICE,
VOICE_LEAD,
+ VOICE_MODEL,
+ voiceInput,
voicePath,
- voicePrompt,
} from "./audio.js";
import { ASSETS } from "./theme.js";
-/** USD per minute of audio, from the gateway's published rates (November 2026). */
-const RATE = { [AUDIO_MODEL]: 0.234375, [TRANSCRIBER]: 0.01 };
+/**
+ * The gateway's rates (November 2026: fal's price plus 25 %): per minute of
+ * audio, per 1000 characters read, and per music track.
+ */
+const PER_MINUTE = { [CLICK_MODEL]: 0.234375, [TRANSCRIBER]: 0.0125 };
+const PER_1000_CHARACTERS = 0.05625;
+const PER_TRACK = 0.125;
const SAMPLE_RATE = 48000;
const MAX_TAKES = 4;
const TAKES = path.join(ASSETS, "takes");
@@ -50,29 +56,32 @@ const MAX_PAUSE = 0.45;
const MIN_BRIGHTNESS = 0.25;
/** A take's file, named by what was asked for, so a changed prompt is made afresh. */
-const takePath = (kind: string, input: Record<string, unknown>, take: number) =>
+const takePath = (
+ kind: string,
+ input: Record<string, unknown>,
+ take: number,
+ ext = "wav",
+) =>
path.join(
TAKES,
- `${kind}-${createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 8)}-${take}.wav`,
+ `${kind}-${createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 8)}-${take}.${ext}`,
);
let spent = 0;
-const bill = (model: keyof typeof RATE, seconds: number) => {
- spent += (RATE[model] * seconds) / 60;
-};
+/** Runs `model` and saves the audio it returns to `dest`. */
async function generate(
+ model: string,
input: Record<string, unknown>,
dest: string,
): Promise<void> {
- const { data } = await ai.subscribe<{ audio: { url: string } }>(AUDIO_MODEL, {
- input: { output_format: "wav", sample_rate: SAMPLE_RATE, ...input },
+ const { data } = await ai.subscribe<{ audio: { url: string } }>(model, {
+ input,
});
const res = await fetch(data.audio.url);
if (!res.ok) throw new Error(`GET ${data.audio.url} → ${res.status}`);
await fs.mkdir(path.dirname(dest), { recursive: true });
await fs.writeFile(dest, Buffer.from(await res.arrayBuffer()));
- bill(AUDIO_MODEL, duration(dest));
}
interface Word {
@@ -93,7 +102,7 @@ async function transcribe(file: string): Promise<Word[]> {
tag_audio_events: false,
},
});
- bill(TRANSCRIBER, duration(file));
+ spent += (PER_MINUTE[TRANSCRIBER] * duration(file)) / 60;
return data.words
.filter((w) => w.type === "word")
.map(({ text, start, end }) => ({ text: text.trim(), start, end }));
@@ -187,9 +196,12 @@ const words = (s: string) =>
async function voiceLine(id: LineId): Promise<AudioManifest["voice"][LineId]> {
const text = LINES[id];
for (let take = 1; take <= MAX_TAKES; take++) {
- const input = { prompt: voicePrompt(text), voice: VOICE };
+ const input = voiceInput(id);
const raw = takePath(`voice-${id}`, input, take);
- if (!existsSync(raw)) await generate(input, raw);
+ if (!existsSync(raw)) {
+ await generate(VOICE_MODEL, input, raw);
+ spent += (PER_1000_CHARACTERS * text.length) / 1000;
+ }
const heard = await transcribe(raw);
const said = heard.map((w) => w.text).join(" ");
const first = heard[0];
@@ -219,7 +231,7 @@ async function voiceLine(id: LineId): Promise<AudioManifest["voice"][LineId]> {
]);
const seconds = duration(voicePath(id));
console.log(`voice ${id} ${seconds.toFixed(2)} s "${said}"`);
- return { text, prompt: input.prompt, heard: said, seconds };
+ return { text, input, heard: said, seconds };
}
throw new Error(
`No take of "${text}" was read as written in ${MAX_TAKES} tries.`,
@@ -227,34 +239,31 @@ async function voiceLine(id: LineId): Promise<AudioManifest["voice"][LineId]> {
}
async function music(): Promise<AudioManifest["music"]> {
- const input = { prompt: MUSIC_PROMPT, multilingual: true };
+ const input = { prompt: MUSIC_PROMPT };
for (let take = 1; take <= MAX_TAKES; take++) {
- const raw = takePath("music", input, take);
- if (!existsSync(raw)) await generate(input, raw);
+ const raw = takePath("music", input, take, "mp3");
+ if (!existsSync(raw)) {
+ await generate(MUSIC_MODEL, input, raw);
+ spent += PER_TRACK;
+ }
+ const length = duration(raw);
const heard = await transcribe(raw);
- const cue = heard[0];
- if (
- heard.length !== 1 ||
- !cue ||
- words(cue.text) !== words(MUSIC_CUE) ||
- cue.start < MUSIC_SECONDS
- ) {
+ if (heard.length > 0 || length < MUSIC_SECONDS) {
console.log(
- `music take ${take} heard "${heard.map((w) => `${w.text}@${w.start.toFixed(1)}`).join(" ")}", retrying`,
+ `music take ${take}: ${length.toFixed(1)} s, heard "${heard.map((w) => w.text).join(" ")}", retrying`,
);
continue;
}
- const end = cue.start - 0.25;
- cut(raw, MUSIC_PATH, 0, end, [
- `afade=t=out:st=${(end - 0.5).toFixed(3)}:d=0.5`,
+ cut(raw, MUSIC_PATH, 0, MUSIC_SECONDS, [
+ `afade=t=out:st=${(MUSIC_SECONDS - 1.5).toFixed(3)}:d=1.5`,
"loudnorm=I=-18:TP=-2:LRA=11",
]);
const seconds = duration(MUSIC_PATH);
- console.log(`music ${seconds.toFixed(1)} s before the cue`);
- return { seconds };
+ console.log(`music ${seconds.toFixed(1)} s of ${length.toFixed(1)} s`);
+ return { prompt: MUSIC_PROMPT, seconds };
}
throw new Error(
- `No music take held its cue past ${MUSIC_SECONDS} s in ${MAX_TAKES} tries.`,
+ `No instrumental music take of ${MUSIC_SECONDS} s or more in ${MAX_TAKES} tries.`,
);
}
@@ -290,7 +299,14 @@ async function click(): Promise<AudioManifest["click"]> {
for (let take = 1; take <= MAX_TAKES; take++) {
const input = { prompt: CLICK_PROMPT };
const raw = takePath("click", input, take);
- if (!existsSync(raw)) await generate(input, raw);
+ if (!existsSync(raw)) {
+ await generate(
+ CLICK_MODEL,
+ { output_format: "wav", sample_rate: SAMPLE_RATE, ...input },
+ raw,
+ );
+ spent += (PER_MINUTE[CLICK_MODEL] * duration(raw)) / 60;
+ }
const { onset, gainDb } = strike(decode(raw));
const from = Math.max(0, onset - 0.002);
cut(raw, CLICK_PATH, from, from + CLICK_SECONDS, [
@@ -332,7 +348,8 @@ async function main(): Promise<void> {
const voice: Partial<AudioManifest["voice"]> = {};
for (const id of LINE_IDS) {
const kept = manifest.voice?.[id];
- if (kept?.prompt === voicePrompt(LINES[id]) && existsSync(voicePath(id))) {
+ const same = JSON.stringify(kept?.input) === JSON.stringify(voiceInput(id));
+ if (kept && same && existsSync(voicePath(id))) {
voice[id] = kept;
continue;
}
@@ -340,7 +357,7 @@ async function main(): Promise<void> {
await save({ ...manifest, voice: voice as AudioManifest["voice"] });
}
manifest.voice = voice as AudioManifest["voice"];
- if (!manifest.music || !existsSync(MUSIC_PATH)) {
+ if (manifest.music?.prompt !== MUSIC_PROMPT || !existsSync(MUSIC_PATH)) {
manifest.music = await music();
await save(manifest);
}
@@ -352,14 +369,15 @@ async function main(): Promise<void> {
}
if (process.argv[2] === "cost") {
+ const characters = Object.values(LINES).join("").length;
const speech = Object.values(LINES).join(" ").split(" ").length / 2.9;
- const generated = speech + MUSIC_SECONDS + 5;
const usd =
- (RATE[AUDIO_MODEL] * generated +
- RATE[TRANSCRIBER] * (speech + MUSIC_SECONDS)) /
- 60;
+ (PER_1000_CHARACTERS * characters) / 1000 +
+ PER_TRACK +
+ (PER_MINUTE[CLICK_MODEL] * 5) / 60 +
+ (PER_MINUTE[TRANSCRIBER] * (speech + 30)) / 60;
console.log(
- `estimate $${usd.toFixed(2)} per pass (${speech.toFixed(0)} s of speech, ${MUSIC_SECONDS} s of music, a click), up to ${MAX_TAKES}× with retries`,
+ `estimate $${usd.toFixed(2)} per pass (${characters} characters of narration, a ${MUSIC_SECONDS} s music track, a click), up to ${MAX_TAKES}× with retries`,
);
} else {
await main();
diff --git a/examples/32_relay/src/titles.ts b/examples/32_relay/src/titles.ts
index d8f4e647..fbf249a1 100644
--- a/examples/32_relay/src/titles.ts
+++ b/examples/32_relay/src/titles.ts
@@ -14,10 +14,15 @@ import { APP, H, INK, REGULAR, SANS, SEMIBOLD, SOFT, sec, W } from "./theme.js";
import { MARK_RADIUS, relayMark } from "./ui/brand.js";
import { shell } from "./ui/handoffs.js";
import { messageRow, REQUEST_MESSAGE } from "./ui/inbox.js";
-import { type Rect, raise, shown, text } from "./ui/kit.js";
+import { type Rect, raise } from "./ui/kit.js";
import { LIST_W, ROW_H } from "./ui/layout.js";
const ARRIVE = "power3.inOut";
+/**
+ * The vertical leg of the flight: later and steeper than the horizontal one,
+ * so the card travels a curve, not a straight line, into its row.
+ */
+const DIVE = "expo.inOut";
/**
* A line whose words rise out of a soft blur one after another, starting at
@@ -82,6 +87,11 @@ export function opening(fly: number, landed: number, row: Rect): LayoutNode[] {
[0, 312],
[fly - 26, 312, "hold"],
[fly - 12, 296, "power2.in"],
+ ])
+ // A slow push while it reads.
+ .keys("scale", [
+ [0, 0.98],
+ [fly - 12, 1.03, "sine.inOut"],
]),
);
const card = raise(
@@ -113,35 +123,46 @@ export function opening(fly: number, landed: number, row: Rect): LayoutNode[] {
[landed, row.x, ARRIVE],
])
.keys("y", [
- [0, y + 24],
- [cardIn, y + 24, "hold"],
- [cardIn + sec(0.7), y, "expo.out"],
- [fly, y, "hold"],
- [landed, row.y, ARRIVE],
+ [0, y + 40],
+ [cardIn, y + 40, "hold"],
+ [cardIn + sec(0.8), y, "expo.out"],
+ // It floats up a little while it waits, then lifts before diving.
+ [fly - 4, y - 10, "sine.inOut"],
+ [landed, row.y, DIVE],
])
.keys("scale", [
- [0, k],
- [fly, k, "hold"],
+ [0, k * 0.92],
+ [cardIn, k * 0.92, "hold"],
+ [cardIn + sec(0.8), k, "expo.out"],
+ [fly, k * 1.02, "sine.inOut"],
[landed, 1, ARRIVE],
+ ])
+ // Banks into the turn and levels out as it lands.
+ .keys("rotation", [
+ [0, 0],
+ [fly, 0, "hold"],
+ [fly + (landed - fly) * 0.45, -2.5, "power2.inOut"],
+ [landed, 0, "power2.out"],
]),
);
return [headline, card];
}
/**
- * One line, three promises, each lighting up as the modal delivers it:
- * dim until its frame in `lights`, then full white.
+ * One line, three promises, each arriving as the modal delivers it: the pill
+ * appears with the first, widens for each next one as its words rise in, and
+ * leaves at `close`.
*/
export function captions(
phrases: string[],
lights: number[],
- open: number,
close: number,
): LayoutNode {
const size = 30;
const gap = 14;
const padX = 32;
const height = 64;
+ const grow = 14;
const widths = phrases.map(
(p) =>
FontManager.measure(p, {
@@ -150,43 +171,90 @@ export function captions(
fontWeight: SEMIBOLD,
}).width,
);
- const width =
- widths.reduce((a, b) => a + b, 0) + gap * (phrases.length - 1) + 2 * padX;
+ // The pill's width once the first `n` phrases are in.
+ const widthWith = (n: number) =>
+ widths.slice(0, n).reduce((a, b) => a + b, 0) + gap * (n - 1) + 2 * padX;
const y = H - 56 - height;
+ const open = lights[0] ?? 0;
let cursor = padX;
const words = phrases.map((p, i) => {
- const node = text(p, {
+ const at = (lights[i] ?? 0) - open;
+ const node = Layer.text(p, {
+ position: "absolute",
x: cursor,
y: 15,
- size,
- strong: true,
- color: "#ffffff",
+ fontFamily: SANS,
+ fontSize: size,
+ fontWeight: SEMIBOLD,
+ fill: "#ffffff",
+ animators: [
+ TextAnimator.waveRise({
+ unit: "word",
+ y: size * 0.5,
+ rotationX: 0,
+ opacity: 0,
+ blur: 6,
+ easing: "expo.in",
+ }),
+ ],
}).animate(
- LayerAnimation.keys("opacity", [
- [0, 0.38],
- [lights[i] ?? 0, 0.38, "hold"],
- [(lights[i] ?? 0) + 8, 1, "power2.out"],
- ]),
+ LayerAnimation.create().kineticSweep(
+ -1,
+ 1,
+ at + 2,
+ at + 16,
+ "power2.out",
+ ),
);
cursor += (widths[i] ?? 0) + gap;
return node;
});
+ const steps = lights
+ .slice(1)
+ .flatMap((light, i): Array<[number, number, string?]> => {
+ const before = widthWith(i + 1);
+ const after = widthWith(i + 2);
+ return [
+ [light - open, before, "hold"],
+ [light - open + grow, after, "expo.out"],
+ ];
+ });
+ const xs = steps.map(([f, w, e]): [number, number, string?] => [
+ f,
+ (W - w) / 2,
+ e,
+ ]);
+ const first = widthWith(1);
return Layer.box({
position: "absolute",
- x: (W - width) / 2,
+ x: (W - first) / 2,
y,
- width,
+ width: first,
height,
borderRadius: height / 2,
background: INK,
+ overflow: "hidden",
+ startFrame: open,
+ durationFrames: close - open + 8,
children: words,
}).animate(
- shown(open, close, 8),
- LayerAnimation.keys("y", [
- [0, y + 14],
- [open, y + 14, "hold"],
- [open + sec(0.5), y, "expo.out"],
- ]),
+ LayerAnimation.create()
+ .keys("width", [[0, first], ...steps])
+ .keys("x", [[0, (W - first) / 2], ...xs])
+ .keys("opacity", [
+ [0, 0],
+ [8, 1, "power2.out"],
+ [close - open, 1, "hold"],
+ [close - open + 8, 0, "power2.in"],
+ ])
+ .keys("y", [
+ [0, y + 18],
+ [sec(0.6), y, "expo.out"],
+ ])
+ .keys("scale", [
+ [0, 0.9],
+ [sec(0.6), 1, "expo.out"],
+ ]),
);
}
@@ -238,7 +306,8 @@ export function handover(
/**
* "Relay. Keep client work moving." The mark is there from the first frame
* (the card has just become it): its dot appears and its arrow draws as the
- * name slides out beside it and the line rises underneath.
+ * name slides out beside it and the line rises underneath, while the whole
+ * card pushes in slowly.
*/
export function endCard(at: number, length: number): LayoutNode {
const mark = endMark();
@@ -286,5 +355,11 @@ export function endCard(at: number, length: number): LayoutNode {
at: sec(0.4),
}),
],
- });
+ }).animate(
+ // A slow push for as long as it holds, so the last shot still moves.
+ LayerAnimation.keys("scale", [
+ [0, 1],
+ [length, 1.05, "power1.out"],
+ ]),
+ );
}
diff --git a/examples/32_relay/src/ui/board.ts b/examples/32_relay/src/ui/board.ts
index ae8efed4..06fa5a1c 100644
--- a/examples/32_relay/src/ui/board.ts
+++ b/examples/32_relay/src/ui/board.ts
@@ -149,10 +149,14 @@ function count(column: number, x: number, t: Timeline): LayoutNode[] {
];
}
+/**
+ * The columns, each settling into place in turn as the board arrives,
+ * Ready first (the new card is headed there) and the rest rippling out.
+ */
function columns(t: Timeline): LayoutNode[] {
- const nodes: LayoutNode[] = [];
const makeRoom = cardLands(t) - sec(0.35);
- COLUMNS.forEach((col, i) => {
+ return COLUMNS.map((col, i) => {
+ const nodes: LayoutNode[] = [];
const x = columnX(i);
const nameWidth = FontManager.measure(col.name, {
fontFamily: SANS,
@@ -192,8 +196,28 @@ function columns(t: Timeline): LayoutNode[] {
}
nodes.push(node);
});
+ const at = boardIn(t) + Math.abs(i - READY) * 3;
+ return Layer.box({
+ position: "absolute",
+ x: 0,
+ y: 0,
+ width: CW,
+ height: CH,
+ children: nodes,
+ }).animate(
+ LayerAnimation.create()
+ .keys("y", [
+ [0, 36],
+ [at, 36, "hold"],
+ [at + sec(0.7), 0, "expo.out"],
+ ])
+ .keys("opacity", [
+ [0, 0],
+ [at, 0, "hold"],
+ [at + 8, 1, "power2.out"],
+ ]),
+ );
});
- return nodes;
}
/**
diff --git a/packages/components/src/focus-zoom/index.ts b/packages/components/src/focus-zoom/index.ts
index 64fd88f7..22069af0 100644
--- a/packages/components/src/focus-zoom/index.ts
+++ b/packages/components/src/focus-zoom/index.ts
@@ -98,6 +98,8 @@ export const focusZoom = /* @__PURE__ */ defineComponent({
smoothing: z.number().min(0).max(1).default(0.5),
/** Space kept around click points / targets. */
padding: z.number().min(0).default(160),
+ /** Slow push while a shot holds: zoom gained per second (0.02 = 2 %). */
+ drift: z.number().min(0).max(0.2).default(0),
/** Never show past the content's edges. */
clamp: z.boolean().default(true),
/** When zoomed, keep the view inside this region (e.g. the app window). */
@@ -202,6 +204,7 @@ export const focusZoom = /* @__PURE__ */ defineComponent({
fps,
shots,
smoothing: props.smoothing,
+ drift: props.drift,
clamp: props.clamp,
bounds,
});
diff --git a/packages/components/src/focus-zoom/plan.test.ts b/packages/components/src/focus-zoom/plan.test.ts
index 199a42af..c1147529 100644
--- a/packages/components/src/focus-zoom/plan.test.ts
+++ b/packages/components/src/focus-zoom/plan.test.ts
@@ -33,6 +33,28 @@ describe("solveCamera", () => {
expect((plan.x[110] ?? 0) + 2 * 1400).toBeCloseTo(960, 0);
});
+ it("keeps pushing in slowly while a shot holds, with drift", () => {
+ const shots = [
+ { start: 0, end: 300, zoom: 1.5, center: [960, 540] as [number, number] },
+ ];
+ const still = solveCamera({ ...base, length: 300, shots });
+ const drifting = solveCamera({ ...base, length: 300, shots, drift: 0.02 });
+ expect(still.states[150]?.zoom).toBeCloseTo(
+ still.states[290]?.zoom ?? 0,
+ 5,
+ );
+ const zooms = drifting.scale.slice(60);
+ // Never stops, never reverses, and gains about 2 % a second.
+ for (let i = 1; i < zooms.length; i++)
+ expect(zooms[i]).toBeGreaterThan(zooms[i - 1] ?? 0);
+ expect(
+ (drifting.states[290]?.zoom ?? 0) / (drifting.states[200]?.zoom ?? 1),
+ ).toBeCloseTo(
+ (1.5 * (1 + 0.02 * (290 / 30))) / (1.5 * (1 + 0.02 * (200 / 30))),
+ 2,
+ );
+ });
+
it("never shows past the edges, or past bounds once zoomed", () => {
const plan = solveCamera({
...base,
diff --git a/packages/components/src/focus-zoom/plan.ts b/packages/components/src/focus-zoom/plan.ts
index fa054cf1..a376f258 100644
--- a/packages/components/src/focus-zoom/plan.ts
+++ b/packages/components/src/focus-zoom/plan.ts
@@ -36,6 +36,11 @@ export interface CameraInput {
/** 0 = quick moves, 1 = slow, soft moves. */
smoothing: number;
clamp: boolean;
+ /**
+ * Slow push while a shot holds: zoom gained per second (0.02 = 2 %), so
+ * the picture keeps moving between moves. 0 holds perfectly still.
+ */
+ drift?: number;
/** The region the view must stay inside when zoomed (default: the whole view). */
bounds?: { x: number; y: number; width: number; height: number };
}
@@ -125,7 +130,8 @@ export function solveCamera(input: CameraInput): CameraPlan {
const s = input.shots[index];
if (!s) return full;
const c = typeof s.center === "function" ? s.center(frame) : s.center;
- return clampState({ cx: c[0], cy: c[1], zoom: s.zoom });
+ const push = 1 + (input.drift ?? 0) * Math.max(0, (frame - s.start) / fps);
+ return clampState({ cx: c[0], cy: c[1], zoom: s.zoom * push });
};
// Move length: a base set by `smoothing`, longer for bigger zoom changes.