Skip to content

Open source CapCut alternative.

TypeScript

feat(site): add a typing caret and press hits [rebuild]

David Le30c01191357e
diff --git a/site/packages/effects-kit/src/anim.ts b/site/packages/effects-kit/src/anim.ts
index c306888d..377bd67c 100644
--- a/site/packages/effects-kit/src/anim.ts
+++ b/site/packages/effects-kit/src/anim.ts
@@ -15,6 +15,8 @@
import {
edgePreset,
EDGE_IDS,
+ hitPreset,
+ HIT_IDS,
holdPreset,
loopPreset,
LOOP_IDS,
@@ -101,6 +103,50 @@ export interface OverlayEdge {
style: OverlayAnimStyle;
seconds: number;
preset?: MotionPreset;
+ /** The typing bar, on an entrance that types (typewriter). Every other
+ * entrance ignores it. */
+ caret?: OverlayCaret;
+}
+
+/**
+ * The bar at the end of the typed text. It holds solid while the entrance
+ * types, then blinks for the rest of the element's life, or for `blinks`
+ * blinks and then stays lit (or goes, with `hide`). With `blink` off it
+ * holds solid after typing, or goes with `hide`.
+ */
+export interface OverlayCaret {
+ blink: boolean;
+ /** How many times it blinks once typing stops; absent = for as long as
+ * the element is up. */
+ blinks?: number;
+ /** Gone after its last blink (or right after typing, with blink off). */
+ hide?: boolean;
+ /** Hex; absent = the text's own color. */
+ color?: string;
+}
+
+/** Half a blink: the bar is lit this long, then dark this long — the rate a
+ * desktop text field blinks at. */
+export const CARET_BLINK_SECONDS = 0.53;
+/** Bar geometry in em of the type size: its width, its height, and the gap
+ * between the last character and the bar. Both painters read these. */
+export const CARET_WIDTH_EM = 0.07;
+export const CARET_HEIGHT_EM = 1;
+export const CARET_GAP_EM = 0.04;
+/** The most blinks the panel and the tools offer. */
+export const CARET_BLINKS_MAX = 20;
+
+/**
+ * A one-shot hit: a catalog motion played once, over `seconds`, starting `at`
+ * seconds into the element (a button press). Unlike the edges it sits inside
+ * the element's life, and unlike a loop it plays exactly once. `darken` lets
+ * the preset's brightness track through; off, the hit only moves.
+ */
+export interface OverlayHit {
+ style: string;
+ at: number;
+ seconds: number;
+ darken?: boolean;
}
/** The cycle an element runs for its whole duration; `speed` multiplies the
@@ -130,6 +176,7 @@ export interface OverlayAnim {
/** Word-by-word emphasis (see words.ts). Text only: every other kind
* ignores it, the way typewriter is offered on titles alone. */
words?: OverlayWords;
+ hit?: OverlayHit;
}
/** The motion a slot plays: the one it carries, or the catalog entry it
@@ -143,6 +190,9 @@ export const loopMotion = (slot: OverlayLoop | undefined): MotionPreset | undefi
export const moveMotion = (slot: OverlayMove | undefined): MotionPreset | undefined =>
slot ? holdPreset(slot.style) : undefined;
+export const hitMotion = (slot: OverlayHit | undefined): MotionPreset | undefined =>
+ slot ? hitPreset(slot.style) : undefined;
+
// ── the registries, read from the catalog ─────────────────────────────────
// A preset added to the motion catalog appears in every menu, tool enum and prompt
// without being named anywhere else. The unions above stay hand-written:
@@ -152,6 +202,18 @@ export const OVERLAY_ANIM_STYLE_IDS = EDGE_IDS as OverlayAnimStyle[];
export const OVERLAY_LOOP_STYLE_IDS = LOOP_IDS as OverlayLoopStyle[];
export const GLYPH_ANIM_STYLE_IDS = PER_UNIT_EDGE_IDS as GlyphAnimStyle[];
export const GLYPH_LOOP_STYLE_IDS = PER_UNIT_LOOP_IDS as GlyphLoopStyle[];
+export const OVERLAY_HIT_STYLE_IDS = HIT_IDS;
+export const OVERLAY_HIT_STYLE_LABELS: Record<string, string> = Object.fromEntries(
+ HIT_IDS.map((id) => [id, MOTION.hits[id].label])
+);
+/** Each hit as "id: note", what tool schemas and prompts teach them from. */
+export const hitNotes = (): string =>
+ HIT_IDS.map((id) => `${id}: ${MOTION.hits[id].note ?? MOTION.hits[id].label}`).join(" ");
+
+/** Hit length bounds, seconds. */
+export const OVERLAY_HIT_MIN_SECONDS = 0.1;
+export const OVERLAY_HIT_MAX_SECONDS = 1;
+export const OVERLAY_HIT_DEFAULT_SECONDS = 0.3;
/** The edges that only play on characters — typing them out, rolling each up
* a reel. Every other kind is offered the rest. */
@@ -283,6 +345,11 @@ export interface OverlayAnimState {
* Renderers turn it into a view with `diveView`, which needs the drawn
* geometry this evaluator never sees. */
dive?: number;
+ /** Whether the typing bar is lit (typewriter with a caret); absent when the
+ * element has none. */
+ caret?: boolean;
+ /** Multiplies the element's colors (a hit that darkens); absent = 1. */
+ brightness?: number;
}
/** How one character sits at a moment: offsets in design px, a scale per axis
@@ -458,9 +525,101 @@ export function evalOverlayAnim(
applyLoop(state, anim.loop, wrap01(tLocal / period), isText);
}
if (anim.move) applyMove(state, anim.move, tLocal, dur);
+ if (anim.hit) applyHit(state, anim.hit, tLocal);
+ if (isText) {
+ const caret = caretOn(anim, tLocal, dur);
+ if (caret !== undefined) state.caret = caret;
+ }
return state;
}
+/** The hit's contribution at `tLocal`: nothing outside its window; inside,
+ * the preset's pose folded onto what the other slots left, as one piece. */
+function applyHit(state: OverlayAnimState, slot: OverlayHit, tLocal: number): void {
+ const preset = hitMotion(slot);
+ const secs = slot.seconds > 0 ? slot.seconds : OVERLAY_HIT_DEFAULT_SECONDS;
+ const q = (tLocal - slot.at) / secs;
+ if (!preset || q <= 0 || q >= 1) return;
+ const pose = evalWhole(preset, q, false);
+ state.dx += pose.dx;
+ state.dy += pose.dy;
+ state.rotate += pose.rotate;
+ state.scale *= (pose.sx + pose.sy) / 2;
+ state.alpha *= pose.alpha;
+ if (slot.darken && pose.brightness !== undefined && pose.brightness !== 1)
+ state.brightness = (state.brightness ?? 1) * pose.brightness;
+}
+
+/** Where the hit plays inside [0, dur], or null when there is none. The frame
+ * samplers draw this window frame by frame. */
+export function hitWindow(
+ anim: OverlayAnim | undefined,
+ dur: number
+): { start: number; end: number } | null {
+ const slot = anim?.hit;
+ if (!slot || !hitMotion(slot)) return null;
+ const secs = slot.seconds > 0 ? slot.seconds : OVERLAY_HIT_DEFAULT_SECONDS;
+ const start = Math.max(0, Math.min(dur, slot.at));
+ const end = Math.max(start, Math.min(dur, slot.at + secs));
+ return end - start > 1e-3 ? { start, end } : null;
+}
+
+/** The largest scale a hit reaches, so the frame sampler can pad its crop. */
+export function hitScale(anim: OverlayAnim | undefined): number {
+ const preset = hitMotion(anim?.hit);
+ let s = 1;
+ for (const key of preset?.animate.scale ?? []) s = Math.max(s, key.v[0], key.v[1]);
+ return s;
+}
+
+/** The element's typing bar, when its entrance types and carries one. */
+export function typeCaret(anim: OverlayAnim | undefined): OverlayCaret | undefined {
+ const edge = anim?.in;
+ if (!edge?.caret) return undefined;
+ return edgeMotion(edge)?.animate.typed ? edge.caret : undefined;
+}
+
+/**
+ * Whether the typing bar is lit at `tLocal`: solid while the entrance types,
+ * then lit and dark by turns, CARET_BLINK_SECONDS each, starting lit.
+ * Undefined when the element has no bar. Pure, so every renderer and every
+ * sampler reads the same blink.
+ */
+export function caretOn(
+ anim: OverlayAnim | undefined,
+ tLocal: number,
+ dur: number
+): boolean | undefined {
+ const caret = typeCaret(anim);
+ if (!caret) return undefined;
+ const typedBy = Math.min(anim!.in!.seconds, dur);
+ if (tLocal < typedBy) return true;
+ if (!caret.blink) return !caret.hide;
+ const u = tLocal - typedBy;
+ const cycle = CARET_BLINK_SECONDS * 2;
+ if (caret.blinks !== undefined && caret.blinks > 0 && u >= caret.blinks * cycle) return !caret.hide;
+ return u % cycle < CARET_BLINK_SECONDS;
+}
+
+/** The moments inside (0, dur) where the bar turns on or off, in order. The
+ * frame samplers cut their still windows here, so a blink costs two pictures
+ * and never a frame-by-frame run. */
+export function caretToggles(anim: OverlayAnim | undefined, dur: number): number[] {
+ const caret = typeCaret(anim);
+ if (!caret) return [];
+ const typedBy = Math.min(anim!.in!.seconds, dur);
+ const out: number[] = [];
+ const half = CARET_BLINK_SECONDS / 2;
+ // Every flip lands on a half-blink boundary after typing stops; each one
+ // is kept when the bar reads differently on its two sides.
+ for (let k = 0; typedBy + k * CARET_BLINK_SECONDS < dur; k++) {
+ const t = typedBy + k * CARET_BLINK_SECONDS;
+ if (t <= 0) continue;
+ if (caretOn(anim, t - half, dur) !== caretOn(anim, t + half, dur)) out.push(t);
+ }
+ return out;
+}
+
/** The move's contribution at `tLocal`: the hold preset sampled across the
* element's whole [0, dur] span, every offset from rest scaled by strength,
* folded onto the state the ramps and the loop left. */
@@ -487,7 +646,9 @@ export function loopPeriod(anim: OverlayAnim | undefined): number | null {
/** Whether any slot is set (an element with an empty anim object is static). */
export function hasOverlayAnim(anim: OverlayAnim | undefined): boolean {
- return !!anim && (!!anim.in || !!anim.out || !!anim.loop || !!anim.move || !!anim.words);
+ return (
+ !!anim && (!!anim.in || !!anim.out || !!anim.loop || !!anim.move || !!anim.words || !!anim.hit)
+ );
}
/**
diff --git a/site/packages/effects-kit/src/caretHit.test.ts b/site/packages/effects-kit/src/caretHit.test.ts
new file mode 100644
index 00000000..a14a3661
--- /dev/null
+++ b/site/packages/effects-kit/src/caretHit.test.ts
@@ -0,0 +1,173 @@
+import { describe, expect, test } from "bun:test";
+import { Canvas } from "skia-canvas";
+import {
+ CARET_BLINK_SECONDS,
+ caretOn,
+ caretToggles,
+ evalOverlayAnim,
+ hasOverlayAnim,
+ hitWindow,
+ OVERLAY_HIT_STYLE_IDS,
+ type OverlayAnim,
+} from "./anim";
+import { evalOverlayFrame } from "./keys";
+import { MOTION, presetsFor } from "./motion/catalog";
+import { darkenCanvas, paintElement, planAnimatedLayers, renderOverlayFrames, type RenderEnv } from "./render";
+import type { ShapeOverlay, TextOverlay } from "./types";
+
+const B = CARET_BLINK_SECONDS;
+const typed = (caret: NonNullable<OverlayAnim["in"]>["caret"]): OverlayAnim => ({
+ in: { style: "typewriter", seconds: 1, caret },
+});
+
+describe("the typing caret", () => {
+ test("holds solid while typing, then blinks lit and dark by turns", () => {
+ const anim = typed({ blink: true });
+ expect(caretOn(anim, 0, 10)).toBe(true);
+ expect(caretOn(anim, 0.99, 10)).toBe(true);
+ expect(caretOn(anim, 1 + B * 0.5, 10)).toBe(true);
+ expect(caretOn(anim, 1 + B * 1.5, 10)).toBe(false);
+ expect(caretOn(anim, 1 + B * 2.5, 10)).toBe(true);
+ expect(caretOn(anim, 1 + B * 3.5, 10)).toBe(false);
+ });
+
+ test("stops after its blinks, lit or gone", () => {
+ const after = 1 + B * 2 * 2 + 0.1;
+ expect(caretOn(typed({ blink: true, blinks: 2 }), after, 10)).toBe(true);
+ expect(caretOn(typed({ blink: true, blinks: 2 }), after + 3, 10)).toBe(true);
+ expect(caretOn(typed({ blink: true, blinks: 2, hide: true }), after, 10)).toBe(false);
+ // Without blinking it holds, or goes the moment typing stops.
+ expect(caretOn(typed({ blink: false }), 5, 10)).toBe(true);
+ expect(caretOn(typed({ blink: false, hide: true }), 5, 10)).toBe(false);
+ expect(caretOn(typed({ blink: false, hide: true }), 0.5, 10)).toBe(true);
+ });
+
+ test("turns exactly where the samplers cut", () => {
+ const toggles = caretToggles(typed({ blink: true, blinks: 2, hide: true }), 10);
+ expect(toggles.map((t) => +t.toFixed(3))).toEqual([1 + B, 1 + 2 * B, 1 + 3 * B].map((t) => +t.toFixed(3)));
+ const forever = caretToggles(typed({ blink: true }), 4);
+ expect(forever.length).toBe(Math.floor((4 - 1 - 1e-9) / B));
+ expect(caretToggles(typed({ blink: false, hide: true }), 4)).toEqual([1]);
+ expect(caretToggles(typed({ blink: false }), 4)).toEqual([]);
+ });
+
+ test("only a typing entrance on a title carries it", () => {
+ expect(caretOn({ in: { style: "fade", seconds: 1, caret: { blink: true } } }, 0.5, 4)).toBeUndefined();
+ expect(evalOverlayAnim(typed({ blink: true }), 0.5, 4, true).caret).toBe(true);
+ expect(evalOverlayAnim(typed({ blink: true }), 0.5, 4, false).caret).toBeUndefined();
+ expect(evalOverlayFrame({ start: 0, end: 4, x: 0.5, y: 0.5, anim: typed({ blink: true }) }, 1 + B * 1.5).caret).toBe(false);
+ });
+
+ test("the canvas export cuts its windows at every turn", () => {
+ const o: TextOverlay = { id: "t", text: "LINK", start: 2, end: 5, x: 0.5, y: 0.5, size: 60, font: "sf", weight: 700, color: "#fff", plate: false, shadow: false, anim: typed({ blink: true }) };
+ const layers = planAnimatedLayers(o, 10);
+ // Every slice of the typing ramp has the bar lit.
+ for (const l of layers.filter((l) => l.end <= 3 + 1e-9)) expect(l.phase?.caret).toBe(true);
+ const rest = layers.filter((l) => l.start >= 3 - 1e-9);
+ expect(rest.length).toBe(caretToggles(o.anim, 3).length + 1);
+ rest.forEach((l, i) => expect(!!l.phase?.caret).toBe(i % 2 === 0));
+ });
+});
+
+const env: RenderEnv = {
+ fontStack: () => "Arial",
+ createCanvas: (w, h) => new Canvas(w, h) as unknown as HTMLCanvasElement,
+ canvasToPngBlob: async (c) => new Blob([new Uint8Array(await (c as unknown as Canvas).toBuffer("png"))]),
+};
+
+describe("the caret on canvas", () => {
+ test("draws a bar past the last character, in the text's color unless set", async () => {
+ const o: TextOverlay = { id: "t", text: "LINK", start: 0, end: 4, x: 0.5, y: 0.5, size: 80, font: "sf", weight: 700, color: "#ffffff", plate: false, shadow: false, anim: typed({ blink: true, color: "#ff0000" }) };
+ const paint = async (caret: boolean) => {
+ const c = new Canvas(1080, 1080);
+ const ctx = c.getContext("2d") as unknown as CanvasRenderingContext2D;
+ await paintElement(ctx, o, { width: 1080, height: 1080, scale: 1, phase: caret ? { caret } : undefined }, env);
+ return ctx.getImageData(0, 0, 1080, 1080).data;
+ };
+ const lit = await paint(true);
+ const dark = await paint(false);
+ let red = 0;
+ let redRight = 0;
+ for (let i = 0; i < lit.length; i += 4) {
+ if (lit[i] > 200 && lit[i + 1] < 60 && lit[i + 2] < 60 && lit[i + 3] > 200) {
+ red++;
+ if ((i / 4) % 1080 > 540) redRight++;
+ }
+ if (dark[i] > 200 && dark[i + 1] < 60 && dark[i + 2] < 60 && dark[i + 3] > 200) throw new Error("bar drawn while dark");
+ }
+ expect(red).toBeGreaterThan(80 * 0.07 * 80 * 0.8);
+ expect(redRight).toBe(red);
+ });
+});
+
+describe("the press", () => {
+ const press = (darken: boolean): OverlayAnim => ({ hit: { style: "press", at: 1, seconds: 0.3, darken } });
+
+ test("is in the catalog every list reads", () => {
+ expect(OVERLAY_HIT_STYLE_IDS).toContain("press");
+ expect(OVERLAY_HIT_STYLE_IDS.sort()).toEqual(Object.keys(MOTION.hits).sort());
+ expect(presetsFor("hit").map((p) => p.id)).toContain("press");
+ for (const p of Object.values(MOTION.hits)) expect(p.note?.length ?? 0).toBeGreaterThan(0);
+ expect(hasOverlayAnim(press(false))).toBe(true);
+ });
+
+ test("dips to about 93% and springs back, inside its own window only", () => {
+ const scale = (t: number) => evalOverlayAnim(press(false), t, 4).scale;
+ expect(scale(0.99)).toBe(1);
+ expect(scale(1.31)).toBe(1);
+ const curve = Array.from({ length: 31 }, (_, i) => scale(1 + i * 0.01));
+ const low = Math.min(...curve);
+ expect(low).toBeGreaterThan(0.92);
+ expect(low).toBeLessThan(0.94);
+ const bottom = curve.indexOf(low);
+ // Down fast, back up slower: monotone on both sides of the bottom.
+ for (let i = 1; i <= bottom; i++) expect(curve[i]).toBeLessThanOrEqual(curve[i - 1] + 1e-9);
+ for (let i = bottom + 1; i < curve.length; i++) expect(curve[i]).toBeGreaterThanOrEqual(curve[i - 1] - 1e-9);
+ expect(bottom / 30).toBeLessThan(0.5);
+ });
+
+ test("darkens only when asked", () => {
+ expect(evalOverlayAnim(press(false), 1.1, 4).brightness).toBeUndefined();
+ const b = evalOverlayAnim(press(true), 1.1, 4).brightness!;
+ expect(b).toBeLessThan(1);
+ expect(b).toBeGreaterThan(0.75);
+ expect(evalOverlayAnim(press(true), 2, 4).brightness).toBeUndefined();
+ });
+
+ test("its window stays inside the element", () => {
+ expect(hitWindow(press(true), 4)).toEqual({ start: 1, end: 1.3 });
+ expect(hitWindow({ hit: { style: "press", at: 3.9, seconds: 0.3 } }, 4)!.end).toBe(4);
+ expect(hitWindow({ hit: { style: "gone", at: 1, seconds: 0.3 } }, 4)).toBeNull();
+ });
+
+ test("darkenCanvas multiplies color and leaves alpha", () => {
+ const c = new Canvas(4, 4);
+ const ctx = c.getContext("2d") as unknown as CanvasRenderingContext2D;
+ ctx.fillStyle = "rgba(200, 100, 50, 1)";
+ ctx.fillRect(0, 0, 2, 4);
+ darkenCanvas(ctx, 4, 4, 0.5);
+ const d = ctx.getImageData(0, 0, 4, 4).data;
+ expect(d[0]).toBeCloseTo(100, -1);
+ expect(d[1]).toBeCloseTo(50, -1);
+ expect(d[3]).toBe(255);
+ expect(d[2 * 4 + 3]).toBe(0);
+ });
+
+ test("the frame sampler plays the press frame by frame and holds the rest still", async () => {
+ const o: ShapeOverlay = { id: "s", kind: "shape", shape: "rect", start: 0, end: 4, x: 0.5, y: 0.5, w: 0.2, h: 0.1, fill: "#ffffff", anim: press(true) };
+ const set = await renderOverlayFrames(o, 540, 960, 30, env);
+ const total = set.entries.reduce((a, e) => a + e.duration, 0);
+ expect(total).toBeCloseTo(4, 6);
+ // Two still pieces share one picture; the 0.3s press is nine frames.
+ expect(set.images.length).toBe(1 + 9);
+ expect(set.entries[0].image).toBe(set.entries[set.entries.length - 1].image);
+ });
+
+ test("the frame sampler draws a blink as two pictures", async () => {
+ const o: TextOverlay = { id: "t", text: "LINK", start: 0, end: 4, x: 0.5, y: 0.5, size: 60, font: "sf", weight: 700, color: "#fff", plate: false, shadow: false, anim: typed({ blink: true }) };
+ const set = await renderOverlayFrames(o, 540, 960, 30, env);
+ expect(set.entries.reduce((a, e) => a + e.duration, 0)).toBeCloseTo(4, 6);
+ // The typing ramp frame by frame, then the middle as lit and dark.
+ expect(set.images.length).toBe(30 + 2);
+ });
+});
diff --git a/site/packages/effects-kit/src/keys.ts b/site/packages/effects-kit/src/keys.ts
index 96aba2e5..5545d14a 100644
--- a/site/packages/effects-kit/src/keys.ts
+++ b/site/packages/effects-kit/src/keys.ts
@@ -325,6 +325,9 @@ function ownFrame(o: Posable, tLocal: number): OverlayFrameState {
...(ev.glyphs ? { glyphs: ev.glyphs } : {}),
...(ev.glyphLoop ? { glyphLoop: ev.glyphLoop } : {}),
...(ev.dive !== undefined ? { dive: ev.dive } : {}),
+ ...(pose.blur ? { blur: pose.blur } : {}),
+ ...(ev.caret !== undefined ? { caret: ev.caret } : {}),
+ ...(ev.brightness !== undefined ? { brightness: ev.brightness } : {}),
};
}
diff --git a/site/packages/effects-kit/src/motion/catalog.ts b/site/packages/effects-kit/src/motion/catalog.ts
index ecb1b669..ef17d5f7 100644
--- a/site/packages/effects-kit/src/motion/catalog.ts
+++ b/site/packages/effects-kit/src/motion/catalog.ts
@@ -1,18 +1,19 @@
/**
* The motion catalog: every animation in the app, as data.
*
- * The three JSON files beside this one hold them in the After Effects
+ * The JSON files beside this one hold them in the After Effects
* text-animator shape — keyframed properties plus a range selector — which is
* what Lottie serializes and what GSAP and Motion reimplement as split-text
* stagger. They are split the way they are used: edges are the entrances and
* exits, loops run for as long as the element does, holds are the moves a
- * pose track is written from. Adding one means adding an entry to its file;
+ * pose track is written from, hits play once at a moment inside the element. Adding one means adding an entry to its file;
* no code lists them.
*/
import edges from "./edges.json";
import loops from "./loops.json";
import holds from "./holds.json";
+import hits from "./hits.json";
import type { MotionCatalog, MotionPreset, MotionSlot } from "./types";
export const MOTION: MotionCatalog = {
@@ -20,15 +21,18 @@ export const MOTION: MotionCatalog = {
edges,
loops,
holds,
+ hits,
} as unknown as MotionCatalog;
export const edgePreset = (id: string): MotionPreset | undefined => MOTION.edges[id];
export const loopPreset = (id: string): MotionPreset | undefined => MOTION.loops[id];
export const holdPreset = (id: string): MotionPreset | undefined => MOTION.holds[id];
+export const hitPreset = (id: string): MotionPreset | undefined => MOTION.hits[id];
export const EDGE_IDS = Object.keys(MOTION.edges);
export const LOOP_IDS = Object.keys(MOTION.loops);
export const HOLD_IDS = Object.keys(MOTION.holds);
+export const HIT_IDS = Object.keys(MOTION.hits);
/** The entrances and exits that move a title character by character. */
export const PER_UNIT_EDGE_IDS = EDGE_IDS.filter((id) => !!MOTION.edges[id].selector);
@@ -37,7 +41,14 @@ export const PER_UNIT_LOOP_IDS = LOOP_IDS.filter((id) => !!MOTION.loops[id].sele
/** Every preset that may fill a slot, for menus and tool enums. */
export function presetsFor(slot: MotionSlot): { id: string; preset: MotionPreset }[] {
- const group = slot === "loop" ? MOTION.loops : slot === "hold" ? MOTION.holds : MOTION.edges;
+ const group =
+ slot === "loop"
+ ? MOTION.loops
+ : slot === "hold"
+ ? MOTION.holds
+ : slot === "hit"
+ ? MOTION.hits
+ : MOTION.edges;
return Object.entries(group)
.filter(([, p]) => p.slots.includes(slot))
.map(([id, preset]) => ({ id, preset }));
diff --git a/site/packages/effects-kit/src/motion/evaluate.ts b/site/packages/effects-kit/src/motion/evaluate.ts
index 461959c8..ad7e81b6 100644
--- a/site/packages/effects-kit/src/motion/evaluate.ts
+++ b/site/packages/effects-kit/src/motion/evaluate.ts
@@ -92,6 +92,7 @@ export function sampleProperties(props: MotionProperties, q: number, dur = 0): M
typed: capped(props.typed, dur),
roll: capped(props.roll, dur),
dive: capped(props.dive, dur),
+ brightness: capped(props.brightness, dur),
};
return samplePropertiesRaw(props, q);
}
@@ -111,6 +112,7 @@ function samplePropertiesRaw(props: MotionProperties, q: number): MotionPose {
...(props.typed ? { typed: sampleTrack(props.typed, q, 1) } : {}),
...(props.roll ? { roll: sampleTrack(props.roll, q, 0) } : {}),
...(props.dive ? { dive: sampleTrack(props.dive, q, 0) } : {}),
+ ...(props.brightness ? { brightness: sampleTrack(props.brightness, q, 1) } : {}),
};
}
diff --git a/site/packages/effects-kit/src/motion/hits.json b/site/packages/effects-kit/src/motion/hits.json
new file mode 100644
index 00000000..77ed5e05
--- /dev/null
+++ b/site/packages/effects-kit/src/motion/hits.json
@@ -0,0 +1,72 @@
+{
+ "press": {
+ "label": "Press",
+ "note": "a finger tapping a button: the element dips to 93% and springs back, and darkens while it is held down when darken is on",
+ "slots": [
+ "hit"
+ ],
+ "animate": {
+ "scale": [
+ {
+ "t": 0,
+ "v": [
+ 1,
+ 1
+ ],
+ "ease": [
+ 0.4,
+ 0,
+ 0.6,
+ 1
+ ]
+ },
+ {
+ "t": 0.35,
+ "v": [
+ 0.93,
+ 0.93
+ ],
+ "ease": [
+ 0.3,
+ 0,
+ 0.2,
+ 1
+ ]
+ },
+ {
+ "t": 1,
+ "v": [
+ 1,
+ 1
+ ]
+ }
+ ],
+ "brightness": [
+ {
+ "t": 0,
+ "v": 1,
+ "ease": [
+ 0.4,
+ 0,
+ 0.6,
+ 1
+ ]
+ },
+ {
+ "t": 0.35,
+ "v": 0.8,
+ "ease": [
+ 0.3,
+ 0,
+ 0.2,
+ 1
+ ]
+ },
+ {
+ "t": 1,
+ "v": 1
+ }
+ ]
+ }
+ }
+}
diff --git a/site/packages/effects-kit/src/motion/types.ts b/site/packages/effects-kit/src/motion/types.ts
index 21e4ccbd..a313b3f7 100644
--- a/site/packages/effects-kit/src/motion/types.ts
+++ b/site/packages/effects-kit/src/motion/types.ts
@@ -65,6 +65,9 @@ export interface MotionProperties {
/** How far the view has flown into the element's deepest ink, 0..1: at 1
* that ink covers the frame (dive). */
dive?: PropertyKey<number>[];
+ /** Multiplies the element's colors, 1 = as drawn; below 1 it darkens
+ * (a pressed button). */
+ brightness?: PropertyKey<number>[];
}
/** What the selector counts as one unit. */
@@ -106,7 +109,7 @@ export interface WigglySelector {
seed?: number;
}
-export type MotionSlot = "in" | "out" | "loop" | "hold";
+export type MotionSlot = "in" | "out" | "loop" | "hold" | "hit";
export interface MotionPreset {
label: string;
@@ -133,6 +136,9 @@ export interface MotionCatalog {
/** Moves an element holds for its whole span, baked onto its own pose
* track rather than played as a preset. */
holds: Record<string, MotionPreset>;
+ /** One-shot hits played once at a chosen moment inside the element (a
+ * button press). */
+ hits: Record<string, MotionPreset>;
}
/** A resolved pose for one unit at one moment. */
@@ -148,6 +154,7 @@ export interface MotionPose {
typed?: number;
roll?: number;
dive?: number;
+ brightness?: number;
}
export const REST_POSE: MotionPose = {
diff --git a/site/packages/effects-kit/src/render.ts b/site/packages/effects-kit/src/render.ts
index fb5b2091..1268f624 100644
--- a/site/packages/effects-kit/src/render.ts
+++ b/site/packages/effects-kit/src/render.ts
@@ -7,12 +7,20 @@
import {
bakesPixels,
+ CARET_GAP_EM,
+ CARET_HEIGHT_EM,
+ CARET_WIDTH_EM,
+ caretOn,
+ caretToggles,
edgeMotion,
glyphStateAt,
hasGlyphMotion,
+ hitScale,
+ hitWindow,
loopExtent,
loopPeriod,
moveExtent,
+ typeCaret,
type GlyphLoopPhase,
type GlyphPhase,
type OverlayAnim,
@@ -44,6 +52,7 @@ import {
wordReach,
wordSampleWindows,
wordSwell,
+ type WordDraw,
} from "./words";
// Text metrics shared by DOM preview and canvas burn-in.
@@ -141,11 +150,34 @@ export interface PaintPhase {
glyphLoop?: GlyphLoopPhase;
/** The view a dive draws the whole picture under (see `diveView`). */
dive?: DiveView;
+ /** The typing bar is lit in this picture (see `caretOn`). */
+ caret?: boolean;
/** A dive's pose is painted before frame clipping, so a half-size title
* still fills the output frame when its flight lands. */
pose?: Pick<OverlayFrameState, "x" | "y" | "dx" | "dy" | "scale" | "rotation">;
}
+/**
+ * Multiply every color already on a surface by `brightness` (0..1), leaving
+ * alpha alone: black laid source-atop at 1 - brightness. The surface has to
+ * hold the element by itself, or whatever sits under it darkens too.
+ */
+export function darkenCanvas(
+ ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+ width: number,
+ height: number,
+ brightness: number
+): void {
+ if (!(brightness < 1)) return;
+ ctx.save();
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.globalCompositeOperation = "source-atop";
+ ctx.globalAlpha = Math.min(1, 1 - Math.max(0, brightness));
+ ctx.fillStyle = "#000000";
+ ctx.fillRect(0, 0, width, height);
+ ctx.restore();
+}
+
/**
* A shape's pixel geometry in the output frame — the one place the fractions
* become pixels, shared by the canvas painter and testable on its own.
@@ -382,23 +414,50 @@ async function paintText(
// and it takes that width in the line — its neighbours move out of its way —
// so the plate, the alignment and the draw all have to read one layout.
const draws = overlay.wordDraw?.length ? overlay.wordDraw : null;
- const fonts = new Map<number, string>();
+ const fonts = new Map<string, string>();
const sizeKey = (mult: number) => Math.round(mult * 1000) / 1000;
- const fontFor = (mult: number) => {
+ // A word set in its own face (an emphasized caption word) measures and
+ // draws in that face; every other word in the line's.
+ const faceKey = (d?: WordDraw) =>
+ d && (d.font !== undefined || d.italic !== undefined || d.weight !== undefined)
+ ? `${d.font ?? ""}|${d.italic === undefined ? "" : d.italic ? 1 : 0}|${d.weight ?? ""}`
+ : "";
+ const fontFor = (mult: number, d?: WordDraw) => {
const key = sizeKey(mult);
- let f = fonts.get(key);
+ const face = faceKey(d);
+ const id = `${key}#${face}`;
+ let f = fonts.get(id);
if (!f) {
- f = key === 1 ? cssFont : textCssFont(overlay, fpx * key, env);
- fonts.set(key, f);
+ f =
+ key === 1 && !face
+ ? cssFont
+ : textCssFont(
+ face && d
+ ? {
+ ...overlay,
+ font: d.font ?? overlay.font,
+ italic: d.italic ?? overlay.italic,
+ weight: d.weight ?? overlay.weight,
+ }
+ : overlay,
+ fpx * key,
+ env
+ );
+ fonts.set(id, f);
}
return f;
};
- // Every size in play has to be resident before measureText, or the layout is
- // measured against a fallback face and the burn-in drifts off the preview.
+ // Every size and face in play has to be resident before measureText, or the
+ // layout is measured against a fallback face and the burn-in drifts off the
+ // preview.
if (draws) {
- const sizes = new Set(draws.flatMap((d) => [sizeKey(d.scale), sizeKey(d.layoutScale)]));
- for (const key of sizes) {
- if (key !== 1) await ensureFontLoaded(fontFor(key), overlay.text);
+ const wanted = new Set<string>();
+ for (const d of draws) {
+ wanted.add(fontFor(d.scale, d));
+ wanted.add(fontFor(d.layoutScale, d));
+ }
+ for (const f of wanted) {
+ if (f !== cssFont) await ensureFontLoaded(f, overlay.text);
}
}
/** Draw and measure at a word's own size, or back at the line's. Tracking is
@@ -408,8 +467,8 @@ async function paintText(
// front of it, so reading the size back out of `ctx.font` returns the weight
// on any bold face; this is the size, tracked where it is set.
let drawSize = fpx;
- const setSize = (mult: number) => {
- ctx.font = fontFor(mult);
+ const setSize = (mult: number, d?: WordDraw) => {
+ ctx.font = fontFor(mult, d);
drawSize = fpx * sizeKey(mult);
if ("letterSpacing" in ctx) ctx.letterSpacing = `${(overlay.letterSpacing ?? 0) * drawSize}px`;
};
@@ -418,9 +477,9 @@ async function paintText(
: (() => {
const spaceW = ctx.measureText(" ").width;
let k = 0;
- const at = (w: string, mult: number) => {
- if (mult === 1) return ctx.measureText(w).width;
- setSize(mult);
+ const at = (w: string, mult: number, d?: WordDraw) => {
+ if (mult === 1 && !faceKey(d)) return ctx.measureText(w).width;
+ setSize(mult, d);
const ww = ctx.measureText(w).width;
setSize(1);
return ww;
@@ -432,8 +491,12 @@ async function paintText(
// into its place, and it draws centered in the room it will fill.
const from = k;
k += words.length;
- const widths = words.map((w, j) => at(w, sizeKey(draws[from + j]?.layoutScale ?? 1)));
- const drawn = words.map((w, j) => at(w, sizeKey(draws[from + j]?.scale ?? 1)));
+ const widths = words.map((w, j) =>
+ at(w, sizeKey(draws[from + j]?.layoutScale ?? 1), draws[from + j])
+ );
+ const drawn = words.map((w, j) =>
+ at(w, sizeKey(draws[from + j]?.scale ?? 1), draws[from + j])
+ );
return {
words,
widths,
@@ -491,6 +554,31 @@ async function paintText(
: cx + maxW / 2 - widthsOf[i] / 2;
const phase = frame.phase;
+ // The typing bar: just past the last character drawn, on the last line, as
+ // tall as the type, centered where the DOM's vertical-align: middle puts
+ // it — half an x-height over the baseline. No shadow, so it reads the same
+ // as the preview's bar.
+ const caretColor = phase?.caret ? (typeCaret(overlay.anim)?.color ?? overlay.color) : null;
+ const alphaOf = ctx.globalAlpha;
+ const drawCaret = (right: number) => {
+ if (!caretColor) return;
+ const y = cy - totalH / 2 + lineH * (lines.length - 0.5);
+ ctx.save();
+ ctx.font = cssFont;
+ ctx.textBaseline = "alphabetic";
+ const base = ctx.measureText("Hg").actualBoundingBoxAscent;
+ const xh = ctx.measureText("x").actualBoundingBoxAscent;
+ ctx.textBaseline = "middle";
+ const mid = ctx.measureText("Hg").actualBoundingBoxAscent;
+ const off =
+ Number.isFinite(base) && Number.isFinite(mid) && Number.isFinite(xh) ? base - mid - xh / 2 : 0;
+ ctx.shadowColor = "transparent";
+ ctx.globalAlpha = alphaOf;
+ ctx.fillStyle = caretColor;
+ const h = CARET_HEIGHT_EM * fpx;
+ ctx.fillRect(right + CARET_GAP_EM * fpx, y + off - h / 2, Math.max(2 * scale, CARET_WIDTH_EM * fpx), h);
+ ctx.restore();
+ };
if (phase && hasGlyphMotion(phase)) {
// A per-glyph ramp or loop draws character by character, each at its own
// place in the motion. The index counts through the whole element, line
@@ -498,6 +586,7 @@ async function paintText(
const total = Math.max(1, [...overlay.text].length);
const alpha0 = ctx.globalAlpha;
let gi = 0;
+ let lastRight = cx;
lines.forEach((line, i) => {
const y = cy - totalH / 2 + lineH * (i + 0.5);
// Characters drawn one by one lose the kerning between them, so the run
@@ -506,6 +595,7 @@ async function paintText(
const chars = [...line];
const widths = chars.map((ch) => ctx.measureText(ch).width);
let x = lineX(i) - widths.reduce((a, b) => a + b, 0) / 2;
+ lastRight = lineX(i) + widths.reduce((a, b) => a + b, 0) / 2;
chars.forEach((ch, ci) => {
const w = widths[ci];
const g = glyphStateAt(phase, gi, total);
@@ -536,6 +626,7 @@ async function paintText(
});
gi++; // the line break takes a turn too
});
+ drawCaret(lastRight);
return;
}
@@ -545,6 +636,7 @@ async function paintText(
// pushed its neighbours aside and they close back up as it shrinks.
if (!runs || !draws) {
lines.forEach((line, i) => drawText(line, lineX(i), cy - totalH / 2 + lineH * (i + 0.5)));
+ drawCaret(lineX(lines.length - 1) + widthsOf[lines.length - 1] / 2);
return;
}
// A word drawn at another size shares the baseline with the words beside it,
@@ -563,6 +655,24 @@ async function paintText(
Number.isFinite(alphabetic) && Number.isFinite(middle) && alphabetic > middle
? alphabetic - middle
: fpx * WORD_BASELINE_DROP;
+ // A word in its own face sits on the line's baseline too, and that face
+ // puts its baseline its own distance below its middle: read once per face,
+ // at the line's size, and scaled with the word.
+ const drops = new Map<string, number>();
+ const dropOf = (d: WordDraw) => {
+ const face = faceKey(d);
+ if (!face) return drop;
+ let v = drops.get(face);
+ if (v === undefined) {
+ setSize(1, d);
+ const a = ascentAt("alphabetic");
+ const m = ascentAt("middle");
+ v = Number.isFinite(a) && Number.isFinite(m) && a > m ? a - m : drop;
+ drops.set(face, v);
+ setSize(1);
+ }
+ return v;
+ };
ctx.textBaseline = "middle";
const alpha0 = ctx.globalAlpha;
let k = 0;
@@ -582,7 +692,7 @@ async function paintText(
// lifted or dropped by exactly as much as sharing the line's baseline
// asks for.
const size = fpx * mult;
- const mid = y + drop * (1 - mult);
+ const mid = y + drop - dropOf(d) * mult;
const drawY = mid;
// Centered in the room the layout left it.
const drawX = x + (width - run.drawn[wi]) / 2;
@@ -597,7 +707,7 @@ async function paintText(
if (d.rotate) ctx.rotate((d.rotate * Math.PI) / 180);
ctx.translate(-wx, -mid);
}
- setSize(mult);
+ setSize(mult, d);
const boxed = d.mark === "box" && !!d.markAlpha;
if (boxed) {
// Accent box behind the word, and the word on top of it — both drawn
@@ -634,6 +744,7 @@ async function paintText(
});
ctx.globalAlpha = alpha0;
ctx.textAlign = "center";
+ drawCaret(cx + (runs[runs.length - 1]?.lineW ?? 0) / 2);
}
/** Paint one element in frame coordinates with no element-level transform —
@@ -1055,6 +1166,8 @@ export async function measureElementBounds(
h += PLATE_PAD_Y * fpx * 2;
}
if (opts?.pad === false) return { cx, cy, w, h };
+ // The typing bar stands past the last character, so the crop leaves it room.
+ if (typeCaret(o.anim)) w += ((CARET_GAP_EM + CARET_WIDTH_EM) * fpx + 2 * frame.scale) * 2;
const strokePad = (o.stroke?.width ?? 0) * fpx * 2;
const shadow = resolveShadow(o.shadow);
const shadowPad = shadow ? (shadow.blur + Math.abs(shadow.offsetY)) * frame.scale : 0;
@@ -1140,6 +1253,18 @@ export function planAnimatedLayers(o: Overlay, end: number): AnimatedLayer[] {
}
add(overlay, start, stop);
};
+ // The typing bar blinks on the element's own clock, so a window it lights
+ // and darkens inside is cut where it turns, and each piece is drawn with the
+ // bar as it stands there.
+ const barred = isText && !!typeCaret(anim);
+ const toggles = barred ? caretToggles(anim, dur).map((t) => o.start + t) : [];
+ const lit = (phase: PaintPhase | undefined, from: number, to: number) =>
+ barred && caretOn(anim, (from + to) / 2 - o.start, dur) ? { ...phase, caret: true } : phase;
+ const pushLit = (from: number, to: number, animPart: OverlayAnim) => {
+ const cuts = [from, ...toggles.filter((t) => t > from && t < to), to];
+ for (let i = 0; i + 1 < cuts.length; i++)
+ push(o, cuts[i], cuts[i + 1], animPart, lit(undefined, cuts[i], cuts[i + 1]));
+ };
/** One window per slice, each drawn at the ramp's position mid-window. The
* slot leaves the window's anim: its motion is in the picture already. */
const slices = (from: number, secs: number, slot: "in" | "out") => {
@@ -1160,29 +1285,97 @@ export function planAnimatedLayers(o: Overlay, end: number): AnimatedLayer[] {
const to = from + ((i + 1) / n) * secs;
if (types) {
const chars = Math.max(1, Math.ceil(p * text.length));
- push({ ...o, text: text.slice(0, chars) } as Overlay, at, to, rest);
+ push({ ...o, text: text.slice(0, chars) } as Overlay, at, to, rest, lit(undefined, at, to));
} else if (perGlyph) {
- push(o, at, to, rest, {
+ push(o, at, to, rest, lit({
glyphs: { style: edge.style, preset: edge.preset, p, exiting: slot === "out" },
- });
+ }, at, to));
} else {
- push(o, at, to, rest, { reveal: p });
+ push(o, at, to, rest, lit({ reveal: p }, at, to));
}
}
};
if (inS > 1e-3) {
if (baked("in")) slices(o.start, inS, "in");
- else push(o, o.start, o.start + inS, anim);
+ else pushLit(o.start, o.start + inS, anim);
}
- push(o, o.start + inS, o.start + dur - outS, anim);
+ pushLit(o.start + inS, o.start + dur - outS, anim);
if (outS > 1e-3) {
if (baked("out")) slices(o.start + dur - outS, outS, "out");
- else push(o, o.start + dur - outS, o.start + dur, anim);
+ else pushLit(o.start + dur - outS, o.start + dur, anim);
}
return out;
}
+/** The element as it paints at one moment, neutral — rotation and opacity
+ * belong to the pose: its words drawn as they stand, the typed share of its
+ * text, and the pixel-level phase of a wipe or a per-glyph move. `typed` says
+ * the typewriter is running. */
+export function elementAtMoment(
+ overlay: Overlay,
+ tLocal: number,
+ ev: OverlayFrameState
+): { el: Overlay; phase?: PaintPhase; typed: boolean } {
+ const dur = Math.max(0.1, overlay.end - overlay.start);
+ let el: Overlay = applyWordDraw({ ...overlay, rotation: undefined, opacity: undefined }, tLocal, dur);
+ const typed = ev.textProgress !== undefined && (overlay.kind ?? "text") === "text";
+ if (typed) {
+ const text = (overlay as TextOverlay).text;
+ const chars = Math.max(0, Math.min(text.length, Math.ceil(ev.textProgress! * text.length)));
+ el = { ...el, text: text.slice(0, chars) } as Overlay;
+ }
+ const phase: PaintPhase | undefined =
+ ev.glyphs || ev.glyphLoop || ev.reveal !== undefined || ev.caret
+ ? {
+ ...(ev.glyphs ? { glyphs: ev.glyphs } : {}),
+ ...(ev.glyphLoop ? { glyphLoop: ev.glyphLoop } : {}),
+ ...(ev.reveal !== undefined ? { reveal: ev.reveal } : {}),
+ ...(ev.caret ? { caret: true } : {}),
+ }
+ : undefined;
+ return { el, phase, typed };
+}
+
+/**
+ * Paint an element, neutral and at rest, into a canvas cropped to its inked
+ * box in a `width` × `height` frame, its shape mask applied. Returns where the
+ * crop's top-left sits in the frame. The preview's motion blur smears this
+ * picture: a few element-sized draws a frame where a frame-sized picture
+ * would cost the whole stage.
+ */
+export async function paintElementCrop(
+ canvas: HTMLCanvasElement,
+ el: Overlay,
+ width: number,
+ height: number,
+ env: RenderEnv,
+ opts: { t: number; phase?: PaintPhase }
+): Promise<{ x: number; y: number }> {
+ const scale = Math.min(width, height) / 1080;
+ const frame: PaintFrame = { width, height, scale, t: opts.t, phase: opts.phase };
+ const b = await measureElementBounds(el, frame, env);
+ // Letters on a per-glyph move and lifted words travel a little past the
+ // resting box.
+ const pad = Math.ceil(4 + 0.1 * Math.max(b.w, b.h) + ((el.kind ?? "text") === "text" ? 1.5 * (el as TextOverlay).size * scale : 0));
+ const x = Math.floor(b.cx - b.w / 2 - pad);
+ const y = Math.floor(b.cy - b.h / 2 - pad);
+ const w = Math.max(1, Math.ceil(b.w + 2 * pad));
+ const h = Math.max(1, Math.ceil(b.h + 2 * pad));
+ if (canvas.width !== w) canvas.width = w;
+ if (canvas.height !== h) canvas.height = h;
+ const ctx = canvas.getContext("2d")!;
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.globalAlpha = 1;
+ ctx.clearRect(0, 0, w, h);
+ ctx.translate(-x, -y);
+ await paintElement(ctx, el, frame, env);
+ if (el.mask && el.mask.kind !== "subject") {
+ applyMaskToCanvas(ctx, newCanvas(env, 1, 1), el.mask, opts.t, frame, { x: el.x, y: el.y }, ctx.getTransform());
+ }
+ return { x, y };
+}
+
/** One animated element as a region-cropped frame sequence: unique pictures
* plus play entries (repeats reference the same picture, so a loop costs one
* cycle of pixels however long the element runs). Entries cover the
@@ -1254,7 +1447,7 @@ export async function renderOverlayFrames(
const extent = poseExtent(overlay);
// A swollen word grows the line past the resting box, so the region carries
// it — width from the swell, and the type's own extra height with it.
- const maxScale = 1.15 * extent.scale * move.scale * wordSwell(accent);
+ const maxScale = 1.15 * extent.scale * move.scale * wordSwell(accent) * hitScale(anim);
const rotates =
!!overlay.rotation ||
loop.rotates ||
@@ -1308,26 +1501,9 @@ export async function renderOverlayFrames(
// A dive's view goes inside the pose, so the type is drawn as vectors at
// the size it is seen and stays sharp however far in the flight gets.
if (focus && ev.dive) applyDiveView(ctx, diveView(ev.dive, focus, ev, overlay, frame));
- let el: Overlay = applyWordDraw(
- { ...overlay, rotation: undefined, opacity: undefined },
- tLocal,
- dur
- );
- if (ev.textProgress !== undefined && (overlay.kind ?? "text") === "text") {
- const text = (overlay as TextOverlay).text;
- const chars = Math.max(0, Math.min(text.length, Math.ceil(ev.textProgress * text.length)));
- el = { ...el, text: text.slice(0, chars) } as Overlay;
- // Typing reveals characters; it must not also fade them.
- ctx.globalAlpha = poseAt(overlay, tLocal).opacity;
- }
- const phase: PaintPhase | undefined =
- ev.glyphs || ev.glyphLoop || ev.reveal !== undefined
- ? {
- ...(ev.glyphs ? { glyphs: ev.glyphs } : {}),
- ...(ev.glyphLoop ? { glyphLoop: ev.glyphLoop } : {}),
- ...(ev.reveal !== undefined ? { reveal: ev.reveal } : {}),
- }
- : undefined;
+ const { el, phase, typed } = elementAtMoment(overlay, tLocal, ev);
+ // Typing reveals characters; it must not also fade them.
+ if (typed) ctx.globalAlpha = poseAt(overlay, tLocal).opacity;
const maskTransform = ctx.getTransform();
await paintElement(ctx, el, { ...frame, t: tLocal, phase }, env);
if (overlay.mask && maskScratch) {
@@ -1343,6 +1519,10 @@ export async function renderOverlayFrames(
maskTransform
);
}
+ // The region holds this element alone, so a hit's darkening lands on its
+ // own pixels.
+ if (ev.brightness !== undefined) darkenCanvas(ctx, rw, rh, ev.brightness);
+ fx.end(target);
return pngBlob(canvas, env);
};
@@ -1354,16 +1534,30 @@ export async function renderOverlayFrames(
};
const step = 1 / fps;
- // A keyframed pose, a move, or a keyframed mask changes on its own schedule
- // across the whole element, so there is no still middle and no cycle to
- // repeat: sample the element frame by frame.
- if (keyed || anim?.move || isMaskAnimated(overlay.mask) || (accent && anim?.loop)) {
+ const period = loopPeriod(anim) ?? lottieDur;
+ // A hit and a blinking bar change the middle at their own moments.
+ const hit = hitWindow(anim, dur);
+ const toggles = (overlay.kind ?? "text") === "text" ? caretToggles(anim, dur) : [];
+ const timed = !!hit || toggles.length > 0;
+ // A keyframed pose, a camera, a move, or a keyframed mask changes on its
+ // own schedule across the whole element, so there is no still middle and no cycle to
+ // repeat: sample the element frame by frame. So does a hit or a blink over
+ // a middle that already cycles.
+ if (
+ keyed ||
+ filmed ||
+ anim?.move ||
+ isMaskAnimated(overlay.mask) ||
+ (accent && anim?.loop) ||
+ (timed && (!!accent || !!period))
+ ) {
const n = Math.max(1, Math.round(dur * fps));
for (let i = 0; i < n; i++) {
await push(i * step, i === n - 1 ? dur - (n - 1) * step : step);
}
- ctx.setTransform(1, 0, 0, 1, 0, 0);
- ctx.clearRect(0, 0, rw, rh);
+ target.setTransform(1, 0, 0, 1, 0, 0);
+ target.clearRect(0, 0, rw, rh);
+ fx.dispose();
return { x: x0, y: y0, w: rw, h: rh, images, blank: await pngBlob(canvas, env), entries };
}
// Head ramp, frame by frame; the last frame absorbs the rounding so the
@@ -1376,8 +1570,35 @@ export async function renderOverlayFrames(
}
const middle = dur - inS - outS;
if (middle > 1e-3) {
- const period = loopPeriod(anim) ?? lottieDur;
- if (accent) {
+ if (timed && !accent && !period) {
+ // The middle holds still between the hit and the bar's turns: one
+ // picture per look (bar lit, bar dark), reused across every still
+ // piece, and the hit frame by frame.
+ const a0 = inS;
+ const b0 = dur - outS;
+ const cuts = [a0, b0, ...toggles];
+ if (hit) cuts.push(hit.start, hit.end);
+ const marks = [...new Set(cuts.filter((t) => t >= a0 && t <= b0))].sort((x, y) => x - y);
+ const stills = new Map<boolean, number>();
+ for (let i = 0; i + 1 < marks.length; i++) {
+ const a = marks[i];
+ const b = marks[i + 1];
+ if (b - a < 1e-4) continue;
+ if (hit && a >= hit.start - 1e-6 && b <= hit.end + 1e-6) {
+ const n = Math.max(1, Math.round((b - a) * fps));
+ for (let j = 0; j < n; j++) await push(a + j * step, j === n - 1 ? b - a - (n - 1) * step : step);
+ continue;
+ }
+ const lit = caretOn(anim, (a + b) / 2, dur) === true;
+ let image = stills.get(lit);
+ if (image === undefined) {
+ images.push(await drawAt((a + b) / 2));
+ image = images.length - 1;
+ stills.set(lit, image);
+ }
+ entries.push({ image, duration: b - a });
+ }
+ } else if (accent) {
// The picture changes as the effect walks the line, and nowhere else:
// one frame per span the engine says holds still covers the middle
// however long it runs. The last one absorbs the rounding, so the
@@ -1414,8 +1635,9 @@ export async function renderOverlayFrames(
}
}
- ctx.setTransform(1, 0, 0, 1, 0, 0);
- ctx.clearRect(0, 0, rw, rh);
+ target.setTransform(1, 0, 0, 1, 0, 0);
+ target.clearRect(0, 0, rw, rh);
+ fx.dispose();
const blank = await pngBlob(canvas, env);
return { x: x0, y: y0, w: rw, h: rh, images, blank, entries };
}
diff --git a/site/packages/effects-kit/src/wordFace.test.ts b/site/packages/effects-kit/src/wordFace.test.ts
new file mode 100644
index 00000000..fe92e010
--- /dev/null
+++ b/site/packages/effects-kit/src/wordFace.test.ts
@@ -0,0 +1,63 @@
+import { expect, test } from "bun:test";
+import { Canvas } from "skia-canvas";
+import { paintElement, type RenderEnv } from "./render";
+import type { TextOverlay } from "./types";
+import { wordDrawsAt, wordFaceDraws } from "./words";
+
+/**
+ * A word set apart in its own face: the painter measures and fills it in that
+ * face, the rest of the line in the line's, and the effect's accent still
+ * blends from the word's own fill.
+ */
+
+const frame = { width: 1080, height: 1920, scale: 1 };
+const env: RenderEnv = {
+ fontStack: (id) => (id === "serif" ? "Georgia" : "Arial"),
+ createCanvas: (w, h) => new Canvas(w, h) as unknown as HTMLCanvasElement,
+};
+const line: TextOverlay = {
+ id: "t",
+ text: "get Donkey free",
+ start: 0,
+ end: 2,
+ x: 0.5,
+ y: 0.5,
+ size: 60,
+ font: "sf",
+ weight: 700,
+ color: "#FFFFFF",
+ plate: false,
+ shadow: false,
+};
+const face = { font: "serif", italic: true, weight: 400 as const, color: "#FFFF00", scale: 1.2 };
+
+test("each word is filled in its own face and color", async () => {
+ const canvas = new Canvas(frame.width, frame.height);
+ const ctx = canvas.getContext("2d") as unknown as CanvasRenderingContext2D;
+ const fills: { text: string; font: string; color: string }[] = [];
+ const fill = ctx.fillText.bind(ctx);
+ ctx.fillText = (text: string, x: number, y: number) => {
+ fills.push({ text, font: ctx.font, color: String(ctx.fillStyle) });
+ fill(text, x, y);
+ };
+ const draws = wordFaceDraws(line.text, line.color, [undefined, face]);
+ await paintElement(ctx, { ...line, wordDraw: draws }, frame, env);
+ expect(fills.map((f) => f.text)).toEqual(["get", "Donkey", "free"]);
+ expect(fills[0].font).toContain("Arial");
+ expect(fills[0].font).not.toContain("italic");
+ expect(fills[1].font).toContain("italic");
+ expect(fills[1].font).toContain("Georgia");
+ expect(fills[1].font).toContain("72px");
+ expect(fills[1].color.toLowerCase()).toContain("ffff00");
+ expect(fills[2].font).toBe(fills[0].font);
+});
+
+test("the accent blends from the face's color, and the face rides every pose", () => {
+ const draws = wordDrawsAt("one two", { style: "color", color: "#FF0000" }, "#FFFFFF", 2, 0.1, [face]);
+ expect(draws[0]).toMatchObject({ font: "serif", italic: true, weight: 400 });
+ expect(draws[0].scale).toBeCloseTo(1.2);
+ expect(draws[0].color.toLowerCase()).not.toBe("#ffff00"); // on its moment: the accent
+ expect(draws[1].font).toBeUndefined();
+ const rest = wordDrawsAt("one two", { style: "color", color: "#FF0000" }, "#FFFFFF", 2, 1.9, [face]);
+ expect(rest[0].color.toLowerCase()).toBe("#ffff00");
+});
diff --git a/site/packages/effects-kit/src/words/engine.ts b/site/packages/effects-kit/src/words/engine.ts
index 5b2ba6fb..f3d9be77 100644
--- a/site/packages/effects-kit/src/words/engine.ts
+++ b/site/packages/effects-kit/src/words/engine.ts
@@ -17,7 +17,7 @@
import { bezierEase } from "../motion/evaluate";
import type { Overlay, TextOverlay } from "../types";
import { wordEffect, wordEffectRamps } from "./catalog";
-import type { OverlayWords, WordDraw, WordEffect, WordPose } from "./types";
+import type { OverlayWords, WordDraw, WordEffect, WordFace, WordPose } from "./types";
/** How far a word's alphabetic baseline sits below the middle of its em box,
* in em. The canvas painter measures this off the face it is drawing and only
@@ -219,7 +219,10 @@ export function wordDrawsAt(
words: OverlayWords,
textColor: string,
dur: number,
- t: number
+ t: number,
+ /** Per display word: the face a word is set apart in, or nothing. The
+ * effect's accent blends from the face's own fill. */
+ faces?: readonly (WordFace | undefined)[]
): WordDraw[] {
const e = wordEffect(words.style);
const accent = wordAccent(words, textColor);
@@ -229,20 +232,56 @@ export function wordDrawsAt(
const target = e.mark === "box" ? markText : accent;
const opts = { swell: wordSwell(words), dim: wordDim(words) };
const restScale = (e.after.scale ?? 1) > 1 ? opts.swell : (e.after.scale ?? 1);
- return wordWindows(text, dur, words.times).map((win) => {
+ return wordWindows(text, dur, words.times).map((win, i) => {
const p = wordPoseAt(e, win, t, opts);
- return {
- color: mixHex(textColor, target, p.accent),
- opacity: p.opacity,
- scale: p.scale,
- layoutScale: Math.max(p.scale, restScale),
- dx: p.dx,
- dy: p.dy,
- rotate: p.rotate,
- ...(e.mark && p.mark > 0.001
- ? { mark: e.mark, markAlpha: p.mark, markColor: accent, markText }
- : {}),
- };
+ const face = faces?.[i];
+ return withFace(
+ {
+ color: mixHex(face?.color ?? textColor, target, p.accent),
+ opacity: p.opacity,
+ scale: p.scale,
+ layoutScale: Math.max(p.scale, restScale),
+ dx: p.dx,
+ dy: p.dy,
+ rotate: p.rotate,
+ ...(e.mark && p.mark > 0.001
+ ? { mark: e.mark, markAlpha: p.mark, markColor: accent, markText }
+ : {}),
+ },
+ face
+ );
+ });
+}
+
+/** A resolved word set in its face: the face's size multiplies the pose's,
+ * and its family, slant and weight ride along for the painters. */
+function withFace(d: WordDraw, face: WordFace | undefined): WordDraw {
+ if (!face) return d;
+ const k = face.scale !== undefined && Number.isFinite(face.scale) && face.scale > 0 ? face.scale : 1;
+ if (k !== 1) {
+ d.scale *= k;
+ d.layoutScale *= k;
+ }
+ if (face.font) d.font = face.font;
+ if (face.italic !== undefined) d.italic = face.italic;
+ if (face.weight !== undefined) d.weight = face.weight;
+ return d;
+}
+
+/**
+ * Every display word at rest, with the faces some of them are set apart in —
+ * a line that runs no word effect but still sets chosen words apart. Nothing
+ * here moves, so a caller may compute it once and hold it for the line's
+ * whole span.
+ */
+export function wordFaceDraws(
+ text: string,
+ textColor: string,
+ faces: readonly (WordFace | undefined)[]
+): WordDraw[] {
+ return displayWords(text).map((_, i) => {
+ const face = faces[i];
+ return withFace({ ...REST_WORD, color: face?.color ?? textColor }, face);
});
}
diff --git a/site/packages/effects-kit/src/words/types.ts b/site/packages/effects-kit/src/words/types.ts
index 9bd111e8..c224558b 100644
--- a/site/packages/effects-kit/src/words/types.ts
+++ b/site/packages/effects-kit/src/words/types.ts
@@ -105,4 +105,22 @@ export interface WordDraw {
markColor?: string;
/** Word color over a box, which needs to read against the fill. */
markText?: string;
+ /** The word's own face, where it is set apart from the line: an emphasized
+ * word in another family, slant or weight. Absent = the line's. */
+ font?: string;
+ italic?: boolean;
+ weight?: 400 | 700;
+}
+
+/** How one word is set apart from the rest of its line: its own fill, face
+ * and size. A caption's emphasized words wear one of these, and the word
+ * effect runs on top of it, so a spoken emphasized word still lights up. */
+export interface WordFace {
+ /** The fill the word rests at; the accent blends from here. */
+ color?: string;
+ font?: string;
+ italic?: boolean;
+ weight?: 400 | 700;
+ /** Type-size multiplier, on top of whatever the effect does to the word. */
+ scale?: number;
}
diff --git a/site/scripts/stage-cut-fonts.mjs b/site/scripts/stage-cut-fonts.mjs
index c2fa04fc..7fe5869b 100644
--- a/site/scripts/stage-cut-fonts.mjs
+++ b/site/scripts/stage-cut-fonts.mjs
@@ -28,7 +28,13 @@ const FAMILIES = [
{ id: "poppins", label: "Poppins", family: "Poppins", weights: [400, 700] },
{ id: "oswald", label: "Oswald", family: "Oswald", weights: [400, 700] },
{ id: "space-grotesk", label: "Space Grotesk", family: "Space Grotesk", weights: [400, 700] },
- { id: "playfair", label: "Playfair Display", family: "Playfair Display", weights: [400, 700] },
+ {
+ id: "playfair",
+ label: "Playfair Display",
+ family: "Playfair Display",
+ weights: [400, 700],
+ italics: [400, 700],
+ },
{ id: "caveat", label: "Caveat", family: "Caveat", weights: [400, 700] },
{ id: "bebas", label: "Bebas Neue", family: "Bebas Neue", weights: [400] },
{
@@ -88,6 +94,12 @@ for (const entry of FAMILIES) {
copyFileSync(path.join(files, `${pkg}-latin-${weight}-normal.woff2`), path.join(dest, name));
return name;
});
+ // Italic faces join the same family; skia reads the slant off the file.
+ for (const weight of entry.italics ?? []) {
+ const name = `${pkg}-${weight}-italic.woff2`;
+ copyFileSync(path.join(files, `${pkg}-latin-${weight}-italic.woff2`), path.join(dest, name));
+ staged.push(name);
+ }
manifest.push({
...(entry.id ? { id: entry.id, label: entry.label } : {}),
family: entry.family,
diff --git a/site/src/cut/components/AnimationTiles.tools.ts b/site/src/cut/components/AnimationTiles.tools.ts
index 880fe3b9..4d2dc047 100644
--- a/site/src/cut/components/AnimationTiles.tools.ts
+++ b/site/src/cut/components/AnimationTiles.tools.ts
@@ -8,7 +8,15 @@
* until every side catches up.
*/
+import { EASE_IDS, ELEMENT_BLUR_MAX } from "@donkeycut/effects-kit";
import {
+ CARET_BLINK_SECONDS,
+ CARET_BLINKS_MAX,
+ hitNotes,
+ OVERLAY_HIT_DEFAULT_SECONDS,
+ OVERLAY_HIT_MAX_SECONDS,
+ OVERLAY_HIT_MIN_SECONDS,
+ OVERLAY_HIT_STYLE_IDS,
OVERLAY_ANIM_DEFAULT_SECONDS,
OVERLAY_ANIM_MAX_SECONDS,
OVERLAY_ANIM_MIN_SECONDS,
@@ -25,7 +33,7 @@ import {
WORD_SWELL_MAX,
WORD_SWELL_MIN,
} from "@donkeycut/effects-kit";
-import { num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
+import { bool, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
import {
MOVE_STRENGTH_MAX,
MOVE_STRENGTH_MIN,
@@ -39,7 +47,7 @@ export const OVERLAY_ANIMATION_TOOLS = [
{
name: "set_overlay_animation",
description:
- `Animate an overlay element (title, shape, or sticker): preset In/Out ramps, a Loop that runs its whole duration, and a Move that says what the element does WHILE it holds. Omitted slots keep their setting; pass "none" to clear one. In/Out styles: ${OVERLAY_ANIM_STYLE_IDS.join(", ")} — slide names are the motion direction; ${TEXT_ONLY_ANIM_STYLE_IDS.join(", ")} animate titles only; ${GLYPH_ANIM_STYLE_IDS.join(", ")} move a title's letters one at a time, and move any other kind as one piece. ${edgeNotes()} ${DIVE_ANIM_STYLE_IDS.join(", ")} as the exit is the zoom-through-a-letter transition: the shot after the element shows once the ink has filled the frame, so it needs no freeze frames, image stills or per-letter keyframes. Loop styles: ${OVERLAY_LOOP_STYLE_IDS.join(", ")} — ${GLYPH_LOOP_STYLE_IDS.join(", ")} carry a title's letters on their own delays, and carry any other kind as one piece. A move is a slot like the others and never touches the element's keyframe track; set_overlay_keyframes is the precise way to move an element and the default reach, so name a move when it is exactly what you mean — the two compose together. Word effects — words_style, titles only — play the line word by word, timed against the cut's transcript when there is one and spread across the element's own span when there is not: an emphasis travels along a line that is fully up, and a build assembles the line as it is spoken. It is the slot for "make each word pop as I say it" and for "have the words appear one by one".`,
+ `Animate an overlay element (title, shape, or sticker): preset In/Out ramps, a Loop that runs its whole duration, and a Move that says what the element does WHILE it holds. Omitted slots keep their setting; pass "none" to clear one. In/Out styles: ${OVERLAY_ANIM_STYLE_IDS.join(", ")} — slide names are the motion direction; ${TEXT_ONLY_ANIM_STYLE_IDS.join(", ")} animate titles only; ${GLYPH_ANIM_STYLE_IDS.join(", ")} move a title's letters one at a time, and move any other kind as one piece. ${edgeNotes()} ${DIVE_ANIM_STYLE_IDS.join(", ")} as the exit is the zoom-through-a-letter transition: the shot after the element shows once the ink has filled the frame, so it needs no freeze frames, image stills or per-letter keyframes. Loop styles: ${OVERLAY_LOOP_STYLE_IDS.join(", ")} — ${GLYPH_LOOP_STYLE_IDS.join(", ")} carry a title's letters on their own delays, and carry any other kind as one piece. A move is a slot like the others and never touches the element's keyframe track; set_overlay_keyframes is the precise way to move an element and the default reach, so name a move when it is exactly what you mean — the two compose together. Word effects — words_style, titles only — play the line word by word, timed against the cut's transcript when there is one and spread across the element's own span when there is not: an emphasis travels along a line that is fully up, and a build assembles the line as it is spoken. It is the slot for "make each word pop as I say it" and for "have the words appear one by one". A typewriter entrance on a title can carry a caret: a thin bar at the end of the typed text, solid while it types, then blinking every ${CARET_BLINK_SECONDS}s — the typed comment or search box of a call-to-action card. A hit plays one motion once at a moment inside the element (hit_at seconds from its start), between its entrance and exit: ${hitNotes()}. Use press for a button being pressed, a tap on a UI element, a like or post button; pair it with a click from stock_search kind "sound" placed at the same moment when the tap should be heard.`,
inputSchema: obj({
id: str("Overlay element id"),
move: {
@@ -80,12 +88,29 @@ export const OVERLAY_ANIMATION_TOOLS = [
words_dim: num(
"How opaque a word is off its moment, 0..1 — 0 means it is not there yet (build), a fraction leaves the line faintly visible (fill, spotlight). Effects that do not fade their words ignore it."
),
+ caret: bool("Typewriter entrance only: show a typing caret at the end of the typed text (false removes it)"),
+ caret_blink: bool("Whether the caret blinks once typing stops (default true); false holds it solid"),
+ caret_blinks: num(
+ `How many times the caret blinks before it stops, 1..${CARET_BLINKS_MAX}; 0 blinks for as long as the element is up (default)`
+ ),
+ caret_hide: bool("Remove the caret after its last blink (or right after typing when it does not blink)"),
+ caret_color: str("Caret color (hex); default is the text color"),
+ hit_style: {
+ type: "string",
+ enum: [...OVERLAY_HIT_STYLE_IDS, "none"],
+ description: 'One-shot hit played once inside the element, or "none" to clear',
+ },
+ hit_at: num("Seconds from the element's start where the hit begins"),
+ hit_seconds: num(
+ `Hit length seconds ${OVERLAY_HIT_MIN_SECONDS}..${OVERLAY_HIT_MAX_SECONDS} (default ${OVERLAY_HIT_DEFAULT_SECONDS})`
+ ),
+ hit_darken: bool("Darken the element while the hit holds it down (default true for a new hit)"),
}, ["id"]),
},
{
name: "set_overlay_keyframes",
description:
- "Give an overlay element a keyframed pose track — the default way to animate one, because a key names the exact pose at the exact second where a preset ramp approximates it. Each key is a whole pose at a time measured in seconds from the element's own start; the pose moves linearly between keys and holds outside them. Omitted fields on a key take the element's current value. Pass an empty list to clear the track and return the element to its resting pose. Preset In/Out/Loop animation still composes on top, so a keyframed title can also fade in.",
+ `Give an overlay element a keyframed pose track — the default way to animate one, because a key names the exact pose at the exact second where a preset ramp approximates it. Each key is a whole pose at a time measured in seconds from the element's own start; the pose moves between keys along the key's ease (${EASE_IDS.join(", ")}; default linear) and holds outside them. Omitted fields on a key take the element's current value. blur softens the element (px at the 1080 short side, 0..${ELEMENT_BLUR_MAX}): keyed with opacity it is depth of field — the card being read sharp, its neighbours blurred and dimmed while a group camera (set_group_camera) pushes in. Pass an empty list to clear the track and return the element to its resting pose. Preset In/Out/Loop animation still composes on top, so a keyframed title can also fade in.`,
inputSchema: obj(
{
id: str("Overlay element id"),
diff --git a/site/src/cut/components/AnimationTiles.tsx b/site/src/cut/components/AnimationTiles.tsx
index 30e72418..e628b672 100644
--- a/site/src/cut/components/AnimationTiles.tsx
+++ b/site/src/cut/components/AnimationTiles.tsx
@@ -18,6 +18,8 @@ import {
OVERLAY_ANIM_STYLE_LABELS,
OVERLAY_LOOP_STYLE_IDS,
OVERLAY_LOOP_STYLE_LABELS,
+ OVERLAY_HIT_STYLE_IDS,
+ OVERLAY_HIT_STYLE_LABELS,
wordDrawsAt,
wordEffectIdsIn,
WORD_ACCENT_DEFAULT,
@@ -93,20 +95,22 @@ const referencePx = (slot: Slot, style: string) =>
? LOOP_REFERENCE_PX
: DEMO_REFERENCE_PX;
-type Slot = "in" | "out" | "loop" | "move" | "words";
+type Slot = "in" | "out" | "loop" | "move" | "words" | "hit";
/** A move runs the element's whole span, so the tile gives it one of its own
* to play in — long enough to read a push, short enough to come round again. */
const MOVE_DEMO_SECONDS = 2.4;
const labelOf = (slot: Slot, style: string) =>
- slot === "words"
- ? (WORD_EFFECT_LABELS[style] ?? style)
- : slot === "move"
- ? (MOTION.holds[style]?.label ?? style)
- : slot === "loop"
- ? (OVERLAY_LOOP_STYLE_LABELS[style as OverlayLoopStyle] ?? style)
- : (OVERLAY_ANIM_STYLE_LABELS[style as OverlayAnimStyle] ?? style);
+ slot === "hit"
+ ? (OVERLAY_HIT_STYLE_LABELS[style] ?? style)
+ : slot === "words"
+ ? (WORD_EFFECT_LABELS[style] ?? style)
+ : slot === "move"
+ ? (MOTION.holds[style]?.label ?? style)
+ : slot === "loop"
+ ? (OVERLAY_LOOP_STYLE_LABELS[style as OverlayLoopStyle] ?? style)
+ : (OVERLAY_ANIM_STYLE_LABELS[style as OverlayAnimStyle] ?? style);
function demoStateAt(
slot: Slot,
@@ -137,6 +141,17 @@ function demoStateAt(
if (slot === "loop") {
return evalOverlayAnim({ loop: { style: style as OverlayLoopStyle, speed } }, t, 60, isText);
}
+ if (slot === "hit") {
+ // A hit is one press and the rest beat after it, darkening as it will on
+ // the element.
+ const cycle = demoCycle(ramp);
+ return evalOverlayAnim(
+ { hit: { style, at: 0, seconds: ramp, darken: true } },
+ t % cycle,
+ cycle,
+ isText
+ );
+ }
const cycle = demoCycle(ramp);
const local = t % cycle;
const anim =
@@ -301,6 +316,7 @@ function LiveName({
}
el.style.clipPath =
st.reveal !== undefined ? `inset(0 ${(1 - st.reveal) * 100}% 0 0)` : "";
+ el.style.filter = st.brightness !== undefined ? `brightness(${st.brightness})` : "";
if (st.textProgress !== undefined) {
const word = el.dataset.word ?? "";
el.textContent = word.slice(0, Math.ceil(st.textProgress * word.length));
diff --git a/site/src/cut/components/Inspector.tsx b/site/src/cut/components/Inspector.tsx
index 9930a781..9621d57f 100644
--- a/site/src/cut/components/Inspector.tsx
+++ b/site/src/cut/components/Inspector.tsx
@@ -41,6 +41,14 @@ import {
OVERLAY_ANIM_DEFAULT_SECONDS,
OVERLAY_ANIM_MAX_SECONDS,
OVERLAY_ANIM_MIN_SECONDS,
+ CARET_BLINKS_MAX,
+ edgeMotion,
+ edgePreset,
+ OVERLAY_HIT_DEFAULT_SECONDS,
+ OVERLAY_HIT_MAX_SECONDS,
+ OVERLAY_HIT_MIN_SECONDS,
+ type OverlayCaret,
+ type OverlayHit,
normalizeSound,
SOUND_COMPRESSOR_DEFAULT,
SOUND_COMPRESSOR_RANGE,
@@ -2589,6 +2597,7 @@ function writeOverlayAnim(o: Overlay, anim: OverlayAnim, patch: Partial<OverlayA
if (!next.loop) delete next.loop;
if (!next.move) delete next.move;
if (!next.words) delete next.words;
+ if (!next.hit) delete next.hit;
const value = hasOverlayAnim(next) ? next : undefined;
const st = useEditor.getState();
// A multi-selection names its targets; a lone grouped element stamps its group.
@@ -2732,8 +2741,187 @@ function WordSettings({ overlay: o, peers }: { overlay: Overlay; peers?: readonl
);
}
+/** A caret with one setting taken out, so an absent field reads as its
+ * default. */
+function caretWithout(caret: OverlayCaret, key: "blinks"): OverlayCaret {
+ const next = { ...caret };
+ delete next[key];
+ return next;
+}
+
+/** The typing bar of a typewriter entrance: on or off, whether it blinks once
+ * typing stops, how many times, whether it goes after, and its color. */
+function CaretSettings({ overlay: o, peers }: { overlay: Overlay; peers?: readonly Overlay[] }) {
+ const ck = useSliderCheckpoint();
+ const anim = o.anim ?? {};
+ const edge = anim.in;
+ if (!edge) return null;
+ const caret = edge.caret;
+ const caretBlink = caret?.blink ?? true;
+ const caretBlinks = caret?.blinks ?? 0;
+ const caretHide = caret?.hide ?? false;
+ const caretColor = caret?.color ?? (isTextOverlay(o) ? o.color : WORD_ACCENT_DEFAULT);
+ const write = (next: OverlayCaret | undefined) => {
+ const nextEdge = { ...edge };
+ if (next) nextEdge.caret = next;
+ else delete nextEdge.caret;
+ writeOverlayAnim(o, anim, { in: nextEdge }, peers);
+ };
+ const toggle = (patch: Partial<OverlayCaret>) => {
+ if (!caret) return;
+ useEditor.getState().pushHistory();
+ write({ ...caret, ...patch });
+ };
+ const setColor = (color: string) => {
+ if (caret) write({ ...caret, color });
+ };
+ const setBlinks = (v: number) => {
+ if (!caret) return;
+ write(v > 0 ? { ...caret, blinks: Math.round(v) } : caretWithout(caret, "blinks"));
+ };
+ return (
+ <>
+ <Row label="Caret">
+ <Switch
+ checked={!!caret}
+ onCheckedChange={(v) => {
+ useEditor.getState().pushHistory();
+ write(v ? { blink: true } : undefined);
+ }}
+ />
+ </Row>
+ {caret && (
+ <Row label="Blink">
+ <Switch checked={caretBlink} onCheckedChange={(v) => toggle({ blink: v })} />
+ </Row>
+ )}
+ {caret && caretBlink && (
+ <Row label="Blinks">
+ <ValueSlider
+ label="Blinks"
+ sliderClassName="data-horizontal:w-24"
+ valueClassName="w-9 text-muted-foreground"
+ value={caretBlinks}
+ min={0}
+ max={CARET_BLINKS_MAX}
+ step={1}
+ format={(v) => (v === 0 ? "All" : `${Math.round(v)}`)}
+ parse={parseNumberInput}
+ onDraft={(v) => {
+ ck.begin();
+ setBlinks(v);
+ }}
+ onCommit={(v) => {
+ setBlinks(v);
+ ck.end();
+ }}
+ />
+ </Row>
+ )}
+ {caret && (!caretBlink || caretBlinks > 0) && (
+ <Row label="Then hide">
+ <Switch checked={caretHide} onCheckedChange={(v) => toggle({ hide: v })} />
+ </Row>
+ )}
+ {caret && (
+ <Row label="Caret color">
+ <ColorField
+ value={caretColor}
+ label="Caret color"
+ onBegin={() => useEditor.getState().pushHistory()}
+ onLive={setColor}
+ onCommit={setColor}
+ />
+ </Row>
+ )}
+ </>
+ );
+}
+
+/** A newly picked hit lands at the playhead when the playhead is inside the
+ * element with room for the hit, and in the element's middle otherwise.
+ * Switching hits keeps the moment, the length and the darkening. */
+function pickedHit(o: Overlay, anim: OverlayAnim, style: string): OverlayHit {
+ if (anim.hit) return { ...anim.hit, style };
+ const dur = Math.max(0.1, o.end - o.start);
+ const seconds = Math.min(OVERLAY_HIT_DEFAULT_SECONDS, dur);
+ const local = playheadAt() - o.start;
+ const at = local >= 0 && local <= dur - seconds ? local : Math.max(0, (dur - seconds) / 2);
+ return { style, at: Math.round(at * 100) / 100, seconds, darken: true };
+}
+
+/** The hit's moment inside the element, its length, and whether it darkens. */
+function HitSettings({ overlay: o, peers }: { overlay: Overlay; peers?: readonly Overlay[] }) {
+ const ck = useSliderCheckpoint();
+ const anim = o.anim ?? {};
+ const hit = anim.hit;
+ const dur = Math.max(0.2, Math.min(...(peers ?? [o]).map((el) => el.end - el.start)));
+ const hitSeconds = hit?.seconds ?? OVERLAY_HIT_DEFAULT_SECONDS;
+ const hitAt = hit?.at ?? 0;
+ const hitDarken = hit?.darken ?? false;
+ const write = (patch: Partial<OverlayHit>) => {
+ if (!hit) return;
+ ck.begin();
+ writeOverlayAnim(o, anim, { hit: { ...hit, ...patch } }, peers);
+ };
+ return (
+ <>
+ <Row label="At">
+ <ValueSlider
+ label="At"
+ sliderClassName="data-horizontal:w-24"
+ valueClassName="w-9 text-muted-foreground"
+ value={hitAt}
+ min={0}
+ max={Math.max(0, dur - hitSeconds)}
+ step={0.05}
+ format={(v) => `${v.toFixed(2)}s`}
+ parse={parseSecondsInput}
+ disabled={!hit}
+ onDraft={(v) => write({ at: v })}
+ onCommit={(v) => {
+ write({ at: v });
+ ck.end();
+ }}
+ />
+ </Row>
+ <Row label="Duration">
+ <ValueSlider
+ label="Duration"
+ sliderClassName="data-horizontal:w-24"
+ valueClassName="w-9 text-muted-foreground"
+ value={hitSeconds}
+ min={OVERLAY_HIT_MIN_SECONDS}
+ max={Math.min(OVERLAY_HIT_MAX_SECONDS, dur)}
+ step={0.05}
+ snap={[OVERLAY_HIT_DEFAULT_SECONDS]}
+ format={(v) => `${v.toFixed(2)}s`}
+ parse={parseSecondsInput}
+ disabled={!hit}
+ onDraft={(v) => write({ seconds: v })}
+ onCommit={(v) => {
+ write({ seconds: v });
+ ck.end();
+ }}
+ />
+ </Row>
+ <Row label="Darken">
+ <Switch
+ checked={hitDarken}
+ disabled={!hit}
+ onCheckedChange={(v) => {
+ if (!hit) return;
+ useEditor.getState().pushHistory();
+ writeOverlayAnim(o, anim, { hit: { ...hit, darken: v } }, peers);
+ }}
+ />
+ </Row>
+ </>
+ );
+}
+
/** The slots the picker fills, in tab order. */
-type AnimSlot = "in" | "out" | "loop" | "move" | "words";
+type AnimSlot = "in" | "out" | "loop" | "move" | "words" | "hit";
/** The pill of slot tabs over an animation grid. A slot that is already set
* reads darker, so switching tabs is not the only way to see what an item is
@@ -2867,14 +3055,30 @@ export function AnimationPanel({ overlay: o, peers }: { overlay: Overlay; peers?
// selecting one hands the picker back to the entrance.
// Word emphasis needs words on every target.
const tabs: AnimSlot[] = (peers ?? [o]).every(isTextOverlay)
- ? ["in", "out", "loop", "move", "words"]
- : ["in", "out", "loop", "move"];
+ ? ["in", "out", "loop", "move", "words", "hit"]
+ : ["in", "out", "loop", "move", "hit"];
const slot = tabs.includes(picked) ? picked : "in";
const tilesScroll = useRememberedScroll(o.id, `anim:${slot}`);
- const active = slot === "move" || slot === "words" ? undefined : anim[slot];
- const seconds = slot === "in" || slot === "out" ? anim[slot]?.seconds : undefined;
+ const active = slot === "move" || slot === "words" || slot === "hit" ? undefined : anim[slot];
+ const seconds =
+ slot === "in" || slot === "out"
+ ? anim[slot]?.seconds
+ : slot === "hit"
+ ? (anim.hit?.seconds ?? OVERLAY_HIT_DEFAULT_SECONDS)
+ : undefined;
const activeMove = anim.move;
+ const hitStyle = anim.hit?.style;
const pick = (style: string | null) => {
+ if (slot === "hit") {
+ if (!style && !anim.hit) return;
+ useEditor.getState().pushHistory();
+ const patch: Partial<OverlayAnim> = { hit: style ? pickedHit(o, anim, style) : undefined };
+ writeOverlayAnim(o, anim, patch, peers);
+ // The pick plays itself on the stage, over its own window.
+ if (style) playAnimPreview({ ...o, anim: { ...anim, ...patch } }, "hit");
+ else stopAnimPreview();
+ return;
+ }
if (slot === "words") {
if (!style && !anim.words) return;
useEditor.getState().pushHistory();
@@ -2927,6 +3131,10 @@ export function AnimationPanel({ overlay: o, peers }: { overlay: Overlay; peers?
[slot]: {
style: style as OverlayAnimStyle,
seconds: seconds ?? OVERLAY_ANIM_DEFAULT_SECONDS,
+ // A caret stays with an entrance that still types.
+ ...(slot === "in" && anim.in?.caret && edgePreset(style)?.animate.typed
+ ? { caret: anim.in.caret }
+ : {}),
},
};
useEditor.getState().pushHistory();
@@ -2975,7 +3183,9 @@ export function AnimationPanel({ overlay: o, peers }: { overlay: Overlay; peers?
? activeMove?.style
: slot === "words"
? anim.words?.style
- : active?.style
+ : slot === "hit"
+ ? hitStyle
+ : active?.style
}
custom={!!active?.preset}
isText={isTextOverlay(o)}
@@ -3014,6 +3224,8 @@ function AnimationToolbar({ overlay: o, slot, peers }: { overlay: Overlay; slot:
if (slot === "words") return bar(<WordSettings overlay={o} peers={peers} />);
+ if (slot === "hit") return bar(<HitSettings overlay={o} peers={peers} />);
+
if (slot === "loop") {
const write = (speed: number) => {
if (!anim.loop) return;
@@ -3049,27 +3261,32 @@ function AnimationToolbar({ overlay: o, slot, peers }: { overlay: Overlay; slot:
ck.begin();
writeOverlayAnim(o, anim, { [slot]: { ...edge, seconds: secs } }, peers);
};
+ // A typing entrance on a title carries its typing bar's settings too.
+ const types = slot === "in" && isTextOverlay(o) && !!edgeMotion(edge)?.animate.typed;
return bar(
- <Row label="Duration">
- <ValueSlider
- label="Duration"
- sliderClassName="data-horizontal:w-24"
- valueClassName="w-9 text-muted-foreground"
- value={edge?.seconds ?? OVERLAY_ANIM_DEFAULT_SECONDS}
- min={OVERLAY_ANIM_MIN_SECONDS}
- max={Math.min(OVERLAY_ANIM_MAX_SECONDS, dur)}
- step={0.05}
- snap={[OVERLAY_ANIM_DEFAULT_SECONDS]}
- format={(v) => `${v.toFixed(2)}s`}
- parse={parseSecondsInput}
- disabled={!edge}
- onDraft={write}
- onCommit={(v) => {
- write(v);
- ck.end();
- }}
- />
- </Row>
+ <>
+ <Row label="Duration">
+ <ValueSlider
+ label="Duration"
+ sliderClassName="data-horizontal:w-24"
+ valueClassName="w-9 text-muted-foreground"
+ value={edge?.seconds ?? OVERLAY_ANIM_DEFAULT_SECONDS}
+ min={OVERLAY_ANIM_MIN_SECONDS}
+ max={Math.min(OVERLAY_ANIM_MAX_SECONDS, dur)}
+ step={0.05}
+ snap={[OVERLAY_ANIM_DEFAULT_SECONDS]}
+ format={(v) => `${v.toFixed(2)}s`}
+ parse={parseSecondsInput}
+ disabled={!edge}
+ onDraft={write}
+ onCommit={(v) => {
+ write(v);
+ ck.end();
+ }}
+ />
+ </Row>
+ {types && <CaretSettings overlay={o} peers={peers} />}
+ </>
);
}
diff --git a/site/src/cut/components/OverlayLayer.tsx b/site/src/cut/components/OverlayLayer.tsx
index 8c4154e1..4f784684 100644
--- a/site/src/cut/components/OverlayLayer.tsx
+++ b/site/src/cut/components/OverlayLayer.tsx
@@ -388,15 +388,19 @@ function WordText({
text,
draws,
lineHeight,
+ tail,
}: {
text: string;
draws: WordDraw[];
lineHeight?: number;
+ /** Drawn at the end of the last line (the typing bar). */
+ tail?: React.ReactNode;
}) {
let k = 0;
+ const lines = text.split("\n");
return (
<>
- {text.split("\n").map((line, li) => (
+ {lines.map((line, li) => (
<span key={li} className="block">
{line
.split(" ")
@@ -1225,15 +1229,19 @@ function OverlayItem({
{o.text}
</div>
) : glyphs ? (
- <GlyphText text={shownText} phase={glyphs} scale={scale} />
+ <GlyphText text={shownText} phase={glyphs} scale={scale} tail={caretBar} />
) : wordDraw ? (
<WordText
text={shownText}
draws={wordDraw}
lineHeight={(isTextOverlay(o) ? o.lineHeight : undefined) ?? LINE_HEIGHT}
+ tail={caretBar}
/>
) : (
- <span>{shownText}</span>
+ <span>
+ {shownText}
+ {caretBar}
+ </span>
)
) : o.kind === "shape" ? (
<ShapeView shape={o} stageWidth={stageWidth} stageHeight={stageHeight} scale={scale} />
@@ -1356,12 +1364,16 @@ function GlyphText({
text,
phase,
scale,
+ tail,
}: {
text: string;
phase: OverlayFrameState;
scale: number;
+ /** Drawn at the end of the last line (the typing bar). */
+ tail?: React.ReactNode;
}) {
const total = Math.max(1, [...text].length);
+ const lineCount = text.split("\n").length;
const seed = slotSeed(text);
let gi = 0;
return (
@@ -1469,7 +1481,7 @@ function DiveCanvas({
height: h,
scale: Math.min(w, h) / 1080,
});
- await paintElementInto(canvas, el, env, { dive: view, pose: next });
+ await paintElementInto(canvas, el, env, { dive: view, pose: next, ...(next.caret ? { caret: true } : {}) });
}
} finally {
busy.current = false;
diff --git a/site/src/cut/components/SubtitlesPanel.tsx b/site/src/cut/components/SubtitlesPanel.tsx
index 8d352a25..c85f03e7 100644
--- a/site/src/cut/components/SubtitlesPanel.tsx
+++ b/site/src/cut/components/SubtitlesPanel.tsx
@@ -692,6 +692,77 @@ function Transcript({ cues }: { cues: SubtitleCue[] }) {
);
}
+/** Characters from the start of `el` to a point inside it. */
+function offsetIn(el: HTMLElement, node: Node, offset: number): number {
+ const range = document.createRange();
+ range.selectNodeContents(el);
+ range.setEnd(node, offset);
+ return range.toString().length;
+}
+
+/** The current selection as character offsets inside `el`, or null when it
+ * lies elsewhere. */
+function selectionIn(el: HTMLElement): [number, number] | null {
+ const sel = window.getSelection();
+ if (!sel || sel.rangeCount === 0) return null;
+ const r = sel.getRangeAt(0);
+ if (!el.contains(r.startContainer) || !el.contains(r.endContainer)) return null;
+ return [offsetIn(el, r.startContainer, r.startOffset), offsetIn(el, r.endContainer, r.endOffset)];
+}
+
+/** The character offset inside `el` under a pointer, or null off its text. */
+function pointIn(el: HTMLElement, x: number, y: number): number | null {
+ let node: Node | null = null;
+ let offset = 0;
+ if ("caretPositionFromPoint" in document) {
+ const p = document.caretPositionFromPoint(x, y);
+ if (p) ({ offsetNode: node, offset } = p);
+ } else if ("caretRangeFromPoint" in document) {
+ const r = (document as Document).caretRangeFromPoint(x, y);
+ if (r) ({ startContainer: node, startOffset: offset } = r);
+ }
+ return node && el.contains(node) ? offsetIn(el, node, offset) : null;
+}
+
+/** Put the selection back at character offsets inside `el`. */
+function selectIn(el: HTMLElement, from: number, to: number) {
+ const at = (target: number): [Node, number] => {
+ const walk = document.createTreeWalker(el, NodeFilter.SHOW_TEXT);
+ let left = target;
+ let last: Text | null = null;
+ for (let n = walk.nextNode() as Text | null; n; n = walk.nextNode() as Text | null) {
+ if (left <= n.length) return [n, left];
+ left -= n.length;
+ last = n;
+ }
+ return last ? [last, last.length] : [el, 0];
+ };
+ const sel = window.getSelection();
+ if (!sel) return;
+ const range = document.createRange();
+ range.setStart(...at(from));
+ range.setEnd(...at(to));
+ sel.removeAllRanges();
+ sel.addRange(range);
+}
+
+/** Emphasized words in the transcript read set apart from the rest. */
+const EM_CLASS = "sub-em rounded-[3px] bg-[#FFE94A]/70 font-semibold italic";
+
+/** Fill the editable caption with its text, each emphasized word in a marked
+ * span. Reading `textContent` back gives the text unchanged. */
+function paintCue(el: HTMLElement, text: string, marked: readonly number[]) {
+ el.replaceChildren(
+ ...emphasisRuns(text, marked).map((run) => {
+ if (!run.em) return document.createTextNode(run.text);
+ const span = document.createElement("span");
+ span.className = EM_CLASS;
+ span.textContent = run.text;
+ return span;
+ })
+ );
+}
+
function caretOffset(el: HTMLElement): number {
const sel = window.getSelection();
if (!sel || sel.rangeCount === 0) return 0;
@@ -805,10 +876,35 @@ const CueSpan = memo(function CueSpan({ cue, gap }: { cue: SubtitleCue; gap: num
data-cue={cue.id}
onFocus={() => setFocused(true)}
onBlur={commit}
+ onContextMenu={(e) => {
+ const el = ref.current;
+ if (!el || readOnly) return;
+ // A selection under the pointer names its words; otherwise the word
+ // the pointer is on.
+ const sel = selectionIn(el);
+ const at = pointIn(el, e.clientX, e.clientY);
+ const range: [number, number] | null =
+ sel && sel[0] !== sel[1] && at !== null && at >= Math.min(...sel) && at <= Math.max(...sel)
+ ? sel
+ : at !== null
+ ? [at, at]
+ : null;
+ const hit = range ? wordsAt(range[0], range[1]) : null;
+ if (!range || !hit) return;
+ e.preventDefault();
+ e.stopPropagation();
+ setMenu({ x: e.clientX, y: e.clientY, from: range[0], to: range[1], label: hit.label, on: hit.on });
+ }}
onKeyDown={(e) => {
e.stopPropagation();
const el = ref.current;
if (!el) return;
+ if ((e.metaKey || e.ctrlKey) && !e.altKey && !e.shiftKey && e.key.toLowerCase() === "b") {
+ e.preventDefault();
+ const sel = selectionIn(el);
+ if (sel && !readOnly) toggle(sel[0], sel[1]);
+ return;
+ }
if (e.key === "Escape") {
e.preventDefault();
el.blur();
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 2a6d4a06..40665c3a 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -72,6 +72,14 @@ import {
type OverlayAnim,
type OverlayAnimStyle,
type OverlayLoopStyle,
+ CARET_BLINKS_MAX,
+ edgeMotion,
+ edgePreset,
+ OVERLAY_HIT_DEFAULT_SECONDS,
+ OVERLAY_HIT_MAX_SECONDS,
+ OVERLAY_HIT_MIN_SECONDS,
+ OVERLAY_HIT_STYLE_IDS,
+ type OverlayCaret,
STROKE_FEATHER_MAX,
STROKE_OFFSET_MAX,
STROKE_STYLES,
@@ -604,9 +612,12 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
throw new ToolError(`Unknown ${slot} style: ${raw}`);
if ((TEXT_ONLY_ANIM_STYLE_IDS as string[]).includes(raw) && (o.kind ?? "text") !== "text")
throw new ToolError(`${raw} animates titles only.`);
+ // A caret stays with an entrance that still types.
+ const caret = edgePreset(raw)?.animate.typed ? anim[slot]?.caret : undefined;
anim[slot] = {
style: raw as OverlayAnimStyle,
seconds: secs ?? anim[slot]?.seconds ?? OVERLAY_ANIM_DEFAULT_SECONDS,
+ ...(caret ? { caret } : {}),
};
} else if (secs !== undefined && anim[slot]) {
anim[slot] = { ...anim[slot]!, seconds: secs };
@@ -676,6 +687,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
...(wordDim !== undefined ? { dim: wordDim } : {}),
};
}
+ applyCaretInput(o, anim, input);
+ applyHitInput(o, anim, input);
s.updateOverlay(o.id, { anim: hasOverlayAnim(anim) ? anim : undefined });
const next = useEditor.getState().overlays.find((x) => x.id === o.id)!;
return { id: next.id, anim: next.anim ?? null };
@@ -6056,6 +6069,98 @@ function soundTarget(s: Editor, id: unknown) {
throw new ToolError(`No video or soundtrack clip with id "${key}".`);
}
+/** sync_audio's bind: line the recording up with the video and bind it, or
+ * refuse with the reason. The clip it was named by aims the match. */
+async function syncVideoSound(s: Editor, video: MediaAsset, audioId: unknown, clip: VideoClip | undefined) {
+ const rec = requireItem(s.assets, audioId, "audio asset");
+ const bound = await bindRecording(video.id, rec.id, clip).catch((e: unknown) => {
+ throw new ToolError(e instanceof Error ? e.message : "Could not line the recording up.");
+ });
+ return {
+ ...bound,
+ note: `Every clip of "${video.name}" (${bound.clips} on the timeline) now plays "${rec.name}"; recording second = video second ${bound.offset < 0 ? "−" : "+"} ${Math.abs(bound.offset)}.${bound.refined ? "" : " Aligned to the nearest 20 ms; the waveforms had no sharp detail to place it closer."}`,
+ };
+}
+
+/** set_overlay_animation's caret fields, written onto the typewriter entrance
+ * in `anim` (which the call has already updated). */
+function applyCaretInput(
+ o: { kind?: string },
+ anim: OverlayAnim,
+ input: Record<string, unknown>
+): void {
+ const named =
+ typeof input.caret === "boolean" ||
+ typeof input.caret_blink === "boolean" ||
+ isNum(input.caret_blinks) ||
+ typeof input.caret_hide === "boolean" ||
+ typeof input.caret_color === "string";
+ if (!named) return;
+ const edge = anim.in;
+ if (input.caret === false) {
+ if (edge?.caret) {
+ const next = { ...edge };
+ delete next.caret;
+ anim.in = next;
+ }
+ return;
+ }
+ if ((o.kind ?? "text") !== "text" || !edge || !edgeMotion(edge)?.animate.typed)
+ throw new ToolError("A caret rides a typewriter entrance on a title: set in_style typewriter on a title first.");
+ const caret: OverlayCaret = { ...(edge.caret ?? { blink: true }) };
+ if (typeof input.caret_blink === "boolean") caret.blink = input.caret_blink;
+ if (isNum(input.caret_blinks)) {
+ const n = Math.round(clamp(input.caret_blinks, 0, CARET_BLINKS_MAX));
+ if (n > 0) caret.blinks = n;
+ else delete caret.blinks;
+ }
+ if (typeof input.caret_hide === "boolean") {
+ if (input.caret_hide) caret.hide = true;
+ else delete caret.hide;
+ }
+ if (typeof input.caret_color === "string") caret.color = input.caret_color;
+ anim.in = { ...edge, caret };
+}
+
+/** set_overlay_animation's hit fields, written onto `anim`. A new hit takes
+ * the default length, darkens, and starts at the element's middle unless
+ * hit_at names the moment; the moment is held so the whole hit fits. */
+function applyHitInput(
+ o: { start: number; end: number },
+ anim: OverlayAnim,
+ input: Record<string, unknown>
+): void {
+ const raw = input.hit_style;
+ const named =
+ typeof raw === "string" ||
+ isNum(input.hit_at) ||
+ isNum(input.hit_seconds) ||
+ typeof input.hit_darken === "boolean";
+ if (!named) return;
+ if (raw === "none") {
+ delete anim.hit;
+ return;
+ }
+ if (typeof raw === "string" && !OVERLAY_HIT_STYLE_IDS.includes(raw))
+ throw new ToolError(`Unknown hit style: ${raw}. Use one of: ${OVERLAY_HIT_STYLE_IDS.join(", ")}.`);
+ const style = typeof raw === "string" ? raw : anim.hit?.style;
+ if (!style) throw new ToolError("Name hit_style to add a hit.");
+ const dur = Math.max(0.1, o.end - o.start);
+ const seconds = Math.min(
+ dur,
+ isNum(input.hit_seconds)
+ ? clamp(input.hit_seconds, OVERLAY_HIT_MIN_SECONDS, OVERLAY_HIT_MAX_SECONDS)
+ : (anim.hit?.seconds ?? OVERLAY_HIT_DEFAULT_SECONDS)
+ );
+ const at = isNum(input.hit_at) ? input.hit_at : (anim.hit?.at ?? (dur - seconds) / 2);
+ anim.hit = {
+ style,
+ at: Math.round(clamp(at, 0, Math.max(0, dur - seconds)) * 1000) / 1000,
+ seconds,
+ darken: typeof input.hit_darken === "boolean" ? input.hit_darken : (anim.hit?.darken ?? true),
+ };
+}
+
function requireItem<T extends { id: string }>(pool: T[], id: unknown, label: string): T {
const item = pool.find((x) => x.id === String(id ?? ""));
if (!item) throw new ToolError(`No ${label} with id ${String(id)}. Call get_state for current ids.`);
diff --git a/site/src/cut/lib/animPreview.ts b/site/src/cut/lib/animPreview.ts
index 158c72e3..1aa7824d 100644
--- a/site/src/cut/lib/animPreview.ts
+++ b/site/src/cut/lib/animPreview.ts
@@ -10,13 +10,13 @@
* frame and nothing else on the page moves.
*
* A run lasts exactly as long as the animation it shows — the In/Out length,
- * or one cycle of a loop. When it ends the element holds its resting pose:
+ * one cycle of a loop, or a hit's own window. When it ends the element holds its resting pose:
* the paused playhead may sit mid-fade or mid-spin, and snapping there would
* leave the element hidden or upside-down right after the pick. The hold
* lifts the moment the preview clock moves.
*/
-import { loopPeriod, type OverlayAnim } from "@donkeycut/effects-kit";
+import { hitWindow, loopPeriod, type OverlayAnim } from "@donkeycut/effects-kit";
import { useSyncExternalStore } from "react";
/** A live rehearsal: which element, where in the slot's own window, and the
@@ -87,7 +87,7 @@ const SPAN_REHEARSAL_MAX = 2.4;
*/
export function playAnimPreview(
o: { id: string; start: number; end: number; anim?: OverlayAnim },
- slot: "in" | "out" | "loop" | "move" | "words"
+ slot: "in" | "out" | "loop" | "move" | "words" | "hit"
): void {
stopAnimPreview();
const anim: OverlayAnim =
@@ -99,16 +99,23 @@ export function playAnimPreview(
? { loop: o.anim?.loop }
: slot === "move"
? { move: o.anim?.move }
- : { words: o.anim?.words };
+ : slot === "hit"
+ ? { hit: o.anim?.hit }
+ : { words: o.anim?.words };
const wholeSpan = slot === "move" || slot === "words";
if (slot === "move" && !anim.move) return;
if (slot === "words" && !anim.words) return;
const dur = Math.max(0.1, o.end - o.start);
- const span = wholeSpan
- ? dur
- : slot === "loop"
- ? (loopPeriod(anim) ?? 0)
- : Math.min((slot === "in" ? anim.in?.seconds : anim.out?.seconds) ?? 0, dur);
+ // A hit plays its own window, wherever in the element it sits.
+ const hit = slot === "hit" ? hitWindow(anim, dur) : null;
+ if (slot === "hit" && !hit) return;
+ const span = hit
+ ? hit.end - hit.start
+ : wholeSpan
+ ? dur
+ : slot === "loop"
+ ? (loopPeriod(anim) ?? 0)
+ : Math.min((slot === "in" ? anim.in?.seconds : anim.out?.seconds) ?? 0, dur);
if (span <= 0) return;
// How long the rehearsal takes on the wall clock. A slot that plays at its
// own speed keeps it; one that runs the whole span is compressed into the
@@ -116,7 +123,7 @@ export function playAnimPreview(
const window = wholeSpan ? Math.min(span, SPAN_REHEARSAL_MAX) : span;
// Where the slot lives inside the element: In, a loop and a move start at
// its head, Out ends at its tail.
- const from = slot === "out" ? dur - span : 0;
+ const from = hit ? hit.start : slot === "out" ? dur - span : 0;
const t0 = performance.now();
const step = () => {
const elapsed = (performance.now() - t0) / 1000;
diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.ts
index 4aafe7be..0d2c36f2 100644
--- a/site/src/cut/lib/audioMix.ts
+++ b/site/src/cut/lib/audioMix.ts
@@ -121,6 +121,10 @@ export interface MixSpec {
items: MixItem[];
/** Audio effect elements over the finished mix, in timeline order. */
effects?: MixEffect[];
+ /** The duck envelope, when it comes from items outside this spec: a stem
+ * holds some of the mix's items and still ducks under all of its
+ * voiceovers. Absent = read from `items`. */
+ ducks?: { start: number; end: number; gain: number }[];
}
export interface MixOptions {
diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.ts
index a526892a..06b394e8 100644
--- a/site/src/cut/lib/behindPass.ts
+++ b/site/src/cut/lib/behindPass.ts
@@ -11,6 +11,7 @@
import {
applyWordDraw,
+ darkenCanvas,
diveView,
divesAt,
evalOverlayFrame,
@@ -74,6 +75,36 @@ function rasterSliceAt(o: Overlay, tLocal: number): number {
return i < 0 ? spans.length - 1 : i;
}
+/**
+ * Which picture an element needs at one moment: its word slice, how many
+ * characters a typewriter has typed (-1 = the whole text), and whether the
+ * typing bar is lit — the same reading the front path's painters take from
+ * the evaluator. One record, rewritten in place every time it is read.
+ */
+const look = { word: -1, chars: -1, caret: false, key: 0 };
+
+/** Fill `look` for `o` at `tLocal` and return its picture key. A whole-text
+ * picture keys at zero or above (two per word slice, bar dark and lit); a
+ * typing picture keys below zero, one per typed count. */
+function lookAt(o: Overlay, tLocal: number, ev: OverlayFrameState): number {
+ look.word = rasterSliceAt(o, tLocal);
+ look.caret = ev.caret === true;
+ const n = isTextOverlay(o) ? o.text.length : 0;
+ look.chars =
+ isTextOverlay(o) && ev.textProgress !== undefined
+ ? Math.max(0, Math.min(n, Math.ceil(ev.textProgress * n)))
+ : -1;
+ const bar = look.caret ? 1 : 0;
+ look.key =
+ look.chars < 0
+ ? (look.word + 1) * 2 + bar
+ : -1 - (((look.word + 1) * (n + 1) + look.chars) * 2 + bar);
+ return look.key;
+}
+
+/** The phase a picture with its typing bar lit is painted at. */
+const CARET_LIT: PaintPhase = { caret: true };
+
/** Segmentation input width — small on purpose; this runs per frame. */
const SEG_WIDTH = 256;
@@ -136,6 +167,20 @@ const entryBytes = (e: { byWord: Map<number, ImageBitmap> }): number => {
return n;
};
+/** One element's pictures: whole-text ones by word slice and bar, and the
+ * typing picture it is on. `want` is the typing picture it is waiting for. */
+interface RasterEntry {
+ of: Overlay;
+ byWord: Map<number, ImageBitmap>;
+ pending: Set<number>;
+ usedAt: number;
+ last: ImageBitmap | null;
+ want: number | null;
+ wantWord: number;
+ wantChars: number;
+ wantCaret: boolean;
+}
+
export class SubjectMaskCompositor {
/** `publishes` marks the live preview's instance, the one whose matte the
* DOM layer reads. */
@@ -159,10 +204,14 @@ export class SubjectMaskCompositor {
* It marks a draw already asked for, and a marker outliving the picture it
* was asked for would leave the element blank for good.
*/
- private rasters = new Map<
- string,
- { of: Overlay; byWord: Map<number, ImageBitmap>; pending: Set<number>; usedAt: number }
- >();
+ private rasters = new Map<string, RasterEntry>();
+ /** Typing pictures are painted one at a time on this surface: a typed
+ * count is a picture shown for a frame or two, so it skips the PNG round
+ * trip, and an element that types faster than the pictures land shows the
+ * latest count it reached. */
+ private typeSurface: HTMLCanvasElement | null = null;
+ private typing = false;
+ private typeArgs: { w: number; h: number; assets: MediaAsset[] } = { w: 1, h: 1, assets: [] };
/** Frames this pass has drawn, which is the clock the pictures are given
* back on: an element drawn on the frame being composited is the one the
* next frame will ask for again. */
@@ -170,9 +219,12 @@ export class SubjectMaskCompositor {
private readonly releaseMemory = holdMemory("overlayRasters", () => {
let n = 0;
for (const e of this.rasters.values()) n += entryBytes(e);
- return n;
+ const dark = this.darkSurface ? this.darkSurface.width * this.darkSurface.height * 4 : 0;
+ return n + this.fx.bytes() + dark;
});
private person: HTMLCanvasElement | null = null;
+ /** Scratch for blurred and motion-blurred behind elements. */
+ private fx = new ElementFx(createRasterCanvas);
private small: HTMLCanvasElement | null = null;
private mask: { at: number; alpha: HTMLCanvasElement | null } = { at: -1e9, alpha: null };
/** A second matte slot for mid-stack clip masks: it snapshots the canvas as
@@ -198,9 +250,21 @@ export class SubjectMaskCompositor {
entry.byWord.clear();
entry.pending.clear();
entry.of = o;
+ entry.last = null;
+ entry.want = null;
}
if (!entry) {
- entry = { of: o, byWord: new Map(), pending: new Set(), usedAt: this.pass };
+ entry = {
+ of: o,
+ byWord: new Map(),
+ pending: new Set(),
+ usedAt: this.pass,
+ last: null,
+ want: null,
+ wantWord: -1,
+ wantChars: -1,
+ wantCaret: false,
+ };
this.rasters.set(o.id, entry);
}
entry.usedAt = this.pass;
@@ -235,44 +299,115 @@ export class SubjectMaskCompositor {
}
}
- private rasterFor(
- o: Overlay,
- w: number,
- h: number,
- assets: MediaAsset[],
- word: number
- ): ImageBitmap | null {
+ /**
+ * The picture `look` names (see `lookAt`), or while it is being drawn the
+ * last one this element landed, so a typing title moves on a frame late
+ * and never blinks out.
+ */
+ private rasterFor(o: Overlay, w: number, h: number, assets: MediaAsset[]): ImageBitmap | null {
const entry = this.entryFor(o);
- const hit = entry.byWord.get(word);
+ const key = look.key;
+ const hit = entry.byWord.get(key);
if (hit) return hit;
- if (!entry.pending.has(word)) {
- entry.pending.add(word);
+ if (key < 0) {
+ if (entry.want !== key) {
+ entry.want = key;
+ entry.wantWord = look.word;
+ entry.wantChars = look.chars;
+ entry.wantCaret = look.caret;
+ }
+ this.typeArgs.w = w;
+ this.typeArgs.h = h;
+ this.typeArgs.assets = assets;
+ this.pumpTyping();
+ } else if (!entry.pending.has(key)) {
+ entry.pending.add(key);
// Neutral picture: position aside, the per-frame pose owns rotation and
// opacity, so baking them here would apply each of them twice.
- void this.drawRaster(o, w, h, assets, word).catch(() => {});
+ void this.drawRaster(o, w, h, assets, key, look.word, look.caret).catch(() => {});
}
- return null;
+ return entry.last;
}
- /** One element picture, with its words drawn as they stand across the span
- * it covers, kept under the element it came from. */
+ /** The element as one picture shows it: its words as they stand across
+ * the span the picture covers, `chars` of its text typed (-1 = all). */
+ private pictureOf(o: Overlay, word: number, chars: number): Overlay {
+ const spans = rasterSpans(o);
+ const span = word >= 0 ? spans?.[word] : undefined;
+ const at = span ? (span.start + span.end) / 2 : 0;
+ const el = applyWordDraw({ ...o, rotation: undefined, opacity: undefined }, at, spanOf(o));
+ return chars >= 0 && isTextOverlay(el) ? { ...el, text: el.text.slice(0, chars) } : el;
+ }
+
+ /** One whole-text picture, kept under the element it came from. */
private async drawRaster(
o: Overlay,
w: number,
h: number,
assets: MediaAsset[],
- word: number
+ key: number,
+ word: number,
+ caret: boolean
): Promise<void> {
- const spans = rasterSpans(o);
- const span = word >= 0 ? spans?.[word] : undefined;
- const at = span ? (span.start + span.end) / 2 : 0;
- const png = await renderElementPng(
- applyWordDraw({ ...o, rotation: undefined, opacity: undefined }, at, spanOf(o)),
- w,
- h,
- assets
+ const png = await renderElementPng(this.pictureOf(o, word, -1), w, h, assets, caret ? CARET_LIT : undefined);
+ this.land(o, key, await createImageBitmap(png));
+ }
+
+ /** One typing picture, painted on the shared surface. */
+ private async drawTyped(
+ o: Overlay,
+ w: number,
+ h: number,
+ assets: MediaAsset[],
+ key: number,
+ word: number,
+ chars: number,
+ caret: boolean
+ ): Promise<void> {
+ this.typeSurface ??= scratchCanvas();
+ if (this.typeSurface.width !== w) this.typeSurface.width = w;
+ if (this.typeSurface.height !== h) this.typeSurface.height = h;
+ await paintElementInto(
+ this.typeSurface,
+ this.pictureOf(o, word, chars),
+ cutRenderEnv(assets),
+ caret ? CARET_LIT : undefined
);
- const bmp = await createImageBitmap(png);
+ this.land(o, key, await createImageBitmap(this.typeSurface));
+ }
+
+ /** Paint the typing pictures elements are waiting on, one after another,
+ * each at the latest count its element asked for. */
+ private pumpTyping(): void {
+ if (this.typing) return;
+ this.typing = true;
+ void (async () => {
+ try {
+ for (;;) {
+ let next: RasterEntry | null = null;
+ for (const e of this.rasters.values()) {
+ if (e.want !== null) {
+ next = e;
+ break;
+ }
+ }
+ if (!next) break;
+ const key = next.want!;
+ next.want = null;
+ const { w, h, assets } = this.typeArgs;
+ await this.drawTyped(next.of, w, h, assets, key, next.wantWord, next.wantChars, next.wantCaret).catch(
+ () => {}
+ );
+ }
+ } finally {
+ this.typing = false;
+ }
+ })();
+ }
+
+ /** File a finished picture. Any typing picture it supersedes goes: an
+ * element holds one typed count at a time, however long it types. */
+ private land(o: Overlay, key: number, bmp: ImageBitmap): void {
// The element may have been edited, or the pictures released, while this
// one was being drawn; either way its entry is gone or stamped with a
// different object, and this picture is of the wrong thing.
@@ -281,8 +416,34 @@ export class SubjectMaskCompositor {
bmp.close();
return;
}
- entry.byWord.get(word)?.close();
- entry.byWord.set(word, bmp);
+ entry.pending.delete(key);
+ for (const [k, b] of entry.byWord) {
+ if (k >= 0 || k === key) continue;
+ b.close();
+ entry.byWord.delete(k);
+ }
+ entry.byWord.get(key)?.close();
+ entry.byWord.set(key, bmp);
+ entry.last = bmp;
+ }
+
+ /** Export path: the picture every behind element needs at `t`, drawn
+ * before the frame is, so no frame of a render shows a stale one. */
+ async ready(overlays: Overlay[], w: number, h: number, assets: MediaAsset[], t: number): Promise<void> {
+ for (const o of overlays) {
+ if (!behindSubjectOverlay(o) || !drawable(o) || t < o.start || t > o.end) continue;
+ const tLocal = Math.max(0, t - o.start);
+ const key = lookAt(o, tLocal, evalOverlayFrame(o, tLocal, w / h));
+ const entry = this.entryFor(o);
+ if (entry.byWord.has(key)) continue;
+ const { word, chars, caret } = look;
+ try {
+ if (key < 0) await this.drawTyped(o, w, h, assets, key, word, chars, caret);
+ else await this.drawRaster(o, w, h, assets, key, word, caret);
+ } catch {
+ // The overlay just draws in front when its raster is missing.
+ }
+ }
}
/** Give back every picture this pass is holding. The pass stays usable and
@@ -295,6 +456,7 @@ export class SubjectMaskCompositor {
/** Give the pictures back and stop reporting what this pass holds. */
dispose(): void {
this.clear();
+ this.fx.dispose();
this.releaseMemory();
}
@@ -315,9 +477,10 @@ export class SubjectMaskCompositor {
// An element that lights its words needs one picture per word, all of
// them resident before the first frame draws.
rasterSlices(o).map(async (word) => {
- if (this.entryFor(o).byWord.has(word)) return;
+ const key = (word + 1) * 2;
+ if (this.entryFor(o).byWord.has(key)) return;
try {
- await this.drawRaster(o, w, h, assets, word);
+ await this.drawRaster(o, w, h, assets, key, word, false);
} catch {
// The overlay just draws in front when its raster is missing.
}
@@ -417,6 +580,8 @@ export class SubjectMaskCompositor {
* the element moves under it. The surfaces live and die with this pass. */
private stampSurface: HTMLCanvasElement | null = null;
private stampScratch: HTMLCanvasElement | null = null;
+ /** Where an element a hit darkens is drawn alone (see `darkTarget`). */
+ private darkSurface: HTMLCanvasElement | null = null;
mattedStamp(
o: { mask?: { invert?: boolean; feather?: number } },
w: number,
@@ -444,6 +609,17 @@ export class SubjectMaskCompositor {
return this.stampSurface;
}
+ /** The darkening surface, cleared and sized to the frame. */
+ private darkTarget(w: number, h: number): CanvasRenderingContext2D {
+ this.darkSurface ??= scratchCanvas();
+ if (this.darkSurface.width !== w) this.darkSurface.width = w;
+ if (this.darkSurface.height !== h) this.darkSurface.height = h;
+ const c = this.darkSurface.getContext("2d")!;
+ c.setTransform(1, 0, 0, 1, 0, 0);
+ c.clearRect(0, 0, w, h);
+ return c;
+ }
+
/**
* Run the behind pass on `canvas` (video layers already drawn): draw the
* behind-tagged rasters, then the segmented person back over them. Also
@@ -505,9 +681,10 @@ export class SubjectMaskCompositor {
const scale = Math.min(W, H) / 1080;
for (const o of active) {
const tLocal = Math.max(0, t - o.start);
- const bmp = this.rasterFor(o, W, H, assets, rasterSliceAt(o, tLocal));
+ const ev = evalOverlayFrame(o, tLocal, W / H);
+ lookAt(o, tLocal, ev);
+ const bmp = this.rasterFor(o, W, H, assets);
if (!bmp) continue;
- const ev = evalOverlayFrame(o, tLocal);
// One cached picture per element here, so a per-glyph ramp or loop runs
// its motion over the whole box as a single letter would.
const g = hasGlyphMotion(ev) ? glyphStateAt(ev, 0, 1) : null;
@@ -515,26 +692,37 @@ export class SubjectMaskCompositor {
if (alphaOf <= 0.001) continue;
const cx = o.x * W;
const cy = o.y * H;
- ctx.save();
- ctx.globalAlpha = alphaOf;
- ctx.translate(
+ // A hit that darkens is drawn alone first, so the darkening lands on
+ // the element's pixels and not on the frame under it. A blurred or
+ // streaking element poses into the scratch and lands soft.
+ const dark = ev.brightness !== undefined && ev.brightness < 1;
+ const c = dark ? this.darkTarget(W, H) : ctx;
+ const target = this.fx.begin(c, W, H, elementLook(ev, scale)) as CanvasRenderingContext2D;
+ target.save();
+ target.globalAlpha = alphaOf;
+ target.translate(
ev.x * W + (ev.dx + (g?.dx ?? 0)) * scale,
ev.y * H + (ev.dy + (g?.dy ?? 0)) * scale
);
- ctx.rotate(((ev.rotation + (g?.rotate ?? 0)) * Math.PI) / 180);
- ctx.scale(ev.scale * (g?.sx ?? 1), ev.scale * (g?.sy ?? 1));
- ctx.translate(-cx, -cy);
+ target.rotate(((ev.rotation + (g?.rotate ?? 0)) * Math.PI) / 180);
+ target.scale(ev.scale * (g?.sx ?? 1), ev.scale * (g?.sy ?? 1));
+ target.translate(-cx, -cy);
// A dive flies the same one picture in, under the view the other
// renderers draw their type under.
const focus = ev.dive && divesAt(o.anim) ? peekDiveFocus(o, W / H, cutRenderEnv(assets)) : undefined;
if (focus) {
const v = diveView(ev.dive!, focus, ev, o, { width: W, height: H, scale });
- ctx.translate(v.tx, v.ty);
- ctx.scale(v.s, v.s);
- ctx.translate(-v.fx, -v.fy);
+ target.translate(v.tx, v.ty);
+ target.scale(v.s, v.s);
+ target.translate(-v.fx, -v.fy);
+ }
+ target.drawImage(bmp, 0, 0, W, H);
+ target.restore();
+ this.fx.end(c);
+ if (dark) {
+ darkenCanvas(c, W, H, ev.brightness!);
+ ctx.drawImage(c.canvas, 0, 0);
}
- ctx.drawImage(bmp, 0, 0, W, H);
- ctx.restore();
}
// The person back on top, its edge softened by the widest feather any
diff --git a/site/src/cut/lib/behindPassTyping.test.ts b/site/src/cut/lib/behindPassTyping.test.ts
new file mode 100644
index 00000000..a1e37953
--- /dev/null
+++ b/site/src/cut/lib/behindPassTyping.test.ts
@@ -0,0 +1,93 @@
+import { beforeAll, describe, expect, test } from "bun:test";
+import { CARET_BLINK_SECONDS, evalOverlayFrame } from "@donkeycut/effects-kit";
+import { SubjectMaskCompositor } from "./behindPass";
+import { installSkiaRaster } from "./headless/skiaRaster";
+import { createRasterCanvas } from "./raster";
+import type { TextOverlay } from "./types";
+
+/**
+ * Elements behind the speaker type their typewriter entrance and show its
+ * caret, the same pictures the front path paints.
+ */
+
+const W = 540;
+const H = 960;
+const title: TextOverlay = {
+ id: "cta",
+ text: "LINK IN BIO",
+ start: 2,
+ end: 6,
+ x: 0.5,
+ y: 0.5,
+ size: 80,
+ font: "sf",
+ weight: 700,
+ color: "#ffffff",
+ plate: false,
+ shadow: false,
+ mask: { kind: "subject", invert: true } as TextOverlay["mask"],
+ anim: { in: { style: "typewriter", seconds: 1, caret: { blink: true, color: "#ff0000" } } },
+};
+
+type Entry = { byWord: Map<number, CanvasImageSource & { width: number; height: number }> };
+const entryOf = (pass: SubjectMaskCompositor) =>
+ (pass as unknown as { rasters: Map<string, Entry> }).rasters.get(title.id)!;
+
+/** Ink in a picture: how many pixels are white, how many are the red bar,
+ * and the rightmost column anything reaches. */
+function ink(picture: CanvasImageSource) {
+ const c = createRasterCanvas(W, H) as HTMLCanvasElement;
+ const ctx = c.getContext("2d")!;
+ ctx.drawImage(picture, 0, 0, W, H);
+ const d = ctx.getImageData(0, 0, W, H).data;
+ let white = 0;
+ let red = 0;
+ for (let i = 0; i < d.length; i += 4) {
+ if (d[i + 3] < 200) continue;
+ if (d[i] > 200 && d[i + 1] > 200) white++;
+ else if (d[i] > 200 && d[i + 1] < 60) red++;
+ }
+ return { white, red };
+}
+
+beforeAll(async () => {
+ expect(await installSkiaRaster()).toBe(true);
+});
+
+describe("the behind pass types", () => {
+ test("a typing moment is a typed picture with the bar lit", async () => {
+ const pass = new SubjectMaskCompositor();
+ const tLocal = 0.4;
+ const ev = evalOverlayFrame(title, tLocal);
+ expect(ev.textProgress).toBeLessThan(1);
+ await pass.ready([title], W, H, [], title.start + tLocal);
+ const typed = [...entryOf(pass).byWord].filter(([k]) => k < 0);
+ expect(typed.length).toBe(1);
+ const part = ink(typed[0][1]);
+ await pass.ready([title], W, H, [], title.start + 0.95);
+ const typedLater = [...entryOf(pass).byWord].filter(([k]) => k < 0);
+ // One typing picture at a time, and it has typed further.
+ expect(typedLater.length).toBe(1);
+ const more = ink(typedLater[0][1]);
+ expect(more.white).toBeGreaterThan(part.white);
+ expect(part.red).toBeGreaterThan(0);
+ pass.dispose();
+ });
+
+ test("after typing the bar blinks: lit and dark pictures of the whole text", async () => {
+ const pass = new SubjectMaskCompositor();
+ const after = title.start + 1;
+ await pass.ready([title], W, H, [], after + CARET_BLINK_SECONDS * 0.5);
+ await pass.ready([title], W, H, [], after + CARET_BLINK_SECONDS * 1.5);
+ const whole = [...entryOf(pass).byWord].filter(([k]) => k >= 0);
+ expect(whole.map(([k]) => k).sort()).toEqual([0, 1]);
+ const dark = ink(whole.find(([k]) => k === 0)![1]);
+ const lit = ink(whole.find(([k]) => k === 1)![1]);
+ expect(dark.red).toBe(0);
+ expect(lit.red).toBeGreaterThan(0);
+ expect(lit.white).toBe(dark.white);
+ // The typing picture is gone once the whole text lands.
+ expect([...entryOf(pass).byWord.keys()].some((k) => k < 0)).toBe(false);
+ pass.dispose();
+ });
+});
diff --git a/site/src/cut/lib/caretHitTool.test.ts b/site/src/cut/lib/caretHitTool.test.ts
new file mode 100644
index 00000000..a9b1aee9
--- /dev/null
+++ b/site/src/cut/lib/caretHitTool.test.ts
@@ -0,0 +1,80 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+import { OVERLAY_HIT_STYLE_IDS } from "@donkeycut/effects-kit";
+import { OVERLAY_ANIMATION_TOOLS } from "@/cut/components/AnimationTiles.tools";
+import { runAiTool } from "./aiTools";
+import { useEditor } from "./store";
+import { emptySubtitles, isTextOverlay } from "./types";
+
+/**
+ * The call-to-action pieces through set_overlay_animation: a caret on a
+ * typewriter entrance, and a press at a moment inside the element.
+ */
+
+const reset = () =>
+ useEditor.setState({
+ projectId: "p1",
+ aspect: "9:16",
+ assets: [],
+ clips: [],
+ overlays: [],
+ subtitles: emptySubtitles(),
+ background: "#000000",
+ });
+
+const overlays = () => useEditor.getState().overlays;
+const aTitle = async () => {
+ await runAiTool("add_text_sequence", {
+ look: "serif-mood",
+ lines: [{ text: "LINK", start: 0, end: 4 }],
+ });
+ return overlays().filter(isTextOverlay)[0];
+};
+const failure = async (args: Record<string, unknown>) => {
+ try {
+ await runAiTool("set_overlay_animation", args);
+ } catch (err) {
+ return err instanceof Error ? err.message : String(err);
+ }
+ return "";
+};
+
+describe("set_overlay_animation caret and hit", () => {
+ beforeEach(reset);
+
+ test("the schema lists every hit the catalog has", () => {
+ const tool = OVERLAY_ANIMATION_TOOLS.find((t) => t.name === "set_overlay_animation")!;
+ const props = (tool.inputSchema as { properties: Record<string, { enum?: string[] }> }).properties;
+ expect(props.hit_style.enum).toEqual([...OVERLAY_HIT_STYLE_IDS, "none"]);
+ for (const key of ["caret", "caret_blink", "caret_blinks", "caret_hide", "caret_color", "hit_at", "hit_seconds", "hit_darken"])
+ expect(props[key]).toBeDefined();
+ expect(tool.description).toContain("press");
+ });
+
+ test("a caret rides the typewriter entrance and its settings land on it", async () => {
+ const o = await aTitle();
+ await runAiTool("set_overlay_animation", { id: o.id, in_style: "typewriter", in_seconds: 1, caret: true, caret_blinks: 3, caret_hide: true });
+ expect(overlays()[0].anim?.in?.caret).toEqual({ blink: true, blinks: 3, hide: true });
+ // Re-picking a typing entrance keeps the bar; clearing it takes it off.
+ await runAiTool("set_overlay_animation", { id: o.id, in_style: "typewriter" });
+ expect(overlays()[0].anim?.in?.caret?.blinks).toBe(3);
+ await runAiTool("set_overlay_animation", { id: o.id, caret: false });
+ expect(overlays()[0].anim?.in?.caret).toBeUndefined();
+ });
+
+ test("a caret without a typing entrance is refused", async () => {
+ const o = await aTitle();
+ await runAiTool("set_overlay_animation", { id: o.id, in_style: "fade" });
+ expect(await failure({ id: o.id, caret: true })).toContain("typewriter");
+ });
+
+ test("a press lands at its moment, held inside the element, darkening by default", async () => {
+ const o = await aTitle();
+ await runAiTool("set_overlay_animation", { id: o.id, hit_style: "press", hit_at: 2.5 });
+ expect(overlays()[0].anim?.hit).toEqual({ style: "press", at: 2.5, seconds: 0.3, darken: true });
+ await runAiTool("set_overlay_animation", { id: o.id, hit_at: 10, hit_darken: false });
+ expect(overlays()[0].anim?.hit).toEqual({ style: "press", at: 3.7, seconds: 0.3, darken: false });
+ expect(await failure({ id: o.id, hit_style: "smash" })).toContain("Unknown hit style");
+ await runAiTool("set_overlay_animation", { id: o.id, hit_style: "none" });
+ expect(overlays()[0].anim?.hit).toBeUndefined();
+ });
+});
diff --git a/site/src/cut/lib/composite.ts b/site/src/cut/lib/composite.ts
index 5af2cf65..1cac4c67 100644
--- a/site/src/cut/lib/composite.ts
+++ b/site/src/cut/lib/composite.ts
@@ -231,7 +231,10 @@ export class FrameCompositor {
| "removalInk"
| "removalLookA"
| "removalLookB"
- | "detailScratch",
+ | "detailScratch"
+ | "cardPop"
+ | "cardLayer"
+ | "cardPost",
w: number,
h: number
): { surface: Surface; resized: boolean } {
diff --git a/site/src/cut/lib/cueChunk.ts b/site/src/cut/lib/cueChunk.ts
index 50145d52..803bcc81 100644
--- a/site/src/cut/lib/cueChunk.ts
+++ b/site/src/cut/lib/cueChunk.ts
@@ -83,6 +83,8 @@ interface StreamWord {
* caption it lands in reuses when it opens on the same word. */
from: number;
id: string;
+ /** The word is emphasized, and stays so in whichever caption it lands. */
+ em: boolean;
}
/** One track's words in order, each carrying where it came from. Times are
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index baed59ec..f7fa6142 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -730,6 +730,11 @@ async function stampText(doc: ExportDoc): Promise<StampedLayer[]> {
class StampCache {
private drawn = new Map<StampedLayer, ImageBitmap>();
private maskFrames = new Map<StampedLayer, { t: number; bitmap: ImageBitmap }>();
+ /** Scratch for blurred and motion-blurred stamps, made on first use. */
+ readonly fx = new ElementFx(createRasterCanvas);
+ /** Where an element a hit darkens is drawn alone, so the darkening lands on
+ * its pixels and not on the frame under it. Made on the first such frame. */
+ private dark: RasterSurface | null = null;
constructor(
private width: number,
@@ -773,6 +778,23 @@ class StampCache {
return bitmap;
}
+ /** What the scratch surfaces hold: the blur and streak scratch, and the
+ * darkening surface once a hit has made it. */
+ scratchBytes(): number {
+ return this.fx.bytes() + (this.dark ? canvasBytes(this.dark.width * this.dark.height) : 0);
+ }
+
+ /** The darkening surface, cleared, at the frame's size. */
+ darkTarget(): CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D {
+ this.dark ??= createRasterCanvas(this.width, this.height);
+ const c = this.dark.getContext("2d") as
+ | CanvasRenderingContext2D
+ | OffscreenCanvasRenderingContext2D;
+ c.setTransform(1, 0, 0, 1, 0, 0);
+ c.clearRect(0, 0, this.width, this.height);
+ return c;
+ }
+
/** Where a diving element flies into, measured once per look. */
diveFocus(o: Overlay): Promise<DiveFocus> {
return measureDiveFocus(o, this.width / this.height, cutRenderEnv(this.assets));
@@ -795,10 +817,12 @@ class StampCache {
}
dispose() {
+ this.dark = null;
for (const bitmap of this.drawn.values()) bitmap.close();
this.drawn.clear();
for (const hit of this.maskFrames.values()) hit.bitmap.close();
this.maskFrames.clear();
+ this.fx.dispose();
}
}
@@ -919,6 +943,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
fadeOut: a.fadeOut,
sound: a.sound,
duck: a.duck,
+ lane: a.lane ?? 0,
});
}
@@ -932,7 +957,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
file: resolve(sp.asset),
in: sp.clip.in,
out: sp.clip.out,
- muted: sp.clip.muted || !!sp.clip.hidden || assetIsSilent(sp.asset),
+ muted: sp.clip.muted || !!sp.clip.hidden || (!sp.sound && assetIsSilent(sp.asset)),
speed: sp.clip.speed,
speedCurve: sp.clip.speedCurve,
reverse: sp.clip.reverse,
@@ -943,6 +968,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
soundCross: sp.soundOut,
soundBack: sp.soundBack,
soundAhead: sp.soundAhead,
+ ...spanSound(sp),
},
]);
@@ -1191,20 +1217,33 @@ async function drawStamps(
const o = layer.overlay as StickerOverlay;
const ev = evalOverlayFrame(
{ ...(layer.source ?? o), anim: layer.anim },
- t - (layer.animStart ?? layer.start)
+ t - (layer.animStart ?? layer.start),
+ canvas.width / canvas.height
);
if (ev.opacity <= 0.001) continue;
const scale = Math.min(canvas.width, canvas.height) / 1080;
const w = Math.max(1, o.w * canvas.width);
const aspect = layer.lottie.width > 0 ? layer.lottie.width / layer.lottie.height : 1;
const h = w / aspect;
- ctx.save();
- ctx.globalAlpha = ev.opacity;
- ctx.translate(ev.x * canvas.width + ev.dx * scale, ev.y * canvas.height + ev.dy * scale);
- ctx.rotate((ev.rotation * Math.PI) / 180);
- ctx.scale(ev.scale, ev.scale);
- ctx.drawImage(layer.lottie.seek(t - (layer.animStart ?? layer.start)), -w / 2, -h / 2, w, h);
- ctx.restore();
+ // A hit that darkens is drawn alone first, so the darkening lands on
+ // the sticker's pixels; a blurred or streaking one poses into the
+ // scratch and lands soft.
+ const look = elementLook(ev, scale);
+ const dark = ev.brightness !== undefined && ev.brightness < 1;
+ const c = dark ? stamps.darkTarget() : ctx;
+ const lc = stamps.fx.begin(c, canvas.width, canvas.height, look);
+ lc.save();
+ lc.globalAlpha = ev.opacity;
+ lc.translate(ev.x * canvas.width + ev.dx * scale, ev.y * canvas.height + ev.dy * scale);
+ lc.rotate((ev.rotation * Math.PI) / 180);
+ lc.scale(ev.scale, ev.scale);
+ lc.drawImage(layer.lottie.seek(t - (layer.animStart ?? layer.start)), -w / 2, -h / 2, w, h);
+ lc.restore();
+ stamps.fx.end(c);
+ if (dark) {
+ darkenCanvas(c, canvas.width, canvas.height, ev.brightness!);
+ ctx.drawImage(c.canvas, 0, 0);
+ }
continue;
}
// Unkeyed, the bitmap bakes the element's own rotation/opacity and only
@@ -1213,7 +1252,11 @@ async function drawStamps(
// way this samples the evaluator the ffmpeg path rasterized into frames.
const tLocal = t - (layer.animStart ?? layer.start);
const ev = layer.anim
- ? evalOverlayFrame({ ...(layer.source ?? layer.overlay), anim: layer.anim }, tLocal)
+ ? evalOverlayFrame(
+ { ...(layer.source ?? layer.overlay), anim: layer.anim },
+ tLocal,
+ canvas.width / canvas.height
+ )
: null;
// A per-glyph loop moves the characters inside the picture, so its window
// is drawn afresh each frame with the loop folded into the layer's phase.
@@ -1269,20 +1312,33 @@ async function drawStamps(
c.translate(-cx, -cy);
c.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
};
- ctx.save();
- ctx.globalAlpha = ev.opacity;
+ // A blurred or streaking element poses into the scratch first and lands
+ // softened; a sharp one draws straight onto the frame. A hit that darkens
+ // needs a surface holding the element alone: the matte's, or the dark one.
+ const look = elementLook(ev, scale);
+ const dark = ev.brightness !== undefined && ev.brightness < 1;
+ const lay = (into: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) => {
+ const target = stamps.fx.begin(into, canvas.width, canvas.height, look);
+ target.save();
+ target.globalAlpha = ev.opacity;
+ posed(target);
+ target.restore();
+ stamps.fx.end(into);
+ if (dark) darkenCanvas(into, canvas.width, canvas.height, ev.brightness!);
+ };
if (subjectFront) {
// Pose the stamp inside the matte surface, so the matte stays put in
- // frame space while the element travels beneath it.
- ctx.drawImage(
- subject!.mattedStamp(layer.overlay, canvas.width, canvas.height, posed),
- 0,
- 0
- );
+ // frame space while the element travels beneath it. The softening
+ // lands first and the matte trims after, so the person's edge stays
+ // sharp over a moving element, as the preview shows it.
+ ctx.drawImage(subject!.mattedStamp(layer.overlay, canvas.width, canvas.height, lay), 0, 0);
+ } else if (dark) {
+ const c = stamps.darkTarget();
+ lay(c);
+ ctx.drawImage(c.canvas, 0, 0);
} else {
- posed(ctx);
+ lay(ctx);
}
- ctx.restore();
}
}
@@ -1299,12 +1355,13 @@ export class FramePainter {
private comp: FrameCompositor;
private stamps: StampCache;
private readers = new Map<string, ClipReader>();
- /** What the open readers are standing on, so a render in the tab shows up
- * beside the preview's own pictures in the memory report. */
+ /** What the open readers and the stamp scratch are standing on, so a
+ * render in the tab shows up beside the preview's own pictures in the
+ * memory report. */
private readonly releaseMemory = holdMemory("exportReaders", () => {
let n = 0;
for (const r of this.readers.values()) n += canvasBytes(readerPoolFrames(r) * r.sourcePixels);
- return n;
+ return n + this.stamps.scratchBytes();
});
/** Frames drawn, which is the clock the readers' eviction order runs on. */
private frameNo = 0;
@@ -1508,16 +1565,16 @@ export class FramePainter {
const mt = Math.max(0, Math.min(sourceTimeAt(span, t) - m.in, dur - 0.001));
const frame = await this.readerFor(asset).frameAt(mt);
if (frame.kind !== "ready") return;
- let scratch = this.matteScratch.get(clip.id);
+ let scratch = this.matteScratch.get(key);
// The matte advances at its own baked rate below the export's, so a
// converted frame is reused until the read crosses into the next one —
// the pixel pass is per matte frame, never per output frame.
const stamp = `${m.assetId}:${Math.floor(mt * MATTE_FPS)}`;
- if (scratch && this.matteStamp.get(clip.id) === stamp) {
- this.matteFrames.set(clip.id, scratch as CanvasImageSource);
+ if (scratch && this.matteStamp.get(key) === stamp) {
+ this.matteFrames.set(key, scratch as CanvasImageSource);
return;
}
- if (!scratch) this.matteScratch.set(clip.id, (scratch = createRasterCanvas(frame.width, frame.height)));
+ if (!scratch) this.matteScratch.set(key, (scratch = createRasterCanvas(frame.width, frame.height)));
if (scratch.width !== frame.width) scratch.width = frame.width;
if (scratch.height !== frame.height) scratch.height = frame.height;
const ctx = scratch.getContext("2d") as CanvasRenderingContext2D | null;
@@ -1527,8 +1584,8 @@ export class FramePainter {
const img = ctx.getImageData(0, 0, scratch.width, scratch.height);
matteLumaToAlpha(img.data);
ctx.putImageData(img, 0, 0);
- this.matteStamp.set(clip.id, stamp);
- this.matteFrames.set(clip.id, scratch as CanvasImageSource);
+ this.matteStamp.set(key, stamp);
+ this.matteFrames.set(key, scratch as CanvasImageSource);
}
/** Draw the whole cut at timeline time `t` onto the canvas. */
@@ -1594,6 +1651,7 @@ export class FramePainter {
// The subject pass: video → behind elements → segmented person, exactly
// the preview's pass, sampled per drawn frame (no mask throttling).
// It also refreshes the matte the front subject-masked stamps read.
+ await this.behind?.ready(doc.overlays, canvas.width, canvas.height, doc.assets, t);
this.behind?.draw(canvas, doc.overlays, doc.assets, t, { minMaskInterval: 0 });
// The stack, bottom up: an effect grades what plays under it, so the
diff --git a/site/src/cut/lib/memoryBudget.ts b/site/src/cut/lib/memoryBudget.ts
index 32f28ae7..023287b5 100644
--- a/site/src/cut/lib/memoryBudget.ts
+++ b/site/src/cut/lib/memoryBudget.ts
@@ -132,7 +132,7 @@ const HOLDERS = {
* shelf of canvases kept for the sources that have stood down. */
previewCanvases: { bucket: "canvases", portion: 0.72 },
/** Readers a render keeps open between frames, each with its own pool of
- * canvases at the source's size. */
+ * canvases at the source's size, and its element scratch surfaces. */
exportReaders: { bucket: "canvases", portion: 0.2 },
/** The sharpen and clarity pass's float framebuffers, at the picture's
* size while detail is in use. */
diff --git a/site/src/cut/lib/pi/cutAgent.ts b/site/src/cut/lib/pi/cutAgent.ts
index a3fba247..d5ff6e67 100644
--- a/site/src/cut/lib/pi/cutAgent.ts
+++ b/site/src/cut/lib/pi/cutAgent.ts
@@ -61,7 +61,7 @@ import { enforceContextBudget } from "./contextBudget";
import { donkeyModel, type ChatThinkingLevel } from "./donkeyModel";
import { isMutatingTool, ledgerText, recordCall, type LedgerRecord } from "./mutationLedger";
import { makeDonkeyStream, type DonkeyToolDetails, type PostFn, type WireCarrier, type WirePart } from "./donkeyStream";
-import { toAgentTools, toToolResult, type ExecTool } from "./tools";
+import { toAgentTools, toToolResult, uiToolOutput, type ExecTool } from "./tools";
import { subscribeUiChunks } from "./uiChunks";
// The chat turn runner on the pi agent harness. Each turn: one typed judgment
@@ -629,7 +629,7 @@ async function runInstantAction(
}
const result = toToolResult(action.tool, output);
emit({ type: "tool-input-available", toolCallId, toolName: action.tool, input: action.args });
- emit({ type: "tool-output-available", toolCallId, output: result.details?.response ?? null });
+ emit({ type: "tool-output-available", toolCallId, output: uiToolOutput(result.details) });
emit({ type: "text-start", id: "t1" });
emit({ type: "text-delta", id: "t1", delta: action.say });
emit({ type: "text-end", id: "t1" });
diff --git a/site/src/cut/lib/pi/donkeyStream.ts b/site/src/cut/lib/pi/donkeyStream.ts
index da000aac..1eec7365 100644
--- a/site/src/cut/lib/pi/donkeyStream.ts
+++ b/site/src/cut/lib/pi/donkeyStream.ts
@@ -37,6 +37,8 @@ export interface WireCarrier {
export interface DonkeyToolDetails {
response: unknown;
mediaParts?: WirePart[];
+ /** What the chat shows beside the result and the model never reads. */
+ display?: ToolDisplay;
}
interface ResponseBody {
diff --git a/site/src/cut/lib/removalVideo.ts b/site/src/cut/lib/removalVideo.ts
index c9efdab8..f4992408 100644
--- a/site/src/cut/lib/removalVideo.ts
+++ b/site/src/cut/lib/removalVideo.ts
@@ -25,7 +25,7 @@ export interface RemovalPieces {
/** Straight-alpha colors survive a getImageData round trip; premultiplied
* draws would darken every soft edge once the engine re-applies the alpha. */
-function splitFrame(
+export function splitFrame(
layer: CanvasImageSource,
lw: number,
lh: number,
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 822e7283..1c050ead 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -585,6 +585,10 @@ export interface EditorState {
* profile the header read — with an undo step. `undefined` returns the
* asset to the detected profile. */
setAssetColorProfile: (id: string, profile: SourceProfile | undefined) => void;
+ /** Bind a separately recorded sound to a video, so every clip of it plays
+ * the recording; `undefined` returns the video to its own track. One undo
+ * step. */
+ setAssetSoundFrom: (id: string, from: SoundFrom | undefined) => void;
/** Write the preview proxy that landed beside a master onto its asset, and
* the address the preview reads it from. The preview retargets on the next
* frame; the document saves the proxy with the asset. No undo step: the
@@ -1034,6 +1038,7 @@ interface SharedFields {
colorSpace: OutputSpace;
beats: Map<string, AssetBeats | undefined>;
profiles: Map<string, SourceProfile | undefined>;
+ sounds: Map<string, SoundFrom | undefined>;
}
/** Each closed timeline's undo and redo stacks: opening a timeline again
* brings its history back. Session only, cleared when a project opens. */
diff --git a/site/src/cut/lib/subtitles.ts b/site/src/cut/lib/subtitles.ts
index 17427d31..dc9b807c 100644
--- a/site/src/cut/lib/subtitles.ts
+++ b/site/src/cut/lib/subtitles.ts
@@ -3,11 +3,15 @@ import {
textRoom,
wrapTextToRoom,
wordDrawsAt,
+ wordFaceDraws,
WORD_ACCENT_DEFAULT,
wordEffect,
wordSampleWindows,
type OverlayWords,
+ type WordDraw,
+ type WordFace,
} from "@donkeycut/effects-kit";
+import { cueEmphasis } from "./captionEmphasis";
import { measureLine, textFontOf } from "./textFit";
import { formatTime } from "./time";
import type {
@@ -448,13 +452,78 @@ export function cueOverlay(
words,
style.color,
Math.max(0, cue.end - cue.start),
- at - cue.start
+ at - cue.start,
+ cueFaces(cue, pos) ?? undefined
),
}
- : {}),
+ : cueFaces(cue, pos)
+ ? { wordDraw: restingDraws(cue, text, style.color, pos) }
+ : {}),
};
}
+/** One cue's emphasis, resolved: the face list its words wear and, for a
+ * caption with no word effect running, the still picture of them. Both are
+ * the same from frame to frame, so they are held per cue and handed back
+ * until the cue or the style changes — the preview asks every frame. */
+interface FaceEntry {
+ face: Required<WordFace>;
+ faces: (WordFace | undefined)[];
+ restText?: string;
+ restColor?: string;
+ rest?: WordDraw[];
+}
+const faceCache = new WeakMap<SubtitleCue, FaceEntry>();
+
+const sameFace = (a: Required<WordFace>, b: CaptionEmphasisOverrides | undefined) => {
+ const c = CAPTION_EMPHASIS_DEFAULT;
+ return (
+ a.font === (b?.emphasisFont ?? c.font) &&
+ a.color === (b?.emphasisColor ?? c.color) &&
+ a.italic === (b?.emphasisItalic ?? c.italic) &&
+ a.weight === (b?.emphasisWeight ?? c.weight) &&
+ a.scale === (b?.emphasisScale ?? c.scale)
+ );
+};
+
+function faceEntry(cue: SubtitleCue, pos: CaptionEmphasisOverrides | undefined): FaceEntry | null {
+ if (!cue.emphasis?.length) return null;
+ const held = faceCache.get(cue);
+ if (held && sameFace(held.face, pos)) return held;
+ const marked = cueEmphasis(cue);
+ if (marked.length === 0) return null;
+ const face = captionEmphasis(pos);
+ const faces: (WordFace | undefined)[] = [];
+ for (const i of marked) faces[i] = face;
+ const entry: FaceEntry = { face, faces };
+ faceCache.set(cue, entry);
+ return entry;
+}
+
+/** Per display word: the emphasis face, or nothing. Null when no word in the
+ * cue is emphasized. */
+export function cueFaces(
+ cue: SubtitleCue,
+ pos: CaptionEmphasisOverrides | undefined
+): (WordFace | undefined)[] | null {
+ return faceEntry(cue, pos)?.faces ?? null;
+}
+
+function restingDraws(
+ cue: SubtitleCue,
+ text: string,
+ color: string,
+ pos: CaptionEmphasisOverrides | undefined
+): WordDraw[] {
+ const entry = faceEntry(cue, pos)!;
+ if (!entry.rest || entry.restText !== text || entry.restColor !== color) {
+ entry.rest = wordFaceDraws(text, color, entry.faces);
+ entry.restText = text;
+ entry.restColor = color;
+ }
+ return entry.rest;
+}
+
/** A caption's pictures are full frames, one per span, and a cut carries one
* of them for every word of speech in it. A ramp is sampled at this rate, and
* never at more than this many steps however long the ramp runs, which is
diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.ts
index 751ceab9..b0f0c004 100644
--- a/site/src/cut/lib/textRender.ts
+++ b/site/src/cut/lib/textRender.ts
@@ -39,6 +39,9 @@ export {
* box-shadow spread, which stays clear of the layout. */
export function wordDrawCss(d: WordDraw, lineHeight = LINE_HEIGHT): CSSProperties {
const css: CSSProperties = { color: d.color };
+ if (d.font !== undefined) css.fontFamily = fontStack(d.font);
+ if (d.italic !== undefined) css.fontStyle = d.italic ? "italic" : "normal";
+ if (d.weight !== undefined) css.fontWeight = d.weight;
if (d.opacity < 0.999) css.opacity = d.opacity;
// The word takes its room at the layout size and draws at its own, which
// differ only while it is growing into its place: real type size for the
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index d6ac364d..b91ea884 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -63,6 +63,7 @@ import {
GLYPH_ANIM_STYLE_IDS,
OVERLAY_ANIM_STYLE_IDS,
OVERLAY_LOOP_STYLE_IDS,
+ OVERLAY_HIT_STYLE_IDS,
SOUND_EQ_BANDS,
TEXT_ONLY_ANIM_STYLE_IDS,
SOURCE_PROFILES,
@@ -388,8 +389,9 @@ Stickers (add_sticker): a project image asset (generated cutouts in the Elements
Custom stickers (create_sticker): one call generates the image, removes the background, adds the die-cut outline, and places it — use it when the user asks for a sticker of something they don't have (it spends credits like image generation).
Effects (add_effect): time-ranged treatments over the finished cut. Three families, one list: the picture treatments (${EFFECT_TREATMENTS.join(", ")}), the graded looks (${LOOK_EFFECTS.join(", ")}), and the audio treatments (${AUDIO_EFFECT_IDS.join(", ")}). A picture effect treats the footage and everything laid over it; an audio effect treats everything audible under its window at once — clip sound, upper-track sound and the soundtrack — so "make this bit sound like a phone call", "put an echo on that line", or "muffle the music here" is an effect element over that stretch, and a per-clip volume, fade or duck is the audio tools instead. A handover on the sound at a cut belongs to the transition on that cut — the audiocross style — not here. Placed like any element and tuned with \`amount\` (update_overlay), and trims like anything else: a picture effect dropped over a clip opens covering that clip, while an audio effect opens at the standard length wherever it was placed — a treatment on the sound is aimed at a moment, so give it the window you mean. One effect at a time reads best, amounts under ~0.5. Effects are stylized post treatments on a stretch of timeline; pure color — exposure, warmth, a cinematic or film palette on one clip — is the per-clip grading tools instead (color-grading skill), whose preset catalog is separate from the look effects.
Advanced text (update_overlay): italic, align (multi-line), letter_spacing (em), line_height, wrap_width (frame-width fraction; side grips wrap text, height follows content; corners scale type proportionally), stroke_color/stroke_width (em outline behind the fill), and richer shadows. Bundled Google families join the system set — ${GOOGLE_FONT_IDS.join(", ")} — and so do the user's own: a .ttf/.otf/.woff/.woff2 file dropped on the Library is offered in every project under a font id like "font:<key>". Editor_state lists the ones this account has under \`fonts\` — use an id from there instead of guessing at a family name, and when someone asks for a typeface that is not in the list, tell them the Library takes font files.
-Animation (set_overlay_animation): every element takes preset In/Out ramps (${OVERLAY_ANIM_STYLE_IDS.join(", ")} — ${TEXT_ONLY_ANIM_STYLE_IDS.join(", ")} for titles only; the per-letter styles sweep a title character by character; ${DIVE_ANIM_STYLE_IDS.join(", ")} flies into the last letter until its ink fills the frame, the built-in zoom-through transition), a Loop (${OVERLAY_LOOP_STYLE_IDS.join(", ")}) that runs its whole duration, and a Move that says what it does WHILE it holds. A title takes one more: Words (${WORD_EFFECT_IDS.join(", ")}), which plays the line word by word — an emphasis travelling along a line that is fully up, or a build assembling the line as it is spoken. Entrances 0.3–0.6s read well; one loop per scene, sparingly; a move on the lines that matter. All of them are slots on the element, applied internally, and they compose over its keyframe track. A slot is the reach when the motion is per-letter or per-word — typewriter, the glyph sweeps, the word effects — which one pose track cannot express; what the whole element does is keyed.
-Keyframes (set_overlay_keyframes): for motion no preset covers — a path across the frame, a slow push in, a drift that holds first. A key is a whole pose (position, scale, rotation, opacity) at a time in seconds from the element's start; the pose moves linearly between keys and holds outside them, and presets still compose on top. This is the default way to animate an element: a key names the exact pose at the exact second, where a named ramp only approximates it. Two or three keys carry almost everything. In the UI the diamond in the inspector adds a key at the playhead, and dragging the element then records into it. set_clip_keyframes is the same track on a video clip (any track): pans, push-ins, picture-in-picture flights, spins, and fades on the footage itself.
+Animation (set_overlay_animation): every element takes preset In/Out ramps (${OVERLAY_ANIM_STYLE_IDS.join(", ")} — ${TEXT_ONLY_ANIM_STYLE_IDS.join(", ")} for titles only; the per-letter styles sweep a title character by character; ${DIVE_ANIM_STYLE_IDS.join(", ")} flies into the last letter until its ink fills the frame, the built-in zoom-through transition), a Loop (${OVERLAY_LOOP_STYLE_IDS.join(", ")}) that runs its whole duration, and a Move that says what it does WHILE it holds. A title takes one more: Words (${WORD_EFFECT_IDS.join(", ")}), which plays the line word by word — an emphasis travelling along a line that is fully up, or a build assembling the line as it is spoken. A typewriter entrance takes a caret (caret, caret_blink, caret_blinks, caret_hide), the blinking bar of a typed comment or search box. Every element takes a Hit (${OVERLAY_HIT_STYLE_IDS.join(", ")}) at hit_at seconds inside it: press is a finger tapping a button — the Post or Like button of a call-to-action card — and a click from stock_search kind "sound" placed at the same moment makes it heard. Entrances 0.3–0.6s read well; one loop per scene, sparingly; a move on the lines that matter. All of them are slots on the element, applied internally, and they compose over its keyframe track. A slot is the reach when the motion is per-letter or per-word — typewriter, the glyph sweeps, the word effects — which one pose track cannot express; what the whole element does is keyed.
+Keyframes (set_overlay_keyframes): for motion no preset covers — a path across the frame, a slow push in, a drift that holds first. A key is a whole pose (position, scale, rotation, opacity, blur) at a time in seconds from the element's start; the pose moves between keys along each key's ease (sine.inOut, power2/power3 in/out, default linear) and holds outside them, and presets still compose on top. This is the default way to animate an element: a key names the exact pose at the exact second, where a named ramp only approximates it. Two or three keys carry almost everything. In the UI the diamond in the inspector adds a key at the playhead, and dragging the element then records into it. set_clip_keyframes is the same track on a video clip (any track): pans, push-ins, picture-in-picture flights, spins, and fades on the footage itself.
+Camera over a board of cards (set_group_camera): lay the cards out across the frame, group them, and key one camera that steps from card to card, pushes onto the one being talked about, pulls back, or whips — the cards keep their own keys inside it. Depth of field is the neighbours' own keys: blur and dim them at the moments the camera pushes in. motion_blur on the camera (or on an element, through update_overlay) streaks fast moves and leaves still frames sharp.
Masks (set_mask): trim any element or video clip to a shape — ${MASK_SHAPE_KINDS.join(", ")} — with feather, invert, rounded corners on a box, and its own keyframe track for reveals and wipes. The pen kind takes an outline as points, for any region the fixed shapes miss: the gap framed between two hands, a doorway, a screen — read the region's corners off a captured frame in frame fractions, and stack a masked copy of the shot (a filter, a title) so it shows through only there. The subject kind trims by the person in the shot (on-device segmentation; preview and export match): inverted sits the item behind the speaker, plain keeps it only on the speaker. Behind-speaker reads well when the speaker is clearly separated from the background; with no detectable person it degrades to the plain picture. Group and Ungroup are in the timeline toolbar (⌘/Ctrl+G and ⇧⌘/Ctrl+G).
Composite builds: these primitives compose. Stacked overlay video copies, masks and their keys, pose keyframes, staggered starts, and shapes construct effects beyond any preset list. Given a reference (a screenshot or clip of an effect the user wants), reverse-engineer it:
1. Read the reference as layers. For each visible piece, decide what it is: live footage, a cropped or repeated slice of the footage, a solid graphic, text, or empty background.
diff --git a/site/src/cut/worker/exportJob.ts b/site/src/cut/worker/exportJob.ts
index b085957d..c270f949 100644
--- a/site/src/cut/worker/exportJob.ts
+++ b/site/src/cut/worker/exportJob.ts
@@ -23,7 +23,7 @@ export interface ExportJobSpec {
/** A render asked for by a client that cannot build a spec — the phone. The
* row carries the size and nothing else; the worker opens the project doc
* and builds the spec itself. */
- fromDoc?: { preset?: string; snapshot: CloudDocSnapshot };
+ fromDoc?: { preset?: string; snapshot: CloudDocSnapshot } & DocExportAudio;
overlays?: { name: string; key: string }[];
/** "overlays": the cut's media came up with the job — a browser-resident
* project borrowing this worker. Its output is the job's scratch too: it