Code-first video & image, rendered natively on WebGPU.
import { describe, expect, it } from "vitest";
import { bind } from "../core/contract.js";
import { evaluateGlyphExpression, type GlyphAnimator } from "../core/glyphs.js";
import { themes } from "../core/theme.js";
import { fakeHost } from "../test/fake-host.js";
import { captions } from "./index.js";
const make = () => bind(fakeHost(), captions);
type Node = {
id?: string;
children?: Node[];
animators?: GlyphAnimator[];
startFrame?: number;
x?: number;
width?: number;
styles?: unknown[];
};
const find = (n: unknown, id: string): Node | undefined => {
const node = n as Node;
if (node.id === id) return node;
for (const c of node.children ?? []) {
const hit = find(c, id);
if (hit) return hit;
}
return undefined;
};
const words = "Ship it faster than ever before today"
.split(" ")
.map((text, i) => ({
text,
start: 0.5 + i * 0.4,
end: 0.8 + i * 0.4,
}));
describe("Captions", () => {
it("an emphasised word on the highlight marker reads in ink, not accent on accent", () => {
// Paper: ink reads on its accent, so only emphasised words switch.
const c = make()({
id: "c",
words,
style: "highlight",
emphasis: ["faster"],
theme: themes.paper,
});
const phrase = find(c, "c-phrase-0");
const onMarker = phrase?.animators?.find((a) => a.name === "on-marker");
expect(onMarker).toBeDefined();
const expr = onMarker?.selectors[0]?.expression ?? "";
const phraseStart = c.cues.phrases[0]?.frame ?? 0;
const local = (c.cues.words[2]?.frame ?? 0) - phraseStart + 3;
// "faster" (word 2) switches on the marker; plain words keep their ink.
expect(evaluateGlyphExpression(expr, { frame: local, wordIndex: 2 })).toBe(
1,
);
const plain = (c.cues.words[1]?.frame ?? 0) - phraseStart + 3;
expect(evaluateGlyphExpression(expr, { frame: plain, wordIndex: 1 })).toBe(
0,
);
});
it("cues every word on round(start × fps), in the parent's timeline", () => {
const c = make()({ id: "c", words, at: 10 });
expect(c.cues.words.map((w) => w.frame)).toEqual(
words.map((w) => 10 + Math.round(w.start * 30)),
);
expect(c.cues.phrases[0]?.frame).toBe(10 + 15);
});
it("karaoke turns each word to ink exactly on its frame", () => {
const c = make()({ id: "c", words, style: "karaoke" });
const spoken = find(c, "c-phrase-0")?.animators?.find(
(a) => a.name === "spoken",
);
const expr = spoken?.selectors[0]?.expression ?? "";
const phraseStart = c.cues.phrases[0]?.frame ?? 0;
const local = (c.cues.words[2]?.frame ?? 0) - phraseStart;
expect(
evaluateGlyphExpression(expr, { frame: local - 1, wordIndex: 2 }),
).toBe(1);
expect(evaluateGlyphExpression(expr, { frame: local, wordIndex: 2 })).toBe(
0,
);
});
it("pulses the active word with the voice level", () => {
// Level = frame / 100: easy to read back.
const level = Array.from({ length: 200 }, (_, f) => f / 100);
const pulse = { fps: 30, frames: 200, bands: [], level, beats: [] };
const c = make()({ id: "c", words, pulse, pulseAmount: 0.2 });
const animator = find(c, "c-phrase-0")?.animators?.find(
(a) => a.name === "pulse",
);
expect(animator?.props.scale).toBeCloseTo(1.2, 6);
const expr = animator?.selectors[0]?.expression ?? "";
const phraseStart = c.cues.phrases[0]?.frame ?? 0;
const word = c.cues.words[1] as { frame: number };
const local = word.frame - phraseStart + 2;
// The active word reads the level of its component frame; others are 0.
expect(
evaluateGlyphExpression(expr, { frame: local, wordIndex: 1 }),
).toBeCloseTo((word.frame + 2) / 100, 6);
expect(evaluateGlyphExpression(expr, { frame: local, wordIndex: 3 })).toBe(
0,
);
expect(() => make()({ words, pulse, fps: 24 })).toThrow(
/analysed at 30 fps/,
);
});
it("keeps every line inside the safe area at the auto-fitted size", () => {
const long =
"a fairly long caption sentence that keeps going on and on for several lines of text"
.split(" ")
.map((text, i) => ({ text, start: i * 0.2, end: i * 0.2 + 0.18 }));
const c = make()({ id: "c", words: long, width: 1080, height: 1920 });
const safe = 1080 * 0.84;
// Fake measure: 0.5 em glyphs, so the longest line × size / 2 must fit.
const size = (find(c, "c-phrase-0") as { fontSize?: number }).fontSize ?? 0;
const longest = Math.max(...phraseLines(c));
expect(longest * size * 0.5).toBeLessThanOrEqual(safe + 1e-6);
});
it("emphasises matching words and supports transcripts", () => {
const srt = "1\n00:00:00,000 --> 00:00:01,000\nmake it faster\n";
const c = make()({ id: "c", transcript: srt, emphasis: ["faster"] });
expect(c.cues.words.map((w) => w.emphasised)).toEqual([false, false, true]);
expect(() => make()({})).toThrow(/give `words` or a `transcript`/);
});
it("puts a cut-out of the speaker over the captions", () => {
const c = make()({ id: "c", words, behind: { video: "talk.mp4" } });
const kids = ((c as unknown as Node).children?.[0]?.children ??
[]) as Node[];
expect(kids[kids.length - 1]?.id).toBe("c-behind");
});
it("outlines with a layer-style stroke, never over the glyphs", () => {
const c = make()({ id: "c", words, style: "outline" });
expect(find(c, "c-phrase-0")?.styles?.[0]).toMatchObject({
stroke: { position: "outside" },
});
});
});
/** Every laid-out line's length, across all phrases. */
function phraseLines(c: unknown): number[] {
const out: number[] = [];
for (let p = 0; ; p++) {
const node = find(c, `c-phrase-${p}`) as { text?: string } | undefined;
if (!node) return out;
out.push(...(node.text ?? "").split("\n").map((l) => l.length));
}
}