Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
// @vitest-environment happy-dom
import { afterEach, describe, expect, it, vi } from "vitest";
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const script = readFileSync(
join(dirname(fileURLToPath(import.meta.url)), "layout-audit.browser.js"),
"utf-8",
);
interface AuditIssue {
code: string;
severity: string;
selector: string;
containerSelector?: string;
message: string;
coveredFraction?: number;
}
interface Box {
left: number;
top: number;
width: number;
height: number;
}
// The <video> fills a 1920x1080 frame from a 192x108 source, so one source pixel is 10x10 px.
const SOURCE_WIDTH = 192;
const SOURCE_HEIGHT = 108;
const FRAME: Box = { left: 0, top: 0, width: 1920, height: 1080 };
const WORD_TOP = 500;
const WORD_HEIGHT = 80;
interface VideoScene {
/** Text elements: id -> words with their left edge and width, all on one line. */
text: Record<string, Array<{ word: string; left: number; width: number }>>;
/** Alpha (0-255) the decoded frame paints at a composition point. */
alphaAt?: (x: number, y: number) => number;
video?: { readyState?: number; seeking?: boolean; readError?: Error; overlapsText?: boolean };
/** Computed style the <video> adds to the base, e.g. a transform or a background. */
videoStyle?: Record<string, string>;
/** Report each element's text as one line box, as Chromium does, rather than one per word. */
wholeLine?: boolean;
textAttrs?: string;
}
function domRect({ left, top, width, height }: Box): DOMRect {
return {
left,
top,
width,
height,
right: left + width,
bottom: top + height,
x: left,
y: top,
toJSON() {
return this;
},
} as DOMRect;
}
const BASE_STYLE: Record<string, string> = {
display: "block",
visibility: "visible",
opacity: "1",
overflow: "visible",
overflowX: "visible",
overflowY: "visible",
backgroundColor: "rgba(0, 0, 0, 0)",
backgroundImage: "none",
objectFit: "fill",
objectPosition: "50% 50%",
fontSize: "36px",
};
function wordBox(left: number, width: number): Box {
return { left, top: WORD_TOP, width, height: WORD_HEIGHT };
}
function installVideoScene(scene: VideoScene): { getImageData: ReturnType<typeof vi.fn> } {
const spans = Object.entries(scene.text)
.map(
([id, words]) =>
`<span id="${id}" ${scene.textAttrs ?? ""}>${words.map(({ word }) => word).join(" ")}</span>`,
)
.join("");
document.body.innerHTML = `
<div id="root" data-composition-id="main" data-width="1920" data-height="1080">
<div id="line">${spans}</div>
<video id="clip"></video>
</div>`;
const video = document.getElementById("clip") as HTMLVideoElement;
Object.defineProperties(video, {
videoWidth: { configurable: true, value: SOURCE_WIDTH },
videoHeight: { configurable: true, value: SOURCE_HEIGHT },
readyState: { configurable: true, value: scene.video?.readyState ?? 4 },
seeking: { configurable: true, value: scene.video?.seeking ?? false },
});
vi.spyOn(window, "getComputedStyle").mockImplementation(
(element) =>
({
...BASE_STYLE,
...(element === video ? scene.videoStyle : {}),
}) as unknown as CSSStyleDeclaration,
);
for (const element of Array.from(document.querySelectorAll("*"))) {
vi.spyOn(element, "getBoundingClientRect").mockReturnValue(domRect(FRAME));
}
const wordRect = (node: Node | null, start: number, end: number): DOMRect[] => {
const owner = node?.parentElement?.id ?? "";
const word = (node?.textContent ?? "").slice(start, end);
const entry = scene.text[owner]?.find((candidate) => candidate.word === word);
return entry ? [domRect(wordBox(entry.left, entry.width))] : [];
};
vi.spyOn(document, "createRange").mockImplementation(() => {
let node: Node | null = null;
let start = 0;
let end: number | null = null;
return {
selectNodeContents(selected: Node) {
node = selected;
},
setStart(selected: Node, offset: number) {
node = selected;
start = offset;
},
setEnd(_selected: Node, offset: number) {
end = offset;
},
getClientRects() {
const owner = node?.parentElement?.id ?? "";
if (end === null) {
const boxes = (scene.text[owner] ?? []).map(({ left, width }) => wordBox(left, width));
if (scene.wholeLine && boxes.length > 0) {
const left = Math.min(...boxes.map((box) => box.left));
const right = Math.max(...boxes.map((box) => box.left + box.width));
return [domRect(wordBox(left, right - left))] as unknown as DOMRectList;
}
return boxes.map(domRect) as unknown as DOMRectList;
}
return wordRect(node, start, end) as unknown as DOMRectList;
},
detach() {},
} as unknown as Range;
});
const overlapsText = scene.video?.overlapsText ?? true;
Object.defineProperty(document, "elementsFromPoint", {
configurable: true,
value: (x: number) => {
const textElement = Object.keys(scene.text)
.map((id) => document.getElementById(id))
.find((element) => {
const words = scene.text[element?.id ?? ""] ?? [];
return words.some(({ left, width }) => x >= left && x < left + width);
});
return [...(overlapsText ? [video] : []), textElement, video].filter(Boolean);
},
});
const alphaAt = scene.alphaAt ?? (() => 255);
const getImageData = vi.fn((_x: number, _y: number, width: number, height: number) => {
if (scene.video?.readError) throw scene.video.readError;
const data = new Uint8ClampedArray(width * height * 4);
for (let row = 0; row < height; row++) {
for (let col = 0; col < width; col++) {
const x = ((col + 0.5) * FRAME.width) / width;
const y = ((row + 0.5) * FRAME.height) / height;
data[(row * width + col) * 4 + 3] = alphaAt(x, y);
}
}
return { data };
});
// getContext's overloads resolve to the WebGPU one; the audit only asks for "2d".
const getContextSpy = vi.spyOn(HTMLCanvasElement.prototype, "getContext") as unknown as {
mockReturnValue(value: CanvasRenderingContext2D): void;
};
getContextSpy.mockReturnValue({
clearRect() {},
drawImage() {},
getImageData,
} as unknown as CanvasRenderingContext2D);
window.eval(script);
return { getImageData };
}
function runAudit(): AuditIssue[] {
const audit = (
window as unknown as {
__hyperframesLayoutAudit: (options: { time: number; tolerance: number }) => AuditIssue[];
}
).__hyperframesLayoutAudit;
return audit({ time: 1, tolerance: 2 });
}
const occlusionOf = (issues: AuditIssue[], id: string) =>
issues.find((issue) => issue.code === "text_occluded" && issue.selector === `#${id}`);
const SHIP = { ship: [{ word: "SHIP", left: 800, width: 200 }] };
const subjectLeftOf = (edge: number) => (x: number) => (x < edge ? 255 : 0);
describe("layout-audit.browser text behind a <video>", () => {
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
Reflect.deleteProperty(document, "elementsFromPoint");
Reflect.deleteProperty(window, "__hyperframesLayoutAudit");
});
it("keeps a fully opaque frame occluding like any raster", () => {
installVideoScene({ text: SHIP });
expect(occlusionOf(runAudit(), "ship")).toMatchObject({
severity: "error",
containerSelector: "#clip",
coveredFraction: 1,
message: "Text is hidden beneath an opaque element.",
});
});
it("does not flag text under the transparent part of a cutout", () => {
installVideoScene({ text: SHIP, alphaAt: subjectLeftOf(400) });
expect(occlusionOf(runAudit(), "ship")).toBeUndefined();
});
it("reports a partly covered word as info with its share under the matte", () => {
installVideoScene({ text: SHIP, alphaAt: subjectLeftOf(860) });
expect(occlusionOf(runAudit(), "ship")).toMatchObject({
severity: "info",
coveredFraction: 0.3,
message: `Text sits partly behind a transparent video's matte; its most covered word "SHIP" is 30% hidden.`,
});
});
it("fails a word that is 80% or more behind the matte", () => {
installVideoScene({ text: SHIP, alphaAt: subjectLeftOf(970) });
expect(occlusionOf(runAudit(), "ship")).toMatchObject({
severity: "error",
coveredFraction: 0.85,
message: `"SHIP" is 85% behind a transparent video's matte.`,
});
});
it("judges each rendered word, so one lost word fails a mostly clear line", () => {
installVideoScene({
text: {
line1: [
{ word: "a", left: 200, width: 40 },
{ word: "big", left: 260, width: 100 },
{ word: "launch", left: 380, width: 240 },
],
},
alphaAt: (x) => (x >= 370 ? 255 : 0),
});
expect(occlusionOf(runAudit(), "line1")).toMatchObject({
severity: "error",
coveredFraction: 1,
message: `"launch" is 100% behind a transparent video's matte.`,
});
});
it("fails two neighbouring words that are each half covered", () => {
installVideoScene({
text: {
pair: [
{ word: "IN", left: 200, width: 100 },
{ word: "SHIP", left: 320, width: 200 },
],
},
alphaAt: (_x, y) => (y >= 530 ? 255 : 0),
});
expect(occlusionOf(runAudit(), "pair")).toMatchObject({
severity: "error",
coveredFraction: 0.63,
message: `Neighbouring words "IN" and "SHIP" are 63% and 63% behind a transparent video's matte.`,
});
});
it("applies the neighbour rule across one-word sibling elements", () => {
installVideoScene({
text: {
first: [{ word: "IN", left: 200, width: 100 }],
second: [{ word: "SHIP", left: 320, width: 200 }],
},
alphaAt: (_x, y) => (y >= 530 ? 255 : 0),
});
const issues = runAudit();
expect(occlusionOf(issues, "first")?.severity).toBe("error");
expect(occlusionOf(issues, "second")?.severity).toBe("error");
});
it("keeps a single half-covered word at info", () => {
installVideoScene({
text: {
first: [{ word: "IN", left: 200, width: 100 }],
second: [{ word: "SHIP", left: 320, width: 200 }],
},
alphaAt: (x, y) => (x < 310 && y >= 530 ? 255 : 0),
});
const issues = runAudit();
expect(occlusionOf(issues, "first")?.severity).toBe("info");
expect(occlusionOf(issues, "second")).toBeUndefined();
});
it("fails a word hidden by a matte feature that falls between two probe columns", () => {
// One 1800px line box: its 20% and 35% probe columns sit at x 360 and 630, the strip at 440-540.
installVideoScene({
text: {
caption: [
{ word: "a", left: 0, width: 40 },
{ word: "gh", left: 460, width: 60 },
{ word: "end", left: 1700, width: 100 },
],
},
wholeLine: true,
alphaAt: (x) => (x >= 440 && x < 540 ? 255 : 0),
});
expect(occlusionOf(runAudit(), "caption")).toMatchObject({
severity: "error",
containerSelector: "#clip",
message: `"gh" is 100% behind a transparent video's matte.`,
});
});
it("reads a mirrored video's alpha on the side it is painted", () => {
installVideoScene({
text: {
shown: [{ word: "SHIP", left: 200, width: 200 }],
hidden: [{ word: "SHIP", left: 1500, width: 200 }],
},
videoStyle: { transform: "matrix(-1, 0, 0, 1, 0, 0)" },
// Source alpha: the subject fills the source's left half, so it paints on the right.
alphaAt: subjectLeftOf(960),
});
const issues = runAudit();
expect(occlusionOf(issues, "shown")).toBeUndefined();
expect(occlusionOf(issues, "hidden")).toMatchObject({ severity: "error", coveredFraction: 1 });
});
it("keeps a video's opaque CSS background occluding behind its clear pixels", () => {
installVideoScene({
text: SHIP,
videoStyle: { backgroundColor: "rgb(18, 52, 86)" },
alphaAt: subjectLeftOf(400),
});
expect(occlusionOf(runAudit(), "ship")).toMatchObject({
severity: "error",
message: "Text is hidden beneath an opaque element.",
});
});
it("respects data-layout-allow-occlusion", () => {
installVideoScene({
text: SHIP,
alphaAt: subjectLeftOf(970),
textAttrs: "data-layout-allow-occlusion",
});
expect(occlusionOf(runAudit(), "ship")).toBeUndefined();
});
it("reads each probed video's frame once per sample and leaves videos under no text alone", () => {
const covering = installVideoScene({ text: SHIP, alphaAt: subjectLeftOf(860) });
runAudit();
expect(covering.getImageData).toHaveBeenCalledTimes(1);
runAudit();
expect(covering.getImageData).toHaveBeenCalledTimes(2);
vi.restoreAllMocks();
const beneath = installVideoScene({ text: SHIP, video: { overlapsText: false } });
runAudit();
expect(beneath.getImageData).not.toHaveBeenCalled();
});
it.each([
{
name: "a video with no decoded frame",
video: { readyState: 1 },
alphaAt: undefined,
reason: "no frame was decoded at this time",
},
{
name: "a video still seeking",
video: { seeking: true },
alphaAt: undefined,
reason: "the seek had not landed",
},
{
name: "a cross-origin video",
video: { readError: Object.assign(new Error("tainted"), { name: "SecurityError" }) },
alphaAt: undefined,
reason: "its source is cross-origin",
},
{
name: "a rotated video",
video: {},
videoStyle: { transform: "matrix(0, 1, -1, 0, 0, 0)" },
alphaAt: subjectLeftOf(400),
reason: "it is rotated, skewed or transformed in 3D",
},
{
name: "a frame that reads back fully transparent",
video: {},
alphaAt: () => 0,
reason: "its frame read back fully transparent",
},
])(
"falls back to opaque with an info note for $name",
({ video, videoStyle, alphaAt, reason }) => {
installVideoScene({ text: SHIP, video, videoStyle, alphaAt });
const issues = runAudit();
expect(occlusionOf(issues, "ship")).toMatchObject({ severity: "error", coveredFraction: 1 });
expect(issues).toContainEqual(
expect.objectContaining({
code: "video_alpha_unread",
severity: "info",
selector: "#clip",
message: `Could not read this video's frame alpha (${reason}), so text under it was judged as if the video were opaque.`,
}),
);
},
);
});