Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
import { END_CARD_HOLD_SEC, type Canvas } from "./motionAudit.js";
import type { CheckBbox, CraftFinding, CraftFrame, CraftResult } from "./checkTypes.js";
/**
* Craft thresholds. Line holds follow the skills' target of 0.5 s + 0.3 s a word and warn
* at three quarters of it; a one- or two-word chapter line may flash for 0.4 s, as the
* official films do. Heights and sizes scale with the frame's short side (1080 px reference).
*/
const CRAFT_THRESHOLDS = {
readHeightAt1080: 30,
readOpacity: 0.5,
shortLineWords: 2,
shortLineFloorSec: 0.4,
holdBaseSec: 0.5,
holdPerWordSec: 0.3,
holdFloorFraction: 0.75,
finalLineFloorSec: 2,
deadStretchSec: 3,
endCardSec: END_CARD_HOLD_SEC,
typeFloorFraction: 0.026,
typeFloorHoldSec: 0.5,
silentFilmSec: 15,
/** One video frame: a line held exactly at its floor measures a hair under it. */
measureToleranceSec: 1 / 30,
/** Halvings of a grid step when re-measuring a line's edges (0.125 s → ~8 ms). */
refineSteps: 4,
maxRefinedLines: 24,
} as const;
const T = CRAFT_THRESHOLDS;
const ZERO_BBOX: CheckBbox = { x: 0, y: 0, width: 0, height: 0 };
const WORD_RUNS = /[\p{L}\p{M}\p{N}%$]+/gu;
// Chinese and Japanese have no spaces; a dictionary segmenter counts their words like an English line's.
const WORD_SEGMENTER = new Intl.Segmenter("ja", { granularity: "word" });
/** Words a viewer reads: letter/number runs, with CJK runs split into dictionary words. */
export function readingWords(text: string): number {
const runs = text.toLowerCase().normalize("NFC").match(WORD_RUNS) ?? [];
return runs.reduce(
(count, run) =>
count + Math.max(1, [...WORD_SEGMENTER.segment(run)].filter((s) => s.isWordLike).length),
0,
);
}
function lineTargetSec(words: number): number {
return T.holdBaseSec + T.holdPerWordSec * words;
}
/** Shortest hold that does not warn; the film's last line needs at least 2 s. */
export function lineFloorSec(words: number, finalLine: boolean): number {
const floor =
words <= T.shortLineWords ? T.shortLineFloorSec : T.holdFloorFraction * lineTargetSec(words);
return finalLine ? Math.max(floor, T.finalLineFloorSec) : floor;
}
export interface CraftAuditInput {
frames: CraftFrame[];
duration: number;
canvas: Canvas;
soundSources: number | null;
/** sweep_static fired, a whole-composition keepsMoving owns stillness, or the root says data-no-timeline. */
stillnessJudgedElsewhere: boolean;
}
/** Seeks to `time` and samples it again, for re-measuring a line's edges between grid samples. */
export type CraftProbe = (time: number) => Promise<CraftFrame>;
export async function auditCraft(input: CraftAuditInput, probe?: CraftProbe): Promise<CraftResult> {
const frames = [...input.frames].sort((a, b) => a.time - b.time);
const step = sampleStep(frames);
const findings: CraftFinding[] = [
...detectFirstFrameEmpty(frames),
...(await detectLineUnread(frames, input.duration, input.canvas, probe)),
...(input.stillnessJudgedElsewhere ? [] : detectDeadStretches(frames, input.duration)),
...detectTextBelowFloor(frames, input.duration, input.canvas),
...detectFilmSilent(input.duration, input.soundSources),
].sort((a, b) => a.time - b.time);
return { enabled: true, step, samples: frames.length, findings };
}
function sampleStep(frames: CraftFrame[]): number {
const gaps = frames
.slice(1)
.map((frame, index) => frame.time - (frames[index]?.time ?? frame.time))
.filter((gap) => gap > 0)
.sort((a, b) => a - b);
return gaps[Math.floor(gaps.length / 2)] ?? 0;
}
function seconds(value: number): string {
return `${value.toFixed(2)} s`;
}
function filmFinding(
code: CraftFinding["code"],
time: number,
message: string,
extra: Partial<CraftFinding> = {},
): CraftFinding {
return {
code,
severity: "warning",
message,
selector: "[data-composition-id]",
dataAttributes: {},
sourceFile: "index.html",
bbox: ZERO_BBOX,
time,
...extra,
};
}
// ---------- first_frame_empty ----------
export function detectFirstFrameEmpty(frames: CraftFrame[]): CraftFinding[] {
const first = frames[0];
if (!first || first.time > 1e-3 || first.painted) return [];
return [
filmFinding(
"first_frame_empty",
0,
"Frame 0 shows only the background, and the first frame is the thumbnail on social players: have the title or key image already on screen at 0 s instead of entering from nothing; see references/review.md.",
),
];
}
// ---------- film_silent ----------
export function detectFilmSilent(duration: number, soundSources: number | null): CraftFinding[] {
if (soundSources === null || soundSources > 0 || duration <= T.silentFilmSec) return [];
return [
filmFinding(
"film_silent",
0,
`This ${Math.round(duration)} s film has no sound source (no <audio>, no <video> with sound): add a music bed and sounds on the visible actions unless the person asked for silence; see references/sound.md.`,
),
];
}
// ---------- dead_stretch ----------
/** Runs of identical liveness signatures longer than 3 s, outside the end card and data-hold. */
export function detectDeadStretches(frames: CraftFrame[], duration: number): CraftFinding[] {
const endCardStart = duration - T.endCardSec;
const findings: CraftFinding[] = [];
let start: number | null = null;
let end = 0;
const flush = () => {
if (start !== null && end - start > T.deadStretchSec) findings.push(deadStretch(start, end));
start = null;
};
for (let index = 1; index < frames.length; index += 1) {
const before = frames[index - 1];
const after = frames[index];
if (!before || !after) continue;
const still =
before.signature !== "" &&
before.signature === after.signature &&
!before.held &&
!after.held &&
before.time < endCardStart;
if (!still) {
flush();
continue;
}
start ??= before.time;
end = Math.min(after.time, endCardStart);
}
flush();
return findings;
}
function deadStretch(start: number, end: number): CraftFinding {
const length = end - start;
return filmFinding(
"dead_stretch",
start,
`Nothing on screen changes for ${seconds(length)} (${start.toFixed(2)}–${end.toFixed(2)} s): start the next move within ${T.deadStretchSec} s, or put data-hold on the element whose stillness is deliberate; see references/motion.md.`,
{ seconds: round(length), floorSeconds: T.deadStretchSec, until: round(end) },
);
}
// ---------- line_unread ----------
interface BlockFrame {
text: string;
rect: CraftRect;
}
interface CraftRect {
left: number;
top: number;
right: number;
bottom: number;
}
interface BlockTrack {
key: number;
selector: string;
sourceFile: string;
/** Frame index → what the block showed there. */
seen: Map<number, BlockFrame>;
}
/** Readable blocks in one frame: non-caption parts at ≥ 0.5 opacity whose union is reading-sized. */
function readableBlocks(frame: CraftFrame, readHeight: number): Map<number, BlockFrame & Anchor> {
const blocks = new Map<number, BlockFrame & Anchor>();
for (const part of frame.parts) {
if (part.captions || part.opacity < T.readOpacity) continue;
const rect = {
left: part.rect.left,
top: part.rect.top,
right: part.rect.left + part.rect.width,
bottom: part.rect.top + part.rect.height,
};
const block = blocks.get(part.block);
if (!block) {
blocks.set(part.block, {
text: part.text,
rect,
selector: part.blockSelector,
sourceFile: part.sourceFile,
});
continue;
}
block.text = `${block.text} ${part.text}`;
block.rect = {
left: Math.min(block.rect.left, rect.left),
top: Math.min(block.rect.top, rect.top),
right: Math.max(block.rect.right, rect.right),
bottom: Math.max(block.rect.bottom, rect.bottom),
};
}
for (const [key, block] of blocks) {
if (block.rect.bottom - block.rect.top < readHeight) blocks.delete(key);
}
return blocks;
}
interface Anchor {
selector: string;
sourceFile: string;
}
function trackBlocks(frames: CraftFrame[], readHeight: number): BlockTrack[] {
const tracks = new Map<number, BlockTrack>();
frames.forEach((frame, index) => {
for (const [key, block] of readableBlocks(frame, readHeight)) {
const track = tracks.get(key) ?? {
key,
selector: block.selector,
sourceFile: block.sourceFile,
seen: new Map<number, BlockFrame>(),
};
track.seen.set(index, { text: block.text, rect: block.rect });
tracks.set(key, track);
}
});
return [...tracks.values()];
}
/** Maximal runs of consecutive frame indices. */
function indexRuns(indices: number[]): [number, number][] {
const sorted = [...indices].sort((a, b) => a - b);
const runs: [number, number][] = [];
for (const index of sorted) {
const last = runs[runs.length - 1];
if (last && index === last[1] + 1) last[1] = index;
else runs.push([index, index]);
}
return runs;
}
/** A line's on-screen window: it starts in (startOut, startIn] and ends in [endIn, endOut). */
interface LineWindow {
startOut: number;
startIn: number;
endIn: number;
endOut: number;
}
function windowSeconds(window: LineWindow): number {
return (window.endIn + window.endOut) / 2 - (window.startOut + window.startIn) / 2;
}
interface LineCandidate {
track: BlockTrack;
words: number;
text: string;
finalLine: boolean;
floor: number;
window: LineWindow;
firstIndex: number;
}
/** A run that reaches the last sample lasts to the film's end; one at sample 0 starts exactly there. */
function runWindow(
frames: CraftFrame[],
[first, last]: [number, number],
duration: number,
): LineWindow {
const timeAt = (index: number) =>
frames[Math.min(Math.max(index, 0), frames.length - 1)]?.time ?? 0;
const reachesEnd = last === frames.length - 1;
return {
startOut: timeAt(first - 1),
startIn: timeAt(first),
endIn: reachesEnd ? duration : timeAt(last),
endOut: reachesEnd ? duration : timeAt(last + 1),
};
}
function lineCandidates(frames: CraftFrame[], duration: number, canvas: Canvas): LineCandidate[] {
const readHeight = T.readHeightAt1080 * (Math.min(canvas.width, canvas.height) / 1080);
const tracks = trackBlocks(frames, readHeight);
const lastEnd = (track: BlockTrack) => Math.max(...track.seen.keys());
const latest = Math.max(-1, ...tracks.map(lastEnd));
const candidates: LineCandidate[] = [];
for (const track of tracks) {
const runs = indexRuns([...track.seen.keys()]);
const finalLine = lastEnd(track) === latest;
const windows = runs.map((run) => runWindow(frames, run, duration));
const chosen = finalLine
? windows.length - 1
: windows.reduce((best, window, index) => {
const bestWindow = windows[best];
return bestWindow && windowSeconds(window) > windowSeconds(bestWindow) ? index : best;
}, 0);
const window = windows[chosen];
const run = runs[chosen];
if (!window || !run) continue;
const texts = [...track.seen.values()].map((seen) => seen.text);
const text = texts.reduce((longest, next) =>
readingWords(next) > readingWords(longest) ? next : longest,
);
const words = readingWords(text);
if (words === 0) continue;
const floor = lineFloorSec(words, finalLine);
// Clearly long enough even at the shortest the grid allows.
if (window.endIn - window.startIn >= floor) continue;
candidates.push({ track, words, text, finalLine, floor, window, firstIndex: run[0] });
}
return candidates;
}
async function refineWindow(
candidate: LineCandidate,
probe: CraftProbe,
canvas: Canvas,
): Promise<LineWindow> {
const readHeight = T.readHeightAt1080 * (Math.min(canvas.width, canvas.height) / 1080);
const visibleAt = async (time: number) =>
readableBlocks(await probe(time), readHeight).has(candidate.track.key);
const window = { ...candidate.window };
for (let step = 0; step < T.refineSteps; step += 1) {
if (window.startIn > window.startOut) {
const mid = (window.startOut + window.startIn) / 2;
if (await visibleAt(mid)) window.startIn = mid;
else window.startOut = mid;
}
if (window.endOut > window.endIn) {
const mid = (window.endIn + window.endOut) / 2;
if (await visibleAt(mid)) window.endIn = mid;
else window.endOut = mid;
}
}
return window;
}
export async function detectLineUnread(
frames: CraftFrame[],
duration: number,
canvas: Canvas,
probe?: CraftProbe,
): Promise<CraftFinding[]> {
const findings: CraftFinding[] = [];
const candidates = lineCandidates(frames, duration, canvas);
for (const [index, candidate] of candidates.entries()) {
const window =
probe && index < T.maxRefinedLines
? await refineWindow(candidate, probe, canvas)
: candidate.window;
const held = windowSeconds(window);
if (held >= candidate.floor - T.measureToleranceSec) continue;
const error = Math.max(window.startIn - window.startOut, window.endOut - window.endIn) / 2;
findings.push(lineUnreadFinding(candidate, window, held, error));
}
return findings;
}
function lineUnreadFinding(
candidate: LineCandidate,
window: LineWindow,
held: number,
error: number,
): CraftFinding {
const { track, words, text, finalLine, floor } = candidate;
const quoted = quote(text, 60);
const wordLabel = `${words} word${words === 1 ? "" : "s"}`;
const measured = `${seconds(held)} (±${Math.max(0.01, error).toFixed(2)} s)`;
const target = lineTargetSec(words);
const message = finalLine
? `The film's last line ${quoted} (${wordLabel}) is on screen ${measured}, under the ${seconds(floor)} a closing line needs: hold it about 0.5 s + 0.3 s a word and at least 2 s before the film ends; see references/motion.md.`
: `${quoted} (${wordLabel}) is on screen ${measured}, under its ${seconds(floor)} floor: hold it about 0.5 s + 0.3 s a word (${seconds(target)} here) before it leaves or changes; see references/motion.md.`;
const firstSeen = track.seen.get(candidate.firstIndex);
const rect = firstSeen?.rect;
return {
code: "line_unread",
severity: "warning",
message,
text,
selector: track.selector,
dataAttributes: {},
sourceFile: track.sourceFile,
bbox: rect
? { x: rect.left, y: rect.top, width: rect.right - rect.left, height: rect.bottom - rect.top }
: ZERO_BBOX,
time: round(window.startIn),
words,
seconds: round(held),
floorSeconds: round(floor),
};
}
// ---------- text_below_floor ----------
interface SmallText {
selector: string;
sourceFile: string;
text: string;
fontPx: number;
seconds: number;
time: number;
bbox: CheckBbox;
}
/** 2.6% of the short side, rounded: 28 px on 1080. */
export function typeFloorPx(canvas: Canvas): number {
return Math.round(T.typeFloorFraction * Math.min(canvas.width, canvas.height));
}
export function detectTextBelowFloor(
frames: CraftFrame[],
duration: number,
canvas: Canvas,
): CraftFinding[] {
const floorPx = typeFloorPx(canvas);
const step = sampleStep(frames);
const sightings = new Map<number, Map<number, CraftFrame["parts"][number]>>();
frames.forEach((frame, index) => {
for (const part of frame.parts) {
if (part.captions || part.opacity < T.readOpacity) continue;
// Half a pixel of slack: a scaled 28 px line measures 27.9.
if (part.fontPx >= floorPx - 0.5 || readingWords(part.text) === 0) continue;
const seen = sightings.get(part.key) ?? new Map();
seen.set(index, part);
sightings.set(part.key, seen);
}
});
const small: SmallText[] = [];
for (const seen of sightings.values()) {
const [first, last] = longestRun(indexRuns([...seen.keys()]));
const start = frames[first]?.time ?? 0;
const end = last === frames.length - 1 ? duration : (frames[last]?.time ?? start) + step;
if (end - start < T.typeFloorHoldSec) continue;
const parts = [...seen.values()];
const part = parts[0];
if (!part) continue;
small.push({
selector: part.selector,
sourceFile: part.sourceFile,
text: part.text,
fontPx: Math.min(...parts.map((entry) => entry.fontPx)),
seconds: end - start,
time: start,
bbox: {
x: part.rect.left,
y: part.rect.top,
width: part.rect.width,
height: part.rect.height,
},
});
}
const shortSide = Math.min(canvas.width, canvas.height);
return groupBySourceFile(small).map((group) => smallTextFinding(group, floorPx, shortSide));
}
function longestRun(runs: [number, number][]): [number, number] {
return runs.reduce((best, run) => (run[1] - run[0] > best[1] - best[0] ? run : best), [0, -1]);
}
/** One finding per file (a scene, with sub-compositions), anchored on its smallest text. */
function groupBySourceFile(small: SmallText[]): SmallText[][] {
const groups = new Map<string, SmallText[]>();
for (const entry of small)
groups.set(entry.sourceFile, [...(groups.get(entry.sourceFile) ?? []), entry]);
return [...groups.values()].map((group) =>
group.sort((a, b) => a.fontPx - b.fontPx || a.time - b.time),
);
}
function quote(text: string, max: number): string {
return `"${text.length > max ? `${text.slice(0, max - 3)}...` : text}"`;
}
function smallTextFinding(group: SmallText[], floorPx: number, shortSide: number): CraftFinding {
const [smallest] = group;
if (!smallest) throw new Error("text_below_floor group is empty");
const subject =
group.length === 1
? `${quote(smallest.text, 48)} renders at ${Math.round(smallest.fontPx)} px for ${seconds(smallest.seconds)}`
: `${group.length} text elements stay under it for 0.5 s or more, the smallest ${quote(smallest.text, 40)} at ${Math.round(smallest.fontPx)} px`;
const byTime = [...group].sort((a, b) => a.time - b.time);
return {
code: "text_below_floor",
severity: "warning",
message: `Type floor is ${floorPx} px (2.6% of the ${shortSide} px short side, what a phone can read) and ${subject}: set reading text to at least ${floorPx} px, or put data-layout-ignore on text that is texture rather than reading; see references/look.md.`,
text: smallest.text,
selector: smallest.selector,
dataAttributes: {},
sourceFile: smallest.sourceFile,
bbox: smallest.bbox,
time: round(smallest.time),
times: byTime.map((entry) => round(entry.time)),
fontPx: round(smallest.fontPx),
floorPx,
seconds: round(smallest.seconds),
elements: group.map((entry) => ({
selector: entry.selector,
text: entry.text,
fontPx: round(entry.fontPx),
time: round(entry.time),
seconds: round(entry.seconds),
})),
};
}
function round(value: number): number {
return Math.round(value * 1000) / 1000;
}