Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
// fallow-ignore-file code-duplication
import type { ExtractedFrames, VideoElement, VideoMetadata } from "@hyperframes/engine";
import { describe, expect, it } from "vitest";
import {
assertVideoFrameCoverage,
computeVideoFrameCoverage,
countAuthoredTimedClips,
expectedFramesForClip,
isVideoFrameCoverageError,
resolveVideoCoverageThreshold,
VideoFrameCoverageError,
} from "./videoFrameCoverage.js";
function makeVideo(overrides: Partial<VideoElement> & { id: string }): VideoElement {
return {
id: overrides.id,
src: overrides.src ?? `${overrides.id}.mp4`,
start: overrides.start ?? 0,
end: overrides.end ?? 1,
mediaStart: overrides.mediaStart ?? 0,
playbackRate: overrides.playbackRate,
loop: overrides.loop ?? false,
hasAudio: overrides.hasAudio ?? false,
};
}
// `durationSeconds` defaults to Infinity, not `delivered / fps` — the two are
// unrelated in production (source duration comes from ffprobe; `delivered`
// is how many frames the extractor happened to deliver) and coupling them
// here made any test modeling a delivery shortfall silently report full
// coverage unless it remembered to override durationSeconds afterward. An
// unbounded default instead falls into computeVideoFrameCoverage's "no
// usable source duration" branch, which requires the full authored slot —
// the same behavior every test had before source-duration crediting
// existed. Tests that care about a specific source duration (the held-tail
// cases below) pass it explicitly.
function makeExtracted(
videoId: string,
delivered: number,
options: {
fps?: number;
durationSeconds?: number;
videoStreamDurationSeconds?: number;
isVFR?: boolean;
} = {},
): ExtractedFrames {
const {
fps = 30,
durationSeconds = Number.POSITIVE_INFINITY,
videoStreamDurationSeconds = durationSeconds,
isVFR = false,
} = options;
const framePaths = new Map<number, string>();
for (let i = 0; i < delivered; i += 1) framePaths.set(i, `/tmp/${videoId}/${i}.jpg`);
const metadata: VideoMetadata = {
durationSeconds,
videoStreamDurationSeconds,
width: 1280,
height: 720,
fps,
videoCodec: "h264",
hasAudio: false,
isVFR,
hasAlpha: false,
colorSpace: null,
};
return {
videoId,
srcPath: `/tmp/${videoId}.mp4`,
outputDir: `/tmp/${videoId}`,
framePattern: `${videoId}-%06d.jpg`,
fps,
totalFrames: delivered,
metadata,
framePaths,
};
}
describe("expectedFramesForClip", () => {
it("returns 0 for invalid inputs", () => {
expect(expectedFramesForClip(Number.NaN, 1, 30)).toBe(0);
expect(expectedFramesForClip(0, 1, 0)).toBe(0);
expect(expectedFramesForClip(0, 1, -1)).toBe(0);
expect(expectedFramesForClip(2, 1, 30)).toBe(0); // negative window collapses to 0
});
it("defaults to fail-closed ceil rounding", () => {
expect(expectedFramesForClip(0, 1, 29.97)).toBe(30);
expect(expectedFramesForClip(0, 5, 30)).toBe(150);
expect(expectedFramesForClip(0, 0.616666, 30)).toBe(19);
expect(expectedFramesForClip(0, 0.316666, 30)).toBe(10);
});
it("matches the CFR fps filter's nearest-boundary rounding when requested", () => {
expect(expectedFramesForClip(0, 0.616666, 30, "nearest")).toBe(18);
expect(expectedFramesForClip(0, 0.316666, 30, "nearest")).toBe(9);
expect(expectedFramesForClip(0, 0.633333, 30, "nearest")).toBe(19);
});
it("uses exact NTSC rationals for short CFR and VFR boundaries", () => {
expect(expectedFramesForClip(0, 0.25025, { num: 30000, den: 1001 }, "nearest")).toBe(8);
expect(expectedFramesForClip(0, 0.125125, { num: 24000, den: 1001 })).toBe(3);
expect(expectedFramesForClip(0, 0.5005, { num: 24000, den: 1001 })).toBe(12);
});
it("requires one frame for every positive sub-frame clip", () => {
expect(expectedFramesForClip(0, 0.001, 30)).toBe(1);
expect(expectedFramesForClip(0, 0.001, 30, "nearest")).toBe(1);
expect(expectedFramesForClip(1, 1, 30)).toBe(0);
});
});
describe("resolveVideoCoverageThreshold", () => {
it("defaults to 0.95 when env is unset or non-numeric", () => {
expect(resolveVideoCoverageThreshold(undefined)).toBe(0.95);
expect(resolveVideoCoverageThreshold("garbage")).toBe(0.95);
});
it("returns null when env is 0 or negative — gate disabled", () => {
expect(resolveVideoCoverageThreshold("0")).toBeNull();
expect(resolveVideoCoverageThreshold("-1")).toBeNull();
});
it("clamps values above 1 to 1", () => {
expect(resolveVideoCoverageThreshold("2")).toBe(1);
});
it("passes through in-range values", () => {
expect(resolveVideoCoverageThreshold("0.8")).toBe(0.8);
expect(resolveVideoCoverageThreshold("0.99")).toBe(0.99);
expect(resolveVideoCoverageThreshold("1")).toBe(1);
});
});
describe("computeVideoFrameCoverage", () => {
it("expects only the source frames consumed by a slowed authored slot", () => {
const videos = [makeVideo({ id: "slow", start: 0, end: 4, playbackRate: 0.5 })];
const extracted = [makeExtracted("slow", 60, { durationSeconds: 4 })];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports[0]).toMatchObject({ expectedFrames: 60, capturedFrames: 60, ratio: 1 });
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
});
it("reports 1.0 ratio when every video delivered its authored window", () => {
const videos = [
makeVideo({ id: "a", start: 0, end: 1 }),
makeVideo({ id: "b", start: 1, end: 3 }),
];
const extracted = [makeExtracted("a", 30), makeExtracted("b", 60)];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports).toHaveLength(2);
expect(reports[0]).toMatchObject({
videoId: "a",
expectedFrames: 30,
capturedFrames: 30,
ratio: 1,
});
expect(reports[1]).toMatchObject({
videoId: "b",
expectedFrames: 60,
capturedFrames: 60,
ratio: 1,
});
});
it("reports full coverage for a short CFR clip matching FFmpeg boundary rounding", () => {
const videos = [makeVideo({ id: "short", start: 0, end: 0.616666 })];
const reports = computeVideoFrameCoverage(videos, [makeExtracted("short", 18)], 30);
expect(reports[0]).toMatchObject({
expectedFrames: 18,
capturedFrames: 18,
ratio: 1,
});
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
});
it("tolerates a single FFmpeg boundary frame on a short 18/19 VFR extraction", () => {
const videos = [makeVideo({ id: "short-vfr", start: 0, end: 0.616666 })];
const reports = computeVideoFrameCoverage(
videos,
[makeExtracted("short-vfr", 18, { isVFR: true })],
30,
);
expect(reports[0]).toMatchObject({
expectedFrames: 19,
capturedFrames: 18,
ratio: 18 / 19,
});
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
});
it("does not reject complete 24000/1001 VFR extraction at an exact boundary", () => {
const videos = [makeVideo({ id: "ntsc-vfr", start: 0, end: 0.125125 })];
const reports = computeVideoFrameCoverage(
videos,
[makeExtracted("ntsc-vfr", 3, { isVFR: true })],
{ num: 24000, den: 1001 },
);
expect(reports[0]).toMatchObject({ expectedFrames: 3, capturedFrames: 3, ratio: 1 });
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
});
it("still fails closed when a positive sub-frame clip captured zero frames", () => {
const videos = [makeVideo({ id: "blank-sub-frame", start: 0, end: 0.001 })];
const reports = computeVideoFrameCoverage(videos, [makeExtracted("blank-sub-frame", 0)], 30);
expect(reports[0]).toMatchObject({
expectedFrames: 1,
capturedFrames: 0,
ratio: 0,
});
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
it("reports 0 capturedFrames when a video was never extracted (injection failure)", () => {
// Field signal ts=1784139267: later-injected video clips silently drop
// out of the extractor under injection-count saturation.
const videos = [makeVideo({ id: "later-injection", start: 0, end: 5 })];
const reports = computeVideoFrameCoverage(videos, [], 30);
expect(reports).toHaveLength(1);
expect(reports[0]).toMatchObject({
videoId: "later-injection",
expectedFrames: 150,
capturedFrames: 0,
ratio: 0,
});
});
it("uses delivered framePaths.size, not the possibly-stale totalFrames field", () => {
const videos = [makeVideo({ id: "a", start: 0, end: 1 })];
// Simulate an extractor whose totalFrames reports 30 but only 5 frames
// landed in framePaths (mid-extraction crash, partial cache read, …).
const partial = makeExtracted("a", 5);
partial.totalFrames = 30;
const reports = computeVideoFrameCoverage(videos, [partial], 30);
expect(reports[0]!.capturedFrames).toBe(5);
expect(reports[0]!.ratio).toBeCloseTo(5 / 30, 5);
});
it("credits a non-looping held tail against the source portion only", () => {
const videos = [makeVideo({ id: "held", start: 0, end: 10 })];
const extracted = [makeExtracted("held", 90, { durationSeconds: 3 })];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports[0]).toMatchObject({ expectedFrames: 90, capturedFrames: 90, ratio: 1 });
});
it("credits a looping short clip against the source portion — the delivered frame set covers every repeat (#2665)", () => {
// Regression #2665: a looping video shorter than its slot delivered all
// its source frames (extractor complete), but the pre-fix gate measured
// 90 unique / 300 slot = 30% and aborted. Every one of the 300 output
// frames maps to one of the 90 source frames — coverage is 100%.
const videos = [makeVideo({ id: "loop", start: 0, end: 10, loop: true })];
const extracted = [makeExtracted("loop", 90, { durationSeconds: 3 })];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports[0]).toMatchObject({ expectedFrames: 90, capturedFrames: 90, ratio: 1 });
});
it.each([
{ loop: false, label: "held tail" },
{ loop: true, label: "loop" },
])(
"credits the playable video stream instead of longer container audio for a $label",
({ loop }) => {
const videos = [makeVideo({ id: "long-audio-mux", start: 0, end: 60, loop })];
const extracted = [
makeExtracted("long-audio-mux", 90, {
durationSeconds: 60,
videoStreamDurationSeconds: 3,
}),
];
expect(computeVideoFrameCoverage(videos, extracted, 30)[0]).toMatchObject({
expectedFrames: 90,
capturedFrames: 90,
ratio: 1,
});
},
);
it("still fails when a looping clip's source extraction is truncated", () => {
// Fail-loud preserved for a genuinely-broken loop: only 60/90 source
// frames arrived, so the delivered set does NOT cover every repeat.
const videos = [makeVideo({ id: "truncated-loop", start: 0, end: 10, loop: true })];
const extracted = [makeExtracted("truncated-loop", 60, { durationSeconds: 3 })];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports[0]).toMatchObject({ expectedFrames: 90, capturedFrames: 60 });
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
it("still fails when extraction is truncated before the held-tail source", () => {
const videos = [makeVideo({ id: "truncated", start: 0, end: 10 })];
const extracted = [makeExtracted("truncated", 60, { durationSeconds: 3 })];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports[0]).toMatchObject({ expectedFrames: 90, capturedFrames: 60 });
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
it("fails closed for invalid or exhausted source durations", () => {
const videos = [
makeVideo({ id: "zero", start: 0, end: 10 }),
makeVideo({ id: "trimmed-away", start: 0, end: 10, mediaStart: 4 }),
];
const extracted = [
makeExtracted("zero", 0, { durationSeconds: 0 }),
makeExtracted("trimmed-away", 30, { durationSeconds: 3 }),
];
const reports = computeVideoFrameCoverage(videos, extracted, 30);
expect(reports[0]).toMatchObject({ expectedFrames: 300, capturedFrames: 0 });
expect(reports[1]).toMatchObject({ expectedFrames: 300, capturedFrames: 30 });
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
});
describe("assertVideoFrameCoverage", () => {
it("does not throw when every clip has full frames", () => {
const reports = [
{ videoId: "a", clipStart: 0, clipEnd: 1, expectedFrames: 30, capturedFrames: 30, ratio: 1 },
{ videoId: "b", clipStart: 1, clipEnd: 2, expectedFrames: 30, capturedFrames: 30, ratio: 1 },
];
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
});
it("fails loudly with VideoFrameCoverageError when any clip has zero frames", () => {
const reports = [
{
videoId: "good",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 30,
ratio: 1,
},
{
videoId: "blank",
clipStart: 5,
clipEnd: 10,
expectedFrames: 150,
capturedFrames: 0,
ratio: 0,
},
];
let caught: unknown = null;
try {
assertVideoFrameCoverage(reports, 0.95);
} catch (err) {
caught = err;
}
expect(caught).toBeInstanceOf(VideoFrameCoverageError);
expect(isVideoFrameCoverageError(caught)).toBe(true);
const err = caught as VideoFrameCoverageError;
expect(err.worst.videoId).toBe("blank");
expect(err.threshold).toBe(0.95);
expect(err.message).toContain("blank");
expect(err.message).toContain("check/snapshot");
expect(err.message).toContain("HF_VIDEO_COVERAGE_THRESHOLD=0");
});
it("fails loudly when a clip is below the threshold (partial coverage)", () => {
// 80% coverage: 24/30 — below the 0.95 default, above zero.
const reports = [
{
videoId: "partial",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 24,
ratio: 0.8,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
it.each([
[13, 14],
[18, 19],
])(
"tolerates exactly one nonzero boundary frame for a short clip (%i/%i)",
(capturedFrames, expectedFrames) => {
const reports = [
{
videoId: "short-boundary",
clipStart: 0,
clipEnd: expectedFrames / 30,
expectedFrames,
capturedFrames,
ratio: capturedFrames / expectedFrames,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
},
);
it("does not treat a one-frame deficit as tolerance when it represents major loss", () => {
const reports = [
{
videoId: "major-loss",
clipStart: 0,
clipEnd: 2 / 30,
expectedFrames: 2,
capturedFrames: 1,
ratio: 0.5,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
it("does not tolerate two missing frames, zero captured frames, or an exact threshold", () => {
const report = {
videoId: "short-incomplete",
clipStart: 0,
clipEnd: 19 / 30,
expectedFrames: 19,
capturedFrames: 17,
ratio: 17 / 19,
};
expect(() => assertVideoFrameCoverage([report], 0.95)).toThrow(VideoFrameCoverageError);
expect(() =>
assertVideoFrameCoverage([{ ...report, capturedFrames: 0, ratio: 0 }], 0.95),
).toThrow(VideoFrameCoverageError);
expect(() =>
assertVideoFrameCoverage([{ ...report, capturedFrames: 18, ratio: 18 / 19 }], 1),
).toThrow(VideoFrameCoverageError);
});
it("does not apply the one-frame tolerance to longer clips", () => {
const reports = [
{
videoId: "long-boundary",
clipStart: 0,
clipEnd: 21 / 30,
expectedFrames: 21,
capturedFrames: 20,
ratio: 20 / 21,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.99)).toThrow(VideoFrameCoverageError);
});
it("still rejects a material long-clip shortfall even when it is five frames", () => {
const reports = [
{
videoId: "long-partial",
clipStart: 0,
clipEnd: 89 / 30,
expectedFrames: 89,
capturedFrames: 84,
ratio: 84 / 89,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(VideoFrameCoverageError);
});
it("respects a threshold override — 0.5 passes 60% coverage", () => {
const reports = [
{
videoId: "partial",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 18,
ratio: 0.6,
},
];
expect(() => assertVideoFrameCoverage(reports, 0.5)).not.toThrow();
});
it("is a no-op when the threshold is null (env opt-out)", () => {
const reports = [
{
videoId: "blank",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 0,
ratio: 0,
},
];
expect(() => assertVideoFrameCoverage(reports, null)).not.toThrow();
});
it("ignores 0-expected-frame windows so the gate never fires on a degenerate clip", () => {
const reports = [
{
videoId: "zero-duration",
clipStart: 3,
clipEnd: 3,
expectedFrames: 0,
capturedFrames: 0,
ratio: 1,
},
];
expect(() => assertVideoFrameCoverage(reports, 1)).not.toThrow();
});
it("cites the worst-ratio clip in the error, not the first-found", () => {
// Field-signal shape: 15 injected videos, several later ones blank. Cite
// the deepest failure so operators fix root cause first, not the closest.
const reports = [
{
videoId: "slightly-low",
clipStart: 0,
clipEnd: 1,
expectedFrames: 30,
capturedFrames: 27,
ratio: 0.9,
},
{
videoId: "totally-blank",
clipStart: 1,
clipEnd: 2,
expectedFrames: 30,
capturedFrames: 0,
ratio: 0,
},
{
videoId: "moderately-low",
clipStart: 2,
clipEnd: 3,
expectedFrames: 30,
capturedFrames: 15,
ratio: 0.5,
},
];
try {
assertVideoFrameCoverage(reports, 0.95);
throw new Error("expected coverage assertion to throw");
} catch (err) {
expect(isVideoFrameCoverageError(err)).toBe(true);
const cov = err as VideoFrameCoverageError;
expect(cov.worst.videoId).toBe("totally-blank");
expect(cov.failedReports.map((r) => r.videoId)).toEqual([
"totally-blank",
"moderately-low",
"slightly-low",
]);
expect(cov.message).toContain("+2 more clip(s) below threshold");
}
});
});
describe("countAuthoredTimedClips", () => {
it("counts every [data-start] element in the compiled HTML", () => {
// Field signal ts=1784144554: 147-clip composition with 130 word-level
// caption divs. Static scan gives a coarse proxy — enough to make a
// 147-clip render distinguishable in telemetry from a 3-clip render.
const html = `<html><body>
<div data-start="0" data-duration="1">a</div>
<div data-start="1" data-duration="1">b</div>
<video data-start="2" data-duration="3" src="v.mp4"></video>
<div>not timed</div>
</body></html>`;
expect(countAuthoredTimedClips(html)).toBe(3);
});
it("returns 0 when no timed clips exist", () => {
expect(countAuthoredTimedClips("<html><body><div>hi</div></body></html>")).toBe(0);
});
});