Skip to content

Write HTML. Render video. Built for agents.

test(producer): run the late-host pip and dogs-captions fixtures in CI (#4625)

Miguel Ángel625b07fe2d49
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index 7a7bbc3fae..0c486b046f 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -12,7 +12,7 @@ env:
# Graphite can update a branch and its PR base separately for the same head
# SHA. Keep only the newest expensive regression run for each PR/ref.
concurrency:
- group: regression-${{ github.event.pull_request.number || github.ref }}
+ group: regression-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'schedule' && '-nightly' || '' }}
# A body or title edit queues behind running checks instead of cancelling them.
cancel-in-progress: ${{ github.event.action != 'edited' || github.event.changes.base != null }}
@@ -31,6 +31,9 @@ on:
push:
branches:
- main
+ schedule:
+ # Daily at 06:17 UTC: the full suite, including fixtures too slow for every PR.
+ - cron: "17 6 * * *"
jobs:
changes:
@@ -38,7 +41,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 2
outputs:
- code: ${{ (github.event_name == 'workflow_dispatch' && inputs.catalog_publish) && 'true' || steps.filter.outputs.code }}
+ code: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.catalog_publish)) && 'true' || steps.filter.outputs.code }}
matrix: ${{ steps.shards.outputs.matrix }}
steps:
# Force git-based change detection instead of the pull_request REST API.
@@ -49,6 +52,7 @@ jobs:
fetch-depth: 0
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4
id: filter
+ if: github.event_name != 'schedule'
with:
token: ""
filters: |
@@ -62,9 +66,11 @@ jobs:
# nor explicitly excluded, so a new fixture cannot silently never run.
- name: Plan regression shards
id: shards
+ env:
+ NIGHTLY: ${{ github.event_name == 'schedule' && '--nightly' || '' }}
run: |
- echo "matrix=$(node packages/producer/scripts/plan-regression-shards.mjs)" >> "$GITHUB_OUTPUT"
- node packages/producer/scripts/plan-regression-shards.mjs --pretty
+ echo "matrix=$(node packages/producer/scripts/plan-regression-shards.mjs $NIGHTLY)" >> "$GITHUB_OUTPUT"
+ node packages/producer/scripts/plan-regression-shards.mjs $NIGHTLY --pretty
preflight:
name: Preflight (lint + format)
diff --git a/packages/producer/scripts/plan-regression-shards.mjs b/packages/producer/scripts/plan-regression-shards.mjs
index a6962a770f..94066ee5ac 100644
--- a/packages/producer/scripts/plan-regression-shards.mjs
+++ b/packages/producer/scripts/plan-regression-shards.mjs
@@ -19,7 +19,7 @@
// Drift becomes a build error instead of silent absence.
//
// Usage:
-// node scripts/plan-regression-shards.mjs [--shards N] [--pretty]
+// node scripts/plan-regression-shards.mjs [--shards N] [--nightly] [--pretty]
import { readdirSync, readFileSync, existsSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
@@ -103,11 +103,13 @@ export function planShards({
testsDir = TESTS_DIR,
scheduleFile = SCHEDULE_FILE,
shardCount,
+ nightly = false,
} = {}) {
const schedule = JSON.parse(readFileSync(scheduleFile, "utf-8"));
const timings = schedule.timings ?? {};
const excluded = schedule.excluded ?? {};
const distributed = schedule.distributed ?? {};
+ const nightlyOnly = schedule.nightly ?? {};
const resolvedShardCount = shardCount ?? schedule.shardCount ?? DEFAULT_SHARD_COUNT;
const resolvedDistributedShardCount =
schedule.distributedShardCount ?? DEFAULT_DISTRIBUTED_SHARD_COUNT;
@@ -149,18 +151,23 @@ export function planShards({
);
}
- // A distributed fixture must also be scheduled — the mode says *how* to run
- // it, not *whether*. Listing one that is excluded or absent is a typo, and a
- // silent one, since the mode map is not consulted when building the shard set.
- const misdeclared = Object.keys(distributed).filter((name) => !(name in timings));
+ // A distributed or nightly fixture must also be scheduled — those maps say
+ // *how* or *when* to run it, not *whether*. Listing one that is excluded or
+ // absent is a typo, and a silent one, since neither map adds to the shard set.
+ const misdeclared = [...Object.keys(distributed), ...Object.keys(nightlyOnly)].filter(
+ (name) => !(name in timings),
+ );
if (misdeclared.length > 0) {
throw new Error(
- `Fixtures marked "distributed" are not scheduled: ${misdeclared.join(", ")}.\n` +
- `Add each to "timings" in ${scheduleFile}, or drop it from "distributed".`,
+ `Fixtures marked "distributed" or "nightly" are not scheduled: ${misdeclared.join(", ")}.\n` +
+ `Add each to "timings" in ${scheduleFile}, or drop it from that map.`,
);
}
- const scheduled = onDisk.filter((name) => !(name in excluded));
+ // Nightly fixtures stay out of per-PR runs; the scheduled run passes --nightly.
+ const scheduled = onDisk.filter(
+ (name) => !(name in excluded) && (nightly || !(name in nightlyOnly)),
+ );
// Harness mode is a per-invocation flag, so a shard cannot mix modes. Pack
// each mode into its own shards rather than trying to interleave them.
@@ -205,7 +212,10 @@ function main() {
throw new Error("--shards must be a positive integer");
}
- const { include, plan, excludedCount } = planShards({ shardCount });
+ const { include, plan, excludedCount } = planShards({
+ shardCount,
+ nightly: argv.includes("--nightly"),
+ });
if (argv.includes("--pretty")) {
const worst = Math.max(...plan.map((row) => row.estimatedMinutes));
diff --git a/packages/producer/src/regression-shard-plan.test.ts b/packages/producer/src/regression-shard-plan.test.ts
index 158154e0d9..aeabc2bae6 100644
--- a/packages/producer/src/regression-shard-plan.test.ts
+++ b/packages/producer/src/regression-shard-plan.test.ts
@@ -26,6 +26,7 @@ function readSchedule(): {
timings?: Record<string, number>;
excluded?: Record<string, string>;
distributed?: Record<string, string>;
+ nightly?: Record<string, string>;
} {
return JSON.parse(readFileSync(join(TESTS_DIR, "shard-schedule.json"), "utf-8"));
}
@@ -55,7 +56,7 @@ describe("shard planner fixture discovery", () => {
.map((suite) => suite.id)
.filter((id) => !excluded.has(id))
.sort();
- const scheduled = planShards()
+ const scheduled = planShards({ nightly: true })
.include.flatMap((row) => row.args.split(" "))
.sort();
expect(scheduled).toEqual(harnessRunnable);
@@ -98,6 +99,19 @@ describe("shard planner fixture discovery", () => {
}
});
+ it("rejects a nightly fixture that is not scheduled", () => {
+ const schedule = readSchedule();
+ const tainted = join(mkdtempSync(join(tmpdir(), "hf-shard-schedule-")), "shard-schedule.json");
+ writeFileSync(
+ tainted,
+ JSON.stringify({
+ ...schedule,
+ nightly: { ...schedule.nightly, "not-a-real-fixture": "typo" },
+ }),
+ );
+ expect(() => planShards({ scheduleFile: tainted })).toThrow(/are not scheduled/);
+ });
+
it("rejects a distributed fixture that is not scheduled", () => {
const schedule = readSchedule();
const tainted = join(mkdtempSync(join(tmpdir(), "hf-shard-schedule-")), "shard-schedule.json");
@@ -156,6 +170,13 @@ describe("shard planner fixture discovery", () => {
}
});
+ it("gives every nightly fixture a written reason", () => {
+ for (const [fixture, reason] of Object.entries(readSchedule().nightly ?? {})) {
+ expect(typeof reason, `${fixture} needs a reason`).toBe("string");
+ expect((reason as string).length, `${fixture} needs a real reason`).toBeGreaterThan(20);
+ }
+ });
+
it("gives every excluded fixture a written reason", () => {
// Exclusions are how a fixture legitimately stays out of CI, so the bar
// is that someone had to type why. This is what stops the excluded list
@@ -229,11 +250,25 @@ describe("planShards()", () => {
});
it("runs every fixture that is not explicitly excluded", () => {
- const { include } = planShards();
+ const { include } = planShards({ nightly: true });
const scheduled = new Set(include.flatMap((row) => row.args.split(" ")));
const excluded = new Set(Object.keys(readSchedule().excluded ?? {}));
for (const fixture of discoverFixtures(TESTS_DIR)) {
expect(scheduled.has(fixture) || excluded.has(fixture)).toBe(true);
}
});
+
+ it("keeps nightly fixtures out of the per-PR plan and runs them in the nightly one", () => {
+ const nightly = Object.keys(readSchedule().nightly ?? {});
+ expect(nightly.length).toBeGreaterThan(0);
+ const fixturesOf = (plan: ReturnType<typeof planShards>) =>
+ plan.include.flatMap((row) => row.args.split(" "));
+ const perPr = fixturesOf(planShards());
+ const full = fixturesOf(planShards({ nightly: true }));
+ for (const fixture of nightly) {
+ expect(perPr).not.toContain(fixture);
+ expect(full).toContain(fixture);
+ }
+ expect(full.length).toBe(perPr.length + nightly.length);
+ });
});
diff --git a/packages/producer/tests/pip-video-late-host/src/compositions/pip.html b/packages/producer/tests/pip-video-late-host/src/compositions/pip.html
index 9a4be1721e..d87d8d9965 100644
--- a/packages/producer/tests/pip-video-late-host/src/compositions/pip.html
+++ b/packages/producer/tests/pip-video-late-host/src/compositions/pip.html
@@ -1,11 +1,11 @@
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/gsap.min.js"></script>
<template id="pip-template">
<div data-composition-id="pip-scene" data-width="1920" data-height="1080">
- <!-- Pip video: data-start="3" is global time matching the host start.
- Before the fix, resolveStartForElement returned 3+3=6, hiding the
- video for the entire host window (3–6s). -->
+ <!-- Pip video: data-start="3" is legacy root-global time matching the host start.
+ Without the global basis it reads as local, 3+3=6, past the host's end. -->
<video
id="pip-video"
+ data-hf-media-start-basis="global"
data-start="3"
data-duration="3"
data-track-index="0"
diff --git a/packages/producer/tests/shard-schedule.json b/packages/producer/tests/shard-schedule.json
index dc1a5a7f40..b6beb922ad 100644
--- a/packages/producer/tests/shard-schedule.json
+++ b/packages/producer/tests/shard-schedule.json
@@ -7,9 +7,10 @@
" Refresh from a green main run when shards drift apart; no test asserts these values.",
"distributed: fixtures run with --mode=distributed-simulated, mapped to why.",
" Mode is a per-invocation flag, so these get their own shards.",
- "excluded: fixtures deliberately not run, each mapped to the reason why."
+ "excluded: fixtures deliberately not run, each mapped to the reason why.",
+ "nightly: scheduled fixtures that run only in the nightly regression job, mapped to why."
],
- "shardCount": 8,
+ "shardCount": 9,
"distributedShardCount": 1,
"timings": {
"animejs-adapter": 16,
@@ -17,6 +18,7 @@
"audio-mux-parity": 16,
"chat": 34,
"css-spinner-render-compat": 14,
+ "dogs-captions": 524,
"font-variant-numeric": 7,
"gsap-letters-render-compat": 9,
"hdr-hlg-regression": 125,
@@ -29,6 +31,7 @@
"mp4-h265-sdr": 4,
"overlay-montage-prod": 204,
"parallel-capture-regression": 18,
+ "pip-video-late-host": 35,
"plan-v2-partial-color": 2,
"png-sequence": 3,
"portrait-edge-bleed": 6,
@@ -70,7 +73,6 @@
"css-import-scoping": "Rejected by the harness at load time (meta.json: maxAudioLagWindows must be >= 1), so it has never run despite looking scheduled-able. Fix the metadata, then schedule it.",
"css-var-fonts": "Rejected by the harness at load time (meta.json: minAudioCorrelation must be between 0 and 1), so it has never run despite looking scheduled-able. Fix the metadata, then schedule it.",
"d3-adapter": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
- "dogs-captions": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"escape-hatch-fatal-fallback": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"fast-capture-gsap": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"google-maps-adapter": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
@@ -82,7 +84,6 @@
"maplibre-adapter": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"nested-subcomp-depth-3": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"page-side-shader-compositor-render-compat": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
- "pip-video-late-host": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"render-video-overlay-stretch": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"software-beginframe-yoyo-compositor": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
"spanish-empire-cdn-inline": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",
@@ -99,5 +100,8 @@
"distributed": {
"style-15-prod": "Renders 12.4x faster chunked (1413s -> 114s at 4 CPUs) because in-process capture is pinned to one worker by the auto-worker derate. Passes the same golden at the same 30dB bar, 0/100 failed frames, psnr min 39.3 vs 40.1 in-process, so the sensitivity cost is 0.8dB.",
"sub-composition-video": "Renders 12.1x faster chunked (1346s -> 111s at 4 CPUs). Slow in-process for a different reason to style-15-prod (video decode rather than feTurbulence), which is why chunking helps both. Passes the same golden, 0/100 failed."
+ },
+ "nightly": {
+ "dogs-captions": "One long fixture: 7 to 10.5 min of rendering in its first two CI runs (timing is their median), and the slowest shard in both, 20s to 1m40s past the next. A single fixture cannot be split across shards, so it runs nightly instead of on every PR."
}
}