Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
test(producer): run the late-host pip and dogs-captions fixtures in CI (#4625)
Miguel Ángel625b07fe2d49
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.ymlindex 7a7bbc3fae..0c486b046f 100644--- a/.github/workflows/regression.yml+++ b/.github/workflows/regression.yml@@ -12,7 +12,7 @@ env:# Graphite can update a branch and its PR base separately for the same head# SHA. Keep only the newest expensive regression run for each PR/ref.concurrency:- group: regression-${{ github.event.pull_request.number || github.ref }}+ group: regression-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'schedule' && '-nightly' || '' }}# A body or title edit queues behind running checks instead of cancelling them.cancel-in-progress: ${{ github.event.action != 'edited' || github.event.changes.base != null }}@@ -31,6 +31,9 @@ on:push:branches:- main+ schedule:+ # Daily at 06:17 UTC: the full suite, including fixtures too slow for every PR.+ - cron: "17 6 * * *"jobs:changes:@@ -38,7 +41,7 @@ jobs:runs-on: ubuntu-latesttimeout-minutes: 2outputs:- code: ${{ (github.event_name == 'workflow_dispatch' && inputs.catalog_publish) && 'true' || steps.filter.outputs.code }}+ code: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.catalog_publish)) && 'true' || steps.filter.outputs.code }}matrix: ${{ steps.shards.outputs.matrix }}steps:# Force git-based change detection instead of the pull_request REST API.@@ -49,6 +52,7 @@ jobs:fetch-depth: 0- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4id: filter+ if: github.event_name != 'schedule'with:token: ""filters: |@@ -62,9 +66,11 @@ jobs:# nor explicitly excluded, so a new fixture cannot silently never run.- name: Plan regression shardsid: shards+ env:+ NIGHTLY: ${{ github.event_name == 'schedule' && '--nightly' || '' }}run: |- echo "matrix=$(node packages/producer/scripts/plan-regression-shards.mjs)" >> "$GITHUB_OUTPUT"- node packages/producer/scripts/plan-regression-shards.mjs --pretty+ echo "matrix=$(node packages/producer/scripts/plan-regression-shards.mjs $NIGHTLY)" >> "$GITHUB_OUTPUT"+ node packages/producer/scripts/plan-regression-shards.mjs $NIGHTLY --prettypreflight:name: Preflight (lint + format)diff --git a/packages/producer/scripts/plan-regression-shards.mjs b/packages/producer/scripts/plan-regression-shards.mjsindex a6962a770f..94066ee5ac 100644--- a/packages/producer/scripts/plan-regression-shards.mjs+++ b/packages/producer/scripts/plan-regression-shards.mjs@@ -19,7 +19,7 @@// Drift becomes a build error instead of silent absence.//// Usage:-// node scripts/plan-regression-shards.mjs [--shards N] [--pretty]+// node scripts/plan-regression-shards.mjs [--shards N] [--nightly] [--pretty]import { readdirSync, readFileSync, existsSync } from "node:fs";import { dirname, join, resolve } from "node:path";@@ -103,11 +103,13 @@ export function planShards({testsDir = TESTS_DIR,scheduleFile = SCHEDULE_FILE,shardCount,+ nightly = false,} = {}) {const schedule = JSON.parse(readFileSync(scheduleFile, "utf-8"));const timings = schedule.timings ?? {};const excluded = schedule.excluded ?? {};const distributed = schedule.distributed ?? {};+ const nightlyOnly = schedule.nightly ?? {};const resolvedShardCount = shardCount ?? schedule.shardCount ?? DEFAULT_SHARD_COUNT;const resolvedDistributedShardCount =schedule.distributedShardCount ?? DEFAULT_DISTRIBUTED_SHARD_COUNT;@@ -149,18 +151,23 @@ export function planShards({);}- // A distributed fixture must also be scheduled — the mode says *how* to run- // it, not *whether*. Listing one that is excluded or absent is a typo, and a- // silent one, since the mode map is not consulted when building the shard set.- const misdeclared = Object.keys(distributed).filter((name) => !(name in timings));+ // A distributed or nightly fixture must also be scheduled — those maps say+ // *how* or *when* to run it, not *whether*. Listing one that is excluded or+ // absent is a typo, and a silent one, since neither map adds to the shard set.+ const misdeclared = [...Object.keys(distributed), ...Object.keys(nightlyOnly)].filter(+ (name) => !(name in timings),+ );if (misdeclared.length > 0) {throw new Error(- `Fixtures marked "distributed" are not scheduled: ${misdeclared.join(", ")}.\n` +- `Add each to "timings" in ${scheduleFile}, or drop it from "distributed".`,+ `Fixtures marked "distributed" or "nightly" are not scheduled: ${misdeclared.join(", ")}.\n` ++ `Add each to "timings" in ${scheduleFile}, or drop it from that map.`,);}- const scheduled = onDisk.filter((name) => !(name in excluded));+ // Nightly fixtures stay out of per-PR runs; the scheduled run passes --nightly.+ const scheduled = onDisk.filter(+ (name) => !(name in excluded) && (nightly || !(name in nightlyOnly)),+ );// Harness mode is a per-invocation flag, so a shard cannot mix modes. Pack// each mode into its own shards rather than trying to interleave them.@@ -205,7 +212,10 @@ function main() {throw new Error("--shards must be a positive integer");}- const { include, plan, excludedCount } = planShards({ shardCount });+ const { include, plan, excludedCount } = planShards({+ shardCount,+ nightly: argv.includes("--nightly"),+ });if (argv.includes("--pretty")) {const worst = Math.max(...plan.map((row) => row.estimatedMinutes));diff --git a/packages/producer/src/regression-shard-plan.test.ts b/packages/producer/src/regression-shard-plan.test.tsindex 158154e0d9..aeabc2bae6 100644--- a/packages/producer/src/regression-shard-plan.test.ts+++ b/packages/producer/src/regression-shard-plan.test.ts@@ -26,6 +26,7 @@ function readSchedule(): {timings?: Record<string, number>;excluded?: Record<string, string>;distributed?: Record<string, string>;+ nightly?: Record<string, string>;} {return JSON.parse(readFileSync(join(TESTS_DIR, "shard-schedule.json"), "utf-8"));}@@ -55,7 +56,7 @@ describe("shard planner fixture discovery", () => {.map((suite) => suite.id).filter((id) => !excluded.has(id)).sort();- const scheduled = planShards()+ const scheduled = planShards({ nightly: true }).include.flatMap((row) => row.args.split(" ")).sort();expect(scheduled).toEqual(harnessRunnable);@@ -98,6 +99,19 @@ describe("shard planner fixture discovery", () => {}});+ it("rejects a nightly fixture that is not scheduled", () => {+ const schedule = readSchedule();+ const tainted = join(mkdtempSync(join(tmpdir(), "hf-shard-schedule-")), "shard-schedule.json");+ writeFileSync(+ tainted,+ JSON.stringify({+ ...schedule,+ nightly: { ...schedule.nightly, "not-a-real-fixture": "typo" },+ }),+ );+ expect(() => planShards({ scheduleFile: tainted })).toThrow(/are not scheduled/);+ });+it("rejects a distributed fixture that is not scheduled", () => {const schedule = readSchedule();const tainted = join(mkdtempSync(join(tmpdir(), "hf-shard-schedule-")), "shard-schedule.json");@@ -156,6 +170,13 @@ describe("shard planner fixture discovery", () => {}});+ it("gives every nightly fixture a written reason", () => {+ for (const [fixture, reason] of Object.entries(readSchedule().nightly ?? {})) {+ expect(typeof reason, `${fixture} needs a reason`).toBe("string");+ expect((reason as string).length, `${fixture} needs a real reason`).toBeGreaterThan(20);+ }+ });+it("gives every excluded fixture a written reason", () => {// Exclusions are how a fixture legitimately stays out of CI, so the bar// is that someone had to type why. This is what stops the excluded list@@ -229,11 +250,25 @@ describe("planShards()", () => {});it("runs every fixture that is not explicitly excluded", () => {- const { include } = planShards();+ const { include } = planShards({ nightly: true });const scheduled = new Set(include.flatMap((row) => row.args.split(" ")));const excluded = new Set(Object.keys(readSchedule().excluded ?? {}));for (const fixture of discoverFixtures(TESTS_DIR)) {expect(scheduled.has(fixture) || excluded.has(fixture)).toBe(true);}});++ it("keeps nightly fixtures out of the per-PR plan and runs them in the nightly one", () => {+ const nightly = Object.keys(readSchedule().nightly ?? {});+ expect(nightly.length).toBeGreaterThan(0);+ const fixturesOf = (plan: ReturnType<typeof planShards>) =>+ plan.include.flatMap((row) => row.args.split(" "));+ const perPr = fixturesOf(planShards());+ const full = fixturesOf(planShards({ nightly: true }));+ for (const fixture of nightly) {+ expect(perPr).not.toContain(fixture);+ expect(full).toContain(fixture);+ }+ expect(full.length).toBe(perPr.length + nightly.length);+ });});diff --git a/packages/producer/tests/pip-video-late-host/src/compositions/pip.html b/packages/producer/tests/pip-video-late-host/src/compositions/pip.htmlindex 9a4be1721e..d87d8d9965 100644--- a/packages/producer/tests/pip-video-late-host/src/compositions/pip.html+++ b/packages/producer/tests/pip-video-late-host/src/compositions/pip.html@@ -1,11 +1,11 @@<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/gsap.min.js"></script><template id="pip-template"><div data-composition-id="pip-scene" data-width="1920" data-height="1080">- <!-- Pip video: data-start="3" is global time matching the host start.- Before the fix, resolveStartForElement returned 3+3=6, hiding the- video for the entire host window (3–6s). -->+ <!-- Pip video: data-start="3" is legacy root-global time matching the host start.+ Without the global basis it reads as local, 3+3=6, past the host's end. --><videoid="pip-video"+ data-hf-media-start-basis="global"data-start="3"data-duration="3"data-track-index="0"diff --git a/packages/producer/tests/shard-schedule.json b/packages/producer/tests/shard-schedule.jsonindex dc1a5a7f40..b6beb922ad 100644--- a/packages/producer/tests/shard-schedule.json+++ b/packages/producer/tests/shard-schedule.json@@ -7,9 +7,10 @@" Refresh from a green main run when shards drift apart; no test asserts these values.","distributed: fixtures run with --mode=distributed-simulated, mapped to why."," Mode is a per-invocation flag, so these get their own shards.",- "excluded: fixtures deliberately not run, each mapped to the reason why."+ "excluded: fixtures deliberately not run, each mapped to the reason why.",+ "nightly: scheduled fixtures that run only in the nightly regression job, mapped to why."],- "shardCount": 8,+ "shardCount": 9,"distributedShardCount": 1,"timings": {"animejs-adapter": 16,@@ -17,6 +18,7 @@"audio-mux-parity": 16,"chat": 34,"css-spinner-render-compat": 14,+ "dogs-captions": 524,"font-variant-numeric": 7,"gsap-letters-render-compat": 9,"hdr-hlg-regression": 125,@@ -29,6 +31,7 @@"mp4-h265-sdr": 4,"overlay-montage-prod": 204,"parallel-capture-regression": 18,+ "pip-video-late-host": 35,"plan-v2-partial-color": 2,"png-sequence": 3,"portrait-edge-bleed": 6,@@ -70,7 +73,6 @@"css-import-scoping": "Rejected by the harness at load time (meta.json: maxAudioLagWindows must be >= 1), so it has never run despite looking scheduled-able. Fix the metadata, then schedule it.","css-var-fonts": "Rejected by the harness at load time (meta.json: minAudioCorrelation must be between 0 and 1), so it has never run despite looking scheduled-able. Fix the metadata, then schedule it.","d3-adapter": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",- "dogs-captions": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","escape-hatch-fatal-fallback": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","fast-capture-gsap": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","google-maps-adapter": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",@@ -82,7 +84,6 @@"maplibre-adapter": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","nested-subcomp-depth-3": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","page-side-shader-compositor-render-compat": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",- "pip-video-late-host": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","render-video-overlay-stretch": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","software-beginframe-yoyo-compositor": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.","spanish-empire-cdn-inline": "Was absent from the hand-written shard matrix, so it has never run in CI and its cost and pass state are unknown. Triage and either schedule it or record a real reason here.",@@ -99,5 +100,8 @@"distributed": {"style-15-prod": "Renders 12.4x faster chunked (1413s -> 114s at 4 CPUs) because in-process capture is pinned to one worker by the auto-worker derate. Passes the same golden at the same 30dB bar, 0/100 failed frames, psnr min 39.3 vs 40.1 in-process, so the sensitivity cost is 0.8dB.","sub-composition-video": "Renders 12.1x faster chunked (1346s -> 111s at 4 CPUs). Slow in-process for a different reason to style-15-prod (video decode rather than feTurbulence), which is why chunking helps both. Passes the same golden, 0/100 failed."+ },+ "nightly": {+ "dogs-captions": "One long fixture: 7 to 10.5 min of rendering in its first two CI runs (timing is their median), and the slowest shard in both, 20s to 1m40s past the next. A single fixture cannot be split across shards, so it runs nightly instead of on every PR."}}