Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(studio): the edit-accuracy gate says the shards did not run, not that accuracy fell (#6069)
Miguel Ángel609ff5a7c013
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex f4f795d32d..309968f40f 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -1310,7 +1310,8 @@ jobs:- name: Fail if the gate failedif: steps.gate.outputs.exit_code != '0'run: |- echo "::error::Edit accuracy fell against the base branch; see the PR comment."+ verdict=$(jq -r '.verdict // empty' /tmp/edit-accuracy-gate/gate.json 2>/dev/null)+ echo "::error::${verdict:-The edit accuracy gate could not finish; see its log.}"exit 1smoke-global-install:diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjsindex 2763e04ebb..99ab1efe6f 100644--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs@@ -157,6 +157,20 @@ const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));/** The grid a shard ran, from its first run's meta ("pr shard 2/5"); re-runs record their filter instead. */const gridOf = (results) => results.find((r) => !r.meta.rerun)?.meta.grid.split(" ")[0];+const VERDICTS = {+ 0: "Edit accuracy holds against the base branch.",+ 1: "Edit accuracy fell against the base branch; see the PR comment.",+};+const notRun = (reported) => `Edit accuracy did not run: ${reported}; see the shard jobs.`;++/** Writes what the workflow posts and prints; the verdict is the only text it shows for a failed gate. */+function report(out, markdown, g) {+ mkdirSync(out, { recursive: true });+ writeFileSync(join(out, "comment.md"), markdown);+ writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1));+ console.log(markdown);+}+// fallow-ignore-next-line complexityfunction main(argv) {const shardsAt = argv.indexOf("--shards");@@ -169,7 +183,18 @@ function main(argv) {}const [headPath, out, ...resultPaths] = rest;const runs = resultPaths.map((p) => readJson(p));- if (!runs.length) throw new Error("no results.json from any shard");+ if (!runs.length) {+ const verdict = notRun(`0 of ${shards ?? "the"} shards reported results`);+ report(+ out,+ `<!-- edit-accuracy -->\n### ${verdict}\n\nNothing was compared with the base branch.\n`,+ {+ ok: false,+ verdict,+ },+ );+ return 2;+ }const cases = runs.flatMap((r) => r.cases);const full = gridOf(runs) === "full";const g = gate(@@ -181,21 +206,21 @@ function main(argv) {);// A shard that died uploads nothing, so the count of first runs is what shows it.const firstRuns = runs.filter((r) => !r.meta.rerun).length;- if (shards !== null && firstRuns !== shards) {- g.reasons.push(`${firstRuns} of ${shards} shards reported results`);- g.ok = false;- }- mkdirSync(out, { recursive: true });- writeFileSync(join(out, "comment.md"), comment(g));- writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1));+ const shortfall =+ shards !== null && firstRuns !== shards && `${firstRuns} of ${shards} shards reported results`;+ // A case that ran and fell is a drop; a shard that did not report is not one.+ const code = g.ok ? (shortfall ? 2 : 0) : 1;+ if (shortfall) g.reasons.push(shortfall);+ g.ok = code === 0;+ g.verdict = code === 2 ? notRun(shortfall) : VERDICTS[code];+ report(out, comment(g), g);// A baseline.json to commit when cases newly pass; only a full grid holds every case.if (full) writeReport(out, { ...runs[0].meta, grid: "full (CI)" }, bankable(cases), 0);- console.log(comment(g));if (g.unbanked.length && process.env.GITHUB_ACTIONS)console.log(`::notice title=Edit accuracy::${g.unbanked.length} newly passing case(s) are not banked in baseline.json; not gated.`,);- return g.ok ? 0 : 1;+ return code;}if (process.argv[1] === fileURLToPath(import.meta.url))diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjsindex 745ca3c2aa..a8aecf024b 100644--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs@@ -1,3 +1,8 @@+import { spawnSync } from "node:child_process";+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";import { describe, expect, it } from "vitest";import { accurate, bankable, comment, flipped, gate, QUARANTINED } from "./ratchet.mjs";import { entry, score } from "./report.mjs";@@ -190,3 +195,63 @@ describe("teleport in the gate", () => {expect(accurate(entry(score({ id: "nudge-x" }, { ...r, teleport: null })))).toBe(true);});});++describe("gate command, as CI runs it", () => {+ const script = fileURLToPath(new URL("./ratchet.mjs", import.meta.url));+ /** Runs `gate --shards <shards>` over the given shard results; returns its exit code, comment and verdict. */+ function runGate(shards, shardCases) {+ const dir = mkdtempSync(join(tmpdir(), "edit-accuracy-gate-"));+ const base = join(dir, "base.json");+ writeFileSync(base, JSON.stringify(baseline({ a: good, b: good })));+ const results = shardCases.map((cases, i) => {+ const path = join(dir, `shard-${i}.json`);+ writeFileSync(path, JSON.stringify({ meta: { grid: `pr shard ${i + 1}/${shards}` }, cases }));+ return path;+ });+ const out = join(dir, "out");+ const run = spawnSync(+ process.execPath,+ [script, "gate", "--shards", String(shards), base, base, out, ...results],+ {+ encoding: "utf8",+ },+ );+ const report = JSON.parse(readFileSync(join(out, "gate.json"), "utf8"));+ return {+ code: run.status,+ comment: readFileSync(join(out, "comment.md"), "utf8"),+ verdict: report.verdict,+ };+ }++ it("says the shards did not run when none reported results, never that accuracy fell", () => {+ const g = runGate(5, []);+ expect(g.code).toBe(2);+ expect(g.verdict).toBe(+ "Edit accuracy did not run: 0 of 5 shards reported results; see the shard jobs.",+ );+ expect(g.comment).toContain("0 of 5 shards reported results");+ expect(g.comment).not.toMatch(/fell|accurate \d/);+ });++ it("says the shards did not all run when the cases that ran hold, so a missing shard is not a drop", () => {+ const g = runGate(2, [[run("a")]]);+ expect(g.code).toBe(2);+ expect(g.verdict).toBe(+ "Edit accuracy did not run: 1 of 2 shards reported results; see the shard jobs.",+ );+ });++ it("still says accuracy fell when a case that ran regressed, even with a shard missing", () => {+ const g = runGate(2, [[run("a", 9), run("a", 9), run("a", 9)]]);+ expect(g.code).toBe(1);+ expect(g.verdict).toBe("Edit accuracy fell against the base branch; see the PR comment.");+ expect(g.comment).toContain("1 of 2 shards reported results");+ });++ it("passes when every shard reported and nothing fell", () => {+ const g = runGate(2, [[run("a")], [run("b")]]);+ expect(g.code).toBe(0);+ expect(g.verdict).toBe("Edit accuracy holds against the base branch.");+ });+});