Skip to content

Write HTML. Render video. Built for agents.

fix(studio): the edit-accuracy gate says the shards did not run, not that accuracy fell (#6069)

Miguel Ángel609ff5a7c013
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f4f795d32d..309968f40f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1310,7 +1310,8 @@ jobs:
- name: Fail if the gate failed
if: steps.gate.outputs.exit_code != '0'
run: |
- echo "::error::Edit accuracy fell against the base branch; see the PR comment."
+ verdict=$(jq -r '.verdict // empty' /tmp/edit-accuracy-gate/gate.json 2>/dev/null)
+ echo "::error::${verdict:-The edit accuracy gate could not finish; see its log.}"
exit 1
smoke-global-install:
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
index 2763e04ebb..99ab1efe6f 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -157,6 +157,20 @@ const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
/** The grid a shard ran, from its first run's meta ("pr shard 2/5"); re-runs record their filter instead. */
const gridOf = (results) => results.find((r) => !r.meta.rerun)?.meta.grid.split(" ")[0];
+const VERDICTS = {
+ 0: "Edit accuracy holds against the base branch.",
+ 1: "Edit accuracy fell against the base branch; see the PR comment.",
+};
+const notRun = (reported) => `Edit accuracy did not run: ${reported}; see the shard jobs.`;
+
+/** Writes what the workflow posts and prints; the verdict is the only text it shows for a failed gate. */
+function report(out, markdown, g) {
+ mkdirSync(out, { recursive: true });
+ writeFileSync(join(out, "comment.md"), markdown);
+ writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1));
+ console.log(markdown);
+}
+
// fallow-ignore-next-line complexity
function main(argv) {
const shardsAt = argv.indexOf("--shards");
@@ -169,7 +183,18 @@ function main(argv) {
}
const [headPath, out, ...resultPaths] = rest;
const runs = resultPaths.map((p) => readJson(p));
- if (!runs.length) throw new Error("no results.json from any shard");
+ if (!runs.length) {
+ const verdict = notRun(`0 of ${shards ?? "the"} shards reported results`);
+ report(
+ out,
+ `<!-- edit-accuracy -->\n### ${verdict}\n\nNothing was compared with the base branch.\n`,
+ {
+ ok: false,
+ verdict,
+ },
+ );
+ return 2;
+ }
const cases = runs.flatMap((r) => r.cases);
const full = gridOf(runs) === "full";
const g = gate(
@@ -181,21 +206,21 @@ function main(argv) {
);
// A shard that died uploads nothing, so the count of first runs is what shows it.
const firstRuns = runs.filter((r) => !r.meta.rerun).length;
- if (shards !== null && firstRuns !== shards) {
- g.reasons.push(`${firstRuns} of ${shards} shards reported results`);
- g.ok = false;
- }
- mkdirSync(out, { recursive: true });
- writeFileSync(join(out, "comment.md"), comment(g));
- writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1));
+ const shortfall =
+ shards !== null && firstRuns !== shards && `${firstRuns} of ${shards} shards reported results`;
+ // A case that ran and fell is a drop; a shard that did not report is not one.
+ const code = g.ok ? (shortfall ? 2 : 0) : 1;
+ if (shortfall) g.reasons.push(shortfall);
+ g.ok = code === 0;
+ g.verdict = code === 2 ? notRun(shortfall) : VERDICTS[code];
+ report(out, comment(g), g);
// A baseline.json to commit when cases newly pass; only a full grid holds every case.
if (full) writeReport(out, { ...runs[0].meta, grid: "full (CI)" }, bankable(cases), 0);
- console.log(comment(g));
if (g.unbanked.length && process.env.GITHUB_ACTIONS)
console.log(
`::notice title=Edit accuracy::${g.unbanked.length} newly passing case(s) are not banked in baseline.json; not gated.`,
);
- return g.ok ? 0 : 1;
+ return code;
}
if (process.argv[1] === fileURLToPath(import.meta.url))
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
index 745ca3c2aa..a8aecf024b 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
@@ -1,3 +1,8 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { accurate, bankable, comment, flipped, gate, QUARANTINED } from "./ratchet.mjs";
import { entry, score } from "./report.mjs";
@@ -190,3 +195,63 @@ describe("teleport in the gate", () => {
expect(accurate(entry(score({ id: "nudge-x" }, { ...r, teleport: null })))).toBe(true);
});
});
+
+describe("gate command, as CI runs it", () => {
+ const script = fileURLToPath(new URL("./ratchet.mjs", import.meta.url));
+ /** Runs `gate --shards <shards>` over the given shard results; returns its exit code, comment and verdict. */
+ function runGate(shards, shardCases) {
+ const dir = mkdtempSync(join(tmpdir(), "edit-accuracy-gate-"));
+ const base = join(dir, "base.json");
+ writeFileSync(base, JSON.stringify(baseline({ a: good, b: good })));
+ const results = shardCases.map((cases, i) => {
+ const path = join(dir, `shard-${i}.json`);
+ writeFileSync(path, JSON.stringify({ meta: { grid: `pr shard ${i + 1}/${shards}` }, cases }));
+ return path;
+ });
+ const out = join(dir, "out");
+ const run = spawnSync(
+ process.execPath,
+ [script, "gate", "--shards", String(shards), base, base, out, ...results],
+ {
+ encoding: "utf8",
+ },
+ );
+ const report = JSON.parse(readFileSync(join(out, "gate.json"), "utf8"));
+ return {
+ code: run.status,
+ comment: readFileSync(join(out, "comment.md"), "utf8"),
+ verdict: report.verdict,
+ };
+ }
+
+ it("says the shards did not run when none reported results, never that accuracy fell", () => {
+ const g = runGate(5, []);
+ expect(g.code).toBe(2);
+ expect(g.verdict).toBe(
+ "Edit accuracy did not run: 0 of 5 shards reported results; see the shard jobs.",
+ );
+ expect(g.comment).toContain("0 of 5 shards reported results");
+ expect(g.comment).not.toMatch(/fell|accurate \d/);
+ });
+
+ it("says the shards did not all run when the cases that ran hold, so a missing shard is not a drop", () => {
+ const g = runGate(2, [[run("a")]]);
+ expect(g.code).toBe(2);
+ expect(g.verdict).toBe(
+ "Edit accuracy did not run: 1 of 2 shards reported results; see the shard jobs.",
+ );
+ });
+
+ it("still says accuracy fell when a case that ran regressed, even with a shard missing", () => {
+ const g = runGate(2, [[run("a", 9), run("a", 9), run("a", 9)]]);
+ expect(g.code).toBe(1);
+ expect(g.verdict).toBe("Edit accuracy fell against the base branch; see the PR comment.");
+ expect(g.comment).toContain("1 of 2 shards reported results");
+ });
+
+ it("passes when every shard reported and nothing fell", () => {
+ const g = runGate(2, [[run("a")], [run("b")]]);
+ expect(g.code).toBe(0);
+ expect(g.verdict).toBe("Edit accuracy holds against the base branch.");
+ });
+});