Skip to content

Write HTML. Render video. Built for agents.

ci(studio): re-run an edit accuracy shard on a fresh runner when its runner is too slow (#5659)

Miguel Ángela403ea1e8b28
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e691070f4a..d91a0eb127 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -264,6 +264,7 @@ jobs:
- "scripts/package-subpaths.mjs"
- "scripts/rewrite-relative-import-extensions.mjs"
- "scripts/ci/base-file.sh"
+ - "scripts/ci/edit-accuracy-shard.sh"
- "scripts/ci/install-workspace-dependencies.sh"
- ".github/actions/prepare-ffmpeg-bin/**"
- "package.json"
@@ -1154,7 +1155,8 @@ jobs:
needs: [changes, build]
if: needs.changes.outputs.edit_accuracy == 'true'
runs-on: ubuntu-latest
- timeout-minutes: 45
+ # Healthy shards take 6-9 min; 25 leaves a cancelled shard's fresh re-run inside the queue's 60 min check wait.
+ timeout-minutes: 25
strategy:
fail-fast: false
matrix:
@@ -1182,15 +1184,14 @@ jobs:
bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
[ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
- name: Run this shard, then re-run twice each case whose verdict differs from the base branch
+ run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${{ strategy.job-total }}"
+ # A slow runner stops the bench (slow-runner.json) or runs into the timeout; either gets one fresh re-run.
+ - name: Mark a slow or cancelled shard for a re-run
+ if: always()
run: |
- set -euo pipefail
- bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
- bench --shard "${{ matrix.shard }}/${{ strategy.job-total }}" --out /tmp/edit-accuracy/run1
- mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
- /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
- if (( ${#FLIPPED[@]} > 0 )); then
- FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"
- for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
+ mkdir -p /tmp/edit-accuracy
+ if [ "${{ job.status }}" = cancelled ] || [ -n "$(find /tmp/edit-accuracy -name slow-runner.json)" ]; then
+ touch /tmp/edit-accuracy/retry
fi
- name: Upload shard evidence
if: always()
@@ -1200,9 +1201,72 @@ jobs:
path: /tmp/edit-accuracy
if-no-files-found: error
+ studio-edit-accuracy-retry-plan:
+ name: "Studio: edit accuracy retry plan"
+ needs: [changes, studio-edit-accuracy]
+ if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' && needs.studio-edit-accuracy.result != 'success' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ outputs:
+ shards: ${{ steps.plan.outputs.shards }}
+ steps:
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ pattern: edit-accuracy-shard-*
+ path: /tmp/edit-accuracy-shards
+ - name: List the shards marked for a re-run
+ id: plan
+ run: |
+ shards=$(find /tmp/edit-accuracy-shards -path '*/edit-accuracy-shard-*/retry' |
+ sed -E 's#.*/edit-accuracy-shard-([0-9]+)/retry$#\1#' | sort -n | paste -sd, -)
+ echo "shards=[${shards}]" >> "$GITHUB_OUTPUT"
+
+ # Each marked shard runs once more on a fresh runner and replaces its evidence before the gate reads it.
+ studio-edit-accuracy-retry:
+ name: "Studio: edit accuracy retry (${{ matrix.shard }})"
+ needs: [changes, build, studio-edit-accuracy-retry-plan]
+ if: ${{ !cancelled() && needs.studio-edit-accuracy-retry-plan.outputs.shards != '[]' && needs.studio-edit-accuracy-retry-plan.outputs.shards != '' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 25
+ strategy:
+ fail-fast: false
+ matrix:
+ shard: ${{ fromJSON(needs.studio-edit-accuracy-retry-plan.outputs.shards) }}
+ env:
+ EDIT_ACCURACY_GRID: ${{ github.event_name == 'merge_group' && 'full' || 'pr' }}
+ SHARDS: ${{ github.event_name == 'merge_group' && 20 || 5 }}
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ with:
+ node-version: 22
+ - uses: ./.github/actions/prepare-ffmpeg-bin
+ - run: bash scripts/ci/install-workspace-dependencies.sh
+ - run: bun run --cwd packages/core build
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: edit-accuracy-cli
+ path: packages/cli/dist
+ - run: node packages/cli/dist/cli.js browser ensure
+ - name: Read the base branch's edit accuracy baseline
+ run: |
+ bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
+ [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
+ - name: Run the shard again
+ run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${SHARDS}"
+ - name: Replace the shard's evidence
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: edit-accuracy-shard-${{ matrix.shard }}
+ path: /tmp/edit-accuracy
+ if-no-files-found: error
+ overwrite: true
+
studio-edit-accuracy-gate:
name: "Studio: edit accuracy gate"
- needs: [changes, studio-edit-accuracy]
+ needs: [changes, studio-edit-accuracy, studio-edit-accuracy-retry]
if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 5
diff --git a/package.json b/package.json
index 3a533543f4..fbea196eb4 100644
--- a/package.json
+++ b/package.json
@@ -53,7 +53,7 @@
"player:perf": "bun run --filter @hyperframes/player perf",
"format:check": "oxfmt --check .",
"knip": "knip",
- "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
+ "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
"test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
"generate:previews": "tsx scripts/generate-template-previews.ts",
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs
index ffb570756b..cfbdcf2f61 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs
@@ -11,6 +11,14 @@ const lateUndo = (r) => Math.max(r.undo.ms ?? 0, r.undo.redoMs ?? 0) > UNDO_WRIT
const DROPPED_FRAME_MS = 25;
/** Smooth means no main-thread frame over this, and no more dropped frames than the blank-page control. */
const WORK_MS = 8;
+/** A runner whose first cases take a median past this (3x a healthy runner's) is too slow to measure on. */
+export const SLOW_RUNNER = { cases: 4, medianSeconds: 60 };
+/** The first cases' median seconds when it marks a slow runner, else null; the blank-page control misses it. */
+export function slowRunner(seconds) {
+ if (seconds.length < SLOW_RUNNER.cases) return null;
+ const median = percentile(seconds.slice(0, SLOW_RUNNER.cases), 50);
+ return median > SLOW_RUNNER.medianSeconds ? median : null;
+}
export const METRICS = [
"tracking",
"press",
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.test.mjs b/packages/studio/tests/e2e/edit-accuracy/report.test.mjs
index a182b3518b..167e1bcd60 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.test.mjs
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import { accurate } from "./ratchet.mjs";
-import { entry, score, UNDO_WRITE_MAX_MS } from "./report.mjs";
+import { entry, score, slowRunner, UNDO_WRITE_MAX_MS } from "./report.mjs";
const frames = { intervals: [16, 17, 16], work: [2, 3, 2] };
const result = (undo) => ({
@@ -60,3 +60,20 @@ describe("a case's verdict", () => {
expect(score({ id: "case" }, { error: "boom" }).pass).toBe(false);
});
});
+
+describe("the slow runner check", () => {
+ it("stops a shard whose first four cases ran about 6x slower than a healthy runner", () => {
+ // A merge queue shard that ran into its timeout: its first cases, in seconds.
+ expect(slowRunner([95.5, 133.7, 57.9, 107.5, 91.6])).toBe(95.5);
+ });
+
+ it("lets a healthy runner through, including one slow case among its first four", () => {
+ // The slowest first four of 20 healthy shards had a 22 s median; one healthy case took 92.7 s.
+ expect(slowRunner([22, 21.4, 19.8, 23.1])).toBeNull();
+ expect(slowRunner([14, 92.7, 16.3, 15.2])).toBeNull();
+ });
+
+ it("waits for four cases before judging", () => {
+ expect(slowRunner([95.5, 133.7, 107.5])).toBeNull();
+ });
+});
diff --git a/packages/studio/tests/e2e/edit-accuracy/run.mjs b/packages/studio/tests/e2e/edit-accuracy/run.mjs
index 5794cfe53f..1c5f37e531 100644
--- a/packages/studio/tests/e2e/edit-accuracy/run.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/run.mjs
@@ -14,7 +14,7 @@ import { resolveHeadlessShellPath } from "../../../../engine/src/index.ts";
import { buildGrid, writeFixture } from "./grid.mjs";
import { killServers, runCase, startServer, stopServer } from "./case.mjs";
import { runSequence } from "./sequences.mjs";
-import { METRICS, score, writeReport } from "./report.mjs";
+import { METRICS, SLOW_RUNNER, score, slowRunner, writeReport } from "./report.mjs";
import { holdProducerBrowser, renderBox } from "./render.mjs";
import { aabb, boxDistance } from "./geometry.mjs";
@@ -152,19 +152,28 @@ function acquireLock(path) {
});
}
+// Set once the first cases show a slow runner: no new case starts, and the run exits 75 (see the end).
+let slow = null;
+
// The suite lock is taken per chunk, so a full grid never holds it for its whole run.
const LOCK_CHUNK = 8;
// fallow-ignore-next-line complexity
async function runChunks(queue, browsers, decoders, results) {
- for (let start = 0; start < queue.length; start += opt.lock ? LOCK_CHUNK : queue.length) {
+ for (
+ let start = 0;
+ start < queue.length && slow === null;
+ start += opt.lock ? LOCK_CHUNK : queue.length
+ ) {
const chunk = queue.slice(start, opt.lock ? start + LOCK_CHUNK : queue.length);
const release = opt.lock ? await acquireLock(opt.lock) : () => undefined;
try {
await Promise.all(
browsers.map(async (browser, i) => {
- for (let spec = chunk.shift(); spec; spec = chunk.shift())
+ for (let spec = chunk.shift(); spec && slow === null; spec = chunk.shift()) {
results.push(await runOne(spec, browser, decoders[i], Number(opt.port) + i));
+ slow = slowRunner(results.map((r) => r.seconds));
+ }
}),
);
} finally {
@@ -228,6 +237,18 @@ try {
} finally {
await Promise.all([releaseProducer(), ...browsers.map((b) => b.close())]);
}
+// CI reads slow-runner.json to re-run the shard once on a fresh runner.
+if (slow !== null) {
+ const seconds = results.slice(0, SLOW_RUNNER.cases).map((r) => r.seconds);
+ writeFileSync(
+ join(out, "slow-runner.json"),
+ JSON.stringify({ seconds, median: slow, ...SLOW_RUNNER }),
+ );
+ console.error(
+ `runner too slow: the first ${SLOW_RUNNER.cases} cases took a median ${slow.toFixed(1)}s, over ${SLOW_RUNNER.medianSeconds}s; stopping`,
+ );
+ process.exit(75);
+}
results.sort((a, b) => a.id.localeCompare(b.id));
const meta = {
studio,
diff --git a/scripts/check-edit-accuracy-filter.test.mjs b/scripts/check-edit-accuracy-filter.test.mjs
index 2c7dcc089b..5bf2124e24 100644
--- a/scripts/check-edit-accuracy-filter.test.mjs
+++ b/scripts/check-edit-accuracy-filter.test.mjs
@@ -180,11 +180,18 @@ class Closure {
}
}
+// A shell script a step runs is read like a step: the scripts and commands it runs count too.
+function seedRun(closure, path) {
+ const fresh = closure.isNew(path);
+ closure.add(path, true);
+ if (fresh && path.endsWith(".sh")) seed(closure, read(path));
+}
+
// Repo files and package scripts the jobs' steps name directly.
function seed(closure, text) {
for (const m of text.matchAll(/uses: \.\/(\S+)/g)) closure.add(`${m[1]}/action.yml`);
for (const m of text.matchAll(/\b(?:bash|sh|node|bun) ((?:scripts|packages)\/\S+)/g))
- closure.add(m[1], true);
+ seedRun(closure, m[1]);
for (const m of text.matchAll(/\bbun run --cwd (\S+) ([\w:-]+)/g)) closure.script(m[1], m[2]);
}
@@ -196,7 +203,12 @@ const cliCommands = (texts) =>
function benchInputs() {
const closure = new Closure();
- const steps = ["build", "studio-edit-accuracy", "studio-edit-accuracy-gate"].map(job);
+ const steps = [
+ "build",
+ "studio-edit-accuracy",
+ "studio-edit-accuracy-retry",
+ "studio-edit-accuracy-gate",
+ ].map(job);
steps.forEach((text) => seed(closure, text));
for (const name of cliCommands([...steps, read(`${BENCH}/case.mjs`)]))
closure.add(`packages/cli/src/commands/${name}.ts`);
diff --git a/scripts/ci/edit-accuracy-shard.sh b/scripts/ci/edit-accuracy-shard.sh
new file mode 100755
index 0000000000..a2c8fdb74a
--- /dev/null
+++ b/scripts/ci/edit-accuracy-shard.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# edit-accuracy-shard.sh <i/n>: runs one edit accuracy shard into /tmp/edit-accuracy, then re-runs twice each case
+# whose verdict differs from the base branch (/tmp/base-edit-accuracy.json). A slow runner makes the bench exit 75.
+set -euo pipefail
+bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
+bench --shard "$1" --out /tmp/edit-accuracy/run1
+mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
+ /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
+if (( ${#FLIPPED[@]} > 0 )); then
+ FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"
+ for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
+fi
diff --git a/scripts/edit-accuracy-retry.test.mjs b/scripts/edit-accuracy-retry.test.mjs
new file mode 100644
index 0000000000..ed7f23b28a
--- /dev/null
+++ b/scripts/edit-accuracy-retry.test.mjs
@@ -0,0 +1,103 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { parse } from "yaml";
+
+// A slow runner must not drop a PR from the merge queue: its shard is marked, re-run once on a fresh runner,
+// and the re-run's evidence is what the gate reads.
+const ci = parse(
+ readFileSync(join(import.meta.dirname, "..", ".github", "workflows", "ci.yml"), "utf8"),
+);
+const jobs = ci.jobs;
+const step = (job, name) => jobs[job].steps.find((s) => s.name === name);
+
+// Runs a step's shell with its fixed paths moved under root; returns what it wrote to GITHUB_OUTPUT.
+function runStep(script, root, env = {}) {
+ const output = join(root, "github-output");
+ writeFileSync(output, "");
+ execFileSync("bash", ["-euo", "pipefail", "-c", script.replaceAll("/tmp/", `${root}/`)], {
+ env: { ...process.env, GITHUB_OUTPUT: output, ...env },
+ });
+ return readFileSync(output, "utf8");
+}
+
+function withTree(fn) {
+ const root = mkdtempSync(join(tmpdir(), "edit-accuracy-retry-"));
+ try {
+ return fn(root);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+}
+
+const markScript = (status) =>
+ step("studio-edit-accuracy", "Mark a slow or cancelled shard for a re-run").run.replaceAll(
+ "${{ job.status }}",
+ status,
+ );
+
+test("a shard is marked for a re-run when it timed out or its bench stopped on a slow runner", () => {
+ withTree((root) => {
+ runStep(markScript("cancelled"), root);
+ assert.ok(existsSync(join(root, "edit-accuracy/retry")));
+ });
+ withTree((root) => {
+ mkdirSync(join(root, "edit-accuracy/run1"), { recursive: true });
+ writeFileSync(join(root, "edit-accuracy/run1/slow-runner.json"), "{}");
+ runStep(markScript("failure"), root);
+ assert.ok(existsSync(join(root, "edit-accuracy/retry")));
+ });
+ withTree((root) => {
+ runStep(markScript("success"), root);
+ assert.ok(!existsSync(join(root, "edit-accuracy/retry")));
+ });
+});
+
+test("the retry plan lists exactly the marked shards", () => {
+ const plan = step("studio-edit-accuracy-retry-plan", "List the shards marked for a re-run").run;
+ withTree((root) => {
+ for (const shard of [7, 12, 3])
+ mkdirSync(join(root, `edit-accuracy-shards/edit-accuracy-shard-${shard}`), {
+ recursive: true,
+ });
+ writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-7/retry"), "");
+ writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-12/retry"), "");
+ assert.equal(runStep(plan, root), "shards=[7,12]\n");
+ });
+ withTree((root) => {
+ mkdirSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-1"), { recursive: true });
+ assert.equal(runStep(plan, root), "shards=[]\n");
+ });
+});
+
+test("each marked shard re-runs once and replaces its evidence before the gate reads it", () => {
+ const retry = jobs["studio-edit-accuracy-retry"];
+ assert.match(retry.needs.join(" "), /studio-edit-accuracy-retry-plan/);
+ assert.match(
+ retry.strategy.matrix.shard,
+ /needs\.studio-edit-accuracy-retry-plan\.outputs\.shards/,
+ );
+ const shardRun = retry.steps.find((s) => s.run?.includes("edit-accuracy-shard.sh"));
+ assert.ok(shardRun, "the retry runs the same shard script as the shard job");
+ assert.ok(
+ jobs["studio-edit-accuracy"].steps.some((s) =>
+ s.run?.includes("scripts/ci/edit-accuracy-shard.sh"),
+ ),
+ );
+ const upload = retry.steps.find((s) => s.uses?.startsWith("actions/upload-artifact@"));
+ assert.equal(upload.with.name, "edit-accuracy-shard-${{ matrix.shard }}");
+ assert.equal(upload.with.overwrite, true);
+ assert.ok(jobs["studio-edit-accuracy-gate"].needs.includes("studio-edit-accuracy-retry"));
+ assert.match(jobs["studio-edit-accuracy-gate"].if, /!cancelled\(\)/);
+});
+
+test("a cancelled shard and its re-run both fit in the merge queue's 60 minute check wait", () => {
+ const shard = jobs["studio-edit-accuracy"]["timeout-minutes"];
+ const plan = jobs["studio-edit-accuracy-retry-plan"]["timeout-minutes"];
+ const retry = jobs["studio-edit-accuracy-retry"]["timeout-minutes"];
+ const gate = jobs["studio-edit-accuracy-gate"]["timeout-minutes"];
+ assert.ok(shard + plan + retry + gate <= 60, `${shard}+${plan}+${retry}+${gate} min`);
+});