Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
ci(studio): re-run an edit accuracy shard on a fresh runner when its runner is too slow (#5659)
Miguel Ángela403ea1e8b28
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex e691070f4a..d91a0eb127 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -264,6 +264,7 @@ jobs:- "scripts/package-subpaths.mjs"- "scripts/rewrite-relative-import-extensions.mjs"- "scripts/ci/base-file.sh"+ - "scripts/ci/edit-accuracy-shard.sh"- "scripts/ci/install-workspace-dependencies.sh"- ".github/actions/prepare-ffmpeg-bin/**"- "package.json"@@ -1154,7 +1155,8 @@ jobs:needs: [changes, build]if: needs.changes.outputs.edit_accuracy == 'true'runs-on: ubuntu-latest- timeout-minutes: 45+ # Healthy shards take 6-9 min; 25 leaves a cancelled shard's fresh re-run inside the queue's 60 min check wait.+ timeout-minutes: 25strategy:fail-fast: falsematrix:@@ -1182,15 +1184,14 @@ jobs:bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json[ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json- name: Run this shard, then re-run twice each case whose verdict differs from the base branch+ run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${{ strategy.job-total }}"+ # A slow runner stops the bench (slow-runner.json) or runs into the timeout; either gets one fresh re-run.+ - name: Mark a slow or cancelled shard for a re-run+ if: always()run: |- set -euo pipefail- bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }- bench --shard "${{ matrix.shard }}/${{ strategy.job-total }}" --out /tmp/edit-accuracy/run1- mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \- /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)- if (( ${#FLIPPED[@]} > 0 )); then- FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"- for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done+ mkdir -p /tmp/edit-accuracy+ if [ "${{ job.status }}" = cancelled ] || [ -n "$(find /tmp/edit-accuracy -name slow-runner.json)" ]; then+ touch /tmp/edit-accuracy/retryfi- name: Upload shard evidenceif: always()@@ -1200,9 +1201,72 @@ jobs:path: /tmp/edit-accuracyif-no-files-found: error+ studio-edit-accuracy-retry-plan:+ name: "Studio: edit accuracy retry plan"+ needs: [changes, studio-edit-accuracy]+ if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' && needs.studio-edit-accuracy.result != 'success' }}+ runs-on: ubuntu-latest+ timeout-minutes: 5+ outputs:+ shards: ${{ steps.plan.outputs.shards }}+ steps:+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ pattern: edit-accuracy-shard-*+ path: /tmp/edit-accuracy-shards+ - name: List the shards marked for a re-run+ id: plan+ run: |+ shards=$(find /tmp/edit-accuracy-shards -path '*/edit-accuracy-shard-*/retry' |+ sed -E 's#.*/edit-accuracy-shard-([0-9]+)/retry$#\1#' | sort -n | paste -sd, -)+ echo "shards=[${shards}]" >> "$GITHUB_OUTPUT"++ # Each marked shard runs once more on a fresh runner and replaces its evidence before the gate reads it.+ studio-edit-accuracy-retry:+ name: "Studio: edit accuracy retry (${{ matrix.shard }})"+ needs: [changes, build, studio-edit-accuracy-retry-plan]+ if: ${{ !cancelled() && needs.studio-edit-accuracy-retry-plan.outputs.shards != '[]' && needs.studio-edit-accuracy-retry-plan.outputs.shards != '' }}+ runs-on: ubuntu-latest+ timeout-minutes: 25+ strategy:+ fail-fast: false+ matrix:+ shard: ${{ fromJSON(needs.studio-edit-accuracy-retry-plan.outputs.shards) }}+ env:+ EDIT_ACCURACY_GRID: ${{ github.event_name == 'merge_group' && 'full' || 'pr' }}+ SHARDS: ${{ github.event_name == 'merge_group' && 20 || 5 }}+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version: 22+ - uses: ./.github/actions/prepare-ffmpeg-bin+ - run: bash scripts/ci/install-workspace-dependencies.sh+ - run: bun run --cwd packages/core build+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ name: edit-accuracy-cli+ path: packages/cli/dist+ - run: node packages/cli/dist/cli.js browser ensure+ - name: Read the base branch's edit accuracy baseline+ run: |+ bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json+ [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json+ - name: Run the shard again+ run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${SHARDS}"+ - name: Replace the shard's evidence+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: edit-accuracy-shard-${{ matrix.shard }}+ path: /tmp/edit-accuracy+ if-no-files-found: error+ overwrite: true+studio-edit-accuracy-gate:name: "Studio: edit accuracy gate"- needs: [changes, studio-edit-accuracy]+ needs: [changes, studio-edit-accuracy, studio-edit-accuracy-retry]if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' }}runs-on: ubuntu-latesttimeout-minutes: 5diff --git a/package.json b/package.jsonindex 3a533543f4..fbea196eb4 100644--- a/package.json+++ b/package.json@@ -53,7 +53,7 @@"player:perf": "bun run --filter @hyperframes/player perf","format:check": "oxfmt --check .","knip": "knip",- "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",+ "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts","typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json","test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'","generate:previews": "tsx scripts/generate-template-previews.ts",diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjsindex ffb570756b..cfbdcf2f61 100644--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs@@ -11,6 +11,14 @@ const lateUndo = (r) => Math.max(r.undo.ms ?? 0, r.undo.redoMs ?? 0) > UNDO_WRITconst DROPPED_FRAME_MS = 25;/** Smooth means no main-thread frame over this, and no more dropped frames than the blank-page control. */const WORK_MS = 8;+/** A runner whose first cases take a median past this (3x a healthy runner's) is too slow to measure on. */+export const SLOW_RUNNER = { cases: 4, medianSeconds: 60 };+/** The first cases' median seconds when it marks a slow runner, else null; the blank-page control misses it. */+export function slowRunner(seconds) {+ if (seconds.length < SLOW_RUNNER.cases) return null;+ const median = percentile(seconds.slice(0, SLOW_RUNNER.cases), 50);+ return median > SLOW_RUNNER.medianSeconds ? median : null;+}export const METRICS = ["tracking","press",diff --git a/packages/studio/tests/e2e/edit-accuracy/report.test.mjs b/packages/studio/tests/e2e/edit-accuracy/report.test.mjsindex a182b3518b..167e1bcd60 100644--- a/packages/studio/tests/e2e/edit-accuracy/report.test.mjs+++ b/packages/studio/tests/e2e/edit-accuracy/report.test.mjs@@ -1,6 +1,6 @@import { describe, expect, it } from "vitest";import { accurate } from "./ratchet.mjs";-import { entry, score, UNDO_WRITE_MAX_MS } from "./report.mjs";+import { entry, score, slowRunner, UNDO_WRITE_MAX_MS } from "./report.mjs";const frames = { intervals: [16, 17, 16], work: [2, 3, 2] };const result = (undo) => ({@@ -60,3 +60,20 @@ describe("a case's verdict", () => {expect(score({ id: "case" }, { error: "boom" }).pass).toBe(false);});});++describe("the slow runner check", () => {+ it("stops a shard whose first four cases ran about 6x slower than a healthy runner", () => {+ // A merge queue shard that ran into its timeout: its first cases, in seconds.+ expect(slowRunner([95.5, 133.7, 57.9, 107.5, 91.6])).toBe(95.5);+ });++ it("lets a healthy runner through, including one slow case among its first four", () => {+ // The slowest first four of 20 healthy shards had a 22 s median; one healthy case took 92.7 s.+ expect(slowRunner([22, 21.4, 19.8, 23.1])).toBeNull();+ expect(slowRunner([14, 92.7, 16.3, 15.2])).toBeNull();+ });++ it("waits for four cases before judging", () => {+ expect(slowRunner([95.5, 133.7, 107.5])).toBeNull();+ });+});diff --git a/packages/studio/tests/e2e/edit-accuracy/run.mjs b/packages/studio/tests/e2e/edit-accuracy/run.mjsindex 5794cfe53f..1c5f37e531 100644--- a/packages/studio/tests/e2e/edit-accuracy/run.mjs+++ b/packages/studio/tests/e2e/edit-accuracy/run.mjs@@ -14,7 +14,7 @@ import { resolveHeadlessShellPath } from "../../../../engine/src/index.ts";import { buildGrid, writeFixture } from "./grid.mjs";import { killServers, runCase, startServer, stopServer } from "./case.mjs";import { runSequence } from "./sequences.mjs";-import { METRICS, score, writeReport } from "./report.mjs";+import { METRICS, SLOW_RUNNER, score, slowRunner, writeReport } from "./report.mjs";import { holdProducerBrowser, renderBox } from "./render.mjs";import { aabb, boxDistance } from "./geometry.mjs";@@ -152,19 +152,28 @@ function acquireLock(path) {});}+// Set once the first cases show a slow runner: no new case starts, and the run exits 75 (see the end).+let slow = null;+// The suite lock is taken per chunk, so a full grid never holds it for its whole run.const LOCK_CHUNK = 8;// fallow-ignore-next-line complexityasync function runChunks(queue, browsers, decoders, results) {- for (let start = 0; start < queue.length; start += opt.lock ? LOCK_CHUNK : queue.length) {+ for (+ let start = 0;+ start < queue.length && slow === null;+ start += opt.lock ? LOCK_CHUNK : queue.length+ ) {const chunk = queue.slice(start, opt.lock ? start + LOCK_CHUNK : queue.length);const release = opt.lock ? await acquireLock(opt.lock) : () => undefined;try {await Promise.all(browsers.map(async (browser, i) => {- for (let spec = chunk.shift(); spec; spec = chunk.shift())+ for (let spec = chunk.shift(); spec && slow === null; spec = chunk.shift()) {results.push(await runOne(spec, browser, decoders[i], Number(opt.port) + i));+ slow = slowRunner(results.map((r) => r.seconds));+ }}),);} finally {@@ -228,6 +237,18 @@ try {} finally {await Promise.all([releaseProducer(), ...browsers.map((b) => b.close())]);}+// CI reads slow-runner.json to re-run the shard once on a fresh runner.+if (slow !== null) {+ const seconds = results.slice(0, SLOW_RUNNER.cases).map((r) => r.seconds);+ writeFileSync(+ join(out, "slow-runner.json"),+ JSON.stringify({ seconds, median: slow, ...SLOW_RUNNER }),+ );+ console.error(+ `runner too slow: the first ${SLOW_RUNNER.cases} cases took a median ${slow.toFixed(1)}s, over ${SLOW_RUNNER.medianSeconds}s; stopping`,+ );+ process.exit(75);+}results.sort((a, b) => a.id.localeCompare(b.id));const meta = {studio,diff --git a/scripts/check-edit-accuracy-filter.test.mjs b/scripts/check-edit-accuracy-filter.test.mjsindex 2c7dcc089b..5bf2124e24 100644--- a/scripts/check-edit-accuracy-filter.test.mjs+++ b/scripts/check-edit-accuracy-filter.test.mjs@@ -180,11 +180,18 @@ class Closure {}}+// A shell script a step runs is read like a step: the scripts and commands it runs count too.+function seedRun(closure, path) {+ const fresh = closure.isNew(path);+ closure.add(path, true);+ if (fresh && path.endsWith(".sh")) seed(closure, read(path));+}+// Repo files and package scripts the jobs' steps name directly.function seed(closure, text) {for (const m of text.matchAll(/uses: \.\/(\S+)/g)) closure.add(`${m[1]}/action.yml`);for (const m of text.matchAll(/\b(?:bash|sh|node|bun) ((?:scripts|packages)\/\S+)/g))- closure.add(m[1], true);+ seedRun(closure, m[1]);for (const m of text.matchAll(/\bbun run --cwd (\S+) ([\w:-]+)/g)) closure.script(m[1], m[2]);}@@ -196,7 +203,12 @@ const cliCommands = (texts) =>function benchInputs() {const closure = new Closure();- const steps = ["build", "studio-edit-accuracy", "studio-edit-accuracy-gate"].map(job);+ const steps = [+ "build",+ "studio-edit-accuracy",+ "studio-edit-accuracy-retry",+ "studio-edit-accuracy-gate",+ ].map(job);steps.forEach((text) => seed(closure, text));for (const name of cliCommands([...steps, read(`${BENCH}/case.mjs`)]))closure.add(`packages/cli/src/commands/${name}.ts`);diff --git a/scripts/ci/edit-accuracy-shard.sh b/scripts/ci/edit-accuracy-shard.shnew file mode 100755index 0000000000..a2c8fdb74a--- /dev/null+++ b/scripts/ci/edit-accuracy-shard.sh@@ -0,0 +1,12 @@+#!/usr/bin/env bash+# edit-accuracy-shard.sh <i/n>: runs one edit accuracy shard into /tmp/edit-accuracy, then re-runs twice each case+# whose verdict differs from the base branch (/tmp/base-edit-accuracy.json). A slow runner makes the bench exit 75.+set -euo pipefail+bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }+bench --shard "$1" --out /tmp/edit-accuracy/run1+mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \+ /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)+if (( ${#FLIPPED[@]} > 0 )); then+ FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"+ for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done+fidiff --git a/scripts/edit-accuracy-retry.test.mjs b/scripts/edit-accuracy-retry.test.mjsnew file mode 100644index 0000000000..ed7f23b28a--- /dev/null+++ b/scripts/edit-accuracy-retry.test.mjs@@ -0,0 +1,103 @@+import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import test from "node:test";+import { parse } from "yaml";++// A slow runner must not drop a PR from the merge queue: its shard is marked, re-run once on a fresh runner,+// and the re-run's evidence is what the gate reads.+const ci = parse(+ readFileSync(join(import.meta.dirname, "..", ".github", "workflows", "ci.yml"), "utf8"),+);+const jobs = ci.jobs;+const step = (job, name) => jobs[job].steps.find((s) => s.name === name);++// Runs a step's shell with its fixed paths moved under root; returns what it wrote to GITHUB_OUTPUT.+function runStep(script, root, env = {}) {+ const output = join(root, "github-output");+ writeFileSync(output, "");+ execFileSync("bash", ["-euo", "pipefail", "-c", script.replaceAll("/tmp/", `${root}/`)], {+ env: { ...process.env, GITHUB_OUTPUT: output, ...env },+ });+ return readFileSync(output, "utf8");+}++function withTree(fn) {+ const root = mkdtempSync(join(tmpdir(), "edit-accuracy-retry-"));+ try {+ return fn(root);+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+}++const markScript = (status) =>+ step("studio-edit-accuracy", "Mark a slow or cancelled shard for a re-run").run.replaceAll(+ "${{ job.status }}",+ status,+ );++test("a shard is marked for a re-run when it timed out or its bench stopped on a slow runner", () => {+ withTree((root) => {+ runStep(markScript("cancelled"), root);+ assert.ok(existsSync(join(root, "edit-accuracy/retry")));+ });+ withTree((root) => {+ mkdirSync(join(root, "edit-accuracy/run1"), { recursive: true });+ writeFileSync(join(root, "edit-accuracy/run1/slow-runner.json"), "{}");+ runStep(markScript("failure"), root);+ assert.ok(existsSync(join(root, "edit-accuracy/retry")));+ });+ withTree((root) => {+ runStep(markScript("success"), root);+ assert.ok(!existsSync(join(root, "edit-accuracy/retry")));+ });+});++test("the retry plan lists exactly the marked shards", () => {+ const plan = step("studio-edit-accuracy-retry-plan", "List the shards marked for a re-run").run;+ withTree((root) => {+ for (const shard of [7, 12, 3])+ mkdirSync(join(root, `edit-accuracy-shards/edit-accuracy-shard-${shard}`), {+ recursive: true,+ });+ writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-7/retry"), "");+ writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-12/retry"), "");+ assert.equal(runStep(plan, root), "shards=[7,12]\n");+ });+ withTree((root) => {+ mkdirSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-1"), { recursive: true });+ assert.equal(runStep(plan, root), "shards=[]\n");+ });+});++test("each marked shard re-runs once and replaces its evidence before the gate reads it", () => {+ const retry = jobs["studio-edit-accuracy-retry"];+ assert.match(retry.needs.join(" "), /studio-edit-accuracy-retry-plan/);+ assert.match(+ retry.strategy.matrix.shard,+ /needs\.studio-edit-accuracy-retry-plan\.outputs\.shards/,+ );+ const shardRun = retry.steps.find((s) => s.run?.includes("edit-accuracy-shard.sh"));+ assert.ok(shardRun, "the retry runs the same shard script as the shard job");+ assert.ok(+ jobs["studio-edit-accuracy"].steps.some((s) =>+ s.run?.includes("scripts/ci/edit-accuracy-shard.sh"),+ ),+ );+ const upload = retry.steps.find((s) => s.uses?.startsWith("actions/upload-artifact@"));+ assert.equal(upload.with.name, "edit-accuracy-shard-${{ matrix.shard }}");+ assert.equal(upload.with.overwrite, true);+ assert.ok(jobs["studio-edit-accuracy-gate"].needs.includes("studio-edit-accuracy-retry"));+ assert.match(jobs["studio-edit-accuracy-gate"].if, /!cancelled\(\)/);+});++test("a cancelled shard and its re-run both fit in the merge queue's 60 minute check wait", () => {+ const shard = jobs["studio-edit-accuracy"]["timeout-minutes"];+ const plan = jobs["studio-edit-accuracy-retry-plan"]["timeout-minutes"];+ const retry = jobs["studio-edit-accuracy-retry"]["timeout-minutes"];+ const gate = jobs["studio-edit-accuracy-gate"]["timeout-minutes"];+ assert.ok(shard + plan + retry + gate <= 60, `${shard}+${plan}+${retry}+${gate} min`);+});