Skip to content

Apache DataFusion SQL Query Engine

ci: cancel obsolete merge queue runs (#26760)

Kumar Ujjawald1684ab9f373
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 9220fe6e06..8a5e8b4284 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -88,7 +88,7 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: "24"
- - run: node --test ci/scripts/test_fork_ci_status.cjs
+ - run: node --test ci/scripts/test_fork_ci_status.cjs ci/scripts/test_cancel_merge_queue_runs.cjs
typos:
name: Spell Check with Typos
diff --git a/.github/workflows/merge_queue_cleanup.yml b/.github/workflows/merge_queue_cleanup.yml
new file mode 100644
index 0000000000..1d6b2fc911
--- /dev/null
+++ b/.github/workflows/merge_queue_cleanup.yml
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Merge queue cleanup
+
+on:
+ # New groups trigger an early scan. Ref deletion can lag this event; the
+ # sweep also catches groups removed without a replacement or missed earlier.
+ merge_group:
+ types: [checks_requested]
+ schedule:
+ - cron: "*/15 * * * *"
+ workflow_dispatch:
+ inputs:
+ dry_run:
+ description: "Report obsolete runs without cancelling them"
+ type: boolean
+ default: true
+
+permissions:
+ contents: read
+ actions: write
+
+# Merge-group scans attach checks to commits that can land on main, so each
+# group SHA gets its own group: a replaced pending scan would show as cancelled.
+# Scheduled and manual scans share one group per mode, so a dry run cannot
+# replace a pending cancellation scan. Do not interrupt running scans.
+concurrency:
+ group: merge-queue-cleanup-${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || (vars.MERGE_QUEUE_CLEANUP_ENABLED != 'true' || (github.event_name == 'workflow_dispatch' && inputs.dry_run)) }}
+ cancel-in-progress: false
+
+jobs:
+ cleanup:
+ if: github.repository == 'apache/datafusion'
+ runs-on: ubuntu-slim
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: ${{ github.event.repository.default_branch }}
+ persist-credentials: false
+ - name: Cancel obsolete merge-group runs
+ # During the PR that introduces this workflow, main has no script yet.
+ # Execute main's copy of the script.
+ if: hashFiles('ci/scripts/cancel_merge_queue_runs.cjs') != ''
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ # Observe candidates for at least a week before enabling cancellation.
+ DRY_RUN: ${{ vars.MERGE_QUEUE_CLEANUP_ENABLED != 'true' || (github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
+ with:
+ script: |
+ const { run } = require('./ci/scripts/cancel_merge_queue_runs.cjs');
+ const dryRun = process.env.DRY_RUN !== 'false';
+ const results = await run({ github, context, core, dryRun });
+ await core.summary
+ .addHeading(dryRun ? 'Would cancel merge-group runs' : 'Requested merge-group cancellations')
+ .addTable([
+ ['Run', 'SHA', 'Attempt', 'Observed at (UTC)', 'Reason'].map(data => ({ data, header: true })),
+ ...results.map(({ id, sha, attempt, observed_at, reason }) => [String(id), sha, String(attempt), observed_at, reason]),
+ ])
+ .write();
diff --git a/ci/scripts/cancel_merge_queue_runs.cjs b/ci/scripts/cancel_merge_queue_runs.cjs
new file mode 100644
index 0000000000..9ebec8942a
--- /dev/null
+++ b/ci/scripts/cancel_merge_queue_runs.cjs
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Called on each new merge group and on a schedule. The queue's exact Git
+// refs, rather than PR state or failing checks, determine which runs are stale.
+const ACTIVE = new Set([
+ "queued",
+ "in_progress",
+ "waiting",
+ "pending",
+ "requested",
+]);
+const CLEANUP_PATH = ".github/workflows/merge_queue_cleanup.yml";
+const QUEUE_REF = /^gh-readonly-queue\/(.+)\/pr-[1-9][0-9]*-[0-9a-f]{40}$/;
+
+function eligible(run, repo, currentRunId) {
+ return (
+ run.id !== currentRunId &&
+ run.event === "merge_group" &&
+ run.run_attempt === 1 &&
+ ACTIVE.has(run.status) &&
+ run.path?.split("@")[0] !== CLEANUP_PATH &&
+ run.repository?.full_name?.toLowerCase() === repo.toLowerCase() &&
+ run.head_repository?.full_name?.toLowerCase() === repo.toLowerCase() &&
+ QUEUE_REF.test(run.head_branch) &&
+ /^[0-9a-f]{40}$/.test(run.head_sha)
+ );
+}
+
+async function refSha(github, repo, branch) {
+ try {
+ const { data } = await github.rest.git.getRef({
+ ...repo,
+ ref: `heads/${branch}`,
+ });
+ // Only a missing ref or a known, different commit proves obsolescence.
+ // Unexpected responses and permission/server errors must not cancel CI.
+ if (
+ data.object?.type !== "commit" ||
+ !/^[0-9a-f]{40}$/.test(data.object.sha)
+ ) {
+ throw new Error(`Invalid Git ref response for ${branch}`);
+ }
+ return data.object.sha;
+ } catch (error) {
+ if (error.status === 404) return null;
+ throw error;
+ }
+}
+
+// Filtered Actions listings are capped at 1,000 results even with pagination.
+// Active merge-group runs stay far below the cap, so a full listing is an error.
+async function listRuns(github, repo, status) {
+ const params = { ...repo, event: "merge_group", status, per_page: 100 };
+ const runs = [];
+ for (let page = 1; ; page++) {
+ const { data } = await github.rest.actions.listWorkflowRunsForRepo({
+ ...params,
+ page,
+ });
+ if (
+ !Number.isInteger(data.total_count) ||
+ data.total_count < 0 ||
+ !Array.isArray(data.workflow_runs)
+ ) {
+ throw new Error("Invalid workflow run listing");
+ }
+ if (data.total_count >= 1000) {
+ throw new Error(
+ `${data.total_count} ${status} runs exceed the listing cap`
+ );
+ }
+ runs.push(...data.workflow_runs);
+ if (data.workflow_runs.length < 100) return runs;
+ }
+}
+
+function reportError(core, context, message) {
+ core.warning(message);
+ // Maintenance errors must not mark a merge-group commit as failing CI.
+ // Scheduled/manual scans still expose broken cleanup as workflow failures.
+ if (context.eventName !== "merge_group") core.setFailed(message);
+}
+
+async function run({ github, context, core, dryRun = true }) {
+ const repo = { owner: context.repo.owner, repo: context.repo.repo };
+ const fullName = `${repo.owner}/${repo.repo}`;
+ if (fullName !== "apache/datafusion") return [];
+
+ // Finish pagination before cancelling: otherwise mutations shrink the active
+ // result sets and can cause later pages to skip runs.
+ const listed = new Map();
+ try {
+ for (const status of ACTIVE) {
+ const runs = await listRuns(github, repo, status);
+ for (const candidate of runs) listed.set(candidate.id, candidate);
+ }
+ } catch (error) {
+ reportError(
+ core,
+ context,
+ `Cannot list merge-group runs: ${error.message}`
+ );
+ return [];
+ }
+
+ const refs = new Map();
+ const comparisons = new Map();
+ const results = [];
+ const errors = [];
+ for (const candidate of listed.values()) {
+ if (!eligible(candidate, fullName, context.runId)) continue;
+ try {
+ // Share the initial lookup among workflows for the same merge group.
+ // Cache failures too: an outage must not multiply requests for this ref.
+ if (!refs.has(candidate.head_branch)) {
+ refs.set(
+ candidate.head_branch,
+ refSha(github, repo, candidate.head_branch)
+ );
+ }
+ if ((await refs.get(candidate.head_branch)) === candidate.head_sha) {
+ core.info(
+ `Kept active group ${JSON.stringify({
+ id: candidate.id,
+ sha: candidate.head_sha,
+ branch: candidate.head_branch,
+ observed_at: new Date().toISOString(),
+ trigger: context.eventName,
+ })}`
+ );
+ continue;
+ }
+
+ const { data: current } = await github.rest.actions.getWorkflowRun({
+ ...repo,
+ run_id: candidate.id,
+ });
+ // A completed run or a newly requested rerun is no longer this candidate.
+ if (
+ !eligible(current, fullName, context.runId) ||
+ current.head_branch !== candidate.head_branch ||
+ current.head_sha !== candidate.head_sha ||
+ current.run_attempt !== candidate.run_attempt
+ )
+ continue;
+
+ const head = await refSha(github, repo, current.head_branch);
+ if (head === current.head_sha) continue;
+ // A successful merge removes the queue ref too. Preserve checks on
+ // commits already in the target branch, including optional checks.
+ // Workflows of one group share the comparison, failures included.
+ const base = QUEUE_REF.exec(current.head_branch)[1];
+ const basehead = `${base}...${current.head_sha}`;
+ if (!comparisons.has(basehead)) {
+ comparisons.set(
+ basehead,
+ github.rest.repos.compareCommitsWithBasehead({
+ ...repo,
+ basehead,
+ per_page: 1,
+ })
+ );
+ }
+ const { data: comparison } = await comparisons.get(basehead);
+ if (["identical", "behind"].includes(comparison.status)) continue;
+ if (!["ahead", "diverged"].includes(comparison.status)) {
+ throw new Error(`Invalid comparison status for ${current.head_sha}`);
+ }
+ const reason = head === null ? "queue ref removed" : "queue ref replaced";
+ const result = {
+ id: current.id,
+ sha: current.head_sha,
+ branch: current.head_branch,
+ attempt: current.run_attempt,
+ reason,
+ observed_at: new Date().toISOString(),
+ trigger: context.eventName,
+ };
+ const description = JSON.stringify(result);
+ if (dryRun) {
+ core.info(`Would cancel ${description}`);
+ } else {
+ try {
+ await github.rest.actions.cancelWorkflowRun({
+ ...repo,
+ run_id: current.id,
+ });
+ } catch (error) {
+ // A 409 can mean a completion race, but also an active run that
+ // GitHub cannot cancel. Only suppress it after confirming the race.
+ if (error.status === 409) {
+ const { data: latest } = await github.rest.actions.getWorkflowRun({
+ ...repo,
+ run_id: current.id,
+ });
+ if (
+ latest.status === "completed" ||
+ latest.run_attempt !== current.run_attempt
+ )
+ continue;
+ }
+ throw error;
+ }
+ core.info(`Requested cancellation of ${description}`);
+ }
+ results.push(result);
+ } catch (error) {
+ // One failed read/cancellation must not leave other obsolete groups running.
+ const message = `Run ${candidate.id}: ${error.message}`;
+ core.warning(message);
+ errors.push(message);
+ }
+ }
+ if (errors.length)
+ reportError(
+ core,
+ context,
+ `${errors.length} merge queue cleanup errors; see warnings.`
+ );
+ return results;
+}
+
+module.exports = { run };
diff --git a/ci/scripts/test_cancel_merge_queue_runs.cjs b/ci/scripts/test_cancel_merge_queue_runs.cjs
new file mode 100755
index 0000000000..64d91ed26f
--- /dev/null
+++ b/ci/scripts/test_cancel_merge_queue_runs.cjs
@@ -0,0 +1,484 @@
+#!/usr/bin/env node
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const assert = require("node:assert/strict");
+const { test } = require("node:test");
+const { run } = require("./cancel_merge_queue_runs.cjs");
+
+const sha = "a".repeat(40);
+const otherSha = "b".repeat(40);
+const branch = `gh-readonly-queue/main/pr-123-${"c".repeat(40)}`;
+const fullName = "apache/datafusion";
+function workflow(id = 1, overrides = {}) {
+ return {
+ id,
+ event: "merge_group",
+ status: "in_progress",
+ path: ".github/workflows/rust.yml",
+ repository: { full_name: fullName },
+ head_repository: { full_name: fullName },
+ head_branch: branch,
+ head_sha: sha,
+ run_attempt: 1,
+ ...overrides,
+ };
+}
+const apiError = (status) =>
+ Object.assign(new Error(`HTTP ${status}`), { status });
+
+async function execute({
+ runs = [workflow()],
+ refs = {},
+ current = {},
+ cancelErrors = {},
+ dryRun = false,
+ omitDryRun = false,
+ owner = "apache",
+ listError,
+ listResponse,
+ comparisons = {},
+ eventName = "schedule",
+} = {}) {
+ const calls = [];
+ const cancellations = [];
+ const failures = [];
+ const warnings = [];
+ const logs = [];
+ const refReads = new Map();
+ const runReads = new Map();
+ const actions = {
+ listWorkflowRunsForRepo: async (params) => {
+ assert.equal(params.event, "merge_group");
+ assert.equal(params.per_page, 100);
+ assert.equal(`${params.owner}/${params.repo}`, fullName);
+ calls.push(`list:${params.status}`);
+ if (params.status === listError) throw apiError(503);
+ if (listResponse) return { data: await listResponse(params) };
+ assert.equal(params.created, undefined);
+ const filtered = runs.filter((r) => r.status === params.status);
+ return {
+ data: {
+ total_count: filtered.length,
+ workflow_runs: filtered.slice(
+ (params.page - 1) * 100,
+ params.page * 100
+ ),
+ },
+ };
+ },
+ getWorkflowRun: async ({ run_id }) => {
+ calls.push(`get:${run_id}`);
+ const index = runReads.get(run_id) ?? 0;
+ runReads.set(run_id, index + 1);
+ let value = current[run_id] ?? runs.find((r) => r.id === run_id);
+ if (Array.isArray(value))
+ value = value[Math.min(index, value.length - 1)];
+ if (value instanceof Error) throw value;
+ return { data: value };
+ },
+ cancelWorkflowRun: async ({ owner, repo, run_id }) => {
+ assert.equal(`${owner}/${repo}`, fullName);
+ calls.push(`cancel:${run_id}`);
+ if (cancelErrors[run_id]) throw cancelErrors[run_id];
+ cancellations.push(run_id);
+ },
+ };
+ const github = {
+ rest: {
+ actions,
+ repos: {
+ compareCommitsWithBasehead: async ({ basehead, owner, repo }) => {
+ assert.equal(`${owner}/${repo}`, fullName);
+ calls.push(`compare:${basehead}`);
+ const response = comparisons[basehead] ?? "diverged";
+ if (response instanceof Error) throw response;
+ return { data: { status: response } };
+ },
+ },
+ git: {
+ getRef: async ({ ref, owner, repo }) => {
+ assert.equal(`${owner}/${repo}`, fullName);
+ calls.push(`ref:${ref}`);
+ const index = refReads.get(ref) ?? 0;
+ refReads.set(ref, index + 1);
+ let value = refs[ref];
+ if (Array.isArray(value))
+ value = value[Math.min(index, value.length - 1)];
+ if (value instanceof Error) throw value;
+ if (value === undefined || value === null) throw apiError(404);
+ return {
+ data:
+ typeof value === "string"
+ ? { object: { type: "commit", sha: value } }
+ : value,
+ };
+ },
+ },
+ },
+ };
+ const results = await run({
+ github,
+ context: { repo: { owner, repo: "datafusion" }, runId: 999, eventName },
+ core: {
+ info: (s) => logs.push(s),
+ warning: (s) => warnings.push(s),
+ setFailed: (s) => failures.push(s),
+ },
+ ...(omitDryRun ? {} : { dryRun }),
+ });
+ return { results, cancellations, calls, failures, warnings, logs };
+}
+
+test("keeps every workflow of an active group, including failed-check siblings", async () => {
+ const result = await execute({
+ runs: [
+ workflow(1),
+ workflow(2, { path: ".github/workflows/extended.yml" }),
+ ],
+ refs: { [`heads/${branch}`]: sha },
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.calls.filter((c) => c.startsWith("ref:")).length, 1);
+});
+
+test("cancels all workflows of a removed group after collecting every active status", async () => {
+ const statuses = ["queued", "in_progress", "waiting", "pending", "requested"];
+ const result = await execute({
+ runs: statuses.map((status, i) => workflow(i + 1, { status })),
+ });
+ assert.deepEqual(result.cancellations.sort(), [1, 2, 3, 4, 5]);
+ assert.deepEqual(
+ result.calls.slice(0, 5),
+ statuses.map((s) => `list:${s}`)
+ );
+ assert.ok(result.results.every((r) => r.reason === "queue ref removed"));
+});
+
+test("cancels an old SHA when its queue ref points at a replacement", async () => {
+ const result = await execute({
+ runs: [workflow(1), workflow(2, { head_sha: otherSha })],
+ refs: { [`heads/${branch}`]: otherSha },
+ });
+ assert.deepEqual(result.cancellations, [1]);
+ assert.equal(result.results[0].reason, "queue ref replaced");
+});
+
+test("new base ref for one PR does not cancel its replacement or another PR", async () => {
+ const replacement = `gh-readonly-queue/main/pr-123-${"d".repeat(40)}`;
+ const unrelated = `gh-readonly-queue/main/pr-456-${"e".repeat(40)}`;
+ const result = await execute({
+ runs: [
+ workflow(1),
+ workflow(2, { head_branch: replacement }),
+ workflow(3, { head_branch: unrelated }),
+ ],
+ refs: { [`heads/${replacement}`]: sha, [`heads/${unrelated}`]: sha },
+ });
+ assert.deepEqual(result.cancellations, [1]);
+});
+
+test("checks the full queue ref, including a base branch containing slashes", async () => {
+ const release = `gh-readonly-queue/release/56/pr-123-${"c".repeat(40)}`;
+ const result = await execute({
+ runs: [workflow(1), workflow(2, { head_branch: release })],
+ refs: { [`heads/${release}`]: sha },
+ });
+ assert.deepEqual(result.cancellations, [1]);
+});
+
+test("ignores non-queue events, foreign repos, malformed refs, and cleanup runs", async () => {
+ const overrides = [
+ { event: "push" },
+ { event: "pull_request" },
+ { event: "workflow_dispatch" },
+ { head_repository: { full_name: "contributor/datafusion" } },
+ { repository: { full_name: "contributor/datafusion" } },
+ { head_repository: null },
+ { head_branch: "main" },
+ { head_branch: "gh-readonly-queue/main/pr-123-invalid" },
+ { head_sha: "invalid" },
+ { path: ".github/workflows/merge_queue_cleanup.yml" },
+ { path: ".github/workflows/merge_queue_cleanup.yml@refs/heads/main" },
+ { id: 999 },
+ { status: "completed" },
+ ];
+ const result = await execute({
+ runs: overrides.map((value, i) => workflow(i + 1, value)),
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.calls.length, 5);
+});
+
+test("does nothing on a fork", async () => {
+ const result = await execute({ owner: "contributor" });
+ assert.deepEqual(result.calls, []);
+});
+
+test("dry run performs the same checks without mutations", async () => {
+ const result = await execute({ dryRun: true });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.results[0].id, 1);
+ assert.equal(result.results[0].sha, sha);
+ assert.equal(result.results[0].attempt, 1);
+ assert.equal(result.results[0].reason, "queue ref removed");
+ assert.ok(Number.isFinite(Date.parse(result.results[0].observed_at)));
+ assert.match(result.logs[0], /Would cancel/);
+});
+
+test("defaults to a dry run when the caller does not specify a mode", async () => {
+ const result = await execute({ omitDryRun: true });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.results.length, 1);
+ assert.match(result.logs[0], /Would cancel/);
+});
+
+for (const status of [401, 403, 429, 500]) {
+ test(`a ${status} ref lookup cannot be mistaken for a removed ref`, async () => {
+ const result = await execute({
+ refs: { [`heads/${branch}`]: apiError(status) },
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.failures.length, 1);
+ });
+}
+
+test("malformed ref responses cannot cancel a run", async () => {
+ for (const response of [
+ {},
+ { object: { type: "tag", sha: otherSha } },
+ { object: { type: "commit", sha: "" } },
+ ]) {
+ const result = await execute({ refs: { [`heads/${branch}`]: response } });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.failures.length, 1);
+ }
+});
+
+test("rechecks refs and keeps a group that is active again", async () => {
+ const result = await execute({ refs: { [`heads/${branch}`]: [null, sha] } });
+ assert.deepEqual(result.cancellations, []);
+});
+
+test("a failed second ref read cannot cancel a run", async () => {
+ const result = await execute({
+ refs: { [`heads/${branch}`]: [null, apiError(503)] },
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.failures.length, 1);
+});
+
+test("does not cancel completed runs, newer attempts, or changed run identities", async () => {
+ for (const update of [
+ { status: "completed" },
+ { run_attempt: 2 },
+ { head_sha: otherSha },
+ { head_branch: `gh-readonly-queue/main/pr-456-${"c".repeat(40)}` },
+ { event: "push" },
+ ]) {
+ const result = await execute({ current: { 1: workflow(1, update) } });
+ assert.deepEqual(result.cancellations, []);
+ }
+});
+
+test("409 on cancellation is a harmless completion race", async () => {
+ const result = await execute({
+ current: { 1: [workflow(), workflow(1, { status: "completed" })] },
+ cancelErrors: { 1: apiError(409) },
+ });
+ assert.deepEqual(result.failures, []);
+ assert.deepEqual(result.results, []);
+});
+
+test("409 for an unchanged active run reports a failed cancellation", async () => {
+ const result = await execute({ cancelErrors: { 1: apiError(409) } });
+ assert.equal(result.failures.length, 1);
+ assert.match(result.warnings[0], /HTTP 409/);
+});
+
+test("409 after a new rerun starts leaves that attempt alone", async () => {
+ const result = await execute({
+ current: { 1: [workflow(), workflow(1, { run_attempt: 2 })] },
+ cancelErrors: { 1: apiError(409) },
+ });
+ assert.deepEqual(result.failures, []);
+ assert.deepEqual(result.results, []);
+});
+
+test("409 on a read is still an error", async () => {
+ const result = await execute({ current: { 1: apiError(409) } });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.failures.length, 1);
+});
+
+test("one cancellation failure does not stop cleanup of other runs", async () => {
+ const result = await execute({
+ runs: [workflow(1), workflow(2)],
+ cancelErrors: { 1: apiError(403) },
+ });
+ assert.deepEqual(result.cancellations, [2]);
+ assert.equal(result.failures.length, 1);
+});
+
+test("one failed group lookup does not stop cleanup of another group", async () => {
+ const result = await execute({
+ runs: [
+ workflow(1),
+ workflow(2, {
+ head_branch: `gh-readonly-queue/main/pr-456-${"c".repeat(40)}`,
+ }),
+ ],
+ refs: { [`heads/${branch}`]: apiError(503) },
+ });
+ assert.deepEqual(result.cancellations, [2]);
+ assert.equal(result.failures.length, 1);
+});
+
+test("a listing failure aborts before any cancellation", async () => {
+ const result = await execute({ listError: "requested" });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.failures.length, 1);
+ assert.match(result.warnings[0], /HTTP 503/);
+});
+
+test("uses all paginated results and deduplicates repeated run IDs", async () => {
+ const runs = Array.from({ length: 205 }, (_, i) => workflow(i + 1));
+ runs.push(runs[0]);
+ const result = await execute({ runs });
+ assert.equal(result.cancellations.length, 205);
+ assert.equal(new Set(result.cancellations).size, 205);
+});
+
+test("keeps merged commits even when their queue ref has disappeared or changed", async () => {
+ for (const status of ["identical", "behind"]) {
+ for (const ref of [null, otherSha]) {
+ const result = await execute({
+ comparisons: { [`main...${sha}`]: status },
+ refs: { [`heads/${branch}`]: ref },
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.deepEqual(result.failures, []);
+ }
+ }
+});
+
+test("compares against the actual target branch, including slashes", async () => {
+ const result = await execute({
+ runs: [
+ workflow(1, {
+ head_branch: `gh-readonly-queue/release/56/pr-123-${sha}`,
+ }),
+ ],
+ comparisons: { [`release/56...${sha}`]: "behind" },
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.ok(result.calls.includes(`compare:release/56...${sha}`));
+});
+
+test("only ahead or diverged comparisons permit cancellation", async () => {
+ for (const status of ["ahead", "diverged"]) {
+ const result = await execute({
+ comparisons: { [`main...${sha}`]: status },
+ });
+ assert.deepEqual(result.cancellations, [1]);
+ }
+ for (const status of [
+ "unknown",
+ "",
+ apiError(404),
+ apiError(403),
+ apiError(503),
+ ]) {
+ const result = await execute({
+ comparisons: { [`main...${sha}`]: status },
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.failures.length, 1);
+ }
+});
+
+test("reruns are exempt across successive scans, not just on attempt changes", async () => {
+ for (let scan = 0; scan < 2; scan++) {
+ const result = await execute({
+ runs: [workflow(1, { run_attempt: 2 }), workflow(2)],
+ });
+ assert.deepEqual(result.cancellations, [2]);
+ assert.ok(!result.calls.includes("get:1"));
+ }
+});
+
+test("merge-group maintenance errors warn without failing the commit", async () => {
+ for (const options of [
+ { listError: "requested" },
+ { refs: { [`heads/${branch}`]: apiError(403) } },
+ { comparisons: { [`main...${sha}`]: apiError(503) } },
+ { cancelErrors: { 1: apiError(409) } },
+ ]) {
+ const result = await execute({ eventName: "merge_group", ...options });
+ assert.deepEqual(result.failures, []);
+ assert.ok(result.warnings.length > 0);
+ assert.deepEqual(result.cancellations, []);
+ }
+});
+
+test("manual cleanup failures remain visible", async () => {
+ const result = await execute({
+ eventName: "workflow_dispatch",
+ cancelErrors: { 1: apiError(409) },
+ });
+ assert.equal(result.failures.length, 1);
+});
+
+test("an unresolved 409 is retried by later scans and is never reported as cancelled", async () => {
+ const first = await execute({ cancelErrors: { 1: apiError(409) } });
+ assert.deepEqual(first.results, []);
+ assert.equal(first.failures.length, 1);
+ const next = await execute();
+ assert.deepEqual(next.cancellations, [1]);
+});
+
+test("a capped listing aborts without cancelling", async () => {
+ const result = await execute({
+ runs: Array.from({ length: 1000 }, (_, i) => workflow(i + 1)),
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.deepEqual(
+ result.calls.filter((c) => !c.startsWith("list:")),
+ []
+ );
+ assert.equal(result.failures.length, 1);
+ assert.match(result.warnings[0], /exceed the listing cap/);
+});
+
+test("workflows of one group share the target-branch comparison", async () => {
+ const result = await execute({
+ runs: [
+ workflow(1),
+ workflow(2, { path: ".github/workflows/extended.yml" }),
+ ],
+ });
+ assert.deepEqual(result.cancellations, [1, 2]);
+ assert.equal(result.calls.filter((c) => c.startsWith("compare:")).length, 1);
+});
+
+test("malformed listing metadata aborts without cancelling", async () => {
+ const result = await execute({
+ listResponse: () => ({ workflow_runs: [workflow()] }),
+ });
+ assert.deepEqual(result.cancellations, []);
+ assert.equal(result.failures.length, 1);
+});
diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh
index b1dddaac3b..116bbf0b75 100755
--- a/dev/rust_lint.sh
+++ b/dev/rust_lint.sh
@@ -139,6 +139,7 @@ declare -a READONLY_STEPS=(
"ci/scripts/check_rust_workflow_skips.py|false"
"ci/scripts/test_rust_workflow_skips.py|false"
"ci/scripts/test_fork_ci_status.cjs|false"
+ "ci/scripts/test_cancel_merge_queue_runs.cjs|false"
"ci/scripts/check_large_files.sh|false"
"ci/scripts/markdown_link_check.sh|false"
"ci/scripts/security_audit.sh|false"
diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md
index a9fc48b4ba..5df074618d 100644
--- a/docs/source/contributor-guide/index.md
+++ b/docs/source/contributor-guide/index.md
@@ -284,3 +284,17 @@ To make this work:
synced fork builds much faster.
3. Rebase your branch before you push. The fork run tests your branch as it
is, while the merge queue tests it merged with `main`.
+
+#### Obsolete Merge Queue Runs
+
+The **Merge queue cleanup** workflow finds unfinished merge queue CI runs whose
+queue branch was removed or now points to a different commit. It scans when a
+new merge group is created and every 15 minutes.
+
+Cleanup keeps runs for active groups, commits already in their target branch,
+and rerun attempts. Contributors can rerun an obsolete job to investigate a
+failure. A failed check alone does not cause cancellation.
+
+The workflow only reports candidates until a maintainer sets the repository
+Actions variable `MERGE_QUEUE_CLEANUP_ENABLED` to `true`. Manual runs default
+to `dry_run`. To stop cancellation, remove the variable or set it to `false`.