Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

perf(ci): start source feedback early and diagnose stalled runs (#3013)

morluto6510d0d57f9d
diff --git a/.github/workflows/ci-timings.yml b/.github/workflows/ci-timings.yml
index 73a9ba4b9..74f27af32 100644
--- a/.github/workflows/ci-timings.yml
+++ b/.github/workflows/ci-timings.yml
@@ -12,12 +12,13 @@ concurrency:
permissions:
contents: read
actions: read
+ pull-requests: read
jobs:
report:
name: Collect recent completed workflow attempts
runs-on: ubuntu-latest
- timeout-minutes: 5
+ timeout-minutes: 10
steps:
# Consume only API data and execute batch code from the trusted default branch.
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
@@ -27,6 +28,18 @@ jobs:
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
+ - name: Cache npm package downloads
+ id: npm-cache
+ continue-on-error: true
+ uses: ./.github/actions/npm-cache
+ - name: Diagnose missing and startup-failed PR CI
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ CI_REPOSITORY: ${{ github.repository }}
+ CI_REPOSITORY_ID: ${{ github.repository_id }}
+ CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ run: node scripts/ci/pr-ci-health.mjs
- name: Record queue, execution, and matched step timing
continue-on-error: true
env:
@@ -44,3 +57,28 @@ jobs:
if-no-files-found: error
overwrite: true
retention-days: 14
+ - name: Install dependencies for static failure attribution
+ id: failure-deps
+ if: ${{ !cancelled() && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }}
+ continue-on-error: true
+ run: npm ci
+ - name: Attribute supported static failures against main
+ if: ${{ !cancelled() && steps.failure-deps.outcome == 'success' && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ CI_REPOSITORY: ${{ github.repository }}
+ CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ run: node scripts/ci/failure-report.mjs
+ - name: Upload CI health and failure diagnostics
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: ci-health-diagnostics
+ path: |
+ .cache/ci-pr-health/
+ .cache/ci-failures/
+ include-hidden-files: true
+ if-no-files-found: warn
+ retention-days: 14
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8da32ee27..b35b6a6e1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -139,6 +139,12 @@ jobs:
- name: Validate workflow syntax and expressions
if: needs.changes.outputs.workflows == 'true'
run: go run github.com/rhysd/actionlint/cmd/[email protected] -shellcheck= -pyflakes=
+ - name: Record static task source and execution identity
+ continue-on-error: true
+ if: needs.changes.outputs.code == 'true'
+ env:
+ CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: npm run ci:static-context
- if: needs.changes.outputs.code == 'true'
run: npm run check:ci
- if: needs.changes.outputs.code == 'true'
@@ -167,6 +173,34 @@ jobs:
if-no-files-found: ignore
retention-days: 7
+ test-source:
+ name: Source tests
+ needs: changes
+ if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+ with:
+ node-version-file: .nvmrc
+ - name: Cache npm package downloads
+ id: npm-cache
+ uses: ./.github/actions/npm-cache
+ - run: npm ci
+ - name: Run source tests without waiting for build
+ run: npm run test:fast -- --reporter=default --reporter=blob
+ env:
+ VITEST_BLOB_LABEL: source
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ if: ${{ !cancelled() }}
+ with:
+ name: vitest-report-source
+ path: .vitest/blob/*
+ include-hidden-files: true
+ if-no-files-found: error
+ retention-days: 7
+
test-shard:
name: Test shard (${{ matrix.shard }})
needs: [changes, build]
@@ -178,7 +212,9 @@ jobs:
# every shard's diagnostics and coverage. Passing PRs run the full suite.
fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }}
matrix:
- shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6]
+ # Fast feedback spends the same six test jobs on one source group and
+ # five built groups; full qualification retains six complete shards.
+ shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
@@ -197,15 +233,16 @@ jobs:
run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
env:
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- - name: Run complete portable suite for fast feedback
+ - name: Run built-dependent tests for fast feedback
if: needs.changes.outputs.tier == 'fast'
- run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
+ run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
env:
+ VITEST_BLOB_LABEL: built
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ !cancelled() }}
with:
- name: vitest-report-${{ strategy.job-index }}
+ name: vitest-report-built-${{ strategy.job-index }}
path: .vitest/blob/*
include-hidden-files: true
if-no-files-found: error
@@ -219,11 +256,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- cache: npm
+ - name: Cache npm package downloads
+ id: npm-cache
+ uses: ./.github/actions/npm-cache
- run: npm ci
- name: Run reverse tools tests with coverage
run: npm run test:coverage:reverse-tools
@@ -316,7 +355,7 @@ jobs:
test:
name: Portable tests
- needs: [changes, test-shard]
+ needs: [changes, test-source, test-shard]
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -340,7 +379,9 @@ jobs:
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
if: >-
!cancelled() && needs.changes.outputs.code == 'true' &&
- (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure')
+ (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' ||
+ (needs.changes.outputs.tier == 'fast' &&
+ (needs.test-source.result == 'success' || needs.test-source.result == 'failure')))
with:
path: .vitest/blob
pattern: vitest-report-*
@@ -368,7 +409,8 @@ jobs:
if: >-
!cancelled() && needs.changes.outputs.code == 'true' &&
needs.changes.outputs.tier == 'fast' &&
- (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure')
+ (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' ||
+ needs.test-source.result == 'success' || needs.test-source.result == 'failure')
run: >-
npm exec -- vitest --merge-reports
--reporter=default
@@ -381,6 +423,8 @@ jobs:
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != ''
env:
SHARD_RESULT: ${{ needs.test-shard.result }}
+ SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
+ SOURCE_RESULT: ${{ needs.test-source.result }}
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
CI_VERIFY_TEST_INVENTORY: "true"
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
@@ -413,7 +457,13 @@ jobs:
env:
CODE_REQUIRED: ${{ needs.changes.outputs.code }}
SHARD_RESULT: ${{ needs.test-shard.result }}
- run: test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success"
+ SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
+ SOURCE_RESULT: ${{ needs.test-source.result }}
+ run: |
+ if [ "${CODE_REQUIRED}" = "true" ]; then
+ test "${SHARD_RESULT}" = "success" || exit 1
+ test "${SOURCE_REQUIRED}" != "true" || test "${SOURCE_RESULT}" = "success"
+ fi
conformance-fixtures:
name: Source-built conformance fixtures
@@ -847,6 +897,7 @@ jobs:
build,
reverse-tools-quality,
test-shard,
+ test-source,
sqlite-runtime,
test,
conformance-fixtures,
diff --git a/.github/workflows/real-cutter.yml b/.github/workflows/real-cutter.yml
index 856777999..91121f207 100644
--- a/.github/workflows/real-cutter.yml
+++ b/.github/workflows/real-cutter.yml
@@ -13,8 +13,8 @@ jobs:
name: Build Windows bridge security boundary
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Install the cross compiler used by the Windows native lane
@@ -44,11 +44,13 @@ jobs:
- platform: Windows
runner: [self-hosted, cutter-upstream, Windows]
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- cache: npm
+ - name: Cache npm package downloads
+ id: npm-cache
+ uses: ./.github/actions/npm-cache
- run: npm ci
- if: matrix.platform == 'Windows'
uses: actions/download-artifact@v4
diff --git a/docs/testing.md b/docs/testing.md
index b543fe89f..aa8d2163f 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -280,8 +280,15 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge.
Contract, integration, and real-provider lanes complement these focused tests;
simulated providers do not establish real-provider behavior.
-CI runs six native Vitest shards without retries. Each shard emits a blob
-report; the merge job produces JUnit and JSON diagnostics, a module timing
+Fast CI starts the source-only projects after classification, independently of
+build and static checks, and runs five built-dependent shards after the build.
+The six test jobs still select every portable test. Full qualification keeps six
+complete Vitest shards with coverage. Neither tier retries tests. Canonical
+project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest,
+CI group execution, and development-test preparation. A test importing compiled
+verifier code belongs to the built group even if its directory is conformance.
+Source and built blobs have separate labels to prevent merge collisions.
+Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing
report, plus aggregate coverage during full qualification. It retains available
reports even when a shard fails, uploads available diagnostics, and writes the
slowest files to the workflow summary. Partial reports are marked diagnostic
@@ -316,6 +323,30 @@ portable test shard. These limits leave headroom for cold setup and full coverag
while preventing a hung job from holding the active main run for hours. They do
not bound runner queue waiting. Timeouts still fail the required gate.
+The separate `CI timings` workflow also diagnoses missing, startup-failed,
+queued/running, approval-required, canceled, and failed PR CI through bounded,
+read-only API scans. Incomplete pagination or changing identity stays unknown.
+It never closes/reopens PRs, reruns workflows, posts comments, or changes gates.
+The timing manifest retains each run's event and head branch, including explicit
+unknowns, so PR runs cannot consume the eligible-main comparison budget.
+Its static failure report compares supported lint/duplication diagnostics with
+failed main runs using exact source, ancestry, task hashes, and toolchain proof.
+A match describes those diagnostics only; it does not establish that every
+failure in a job came from main. Duplication reports retain the actual checker
+exit result, so passing duplication checks preserve lint failures and caller
+thresholds remain owned by the producer. Missing checker outcomes leave
+those duplication diagnostics unclassified. Missing historical receipts remain unknown.
+The reporter executes only trusted default-branch code and parses whitelisted
+JSON artifact members without executing artifact content. Reports and limits
+are retained in `ci-health-diagnostics`; its ten-minute job limit covers the
+existing four-minute timing collection plus the bounded diagnostic stages.
+
+Shard-count reduction remains deferred: the available successful timing sample
+covers an older inventory, and file phase weights are not job wall time. The
+one-source/five-built split retains six test jobs; it makes no hosted speedup
+claim. Current complete timings and completed candidate hosted runs are needed
+before enabling a smaller adaptive matrix.
+
The separate `CI timings` workflow batches recent completed CI and Release runs
hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and
a normalized point-in-time report for each collected workflow attempt, plus a
diff --git a/package.json b/package.json
index dff275eb4..e09e51366 100644
--- a/package.json
+++ b/package.json
@@ -118,7 +118,9 @@
"precheck:pr": "npm run deps:check",
"check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos",
"precheck:ci": "npm run deps:check",
- "check:ci": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos && npm run verify:architecture-guards",
+ "check:ci": "node scripts/run-exclusive.mjs artifacts npm run check:ci:tasks && npm run verify:architecture-guards",
+ "check:ci:tasks": "turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos",
+ "ci:static-context": "node scripts/ci/static-task-context.mjs",
"build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills",
"build:termux": "node scripts/run-exclusive.mjs artifacts npm run build:termux:run",
"build:termux:run": "npm run compile:termux && npm run artifacts:skills",
@@ -138,7 +140,8 @@
"test:run": "vitest run",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
"test:local": "node scripts/run-development-tests.mjs local",
- "test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
+ "test:fast": "node scripts/ci/test-group.mjs source",
+ "test:built": "node scripts/ci/test-group.mjs built",
"pretest:boundary": "npm run test:prepare",
"test:boundary": "vitest run --project boundary --project cli-boundary --project process-boundary --project mcp-boundary --project process-global",
"pretest:mcp": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills artifacts:mcp-catalog",
diff --git a/scripts/ci/artifacts.d.mts b/scripts/ci/artifacts.d.mts
new file mode 100644
index 000000000..93cf2f701
--- /dev/null
+++ b/scripts/ci/artifacts.d.mts
@@ -0,0 +1,67 @@
+import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
+import type { StaticTaskContext } from "./static-task-context.mjs";
+export type { StaticTaskContext } from "./static-task-context.mjs";
+
+export interface StaticDiagnostic {
+ readonly kind: "lint" | "jscpd";
+ readonly path?: string;
+ readonly paths?: readonly string[];
+ readonly code?: string;
+ readonly message?: string;
+ readonly severity?: string;
+}
+
+export function classifyFailure(input: {
+ repository: string;
+ defaultBranch: string;
+ run: Record<string, unknown> & NormalizedWorkflowRun;
+ job: Record<string, unknown>;
+ diagnostics: readonly StaticDiagnostic[];
+ context: StaticTaskContext;
+ mainRun: Record<string, unknown> & NormalizedWorkflowRun;
+ mainJob: Record<string, unknown>;
+ mainDiagnostics: readonly StaticDiagnostic[];
+ mainContext: StaticTaskContext;
+ executionSourcesVerified: boolean;
+ ancestry: { status: string; mergeBaseSha: string | null };
+ blobs: Readonly<Record<string, boolean>>;
+}): {
+ classification: "matching-main-diagnostics" | "unknown";
+ reason: string;
+};
+
+export function diagnosticPaths(item: StaticDiagnostic): readonly string[];
+export function isFailedConclusion(
+ value: unknown,
+): value is "failure" | "timed_out";
+export function isEligibleMainRun(
+ run: Record<string, unknown>,
+ defaultBranch: string,
+ repository: string,
+): boolean;
+export function pullRequestBaseSha(
+ run: Record<string, unknown>,
+ repository: string,
+): string | null;
+export function safeSha(value: unknown): string | null;
+export function validRepoPath(value: unknown): value is string;
+export function selectStaticCheckArtifact(
+ page: unknown,
+ run: { id?: string | number; head_sha?: string },
+ runAttempt: number,
+): unknown | null;
+
+export function readStaticCheckArtifact(options: {
+ repository: string;
+ artifactId: number | string;
+ token: string;
+ fetchImpl?: typeof fetch;
+}): Promise<{
+ context: StaticTaskContext;
+ diagnostics: readonly StaticDiagnostic[];
+}>;
+
+export function parseStaticArchive(archivePath: string): Promise<{
+ context: StaticTaskContext;
+ diagnostics: readonly StaticDiagnostic[];
+}>;
diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs
new file mode 100644
index 000000000..9a7cc1c21
--- /dev/null
+++ b/scripts/ci/artifacts.mjs
@@ -0,0 +1,436 @@
+import { execFile } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { isAbsolute, join } from "node:path";
+import { promisify } from "node:util";
+
+import { parseStaticTaskContext } from "./static-task-context.mjs";
+
+const execute = promisify(execFile);
+const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024;
+const MAX_MEMBER_BYTES = 4 * 1024 * 1024;
+const MAX_DIAGNOSTICS = 10_000;
+const API_ROOT = "https://api.github.com/repos";
+const SHA = /^[0-9a-f]{40}$/iu;
+const TASK_HASH = /^[a-f0-9]{16}$/u;
+
+/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */
+export function classifyFailure({
+ repository,
+ defaultBranch,
+ run,
+ job,
+ diagnostics,
+ context,
+ mainRun,
+ mainJob,
+ mainDiagnostics,
+ mainContext,
+ executionSourcesVerified,
+ ancestry,
+ blobs,
+}) {
+ const failedGate = firstFailure([
+ () => [
+ !isCompletedFailure(run) ||
+ job?.status !== "completed" ||
+ !isFailedConclusion(job?.conclusion),
+ "run_or_job_not_a_completed_failure",
+ ],
+ () => [
+ !isEligiblePullRun(run, repository) ||
+ !isEligibleMainRun(mainRun, defaultBranch, repository),
+ "run_event_or_repository_identity_unverified",
+ ],
+ () => [
+ job?.name !== mainJob?.name || job?.name !== "Static checks",
+ "failed_job_not_the_same_static_gate",
+ ],
+ () => [
+ !Array.isArray(diagnostics) ||
+ diagnostics.length === 0 ||
+ !Array.isArray(mainDiagnostics),
+ "structured_diagnostics_missing_or_invalid",
+ ],
+ () => [
+ !validContext(context) || !validContext(mainContext),
+ "static_task_context_missing_or_invalid",
+ ],
+ () => [
+ executionSourcesVerified !== true,
+ "checkout_sources_not_verified_against_api_commits",
+ ],
+ () => [
+ !sameExecutionContext(context, mainContext),
+ "task_or_toolchain_identity_differs",
+ ],
+ () => [
+ ancestry?.status !== "ahead" && ancestry?.status !== "identical",
+ "main_head_not_proven_ancestor_of_tested_base",
+ ],
+ () => [
+ ancestry?.mergeBaseSha !== mainRun?.head_sha,
+ "main_head_is_not_the_verified_merge_base",
+ ],
+ () => [
+ !sameTaskHashes(diagnostics, context, mainContext),
+ "owning_task_hash_differs",
+ ],
+ ]);
+ if (failedGate) return outcome("unknown", failedGate.reason);
+ const keys = diagnostics.map(diagnosticKey);
+ const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean));
+ if (keys.some((key) => key === null))
+ return outcome("unknown", "diagnostic_shape_not_supported");
+ if (!keys.every((key) => mainKeys.has(key)))
+ return outcome(
+ "unknown",
+ "main_run_does_not_contain_the_same_structured_diagnostic",
+ );
+ if (!sameBlobEvidence(diagnostics, blobs))
+ return outcome(
+ "unknown",
+ "diagnostic_source_content_not_identical_at_main_and_tested_base",
+ );
+ return outcome(
+ "matching-main-diagnostics",
+ "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
+ );
+}
+
+export function diagnosticPaths(item) {
+ return item.kind === "jscpd" ? item.paths : [item.path];
+}
+export function isFailedConclusion(value) {
+ return value === "failure" || value === "timed_out";
+}
+export function isEligibleMainRun(run, defaultBranch, repository) {
+ return (
+ isEligibleStaticRun(run) &&
+ run.repository.full_name === repository &&
+ run.head_branch === defaultBranch &&
+ ["push", "schedule", "workflow_dispatch"].includes(run.event)
+ );
+}
+export function pullRequestBaseSha(run, repository) {
+ if (
+ run?.event !== "pull_request" ||
+ run.repository?.full_name !== repository ||
+ !Array.isArray(run.pull_requests) ||
+ run.pull_requests.length !== 1
+ )
+ return null;
+ const base = run.pull_requests[0].base;
+ return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "")
+ ? base.sha
+ : null;
+}
+
+export function safeSha(value) {
+ return typeof value === "string" && SHA.test(value)
+ ? value.toLowerCase()
+ : null;
+}
+export function selectStaticCheckArtifact(page, run, runAttempt) {
+ if (runAttempt !== 1) return null;
+ if (
+ !Array.isArray(page?.artifacts) ||
+ !Number.isSafeInteger(page.total_count) ||
+ page.total_count !== page.artifacts.length ||
+ page.artifacts.length >= 100
+ )
+ return null;
+ const matches = page.artifacts.filter(
+ (artifact) => artifact.name === "static-check-reports",
+ );
+ if (matches.length !== 1) return null;
+ const [artifact] = matches;
+ if (
+ artifact.expired !== false ||
+ String(artifact.workflow_run?.id) !== String(run.id) ||
+ artifact.workflow_run?.head_sha !== run.head_sha
+ )
+ return null;
+ return artifact;
+}
+export function validRepoPath(value) {
+ return (
+ typeof value === "string" &&
+ value.length > 0 &&
+ !isAbsolute(value) &&
+ !value.includes("\\") &&
+ value
+ .split("/")
+ .every((part) => part !== "" && part !== "." && part !== "..") &&
+ !value.startsWith("/")
+ );
+}
+
+/** Download one named static report archive and parse only whitelisted JSON members. */
+export async function readStaticCheckArtifact({
+ repository,
+ artifactId,
+ token,
+ fetchImpl = fetch,
+}) {
+ if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? ""))
+ throw new Error("artifact repository identity is invalid");
+ if (!token) throw new Error("artifact download requires a GitHub token");
+ const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`;
+ let response = await fetchImpl(url, {
+ headers: {
+ Accept: "application/vnd.github+json",
+ Authorization: `Bearer ${token}`,
+ "X-GitHub-Api-Version": "2022-11-28",
+ },
+ signal: AbortSignal.timeout(10_000),
+ redirect: "manual",
+ });
+ if ([301, 302, 303, 307, 308].includes(response.status)) {
+ const location = response.headers.get("location");
+ if (!location || new URL(location).protocol !== "https:")
+ throw new Error("static_artifact_redirect_invalid");
+ response = await fetchImpl(location, {
+ signal: AbortSignal.timeout(10_000),
+ redirect: "error",
+ });
+ }
+ if (!response.ok) throw new Error(`github_http_${response.status}`);
+ const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES);
+ const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-"));
+ const archive = join(directory, "artifact.zip");
+ try {
+ await writeFile(archive, bytes, { flag: "wx" });
+ return await parseStaticArchive(archive);
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+}
+
+/** Accept only the three named producer outputs; no archive paths are written to disk. */
+export async function parseStaticArchive(archivePath) {
+ const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], {
+ encoding: "utf8",
+ maxBuffer: 128 * 1024,
+ timeout: 5_000,
+ });
+ const allowed = new Set([
+ "oxlint.json",
+ "jscpd/jscpd-report.json",
+ "static-task-context.json",
+ ]);
+ const members = new Map();
+ for (const name of listing.split(/\r?\n/u).filter(Boolean)) {
+ const prefix = ".cache/static-checks/";
+ const member = name.startsWith(prefix) ? name.slice(prefix.length) : name;
+ if (!allowed.has(member)) continue;
+ if (members.has(member))
+ throw new Error("static artifact contains duplicate report members");
+ const { stdout } = await execute("unzip", ["-p", archivePath, name], {
+ encoding: "buffer",
+ maxBuffer: MAX_MEMBER_BYTES,
+ timeout: 5_000,
+ });
+ members.set(member, JSON.parse(stdout.toString("utf8")));
+ }
+ const context = parseStaticTaskContext(
+ members.get("static-task-context.json"),
+ );
+ const diagnostics = normalizeDiagnostics(
+ members.get("oxlint.json"),
+ members.get("jscpd/jscpd-report.json"),
+ );
+ if (diagnostics.length > MAX_DIAGNOSTICS)
+ throw new Error("static diagnostic count exceeds the reporting limit");
+ return { context, diagnostics };
+}
+
+function normalizeDiagnostics(lint, duplicate) {
+ return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)];
+}
+
+function normalizeLint(report) {
+ if (report === undefined) return [];
+ if (!Array.isArray(report?.diagnostics))
+ throw new Error("oxlint diagnostics are malformed");
+ return report.diagnostics
+ .filter((item) => item.severity === "error")
+ .map((item) => {
+ if (
+ typeof item.filename !== "string" ||
+ !repoPath(item.filename) ||
+ typeof item.code !== "string" ||
+ typeof item.message !== "string"
+ )
+ throw new Error("oxlint error diagnostic is malformed");
+ return {
+ kind: "lint",
+ path: item.filename,
+ code: item.code,
+ message: item.message,
+ severity: "error",
+ };
+ });
+}
+
+function normalizeDuplicates(report) {
+ if (report === undefined) return [];
+ if (
+ !Array.isArray(report?.duplicates) ||
+ !Number.isFinite(report.statistics?.total?.percentage) ||
+ report.statistics.total.percentage < 0
+ )
+ throw new Error("jscpd report is malformed");
+ // The producer owns the checker outcome; do not copy or infer its threshold.
+ const exitCode = report.reaCheck?.exitCode;
+ if (exitCode === undefined || exitCode === 0) return [];
+ if (!Number.isSafeInteger(exitCode) || exitCode < 0)
+ throw new Error("jscpd checker outcome is malformed");
+ if (exitCode !== 1) return [];
+ return report.duplicates.map((item) => {
+ const paths = [item.firstFile?.name, item.secondFile?.name].sort();
+ if (paths.some((path) => !repoPath(path)))
+ throw new Error("jscpd duplicate paths are malformed");
+ return { kind: "jscpd", paths };
+ });
+}
+
+async function readLimitedBytes(response, limit) {
+ const reader = response.body?.getReader();
+ if (!reader) throw new Error("static artifact response has no body");
+ const chunks = [];
+ let size = 0;
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ size += value.byteLength;
+ if (size > limit) {
+ await reader.cancel();
+ throw new Error("static artifact exceeds the download limit");
+ }
+ chunks.push(value);
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ const result = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of chunks) {
+ result.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return result;
+}
+
+function repoPath(value) {
+ return (
+ typeof value === "string" &&
+ value.length > 0 &&
+ !value.startsWith("/") &&
+ !value.includes("\\") &&
+ value
+ .split("/")
+ .every((part) => part !== "" && part !== "." && part !== "..")
+ );
+}
+
+function positiveId(value) {
+ const id = Number(value);
+ if (!Number.isSafeInteger(id) || id < 1)
+ throw new Error("artifact ID must be a positive integer");
+ return String(id);
+}
+
+function diagnosticKey(item) {
+ if (
+ item?.kind === "lint" &&
+ validRepoPath(item.path) &&
+ typeof item.code === "string" &&
+ typeof item.message === "string" &&
+ item.severity === "error"
+ )
+ return JSON.stringify([
+ item.kind,
+ item.path,
+ item.code,
+ item.message,
+ item.severity,
+ ]);
+ if (
+ item?.kind === "jscpd" &&
+ Array.isArray(item.paths) &&
+ item.paths.length === 2 &&
+ item.paths.every(validRepoPath)
+ )
+ return JSON.stringify([item.kind, ...[...item.paths].sort()]);
+ return null;
+}
+function sameBlobEvidence(diagnostics, blobs) {
+ const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];
+ return paths.length > 0 && paths.every((path) => blobs?.[path] === true);
+}
+function sameTaskHashes(diagnostics, left, right) {
+ return diagnostics.every((diagnostic) => {
+ const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built";
+ return (
+ TASK_HASH.test(left.taskHashes[task] ?? "") &&
+ left.taskHashes[task] === right.taskHashes[task]
+ );
+ });
+}
+function validContext(value) {
+ return (
+ value?.version === 1 &&
+ SHA.test(value.sourceSha ?? "") &&
+ value.sourceClean === true &&
+ typeof value.nodeVersion === "string" &&
+ typeof value.npmVersion === "string" &&
+ typeof value.turboVersion === "string" &&
+ typeof value.platform === "string" &&
+ typeof value.architecture === "string" &&
+ value.taskHashes &&
+ typeof value.taskHashes === "object" &&
+ !Array.isArray(value.taskHashes)
+ );
+}
+function sameExecutionContext(left, right) {
+ return [
+ "nodeVersion",
+ "npmVersion",
+ "turboVersion",
+ "platform",
+ "architecture",
+ ].every((key) => left[key] === right[key]);
+}
+function isEligiblePullRun(run, repository) {
+ return (
+ isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null
+ );
+}
+function isEligibleStaticRun(run) {
+ return (
+ isRecord(run) &&
+ run.status === "completed" &&
+ isFailedConclusion(run.conclusion) &&
+ SHA.test(run.head_sha ?? "") &&
+ typeof run.repository?.full_name === "string" &&
+ run.workflowPath?.file === ".github/workflows/ci.yml"
+ );
+}
+function isCompletedFailure(run) {
+ return run?.status === "completed" && isFailedConclusion(run.conclusion);
+}
+function isRecord(value) {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function firstFailure(checks) {
+ for (const check of checks) {
+ const [failed, reason] = check();
+ if (failed) return { reason };
+ }
+ return null;
+}
+function outcome(classification, reason) {
+ return { classification, reason };
+}
diff --git a/scripts/ci/failure-report.d.mts b/scripts/ci/failure-report.d.mts
new file mode 100644
index 000000000..47ac65745
--- /dev/null
+++ b/scripts/ci/failure-report.d.mts
@@ -0,0 +1,159 @@
+import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
+import type { StaticDiagnostic } from "./artifacts.mjs";
+export type { StaticDiagnostic } from "./artifacts.mjs";
+import type { StaticTaskContext } from "./static-task-context.mjs";
+export type { StaticTaskContext } from "./static-task-context.mjs";
+export interface CiRun {
+ readonly id?: string | number;
+ readonly run_attempt?: number;
+ readonly status?: string;
+ readonly conclusion?: string | null;
+ readonly event?: string;
+ readonly path?: string;
+ readonly head_sha?: string;
+ readonly head_branch?: string;
+ readonly repository?: { readonly full_name?: string };
+ readonly pull_requests?: readonly {
+ readonly base?: {
+ readonly sha?: string;
+ readonly repo?: { readonly full_name?: string };
+ };
+ }[];
+}
+
+export interface CiJob {
+ readonly name?: string;
+ readonly status?: string;
+ readonly conclusion?: string | null;
+ readonly run_id?: number;
+ readonly run_attempt?: number;
+}
+
+export interface StaticEvidence {
+ readonly context: StaticTaskContext;
+ readonly diagnostics: readonly StaticDiagnostic[];
+}
+
+export interface CiFailureReport {
+ readonly version: 1;
+ readonly observedAt: string;
+ readonly repository: string;
+ readonly defaultBranch: string;
+ readonly informationalOnly: true;
+ readonly gateEffect: string;
+ readonly boundedInput: {
+ readonly candidateRuns: number;
+ readonly limit: number;
+ readonly mainCandidateLimit: number;
+ readonly mainCandidateRuns: number;
+ readonly mainRunLookupLimit: number;
+ readonly manifestTruncated: boolean;
+ };
+ readonly counts: Readonly<
+ Record<"matching-main-diagnostics" | "unknown", number>
+ >;
+ readonly failures: readonly {
+ readonly runId: number | string;
+ readonly runAttempt: number;
+ readonly classification: "matching-main-diagnostics" | "unknown";
+ readonly reason: string;
+ readonly candidateFailures?: readonly {
+ readonly runId: number | string;
+ readonly runAttempt: number;
+ readonly reason: string;
+ }[];
+ readonly runHead?: string | null;
+ readonly event?: string | null;
+ readonly job?: string;
+ readonly conclusion?: string | null;
+ }[];
+}
+
+export interface FailureReportApi {
+ getRun(repository: string, id: string | number): Promise<CiRun>;
+ getJobs(
+ repository: string,
+ id: string | number,
+ attempt: number,
+ ): Promise<readonly CiJob[]>;
+ getStaticEvidence(
+ repository: string,
+ run: CiRun,
+ job: CiJob,
+ ): Promise<StaticEvidence | null>;
+ getAncestry(
+ repository: string,
+ ancestor: string,
+ descendant: string,
+ ): Promise<{ status: string; mergeBaseSha: string | null }>;
+ verifyExecutionSource(
+ repository: string,
+ run: CiRun,
+ context: StaticTaskContext,
+ ): Promise<boolean>;
+ getBlobIdentity(
+ repository: string,
+ ancestor: string,
+ descendant: string,
+ paths: readonly string[],
+ ): Promise<Readonly<Record<string, boolean>>>;
+}
+
+export function classifyFailure(input: {
+ repository: string;
+ defaultBranch: string;
+ run: CiRun & NormalizedWorkflowRun;
+ job: CiJob;
+ diagnostics: readonly StaticDiagnostic[];
+ context: StaticTaskContext;
+ mainRun: CiRun & NormalizedWorkflowRun;
+ mainJob: CiJob;
+ mainDiagnostics: readonly StaticDiagnostic[];
+ mainContext: StaticTaskContext;
+ executionSourcesVerified: boolean;
+ ancestry: { status: string; mergeBaseSha: string | null };
+ blobs: Readonly<Record<string, boolean>>;
+}): {
+ classification: "matching-main-diagnostics" | "unknown";
+ reason: string;
+};
+
+export function reportCiFailures(options: {
+ repository: string;
+ defaultBranch: string;
+ batchManifest: {
+ readonly repository: string;
+ readonly runs: readonly {
+ readonly id: number | string;
+ readonly runAttempt: number;
+ readonly workflow: string;
+ readonly event?: string | null;
+ readonly headBranch?: string | null;
+ readonly conclusion: string | null;
+ readonly status: string;
+ }[];
+ readonly truncation?: readonly unknown[];
+ readonly coverage?: { readonly exhaustive?: boolean };
+ readonly selection?: { readonly exhaustive?: boolean };
+ };
+ api: FailureReportApi;
+ outputDirectory?: string;
+ summaryPath?: string;
+ observedAt?: string;
+}): Promise<CiFailureReport>;
+
+export function createGitHubApi(options?: {
+ token?: string;
+ fetchImpl?: typeof fetch;
+ apiRequest?: (
+ method: string,
+ endpoint: string,
+ budget: { requests: number; startedAt: number; limitMs: number },
+ ) => Promise<unknown>;
+}): FailureReportApi;
+
+export function writeFailureReport(
+ report: CiFailureReport,
+ outputDirectory?: string,
+ summaryPath?: string,
+): Promise<{ jsonPath: string; markdownPath: string }>;
diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs
new file mode 100644
index 000000000..2ce6d813c
--- /dev/null
+++ b/scripts/ci/failure-report.mjs
@@ -0,0 +1,529 @@
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import {
+ classifyFailure,
+ diagnosticPaths,
+ isEligibleMainRun,
+ isFailedConclusion,
+ pullRequestBaseSha,
+ readStaticCheckArtifact,
+ safeSha,
+ selectStaticCheckArtifact,
+ validRepoPath,
+} from "./artifacts.mjs";
+import { createGhApiRequest } from "./github-api.mjs";
+import { normalizeWorkflowRun } from "./workflow-run.mjs";
+
+const OUTPUT_DIRECTORY = join(".cache", "ci-failures");
+const BATCH_DIRECTORY = join(".cache", "ci-job-timings");
+const MAX_JSON_BYTES = 8 * 1024 * 1024;
+const MAX_FAILED_RUNS = 20;
+const MAX_MAIN_CANDIDATES = 5;
+const MAX_MAIN_RUN_LOOKUPS = 100;
+const MAX_PATHS_PER_COMPARISON = 50;
+const MAX_API_REQUESTS = 250;
+
+export { classifyFailure } from "./artifacts.mjs";
+
+/** Uses only failed CI candidates present in the supplied bounded timing manifest. */
+export async function reportCiFailures({
+ repository,
+ defaultBranch,
+ batchManifest,
+ api,
+ outputDirectory = OUTPUT_DIRECTORY,
+ summaryPath,
+ observedAt = new Date().toISOString(),
+}) {
+ validateRepository(repository);
+ if (typeof defaultBranch !== "string" || defaultBranch.length === 0)
+ throw new Error("defaultBranch is required");
+ if (
+ !batchManifest ||
+ batchManifest.repository !== repository ||
+ !Array.isArray(batchManifest.runs)
+ )
+ throw new Error("timing manifest repository or runs are invalid");
+
+ const failedRuns = batchManifest.runs.filter(
+ (entry) =>
+ entry.workflow === "CI" &&
+ entry.status === "collected" &&
+ isFailedConclusion(entry.conclusion),
+ );
+ const prRuns = failedRuns.filter((entry) => entry.event === "pull_request");
+ const candidates = prRuns.slice(0, MAX_FAILED_RUNS);
+ const mainEntries = failedRuns.filter(
+ (entry) =>
+ entry.headBranch === defaultBranch &&
+ ["push", "schedule", "workflow_dispatch"].includes(entry.event),
+ );
+ const lookupRun = verifiedRunLookup(repository, api);
+ const mainCandidates = [];
+ for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) {
+ try {
+ const run = await lookupRun(entry);
+ if (isEligibleMainRun(run, defaultBranch, repository))
+ mainCandidates.push({
+ run,
+ getEvidence: mainEvidenceLookup(repository, api, entry, run),
+ });
+ if (mainCandidates.length === MAX_MAIN_CANDIDATES) break;
+ } catch {
+ // Unverified metadata cannot consume the eligible-main candidate budget.
+ }
+ }
+ const entries = [];
+ for (const entry of candidates) {
+ try {
+ entries.push(
+ ...(await analyzeFailedRun({
+ repository,
+ defaultBranch,
+ entry,
+ run: await lookupRun(entry),
+ mainCandidates,
+ api,
+ })),
+ );
+ } catch (error) {
+ entries.push({
+ runId: entry.id,
+ runAttempt: entry.runAttempt,
+ classification: "unknown",
+ reason: errorCode(error),
+ });
+ }
+ }
+ const report = {
+ version: 1,
+ observedAt,
+ repository,
+ defaultBranch,
+ informationalOnly: true,
+ gateEffect: "none; this report never changes CI conclusions",
+ boundedInput: {
+ candidateRuns: candidates.length,
+ limit: MAX_FAILED_RUNS,
+ mainCandidateLimit: MAX_MAIN_CANDIDATES,
+ mainCandidateRuns: mainCandidates.length,
+ mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS,
+ manifestTruncated:
+ prRuns.length > MAX_FAILED_RUNS ||
+ batchManifest.truncation?.length > 0 ||
+ batchManifest.selection?.exhaustive === false ||
+ batchManifest.coverage?.exhaustive === false,
+ },
+ counts: Object.fromEntries(
+ ["matching-main-diagnostics", "unknown"].map((category) => [
+ category,
+ entries.filter((item) => item.classification === category).length,
+ ]),
+ ),
+ failures: entries,
+ };
+ await writeFailureReport(report, outputDirectory, summaryPath);
+ return report;
+}
+
+async function analyzeFailedRun({
+ repository,
+ defaultBranch,
+ entry,
+ run,
+ mainCandidates,
+ api,
+}) {
+ const jobs = await api.getJobs(repository, entry.id, entry.runAttempt);
+ if (jobs.some((job) => !hasJobIdentity(job, entry)))
+ throw new Error("workflow_job_identity_mismatch");
+ const failedJobs = jobs.filter(
+ (job) => job.status === "completed" && isFailedConclusion(job.conclusion),
+ );
+ if (failedJobs.length === 0)
+ return [
+ {
+ runId: entry.id,
+ runAttempt: entry.runAttempt,
+ classification: "unknown",
+ reason: "failed_jobs_not_present_in_api_snapshot",
+ },
+ ];
+ const failures = [];
+ for (const job of failedJobs) {
+ const evidence = await api.getStaticEvidence(repository, run, job);
+ const classification = await attributeToMain({
+ repository,
+ defaultBranch,
+ run,
+ job,
+ evidence,
+ mainCandidates,
+ api,
+ });
+ failures.push({
+ runId: entry.id,
+ runAttempt: entry.runAttempt,
+ runHead: safeSha(run.head_sha),
+ event: run.event ?? null,
+ job: job.name,
+ conclusion: job.conclusion,
+ ...classification,
+ });
+ }
+ return failures;
+}
+
+async function attributeToMain({
+ repository,
+ defaultBranch,
+ run,
+ job,
+ evidence,
+ mainCandidates,
+ api,
+}) {
+ const baseSha = pullRequestBaseSha(run, repository);
+ if (baseSha === null || evidence === null)
+ return outcome("unknown", "no_matching_verified_main_failure");
+ let result = outcome("unknown", "no_matching_verified_main_failure");
+ const candidateFailures = [];
+ for (const candidate of mainCandidates) {
+ try {
+ const main = await compareCandidate({
+ repository,
+ defaultBranch,
+ run,
+ job,
+ evidence,
+ baseSha,
+ candidate,
+ api,
+ });
+ if (main !== null) result = main;
+ if (main?.classification === "matching-main-diagnostics") break;
+ } catch (error) {
+ candidateFailures.push({
+ runId: candidate.run.id,
+ runAttempt: candidate.run.run_attempt,
+ reason: errorCode(error),
+ });
+ }
+ }
+ return {
+ ...result,
+ ...(candidateFailures.length > 0 ? { candidateFailures } : {}),
+ };
+}
+
+async function compareCandidate({
+ repository,
+ defaultBranch,
+ run,
+ job,
+ evidence,
+ baseSha,
+ candidate,
+ api,
+}) {
+ const mainRun = candidate.run;
+ const main = await candidate.getEvidence(job.name);
+ if (!main?.evidence) return null;
+ const { job: mainJob, evidence: mainEvidence } = main;
+ const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha);
+ const paths = [
+ ...new Set(evidence.diagnostics.flatMap(diagnosticPaths)),
+ ].slice(0, MAX_PATHS_PER_COMPARISON);
+ const blobs = await api.getBlobIdentity(
+ repository,
+ mainRun.head_sha,
+ baseSha,
+ paths,
+ );
+ const executionSourcesVerified =
+ (await api.verifyExecutionSource(repository, run, evidence.context)) &&
+ (await api.verifyExecutionSource(
+ repository,
+ mainRun,
+ mainEvidence.context,
+ ));
+ return classifyFailure({
+ repository,
+ defaultBranch,
+ run,
+ job,
+ diagnostics: evidence.diagnostics,
+ context: evidence.context,
+ mainRun,
+ mainJob,
+ mainDiagnostics: mainEvidence.diagnostics,
+ mainContext: mainEvidence.context,
+ executionSourcesVerified,
+ ancestry,
+ blobs,
+ });
+}
+
+// Each verified main run/attempt owns its lazy jobs and parsed evidence for this
+// report only. Cache nulls and rejected reads too; retries cannot spend the budget
+// repeatedly on an immutable observation that was unavailable or invalid.
+function mainEvidenceLookup(repository, api, entry, run) {
+ let jobs;
+ const evidence = new Map();
+ return (name) => {
+ jobs ??= Promise.resolve().then(async () => {
+ const items = await api.getJobs(repository, entry.id, entry.runAttempt);
+ return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items;
+ });
+ if (!evidence.has(name))
+ evidence.set(
+ name,
+ jobs.then(async (items) => {
+ const job = items.find(
+ (item) =>
+ item.name === name &&
+ item.status === "completed" &&
+ isFailedConclusion(item.conclusion),
+ );
+ if (!job) return null;
+ return {
+ job,
+ evidence: await api.getStaticEvidence(repository, run, job),
+ };
+ }),
+ );
+ return evidence.get(name);
+ };
+}
+
+// Selection and attribution share one immutable API observation per run/attempt.
+function verifiedRunLookup(repository, api) {
+ const runs = new Map();
+ return (entry) => {
+ const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`;
+ if (!runs.has(key))
+ runs.set(
+ key,
+ Promise.resolve().then(async () => {
+ const run = normalizeWorkflowRun(
+ await api.getRun(repository, entry.id),
+ );
+ if (!hasRunIdentity(run, entry))
+ throw new Error("workflow_run_identity_mismatch");
+ return run;
+ }),
+ );
+ return runs.get(key);
+ };
+}
+
+function hasRunIdentity(run, entry) {
+ return (
+ String(run?.id) === String(entry.id) &&
+ Number(run.run_attempt) === Number(entry.runAttempt) &&
+ run.workflowPath?.file === ".github/workflows/ci.yml"
+ );
+}
+function hasJobIdentity(job, entry) {
+ return (
+ String(job.run_id) === String(entry.id) &&
+ Number(job.run_attempt) === Number(entry.runAttempt)
+ );
+}
+
+export function createGitHubApi({
+ token = process.env.GH_TOKEN,
+ fetchImpl = fetch,
+ apiRequest = createGhApiRequest(),
+} = {}) {
+ if (!token) throw new Error("GH_TOKEN is required");
+ const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 };
+ const request = async (endpoint) => {
+ if (budget.requests >= MAX_API_REQUESTS)
+ throw new Error("github_api_request_budget_exhausted");
+ budget.requests += 1;
+ return await apiRequest("GET", endpoint, budget);
+ };
+ return {
+ async getRun(repository, id) {
+ return await request(
+ `repos/${repository}/actions/runs/${positiveId(id)}`,
+ );
+ },
+ async getJobs(repository, id, attempt) {
+ const page = await request(
+ `repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`,
+ );
+ if (!Array.isArray(page.jobs) || page.jobs.length >= 100)
+ throw new Error("job_list_incomplete_or_malformed");
+ return page.jobs;
+ },
+ async getStaticEvidence(repository, run, job) {
+ if (job.name !== "Static checks") return null;
+ if (run.run_attempt !== 1) return null;
+ const page = await request(
+ `repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`,
+ );
+ const artifact = selectStaticCheckArtifact(page, run, run.run_attempt);
+ if (!artifact) return null;
+ return await readStaticCheckArtifact({
+ repository,
+ artifactId: artifact.id,
+ token,
+ fetchImpl,
+ });
+ },
+ async getAncestry(repository, ancestor, descendant) {
+ const comparison = await request(
+ `repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`,
+ );
+ return {
+ status: comparison.status,
+ mergeBaseSha: safeSha(comparison.merge_base_commit?.sha),
+ };
+ },
+ async verifyExecutionSource(repository, run, context) {
+ const sourceSha = safeSha(context.sourceSha);
+ if (!sourceSha) return false;
+ const pr =
+ run.event === "pull_request"
+ ? pullRequestBaseSha(run, repository)
+ : null;
+ if (
+ run.event === "pull_request" &&
+ (pr === null || context.prHeadSha !== run.head_sha)
+ )
+ return false;
+ const commit = await request(`repos/${repository}/commits/${sourceSha}`);
+ if (safeSha(commit.sha) !== sourceSha) return false;
+ if (sourceSha === run.head_sha) return true;
+ if (
+ pr === null ||
+ !Array.isArray(commit.parents) ||
+ commit.parents.length !== 2
+ )
+ return false;
+ return (
+ safeSha(commit.parents[0]?.sha) === pr &&
+ safeSha(commit.parents[1]?.sha) === run.head_sha
+ );
+ },
+ async getBlobIdentity(repository, ancestor, descendant, paths) {
+ const result = {};
+ for (const path of paths) {
+ if (!validRepoPath(path)) continue;
+ const [left, right] = await Promise.all([
+ getContent(repository, ancestor, path, request),
+ getContent(repository, descendant, path, request),
+ ]);
+ result[path] = left !== null && left === right;
+ }
+ return result;
+ },
+ };
+}
+
+async function getContent(repository, ref, path, request) {
+ try {
+ const encodedPath = path.split("/").map(encodeURIComponent).join("/");
+ const value = await request(
+ `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
+ );
+ return typeof value.sha === "string" ? value.sha : null;
+ } catch {
+ return null;
+ }
+}
+
+function outcome(classification, reason) {
+ return { classification, reason };
+}
+function validateRepository(value) {
+ if (
+ typeof value !== "string" ||
+ !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)
+ )
+ throw new Error("repository must be owner/name");
+}
+function positiveId(value) {
+ if (!Number.isSafeInteger(Number(value)) || Number(value) < 1)
+ throw new Error(
+ "GitHub run and attempt IDs must be positive safe integers",
+ );
+ return String(Number(value));
+}
+function errorCode(error) {
+ return error instanceof Error ? error.message.slice(0, 160) : "unknown_error";
+}
+
+export async function writeFailureReport(
+ report,
+ outputDirectory = OUTPUT_DIRECTORY,
+ summaryPath,
+) {
+ await mkdir(outputDirectory, { recursive: true });
+ const jsonPath = join(outputDirectory, "report.json");
+ const markdownPath = join(outputDirectory, "report.md");
+ const matchedDiagnostics = report.failures.filter(
+ (item) => item.classification === "matching-main-diagnostics",
+ );
+ const lines = [
+ "# CI failure attribution",
+ "",
+ "Informational only. This report never changes a CI conclusion or required gate.",
+ "",
+ `Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`,
+ "",
+ ];
+ for (const item of report.failures)
+ lines.push(
+ `- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`,
+ );
+ if (report.failures.length === 0)
+ lines.push(
+ "- No completed failed CI attempts were present in the supplied timing snapshot.",
+ );
+ await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+ await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8");
+ if (summaryPath)
+ await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8");
+ return { jsonPath, markdownPath };
+}
+
+async function main() {
+ const repository = process.env.CI_REPOSITORY;
+ const defaultBranch = process.env.CI_DEFAULT_BRANCH;
+ const batchPath =
+ process.env.CI_TIMING_MANIFEST ??
+ join(BATCH_DIRECTORY, "batch-manifest.json");
+ const batchManifest = JSON.parse(await readBoundedFile(batchPath));
+ const api = createGitHubApi();
+ await reportCiFailures({
+ repository,
+ defaultBranch,
+ batchManifest,
+ api,
+ outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY,
+ summaryPath: process.env.GITHUB_STEP_SUMMARY,
+ });
+}
+async function readBoundedFile(path) {
+ const bytes = await readFile(path);
+ if (bytes.byteLength > MAX_JSON_BYTES)
+ throw new Error("timing manifest exceeds size limit");
+ return bytes.toString("utf8");
+}
+
+if (
+ process.argv[1] &&
+ resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+ try {
+ await main();
+ } catch (error) {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exitCode = 1;
+ }
+}
diff --git a/scripts/ci/github-api.d.mts b/scripts/ci/github-api.d.mts
new file mode 100644
index 000000000..67e157e6d
--- /dev/null
+++ b/scripts/ci/github-api.d.mts
@@ -0,0 +1,16 @@
+export interface GitHubApiBudget {
+ requests: number;
+ startedAt: number;
+ limitMs: number;
+}
+
+export type GitHubApiRequest = (
+ method: string,
+ endpoint: string,
+ budget: GitHubApiBudget,
+) => Promise<unknown>;
+
+export function createGhApiRequest(options?: {
+ env?: NodeJS.ProcessEnv;
+ now?: () => number;
+}): GitHubApiRequest;
diff --git a/scripts/ci/github-api.mjs b/scripts/ci/github-api.mjs
new file mode 100644
index 000000000..9567bf93b
--- /dev/null
+++ b/scripts/ci/github-api.mjs
@@ -0,0 +1,33 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+const MAX_BUFFER = 8 * 1024 * 1024;
+const REQUEST_TIMEOUT_MS = 10_000;
+
+/** Create a bounded GitHub CLI JSON transport with GET-only authority. */
+export function createGhApiRequest({ env = process.env, now = Date.now } = {}) {
+ return async (method, endpoint, budget) => {
+ if (method !== "GET")
+ throw new Error(
+ "GitHub API transport supports read-only GET requests only",
+ );
+ const remaining = budget.limitMs - (now() - budget.startedAt);
+ if (remaining <= 0) throw new Error("lookup_time_budget_exhausted");
+ const { stdout } = await execFileAsync(
+ "gh",
+ ["api", "--method", "GET", endpoint],
+ {
+ env,
+ encoding: "utf8",
+ maxBuffer: MAX_BUFFER,
+ timeout: Math.min(REQUEST_TIMEOUT_MS, remaining),
+ },
+ );
+ try {
+ return JSON.parse(stdout);
+ } catch (cause) {
+ throw new Error("github_api_malformed_json", { cause });
+ }
+ };
+}
diff --git a/scripts/ci/job-timings-batch.mjs b/scripts/ci/job-timings-batch.mjs
index 842e884f0..f2dca7143 100644
--- a/scripts/ci/job-timings-batch.mjs
+++ b/scripts/ci/job-timings-batch.mjs
@@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({
id: run.id,
runAttempt: run.run_attempt,
workflow: workflow.name,
+ event: typeof run.event === "string" ? run.event : null,
+ headBranch:
+ typeof run.head_branch === "string" ? run.head_branch : null,
conclusion: run.conclusion ?? null,
createdAt: run.created_at ?? null,
});
diff --git a/scripts/ci/pr-ci-health.d.mts b/scripts/ci/pr-ci-health.d.mts
new file mode 100644
index 000000000..0db2454bd
--- /dev/null
+++ b/scripts/ci/pr-ci-health.d.mts
@@ -0,0 +1,95 @@
+export interface PrCiHead {
+ repositoryId: number | null;
+ repository?: string | null;
+ ref: string | null;
+ sha: string | null;
+}
+
+export interface PrCiPullRequest {
+ number: number | null;
+ draft?: boolean;
+ identityValid: boolean;
+ mergeable?: boolean;
+ head: PrCiHead;
+ [key: string]: unknown;
+}
+
+export interface PrCiWorkflowRun {
+ id?: number;
+ run_attempt?: number;
+ event?: string;
+ head_branch?: string;
+ head_repository?: { id?: number | null } | null;
+ head_sha?: string;
+ status?: string;
+ conclusion?: string | null;
+ created_at?: string;
+ html_url?: string;
+ pull_requests?: Array<{
+ number?: number;
+ head?: { sha?: string; repo?: { id?: number | null } } | null;
+ }>;
+ [key: string]: unknown;
+}
+
+export interface PrCiClassification {
+ category: string;
+ reason: string;
+ runs: Array<Record<string, unknown>>;
+}
+
+export interface PrCiHealthReport {
+ schemaVersion: 1;
+ observedAt: string;
+ repository: string;
+ repositoryId: number | null;
+ defaultBranch: string;
+ lookbackDays: number;
+ scan: {
+ status: "complete" | "partial";
+ cutoff: string;
+ pullRequestPages: number;
+ workflowRunPages: number;
+ pullRequestInventoryComplete: boolean;
+ workflowRunInventoryComplete: boolean;
+ pullRequestDetailsFetched: number;
+ pullRequestDetailsLimit: number;
+ apiRequests: number;
+ apiRequestLimit: number;
+ errors: string[];
+ };
+ counts: Record<string, number>;
+ pullRequests: Array<PrCiPullRequest & { ci: PrCiClassification }>;
+}
+
+export function collectPrCiHealth(options: {
+ repository: string;
+ repositoryId?: number | string | null;
+ defaultBranch: string;
+ lookbackDays?: number;
+ now?: number;
+ apiRequest?: GitHubApiRequest;
+}): Promise<PrCiHealthReport>;
+
+export function classifyPrCi(input: {
+ pull: PrCiPullRequest;
+ runs: Array<{ run: PrCiWorkflowRun; basis: string }>;
+ runsComplete: boolean;
+ pullRequestInventoryComplete: boolean;
+ ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>;
+}): PrCiClassification;
+
+export function matchRunToPullRequest(input: {
+ run: PrCiWorkflowRun;
+ pull: PrCiPullRequest;
+ identityCounts: Map<string, number>;
+ inventoryComplete?: boolean;
+}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string };
+
+export function writePrCiHealthReport(
+ report: PrCiHealthReport,
+ outputDirectory?: string,
+): Promise<{ jsonPath: string; markdownPath: string }>;
+
+export function renderMarkdownSummary(report: PrCiHealthReport): string;
+import type { GitHubApiRequest } from "./github-api.mjs";
diff --git a/scripts/ci/pr-ci-health.mjs b/scripts/ci/pr-ci-health.mjs
new file mode 100644
index 000000000..d312dcecd
--- /dev/null
+++ b/scripts/ci/pr-ci-health.mjs
@@ -0,0 +1,718 @@
+import { appendFile, mkdir, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { createGhApiRequest } from "./github-api.mjs";
+
+const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health");
+const PAGE_SIZE = 100;
+const MAX_PULL_REQUEST_PAGES = 3;
+const MAX_WORKFLOW_RUN_PAGES = 5;
+const MAX_PULL_REQUEST_DETAILS = 40;
+const MAX_API_REQUESTS =
+ MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS;
+const LOOKBACK_DAYS = 14;
+const MAX_LOOKBACK_DAYS = 30;
+const MAX_TOTAL_TIME_MS = 60_000;
+const SHA_PATTERN = /^[0-9a-f]{40}$/iu;
+const COMPLETED_CATEGORIES = new Map([
+ ["failure", "completed_failure"],
+ ["timed_out", "completed_failure"],
+ ["cancelled", "completed_canceled"],
+ ["success", "completed_success"],
+]);
+
+/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */
+export async function collectPrCiHealth({
+ repository,
+ repositoryId,
+ defaultBranch,
+ lookbackDays = LOOKBACK_DAYS,
+ now = Date.now(),
+ apiRequest = createGhApiRequest(),
+}) {
+ const identity = validateInputs({
+ repository,
+ repositoryId,
+ defaultBranch,
+ lookbackDays,
+ now,
+ });
+ const observedAt = new Date(now).toISOString();
+ const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000;
+ const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff });
+ const classified = classifyPullRequests(snapshot);
+ const { pulls, runs, budget } = snapshot;
+ const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter(
+ Boolean,
+ );
+ return {
+ schemaVersion: 1,
+ observedAt,
+ repository: identity.repository,
+ repositoryId: identity.repositoryId,
+ defaultBranch: identity.defaultBranch,
+ lookbackDays: identity.lookbackDays,
+ scan: {
+ status:
+ errors.length === 0 && pulls.complete && runs.complete
+ ? "complete"
+ : "partial",
+ cutoff: new Date(cutoff).toISOString(),
+ pullRequestPages: pulls.pages,
+ workflowRunPages: runs.pages,
+ pullRequestInventoryComplete: pulls.complete,
+ workflowRunInventoryComplete: runs.complete,
+ pullRequestDetailsFetched: snapshot.detailsFetched,
+ pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS,
+ apiRequests: budget.requests,
+ apiRequestLimit: MAX_API_REQUESTS,
+ errors,
+ },
+ counts: countCategories(classified),
+ pullRequests: classified,
+ };
+}
+
+async function collectApiSnapshot({ apiRequest, identity, cutoff }) {
+ const budget = {
+ requests: 0,
+ startedAt: Date.now(),
+ limitMs: MAX_TOTAL_TIME_MS,
+ };
+ const requestGet = async (endpoint) => {
+ if (budget.requests >= MAX_API_REQUESTS)
+ throw new Error("lookup_request_budget_exhausted");
+ if (Date.now() - budget.startedAt >= budget.limitMs)
+ throw new Error("lookup_time_budget_exhausted");
+ budget.requests += 1;
+ return await apiRequest("GET", endpoint, budget);
+ };
+ const pulls = await listOpenPullRequests({
+ requestGet,
+ identity,
+ });
+ const candidates = pulls.items.filter(
+ (pull) =>
+ !pull.draft &&
+ Number.isFinite(Date.parse(pull.createdAt ?? "")) &&
+ Date.parse(pull.createdAt ?? "") >= cutoff &&
+ pull.base.ref === identity.defaultBranch &&
+ (identity.repositoryId === null ||
+ pull.base.repositoryId === identity.repositoryId),
+ );
+ const runs = await listRecentWorkflowRuns({
+ requestGet,
+ identity,
+ cutoff,
+ });
+
+ const detailResult = await resolveMergeability({
+ candidates,
+ requestGet,
+ identity,
+ });
+ return {
+ budget,
+ pulls,
+ runs,
+ candidates,
+ detailsFetched: detailResult.fetched,
+ detailsErrors: detailResult.errors,
+ };
+}
+
+function classifyPullRequests(snapshot) {
+ const { candidates, pulls, runs } = snapshot;
+ const identityCounts = countHeadIdentities(pulls.items);
+ return candidates.map((pull) => {
+ const relevant = [];
+ const ambiguousRuns = [];
+ for (const run of runs.items) {
+ const match = matchRunToPullRequest({
+ run,
+ pull,
+ identityCounts,
+ inventoryComplete: pulls.complete,
+ });
+ if (match.kind === "match") relevant.push({ run, basis: match.basis });
+ if (match.kind === "ambiguous")
+ ambiguousRuns.push({ run, basis: "ambiguous_head_identity" });
+ }
+ return {
+ ...pull,
+ ci: classifyPrCi({
+ pull,
+ runs: relevant,
+ runsComplete: runs.complete,
+ pullRequestInventoryComplete: pulls.complete,
+ ambiguousRuns,
+ }),
+ };
+ });
+}
+
+async function resolveMergeability({ candidates, requestGet, identity }) {
+ let detailsFetched = 0;
+ const errors = new Set();
+ for (const pull of candidates) {
+ if (pull.mergeable !== undefined) continue;
+ if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) {
+ errors.add("pull_request_mergeability_lookup_budget_exhausted");
+ continue;
+ }
+ const result = await getPullRequestDetails({ requestGet, identity, pull });
+ detailsFetched += 1;
+ if (result.error !== null) {
+ errors.add(result.error);
+ continue;
+ }
+ const { detail } = result;
+ const identityChanged =
+ detail.headSha !== pull.head.sha ||
+ detail.headRef !== pull.head.ref ||
+ detail.headRepositoryId !== pull.head.repositoryId ||
+ detail.baseRef !== pull.base.ref ||
+ detail.baseRepositoryId !== pull.base.repositoryId;
+ if (identityChanged) {
+ pull.identityValid = false;
+ pull.identityChangedDuringScan = true;
+ errors.add("pull_request_identity_changed_during_scan");
+ } else {
+ pull.mergeable = detail.mergeable;
+ pull.detailFetched = true;
+ if (detail.mergeable === undefined)
+ errors.add("pull_request_mergeability_unknown");
+ }
+ }
+ return { fetched: detailsFetched, errors: [...errors] };
+}
+
+function countCategories(pulls) {
+ const categories = [
+ "missing",
+ "startup_failure",
+ "queued_pending_executing",
+ "approval_required",
+ "completed_success",
+ "completed_failure",
+ "completed_canceled",
+ "completed_other",
+ "merge_conflict",
+ "unknown",
+ "deferred",
+ ];
+ return Object.fromEntries(
+ categories.map((category) => [
+ category,
+ pulls.filter(({ ci }) => ci.category === category).length,
+ ]),
+ );
+}
+
+/** Pure status classifier; missing requires complete, identity-safe inventories. */
+export function classifyPrCi({
+ pull,
+ runs,
+ runsComplete,
+ pullRequestInventoryComplete,
+ ambiguousRuns,
+}) {
+ if (!pull.identityValid)
+ return result("unknown", "pull_request_head_identity_invalid", []);
+ const evidence = [...runs, ...ambiguousRuns];
+ if (pull.mergeable === false)
+ return result("merge_conflict", "github_reports_merge_conflict", evidence);
+ // The newest collected run is not proven current until pagination completes:
+ // an older-created run outside this snapshot may have been retried recently.
+ if (!runsComplete)
+ return result(
+ evidence.length > 0 ? "unknown" : "deferred",
+ evidence.length > 0
+ ? "workflow_run_inventory_incomplete"
+ : "lookup_inventory_incomplete",
+ evidence,
+ );
+ const selection = newestRun(runs);
+ if (!selection.known)
+ return result("unknown", "workflow_run_order_unknown", evidence);
+ const latest = selection.latest;
+ if (
+ ambiguousRuns.some((candidate) => {
+ const order = latest ? compareRunEvidence(candidate, latest) : null;
+ return order === null || order >= 0;
+ })
+ )
+ return result("unknown", "workflow_run_identity_ambiguous", evidence);
+ if (latest !== undefined) {
+ const run = latest.run;
+ if (run.conclusion === "startup_failure")
+ return result("startup_failure", "workflow_startup_failure", evidence);
+ if (run.status === "waiting" || run.conclusion === "action_required")
+ return result(
+ "approval_required",
+ "workflow_waiting_for_approval_or_gate",
+ evidence,
+ );
+ if (["queued", "requested", "pending", "in_progress"].includes(run.status))
+ return result(
+ "queued_pending_executing",
+ `workflow_${run.status}`,
+ evidence,
+ );
+ if (run.status === "completed")
+ return result(
+ COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other",
+ run.conclusion === "success"
+ ? "workflow_succeeded"
+ : `workflow_${run.conclusion ?? "unknown_conclusion"}`,
+ evidence,
+ );
+ return result("unknown", "workflow_status_unrecognized", evidence);
+ }
+ if (!pullRequestInventoryComplete)
+ return result("deferred", "lookup_inventory_incomplete", []);
+ if (pull.mergeable !== true)
+ return result("unknown", "pull_request_mergeability_unknown", []);
+ return result("missing", "no_pull_request_event_run_for_current_head", []);
+}
+
+/** Match only a pull_request event proven to belong to this immutable PR head. */
+export function matchRunToPullRequest({
+ run,
+ pull,
+ identityCounts,
+ inventoryComplete = true,
+}) {
+ if (run?.event !== "pull_request") return { kind: "none" };
+ if (!pull.identityValid || !isRecord(run)) return { kind: "none" };
+ const association = matchExplicitPullAssociation(run, pull);
+ if (association === "match")
+ return { kind: "match", basis: "event_pr_number_and_head_sha" };
+ if (association === "ambiguous") return { kind: "ambiguous" };
+ if (association === "other" || !sameHeadRepositoryAndRef(run, pull))
+ return { kind: "none" };
+ if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete))
+ return { kind: "ambiguous" };
+ if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha)
+ return { kind: "match", basis: "event_unique_head_ref_and_repository_id" };
+ return { kind: "none" };
+}
+
+function matchExplicitPullAssociation(run, pull) {
+ const associations = Array.isArray(run.pull_requests)
+ ? run.pull_requests
+ : [];
+ if (associations.length === 0) return "absent";
+ const matchingNumber = associations.filter(
+ (item) => item?.number === pull.number,
+ );
+ if (matchingNumber.length === 0) return "other";
+ const exactHead = matchingNumber.find(
+ (item) => item?.head?.sha === pull.head.sha,
+ );
+ if (exactHead === undefined) {
+ return matchingNumber.some(
+ (item) =>
+ typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha,
+ )
+ ? "other"
+ : "absent";
+ }
+ const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id);
+ const runHeadRepositoryId = positiveInteger(run.head_repository?.id);
+ if (associatedHeadRepositoryId === null || runHeadRepositoryId === null)
+ return "ambiguous";
+ if (
+ associatedHeadRepositoryId !== pull.head.repositoryId ||
+ runHeadRepositoryId !== pull.head.repositoryId
+ )
+ return "other";
+ const baseRefMatches =
+ typeof exactHead.base?.ref !== "string" ||
+ exactHead.base.ref === pull.base.ref;
+ const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id);
+ const baseRepositoryMatches =
+ baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId;
+ return baseRefMatches && baseRepositoryMatches ? "match" : "other";
+}
+
+function sameHeadRepositoryAndRef(run, pull) {
+ return (
+ run.head_branch === pull.head.ref &&
+ run.head_repository?.id === pull.head.repositoryId
+ );
+}
+
+function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) {
+ if (!inventoryComplete) return false;
+ const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
+ const unknownRepositoryKey = headIdentityKey(null, pull.head.ref);
+ return (
+ identityCounts.get(key) === 1 &&
+ (identityCounts.get(unknownRepositoryKey) ?? 0) === 0
+ );
+}
+
+function normalizePullRequest(raw, identity) {
+ const number = positiveInteger(raw?.number);
+ const base = normalizeBase(raw?.base);
+ const head = normalizeHead(raw?.head);
+ const draft = raw?.draft === true;
+ return {
+ number,
+ draft,
+ createdAt: typeof raw?.created_at === "string" ? raw.created_at : null,
+ url: typeof raw?.html_url === "string" ? raw.html_url : null,
+ base,
+ head,
+ mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined,
+ identityValid: hasValidPullIdentity({ raw, identity, number, base, head }),
+ };
+}
+
+function normalizeBase(raw) {
+ return {
+ ref: typeof raw?.ref === "string" ? raw.ref : null,
+ repositoryId: positiveInteger(raw?.repo?.id),
+ };
+}
+
+function normalizeHead(raw) {
+ return {
+ repositoryId: positiveInteger(raw?.repo?.id),
+ repository:
+ typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null,
+ ref: typeof raw?.ref === "string" ? raw.ref : null,
+ sha:
+ typeof raw?.sha === "string" && isSha(raw.sha)
+ ? raw.sha.toLowerCase()
+ : null,
+ };
+}
+
+function hasValidPullIdentity({ raw, identity, number, base, head }) {
+ return (
+ number !== null &&
+ typeof raw?.draft === "boolean" &&
+ base.ref === identity.defaultBranch &&
+ (identity.repositoryId === null ||
+ base.repositoryId === identity.repositoryId) &&
+ head.repositoryId !== null &&
+ head.sha !== null &&
+ head.ref !== null
+ );
+}
+
+function countHeadIdentities(pulls) {
+ const counts = new Map();
+ for (const pull of pulls) {
+ if (typeof pull.head.ref !== "string") continue;
+ const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
+ counts.set(key, (counts.get(key) ?? 0) + 1);
+ }
+ return counts;
+}
+
+function headIdentityKey(repositoryId, ref) {
+ return `${String(repositoryId)}\u0000${ref}`;
+}
+
+function newestRun(runs) {
+ let latest = runs[0];
+ for (const candidate of runs) {
+ if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0)
+ latest = candidate;
+ }
+ const known = runs.every((candidate) => {
+ if (candidate === latest) return true;
+ const order = compareRunEvidence(candidate, latest);
+ return order !== null && order <= 0;
+ });
+ return { known, latest };
+}
+
+// Attempt ordering is meaningful within one run; different run IDs are not clocks.
+function compareRunEvidence(left, right) {
+ const leftRun = left.run;
+ const rightRun = right.run;
+ const id = positiveInteger(leftRun.id);
+ if (id !== null && id === positiveInteger(rightRun.id)) {
+ const leftAttempt = positiveInteger(leftRun.run_attempt);
+ const rightAttempt = positiveInteger(rightRun.run_attempt);
+ if (leftAttempt === null || rightAttempt === null) return null;
+ if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt;
+ return leftRun.status === rightRun.status &&
+ leftRun.conclusion === rightRun.conclusion
+ ? 0
+ : null;
+ }
+ // A retry may start long after its original creation; without an attempt
+ // clock, creation dates only order first attempts from different runs.
+ if (
+ positiveInteger(leftRun.run_attempt) !== 1 ||
+ positiveInteger(rightRun.run_attempt) !== 1
+ )
+ return null;
+ const time =
+ Date.parse(leftRun.created_at ?? "") -
+ Date.parse(rightRun.created_at ?? "");
+ return Number.isFinite(time) && time !== 0 ? time : null;
+}
+
+function result(category, reason, runs) {
+ return {
+ category,
+ reason,
+ runs: runs.map(({ run, basis }) => ({
+ id: positiveInteger(run.id),
+ attempt: positiveInteger(run.run_attempt),
+ status: typeof run.status === "string" ? run.status : null,
+ conclusion: typeof run.conclusion === "string" ? run.conclusion : null,
+ headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null,
+ createdAt: typeof run.created_at === "string" ? run.created_at : null,
+ url: typeof run.html_url === "string" ? run.html_url : null,
+ identityBasis: basis,
+ })),
+ };
+}
+
+async function listOpenPullRequests({ requestGet, identity }) {
+ const items = [];
+ let complete = false;
+ let pages = 0;
+ let error = null;
+ for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) {
+ try {
+ const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`;
+ const response = await requestGet(endpoint);
+ if (!Array.isArray(response))
+ throw new Error("pull_request_page_malformed");
+ pages += 1;
+ const normalized = response.map((raw) =>
+ normalizePullRequest(raw, identity),
+ );
+ if (
+ normalized.some(
+ (pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")),
+ )
+ )
+ throw new Error("pull_request_page_malformed");
+ items.push(...normalized);
+ if (response.length < PAGE_SIZE) {
+ complete = true;
+ break;
+ }
+ } catch (cause) {
+ error = safeErrorCode(cause);
+ break;
+ }
+ }
+ return { items, pages, complete, error };
+}
+
+async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) {
+ const items = [];
+ let complete = false;
+ let pages = 0;
+ let error = null;
+ for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) {
+ try {
+ const cutoffDate = new Date(cutoff).toISOString().slice(0, 10);
+ const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`;
+ const response = await requestGet(endpoint);
+ if (!isRecord(response) || !Array.isArray(response.workflow_runs))
+ throw new Error("workflow_run_page_malformed");
+ pages += 1;
+ if (!response.workflow_runs.every(isRecord))
+ throw new Error("workflow_run_page_malformed");
+ items.push(...response.workflow_runs);
+ if (response.workflow_runs.length < PAGE_SIZE) {
+ complete = true;
+ break;
+ }
+ } catch (cause) {
+ error = safeErrorCode(cause);
+ break;
+ }
+ }
+ return { items, pages, complete, error };
+}
+
+async function getPullRequestDetails({ requestGet, identity, pull }) {
+ if (pull.number === null)
+ return { detail: null, error: "pull_request_detail_identity_missing" };
+ try {
+ const response = await requestGet(
+ `repos/${identity.repository}/pulls/${pull.number}`,
+ );
+ if (!isRecord(response))
+ return { detail: null, error: "pull_request_detail_malformed" };
+ return {
+ detail: {
+ mergeable:
+ typeof response.mergeable === "boolean"
+ ? response.mergeable
+ : undefined,
+ headSha: isSha(response.head?.sha)
+ ? response.head.sha.toLowerCase()
+ : null,
+ headRef:
+ typeof response.head?.ref === "string" ? response.head.ref : null,
+ headRepositoryId: positiveInteger(response.head?.repo?.id),
+ baseRef:
+ typeof response.base?.ref === "string" ? response.base.ref : null,
+ baseRepositoryId: positiveInteger(response.base?.repo?.id),
+ },
+ error: null,
+ };
+ } catch (cause) {
+ return { detail: null, error: safeErrorCode(cause) };
+ }
+}
+
+function validateInputs({
+ repository,
+ repositoryId,
+ defaultBranch,
+ lookbackDays,
+ now,
+}) {
+ if (
+ typeof repository !== "string" ||
+ !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
+ )
+ throw new Error(
+ "CI_REPOSITORY must be an owner/name repository identifier",
+ );
+ if (
+ typeof defaultBranch !== "string" ||
+ defaultBranch.length === 0 ||
+ /[\r\n]/u.test(defaultBranch)
+ )
+ throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name");
+ const parsedRepositoryId =
+ repositoryId === undefined || repositoryId === null || repositoryId === ""
+ ? null
+ : positiveInteger(repositoryId);
+ if (
+ repositoryId !== undefined &&
+ repositoryId !== null &&
+ repositoryId !== "" &&
+ parsedRepositoryId === null
+ )
+ throw new Error("CI_REPOSITORY_ID must be a positive safe integer");
+ if (
+ !Number.isInteger(lookbackDays) ||
+ lookbackDays < 1 ||
+ lookbackDays > MAX_LOOKBACK_DAYS
+ )
+ throw new Error(
+ `lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`,
+ );
+ if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp");
+ return {
+ repository,
+ repositoryId: parsedRepositoryId,
+ defaultBranch,
+ lookbackDays,
+ };
+}
+
+function positiveInteger(value) {
+ const number = typeof value === "number" ? value : Number(value);
+ return Number.isSafeInteger(number) && number > 0 ? number : null;
+}
+
+function isSha(value) {
+ return typeof value === "string" && SHA_PATTERN.test(value);
+}
+
+function isRecord(value) {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function safeErrorCode(cause) {
+ if (cause?.message === "lookup_time_budget_exhausted")
+ return "lookup_time_budget_exhausted";
+ if (cause?.message === "lookup_request_budget_exhausted")
+ return "lookup_request_budget_exhausted";
+ if (cause?.message === "github_api_malformed_json")
+ return "github_api_malformed_json";
+ if (cause?.message?.endsWith("_malformed")) return cause.message;
+ if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout";
+ return "github_api_request_failed";
+}
+
+export async function writePrCiHealthReport(
+ report,
+ outputDirectory = OUTPUT_DIRECTORY,
+) {
+ const jsonPath = join(outputDirectory, "report.json");
+ const markdownPath = join(outputDirectory, "summary.md");
+ await mkdir(outputDirectory, { recursive: true });
+ await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+ await writeFile(markdownPath, renderMarkdownSummary(report), "utf8");
+ return { jsonPath, markdownPath };
+}
+
+export function renderMarkdownSummary(report) {
+ const rows = report.pullRequests.map(
+ (pull) =>
+ `| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`,
+ );
+ return [
+ "# Pull request CI health snapshot",
+ "",
+ `Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`,
+ "",
+ "This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.",
+ "",
+ `API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`,
+ "",
+ "| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |",
+ "| ---: | --- | --- | --- | --- | --- |",
+ ...rows,
+ "",
+ "Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.",
+ "",
+ ].join("\n");
+}
+
+function escapeCell(value) {
+ return String(value)
+ .replaceAll("|", "\\|")
+ .replaceAll("\n", " ")
+ .replaceAll("\r", " ");
+}
+
+if (
+ process.argv[1] !== undefined &&
+ resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+ try {
+ const report = await collectPrCiHealth({
+ repository: process.env.CI_REPOSITORY,
+ repositoryId: process.env.CI_REPOSITORY_ID,
+ defaultBranch: process.env.CI_DEFAULT_BRANCH,
+ });
+ const paths = await writePrCiHealthReport(
+ report,
+ process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY,
+ );
+ if (process.env.GITHUB_STEP_SUMMARY)
+ await appendFile(
+ process.env.GITHUB_STEP_SUMMARY,
+ renderMarkdownSummary(report),
+ "utf8",
+ );
+ process.stdout.write(
+ `Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`,
+ );
+ } catch (cause) {
+ process.stderr.write(
+ `PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`,
+ );
+ process.exitCode = 1;
+ }
+}
diff --git a/scripts/ci/qualification.mjs b/scripts/ci/qualification.mjs
index ab47c168f..8dfb3f20c 100644
--- a/scripts/ci/qualification.mjs
+++ b/scripts/ci/qualification.mjs
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
+import { normalizeWorkflowRun } from "./workflow-run.mjs";
const ARTIFACT_NAME = "ci-full-qualification-v1";
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
@@ -119,8 +120,8 @@ function admit() {
// Artifact metadata narrows the search before run lookups. Never scan an
// unbounded history or treat a missing receipt as a successful qualification.
for (const artifact of candidates.slice(0, 3)) {
- const run = api(
- `repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
+ const run = normalizeWorkflowRun(
+ api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
);
// An old run can upload a new artifact when only its gate is rerun.
// Bound the original run age as well so retries cannot renew stale proof.
@@ -133,7 +134,7 @@ function admit() {
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
- run.path === ".github/workflows/ci.yml" &&
+ run.workflowPath?.file === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_sha === head &&
run.head_branch === branch &&
diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs
index 20fbac23f..1839a1149 100644
--- a/scripts/ci/required.mjs
+++ b/scripts/ci/required.mjs
@@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
const required = reused ? ["changes"] : ["changes", "static", "test"];
const lanes = {
- code: ["build", "test-shard"],
+ code: [
+ "build",
+ "test-shard",
+ ...(outputs.tier === "fast" ? ["test-source"] : []),
+ ],
reverse_tools: ["reverse-tools-quality"],
runtime: ["windows-native-build", "package-e2e"],
package: ["windows-curated"],
diff --git a/scripts/ci/static-check-report.mjs b/scripts/ci/static-check-report.mjs
index 04460b2c2..eecaf750a 100644
--- a/scripts/ci/static-check-report.mjs
+++ b/scripts/ci/static-check-report.mjs
@@ -133,6 +133,8 @@ try {
for (const clone of report.duplicates)
for (const file of [clone.firstFile, clone.secondFile])
file.name = relative(process.cwd(), file.name).split(sep).join("/");
+ // Record the actual checker result, including caller-selected thresholds.
+ report.reaCheck = { exitCode: result.status };
await writeFile(reportPath, JSON.stringify(report, null, 2));
const { sources, clones, percentage } = report.statistics.total;
console.log(
diff --git a/scripts/ci/static-task-context.d.mts b/scripts/ci/static-task-context.d.mts
new file mode 100644
index 000000000..164756b15
--- /dev/null
+++ b/scripts/ci/static-task-context.d.mts
@@ -0,0 +1,18 @@
+export interface StaticTaskContext {
+ readonly version: 1;
+ readonly sourceSha: string;
+ readonly sourceClean: boolean;
+ readonly prHeadSha?: string;
+ readonly nodeVersion: string;
+ readonly npmVersion: string;
+ readonly turboVersion: string;
+ readonly platform: string;
+ readonly architecture: string;
+ readonly taskHashes: Readonly<Record<string, string>>;
+}
+export function parseStaticTaskContext(value: unknown): StaticTaskContext;
+export function recordStaticTaskContext(options?: {
+ root?: string;
+ npmCli?: string;
+ outputDirectory?: string;
+}): Promise<StaticTaskContext>;
diff --git a/scripts/ci/static-task-context.mjs b/scripts/ci/static-task-context.mjs
new file mode 100644
index 000000000..879c568d1
--- /dev/null
+++ b/scripts/ci/static-task-context.mjs
@@ -0,0 +1,104 @@
+import { execFile } from "node:child_process";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { z } from "zod";
+
+const execute = promisify(execFile);
+const contextSchema = z.object({
+ version: z.literal(1),
+ sourceSha: z.string().regex(/^[a-f0-9]{40}$/u),
+ sourceClean: z.boolean(),
+ prHeadSha: z
+ .string()
+ .regex(/^[a-f0-9]{40}$/u)
+ .optional(),
+ nodeVersion: z.string().min(1),
+ npmVersion: z.string().min(1),
+ turboVersion: z.string().min(1),
+ platform: z.string().min(1),
+ architecture: z.string().min(1),
+ taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)),
+});
+
+/** Parse the producing check graph's receipt; incomplete provenance is unknown. */
+export const parseStaticTaskContext = (value) => contextSchema.parse(value);
+
+/** Retain only named task hashes and tool/source identity, never the environment. */
+export async function recordStaticTaskContext({
+ root = process.cwd(),
+ npmCli = process.env.npm_execpath,
+ outputDirectory = join(root, ".cache/static-checks"),
+} = {}) {
+ if (typeof npmCli !== "string" || !npmCli)
+ throw new Error(
+ "Run static context recording through npm run ci:static-context",
+ );
+ const options = {
+ cwd: root,
+ encoding: "utf8",
+ timeout: 30_000,
+ maxBuffer: 32 * 1024 * 1024,
+ };
+ const [graph, source, status, npmVersion, turbo] = await Promise.all([
+ execute(
+ process.execPath,
+ [npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"],
+ options,
+ ),
+ execute("git", ["rev-parse", "HEAD"], options),
+ execute(
+ "git",
+ ["status", "--porcelain", "--untracked-files=normal"],
+ options,
+ ),
+ execute(process.execPath, [npmCli, "--version"], options),
+ readFile(join(root, "node_modules/turbo/package.json"), "utf8"),
+ ]);
+ const tasks = z
+ .object({
+ tasks: z
+ .array(
+ z.object({
+ task: z.string().min(1),
+ hash: z.string().regex(/^[a-f0-9]{16}$/u),
+ }),
+ )
+ .min(1),
+ })
+ .parse(JSON.parse(graph.stdout)).tasks;
+ if (new Set(tasks.map(({ task }) => task)).size !== tasks.length)
+ throw new Error("Static graph contains duplicate task identities");
+ const context = parseStaticTaskContext({
+ version: 1,
+ sourceSha: source.stdout.trim(),
+ sourceClean: status.stdout.length === 0,
+ ...(process.env.CI_PR_HEAD_SHA
+ ? { prHeadSha: process.env.CI_PR_HEAD_SHA }
+ : {}),
+ nodeVersion: process.version,
+ npmVersion: npmVersion.stdout.trim(),
+ turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo))
+ .version,
+ platform: process.platform,
+ architecture: process.arch,
+ taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])),
+ });
+ if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha)
+ throw new Error(
+ "Static context source does not match the workflow revision",
+ );
+ await mkdir(outputDirectory, { recursive: true });
+ await writeFile(
+ join(outputDirectory, "static-task-context.json"),
+ JSON.stringify(context, null, 2) + "\n",
+ );
+ return context;
+}
+
+if (
+ process.argv[1] &&
+ resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+)
+ await recordStaticTaskContext();
diff --git a/scripts/ci/test-group.mjs b/scripts/ci/test-group.mjs
new file mode 100644
index 000000000..7d99175b4
--- /dev/null
+++ b/scripts/ci/test-group.mjs
@@ -0,0 +1,46 @@
+import { spawn } from "node:child_process";
+import { resolve } from "node:path";
+
+import {
+ BUILT_TEST_PROJECTS,
+ SOURCE_TEST_PROJECTS,
+} from "../lib/test-projects.mjs";
+
+const [group, ...arguments_] = process.argv.slice(2);
+const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
+ group
+];
+if (!projects) throw new Error("Expected source or built test group");
+// The group owns project selection; callers can request reporters or shards.
+if (
+ arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
+)
+ throw new Error("Test groups cannot override their project inventory");
+const child = spawn(
+ process.execPath,
+ [
+ resolve("node_modules/vitest/vitest.mjs"),
+ "run",
+ ...projects.flatMap(({ name }) => ["--project", name]),
+ ...arguments_,
+ ],
+ { stdio: "inherit" },
+);
+let shutdownSignal;
+for (const signal of ["SIGINT", "SIGTERM"])
+ process.on(signal, () => {
+ shutdownSignal ??= signal;
+ child.kill(signal);
+ });
+child.once("error", (error) => {
+ console.error(error.message);
+ process.exitCode = 1;
+});
+child.once("close", (code) => {
+ process.exitCode =
+ shutdownSignal === "SIGINT"
+ ? 130
+ : shutdownSignal === "SIGTERM"
+ ? 143
+ : (code ?? 1);
+});
diff --git a/scripts/ci/test-history.mjs b/scripts/ci/test-history.mjs
index 4386f48af..14dbb582b 100644
--- a/scripts/ci/test-history.mjs
+++ b/scripts/ci/test-history.mjs
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
+import { normalizeWorkflowRun } from "./workflow-run.mjs";
const ARTIFACT_NAME = "test-timings";
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
@@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
.slice(0, MAX_CANDIDATES);
for (const artifact of candidates) {
- const run = api(
- `repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
+ const run = normalizeWorkflowRun(
+ api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
);
const runAge = now - Date.parse(run?.created_at);
if (
@@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
- run.path === ".github/workflows/ci.yml" &&
+ run.workflowPath?.file === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_branch === branch &&
run.head_repository?.full_name === repository
diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs
index 035a01990..7f85c6ec5 100644
--- a/scripts/ci/test-report.mjs
+++ b/scripts/ci/test-report.mjs
@@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => {
throw new Error(`Test report path is outside the checkout: ${name}`);
return { name: path.split(sep).join("/"), durationMs, state };
});
-const complete = process.env.SHARD_RESULT === "success";
+// Keep group outcomes intact: an unavailable group is not a completed failure.
+const groupResults = [
+ process.env.SHARD_RESULT,
+ ...(process.env.SOURCE_REQUIRED === "true"
+ ? [process.env.SOURCE_RESULT]
+ : []),
+];
+const groupsCompleted = groupResults.every((state) =>
+ ["success", "failure"].includes(state),
+);
const passed =
- complete && process.env.MERGE_RESULT === "success" && report.success;
+ groupResults.every((state) => state === "success") &&
+ process.env.MERGE_RESULT === "success" &&
+ report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
@@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) {
comparison.complete &&
report.unhandledErrors === 0 &&
["passed", "failed"].includes(report.reason) &&
- ["success", "failure"].includes(process.env.SHARD_RESULT) &&
+ groupsCompleted &&
["success", "failure"].includes(process.env.MERGE_RESULT),
executionSha: process.env.GITHUB_SHA ?? null,
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
@@ -91,7 +102,9 @@ if (passed && verifyInventory) {
);
}
const summary = [
- complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
+ groupsCompleted
+ ? "## Merged test diagnostics"
+ : "## Partial test diagnostics",
"",
...(selection
? [
@@ -99,11 +112,14 @@ const summary = [
"",
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
- "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
+ "This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
"",
]
: []),
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
+ ...(process.env.SOURCE_REQUIRED === "true"
+ ? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`]
+ : []),
passed
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
diff --git a/scripts/ci/workflow-run.d.mts b/scripts/ci/workflow-run.d.mts
new file mode 100644
index 000000000..b0e626378
--- /dev/null
+++ b/scripts/ci/workflow-run.d.mts
@@ -0,0 +1,10 @@
+export interface WorkflowPath {
+ readonly file: string;
+ readonly ref: string | null;
+}
+export interface NormalizedWorkflowRun {
+ readonly workflowPath: WorkflowPath | null;
+}
+export function normalizeWorkflowRun<T extends Record<string, unknown>>(
+ value: T,
+): Omit<T, "workflowPath"> & NormalizedWorkflowRun;
diff --git a/scripts/ci/workflow-run.mjs b/scripts/ci/workflow-run.mjs
new file mode 100644
index 000000000..8c33214bf
--- /dev/null
+++ b/scripts/ci/workflow-run.mjs
@@ -0,0 +1,15 @@
+/** Keep the API path as evidence and parse its optional ref at the boundary. */
+export function normalizeWorkflowRun(value) {
+ if (typeof value !== "object" || value === null || Array.isArray(value))
+ throw new Error("workflow_run_malformed");
+ const match =
+ typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path)
+ ? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec(
+ value.path,
+ )
+ : null;
+ return {
+ ...value,
+ workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null,
+ };
+}
diff --git a/scripts/lib/development-tests.mjs b/scripts/lib/development-tests.mjs
index 8ddebe115..6aaf10768 100644
--- a/scripts/lib/development-tests.mjs
+++ b/scripts/lib/development-tests.mjs
@@ -1,17 +1,4 @@
-const sourceProjects = [
- "domain",
- "services",
- "adapters",
- "composition",
- "conformance",
- "evaluation",
-];
-
-const sourceTest = (path) =>
- path.startsWith("src/") ||
- ["composition", "conformance", "evaluation"].some((project) =>
- path.startsWith(`tests/${project}/`),
- );
+import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs";
/** Parse explicit test paths and an optional branch-comparison base. */
export const parseDevelopmentTestRequest = (mode, arguments_) => {
@@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => {
path.split("/").some((part) => part === ".." || part === ".")
)
throw new Error(`Expected an exact repository test path: ${argument}`);
- if (mode !== "focused" && !sourceTest(path))
- throw new Error(
- `Use test:focused for compiled boundary tests: ${path}`,
- );
+ const project = projectForTest(path);
+ if (mode !== "focused" && project.group !== "source")
+ throw new Error(`Use test:focused for built test projects: ${path}`);
paths.push(path);
}
}
@@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => {
!/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "")
)
throw new Error("Changed tests require a resolved Git merge base");
- const projects = request.mode === "focused" ? [] : sourceProjects;
+ const projects =
+ request.mode === "focused"
+ ? []
+ : SOURCE_TEST_PROJECTS.map(({ name }) => name);
return {
- needsBuild: request.paths.some((path) => !sourceTest(path)),
+ needsBuild: request.paths.some(
+ (path) => projectForTest(path).group === "built",
+ ),
artifactTasks: Object.entries(artifactConsumers).flatMap(
([task, consumers]) =>
request.paths.some((path) => consumers.includes(path)) ? [task] : [],
diff --git a/scripts/lib/test-projects.d.mts b/scripts/lib/test-projects.d.mts
new file mode 100644
index 000000000..a9d6ae356
--- /dev/null
+++ b/scripts/lib/test-projects.d.mts
@@ -0,0 +1,10 @@
+export interface TestProjectDefinition {
+ readonly name: string;
+ readonly group: "source" | "built";
+ readonly include: readonly string[];
+ readonly exclude?: readonly string[];
+}
+
+export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[];
+export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[];
+export function projectForTest(testPath: string): TestProjectDefinition;
diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs
new file mode 100644
index 000000000..68fb90dbd
--- /dev/null
+++ b/scripts/lib/test-projects.mjs
@@ -0,0 +1,107 @@
+import { matchesGlob } from "node:path";
+
+/** Canonical file ownership shared by Vitest and development-test planning. */
+export const SOURCE_TEST_PROJECTS = [
+ {
+ name: "domain",
+ group: "source",
+ include: ["src/{contracts,domain}/**/*.test.ts"],
+ },
+ {
+ name: "services",
+ group: "source",
+ include: ["src/application/**/*.test.ts"],
+ },
+ {
+ name: "adapters",
+ group: "source",
+ include: [
+ "src/*.test.ts",
+ "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
+ ],
+ },
+ {
+ name: "composition",
+ group: "source",
+ include: ["tests/composition/**/*.test.ts"],
+ },
+ {
+ name: "conformance",
+ group: "source",
+ include: ["tests/conformance/**/*.test.ts"],
+ exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"],
+ },
+ {
+ name: "evaluation",
+ group: "source",
+ include: ["tests/evaluation/**/*.test.ts"],
+ },
+];
+
+export const BUILT_TEST_PROJECTS = [
+ {
+ name: "boundary",
+ group: "built",
+ include: [
+ "tests/boundary/**/*.test.ts",
+ "tests/conformance/providers/hopper/realHopperSearch.test.ts",
+ ],
+ exclude: [
+ "tests/boundary/cli/**/*.test.ts",
+ "tests/boundary/mcp/**/*.test.ts",
+ "tests/boundary/process/**/*.test.ts",
+ ],
+ },
+ {
+ name: "cli-boundary",
+ group: "built",
+ include: ["tests/boundary/cli/**/*.test.ts"],
+ },
+ {
+ name: "process-boundary",
+ group: "built",
+ include: ["tests/boundary/process/**/*.test.ts"],
+ },
+ {
+ name: "mcp-boundary",
+ group: "built",
+ include: ["tests/boundary/mcp/**/*.test.ts"],
+ },
+ {
+ name: "acceptance",
+ group: "built",
+ include: ["tests/acceptance/**/*.test.ts"],
+ },
+ {
+ name: "process-global",
+ group: "built",
+ include: ["tests/process-global/**/*.test.ts"],
+ },
+];
+
+const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
+
+/** Return the unique Vitest owner for an exact repository-relative test path. */
+export const projectForTest = (testPath) => {
+ if (
+ typeof testPath !== "string" ||
+ testPath.startsWith("/") ||
+ testPath.includes("\\") ||
+ testPath.split("/").some((part) => part === ".." || part === ".") ||
+ !testPath.endsWith(".test.ts")
+ )
+ throw new Error(`Expected a normalized repository test path: ${testPath}`);
+
+ const owners = ALL_TEST_PROJECTS.filter(
+ ({ include, exclude = [] }) =>
+ include.some((pattern) => matchesGlob(testPath, pattern)) &&
+ !exclude.some((pattern) => matchesGlob(testPath, pattern)),
+ );
+ if (owners.length !== 1)
+ throw new Error(
+ owners.length === 0
+ ? `No test project owns ${testPath}`
+ : `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`,
+ );
+ return owners[0];
+};
diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts
index 3c14636d7..5939d458a 100644
--- a/src/domain/analysisView/analysisView.test.ts
+++ b/src/domain/analysisView/analysisView.test.ts
@@ -961,7 +961,7 @@ it("does not impose a message-count cutoff below the byte budget", () => {
);
expect(
boundedViewMessages(messages, `ev_${"a".repeat(64)}`, "limitations"),
- ).toEqual(messages);
+ ).toEqual(messages.toSorted());
});
it("bounds repeated view messages even when fewer than 64 source strings are huge", () => {
diff --git a/tests/boundary/cli/developmentTests.test.ts b/tests/boundary/cli/developmentTests.test.ts
index 2987206c8..7becdd4e7 100644
--- a/tests/boundary/cli/developmentTests.test.ts
+++ b/tests/boundary/cli/developmentTests.test.ts
@@ -45,6 +45,7 @@ const fixture = async () => {
for (const path of [
"scripts/run-development-tests.mjs",
"scripts/lib/development-tests.mjs",
+ "scripts/lib/test-projects.mjs",
])
await copyFile(join(process.cwd(), path), join(root, path));
await symlink(
diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts
new file mode 100644
index 000000000..26feb3ee2
--- /dev/null
+++ b/tests/boundary/filesystem/ciFailureReport.test.ts
@@ -0,0 +1,419 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect, it } from "vitest";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs";
+
+import {
+ readStaticCheckArtifact,
+ selectStaticCheckArtifact,
+} from "../../../scripts/ci/artifacts.mjs";
+import {
+ classifyFailure,
+ createGitHubApi,
+ reportCiFailures,
+ type CiRun,
+ type CiJob,
+ type FailureReportApi,
+} from "../../../scripts/ci/failure-report.mjs";
+
+const sha = (digit: string) => digit.repeat(40);
+const diagnostic = {
+ kind: "lint",
+ path: "src/example.ts",
+ code: "no-debugger",
+ message: "Unexpected debugger statement.",
+ severity: "error",
+} as const;
+const context = (
+ sourceSha: string,
+ taskHash = "a".repeat(16),
+ prHeadSha?: string,
+) => ({
+ version: 1 as const,
+ sourceSha,
+ ...(prHeadSha ? { prHeadSha } : {}),
+ sourceClean: true,
+ nodeVersion: "v24.18.0",
+ npmVersion: "11.16.0",
+ turboVersion: "2.11.7",
+ platform: "linux",
+ architecture: "x64",
+ taskHashes: { "lint:built": taskHash },
+});
+const run = (overrides: Partial<CiRun> = {}) =>
+ normalizeWorkflowRun({
+ id: 10,
+ run_attempt: 1,
+ status: "completed",
+ conclusion: "failure",
+ event: "pull_request",
+ path: ".github/workflows/ci.yml@main",
+ head_sha: sha("a"),
+ head_branch: "feature",
+ repository: { full_name: "owner/rea" },
+ pull_requests: [
+ { base: { sha: sha("b"), repo: { full_name: "owner/rea" } } },
+ ],
+ ...overrides,
+ });
+const job = (overrides: Partial<CiJob> = {}) => ({
+ name: "Static checks",
+ status: "completed",
+ conclusion: "failure",
+ ...overrides,
+});
+
+const comparison = (): Parameters<typeof classifyFailure>[0] => ({
+ repository: "owner/rea",
+ defaultBranch: "main",
+ run: run(),
+ job: job(),
+ diagnostics: [diagnostic],
+ context: context(sha("a")),
+ mainRun: run({
+ id: 20,
+ event: "push",
+ head_sha: sha("c"),
+ head_branch: "main",
+ pull_requests: [],
+ }),
+ mainJob: job(),
+ mainDiagnostics: [diagnostic],
+ mainContext: context(sha("c")),
+ executionSourcesVerified: true,
+ ancestry: { status: "ahead", mergeBaseSha: sha("c") },
+ blobs: { "src/example.ts": true },
+});
+
+it("attributes only an identical structured lint failure with matching task, context, ancestry, and source blob", () => {
+ const result = classifyFailure(comparison());
+ expect(result).toEqual({
+ classification: "matching-main-diagnostics",
+ reason:
+ "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
+ });
+});
+
+it.each([
+ ["task hash", { mainContext: context(sha("c"), "b".repeat(16)) }],
+ ["dirty source", { context: { ...context(sha("a")), sourceClean: false } }],
+ [
+ "toolchain",
+ { mainContext: { ...context(sha("c")), turboVersion: "other" } },
+ ],
+ [
+ "source sha",
+ { context: context(sha("d")), executionSourcesVerified: false },
+ ],
+ [
+ "diagnostic",
+ { mainDiagnostics: [{ ...diagnostic, message: "different message" }] },
+ ],
+ ["content", { blobs: { "src/example.ts": false } }],
+ ["ancestry", { ancestry: { status: "diverged", mergeBaseSha: sha("e") } }],
+ ["canceled job", { job: job({ conclusion: "cancelled" }) }],
+ ["not a static job", { job: job({ name: "Test shard (1)" }) }],
+] as const)("does not attribute on mismatched %s", (_label, overrides) => {
+ const result = classifyFailure({ ...comparison(), ...overrides });
+ expect(result.classification).toBe("unknown");
+});
+
+it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => {
+ const directory = await createTestTempDirectory("rea-ci-failure-test-");
+ const calls: string[] = [];
+ const manifest = {
+ repository: "owner/rea",
+ truncation: [],
+ coverage: { exhaustive: false },
+ runs: [
+ {
+ id: 10,
+ runAttempt: 1,
+ workflow: "CI",
+ event: "pull_request",
+ conclusion: "failure",
+ status: "collected",
+ },
+ {
+ id: 11,
+ runAttempt: 1,
+ workflow: "CI",
+ event: "pull_request",
+ conclusion: "success",
+ status: "collected",
+ },
+ ],
+ };
+ const failedRun = run();
+ const api: FailureReportApi = {
+ async getRun(_repository, id) {
+ calls.push(`run:${id}`);
+ return failedRun;
+ },
+ async getJobs(_repository, id) {
+ calls.push(`jobs:${id}`);
+ return [{ ...job(), run_id: Number(id), run_attempt: 1 }];
+ },
+ async getStaticEvidence() {
+ calls.push("artifact");
+ return null;
+ },
+ async getAncestry() {
+ throw new Error("must not compare without evidence");
+ },
+ async verifyExecutionSource() {
+ throw new Error("must not verify source without evidence");
+ },
+ async getBlobIdentity() {
+ throw new Error("must not inspect blobs without evidence");
+ },
+ };
+ const report = await reportCiFailures({
+ repository: "owner/rea",
+ defaultBranch: "main",
+ batchManifest: manifest,
+ api,
+ outputDirectory: directory,
+ });
+ expect(calls).toEqual(["run:10", "jobs:10", "artifact"]);
+ expect(report).toMatchObject({
+ informationalOnly: true,
+ gateEffect: "none; this report never changes CI conclusions",
+ counts: { unknown: 1 },
+ });
+ const json = JSON.parse(
+ await readFile(join(directory, "report.json"), "utf8"),
+ );
+ const markdown = await readFile(join(directory, "report.md"), "utf8");
+ expect(json.failures[0]).toMatchObject({
+ classification: "unknown",
+ reason: "no_matching_verified_main_failure",
+ });
+ expect(markdown).toContain("Informational only");
+ expect(markdown).toContain("Unknown means evidence was incomplete");
+});
+
+it.each([
+ [22, false],
+ [6, true],
+] as const)(
+ "finds verified main evidence after %s newer PR runs (extra incorrect main hint: %s)",
+ async (prCount, incorrectMainHints) => {
+ const directory = await createTestTempDirectory("rea-ci-main-proof-");
+ const mainSha = sha("c");
+ const prHeadSha = sha("a");
+ const mergeSha = sha("d");
+ const mainRun = run({
+ id: 20,
+ event: "push",
+ head_sha: mainSha,
+ head_branch: "main",
+ pull_requests: [],
+ });
+ const prRun = run({
+ head_sha: prHeadSha,
+ pull_requests: [
+ { base: { sha: mainSha, repo: { full_name: "owner/rea" } } },
+ ],
+ });
+ const evidence = (sourceSha: string, prHead?: string) => ({
+ context: context(sourceSha, undefined, prHead),
+ diagnostics: [diagnostic],
+ });
+ const lookups: number[] = [];
+ const api: FailureReportApi = {
+ async getRun(_repository, id) {
+ lookups.push(Number(id));
+ return Number(id) === 20 ? mainRun : { ...prRun, id };
+ },
+ async getJobs(_repository, id) {
+ return [{ ...job(), run_id: Number(id), run_attempt: 1 }];
+ },
+ async getStaticEvidence(_repository, selectedRun) {
+ return Number(selectedRun.id) === 20
+ ? evidence(mainSha)
+ : evidence(mergeSha, prHeadSha);
+ },
+ async verifyExecutionSource(_repository, selectedRun, sourceContext) {
+ return selectedRun.event === "pull_request"
+ ? sourceContext.sourceSha === mergeSha &&
+ sourceContext.prHeadSha === selectedRun.head_sha
+ : sourceContext.sourceSha === selectedRun.head_sha;
+ },
+ async getAncestry() {
+ return { status: "ahead", mergeBaseSha: mainSha };
+ },
+ async getBlobIdentity() {
+ return { "src/example.ts": true };
+ },
+ };
+ const result = await reportCiFailures({
+ repository: "owner/rea",
+ defaultBranch: "main",
+ batchManifest: {
+ repository: "owner/rea",
+ runs: [
+ ...(incorrectMainHints
+ ? [
+ {
+ id: 30,
+ runAttempt: 1,
+ workflow: "CI",
+ event: "push",
+ headBranch: "main",
+ conclusion: "failure",
+ status: "collected",
+ },
+ ]
+ : []),
+ ...Array.from({ length: prCount }, (_, index) => ({
+ id: 100 + index,
+ runAttempt: 1,
+ workflow: "CI",
+ event: "pull_request",
+ headBranch: "feature",
+ conclusion: "failure",
+ status: "collected",
+ })),
+ {
+ id: 20,
+ runAttempt: 1,
+ workflow: "CI",
+ event: "push",
+ headBranch: "main",
+ conclusion: "failure",
+ status: "collected",
+ },
+ ],
+ },
+ api,
+ outputDirectory: directory,
+ });
+ expect(result.failures[0]).toMatchObject({
+ runId: 100,
+ classification: "matching-main-diagnostics",
+ reason:
+ "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
+ });
+ expect(lookups).toHaveLength(
+ Math.min(prCount, 20) + 1 + Number(incorrectMainHints),
+ );
+ expect(lookups[0]).toBe(incorrectMainHints ? 30 : 20);
+ expect(new Set(lookups).size).toBe(lookups.length);
+ expect(result.boundedInput.mainCandidateRuns).toBe(1);
+ expect(result.failures).toHaveLength(Math.min(prCount, 20));
+ },
+);
+
+it("proves checkout merge parents in base-then-PR-head order and rejects false identities", async () => {
+ const baseSha = sha("b");
+ const headSha = sha("a");
+ const mergeSha = sha("c");
+ const prRun = run({
+ head_sha: headSha,
+ pull_requests: [
+ { base: { sha: baseSha, repo: { full_name: "owner/rea" } } },
+ ],
+ });
+ let parents = [baseSha, headSha];
+ const calls: string[] = [];
+ const api = createGitHubApi({
+ token: "fixture-token",
+ apiRequest: async (_method, endpoint) => {
+ calls.push(endpoint);
+ return {
+ sha: mergeSha,
+ parents: parents.map((parentSha) => ({ sha: parentSha })),
+ };
+ },
+ });
+ const mergeContext = context(mergeSha, undefined, headSha);
+ expect(
+ await api.verifyExecutionSource("owner/rea", prRun, mergeContext),
+ ).toBe(true);
+ expect(calls).toEqual([`repos/owner/rea/commits/${mergeSha}`]);
+
+ parents = [headSha, baseSha];
+ expect(
+ await api.verifyExecutionSource("owner/rea", prRun, mergeContext),
+ ).toBe(false);
+ expect(
+ await api.verifyExecutionSource("owner/rea", prRun, {
+ ...mergeContext,
+ prHeadSha: sha("e"),
+ }),
+ ).toBe(false);
+});
+
+it("uses static artifacts only when listing is complete, unique, and tied to this run attempt", () => {
+ const artifact = {
+ id: 12,
+ name: "static-check-reports",
+ expired: false,
+ workflow_run: { id: 10, head_sha: sha("a") },
+ };
+ const runRecord = { id: 10, head_sha: sha("a") };
+ expect(
+ selectStaticCheckArtifact(
+ { total_count: 1, artifacts: [artifact] },
+ runRecord,
+ 1,
+ ),
+ ).toBe(artifact);
+ expect(
+ selectStaticCheckArtifact(
+ { total_count: 2, artifacts: [artifact] },
+ runRecord,
+ 1,
+ ),
+ ).toBeNull();
+ expect(
+ selectStaticCheckArtifact(
+ { total_count: 2, artifacts: [artifact, artifact] },
+ runRecord,
+ 1,
+ ),
+ ).toBeNull();
+ expect(
+ selectStaticCheckArtifact(
+ {
+ total_count: 1,
+ artifacts: [
+ { ...artifact, workflow_run: { id: 11, head_sha: sha("a") } },
+ ],
+ },
+ runRecord,
+ 1,
+ ),
+ ).toBeNull();
+ expect(
+ selectStaticCheckArtifact(
+ { total_count: 1, artifacts: [artifact] },
+ runRecord,
+ 2,
+ ),
+ ).toBeNull();
+});
+
+it("rejects artifact redirects that could send the bearer token to a non-HTTPS host", async () => {
+ const calls: string[] = [];
+ await expect(
+ readStaticCheckArtifact({
+ repository: "owner/rea",
+ artifactId: 12,
+ token: "fixture-token",
+ fetchImpl: (async (url: string | URL | Request) => {
+ calls.push(String(url));
+ return new Response(null, {
+ status: 302,
+ headers: { location: "http://untrusted.example/artifact.zip" },
+ });
+ }) as typeof fetch,
+ }),
+ ).rejects.toThrow("static_artifact_redirect_invalid");
+ expect(calls).toEqual([
+ "https://api.github.com/repos/owner/rea/actions/artifacts/12/zip",
+ ]);
+});
diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.ts
new file mode 100644
index 000000000..92fe3e4ae
--- /dev/null
+++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts
@@ -0,0 +1,290 @@
+import { expect, it } from "vitest";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import {
+ reportCiFailures,
+ type CiRun,
+ type FailureReportApi,
+ type StaticDiagnostic,
+ type StaticEvidence,
+} from "../../../scripts/ci/failure-report.mjs";
+
+const repository = "owner/rea";
+const mainSha = "c".repeat(40);
+const prSha = "a".repeat(40);
+const diagnostic: StaticDiagnostic = {
+ kind: "lint",
+ path: "src/example.ts",
+ code: "no-debugger",
+ message: "Unexpected debugger statement.",
+ severity: "error",
+};
+const evidence = (
+ run: CiRun,
+ diagnostics: readonly StaticDiagnostic[],
+): StaticEvidence => ({
+ context: {
+ version: 1,
+ sourceSha: run.head_sha!,
+ sourceClean: true,
+ nodeVersion: "v24.18.0",
+ npmVersion: "11.16.0",
+ turboVersion: "2.11.7",
+ platform: "linux",
+ architecture: "x64",
+ taskHashes: { "lint:built": "a".repeat(16) },
+ },
+ diagnostics,
+});
+const entry = (id: number, event = "pull_request", runAttempt = 1) => ({
+ id,
+ runAttempt,
+ workflow: "CI",
+ event,
+ headBranch: event === "pull_request" ? "feature" : "main",
+ status: "collected",
+ conclusion: "failure",
+});
+
+function fixture(mainIds: readonly number[], prCount: number) {
+ const jobs: number[] = [];
+ const artifacts: number[] = [];
+ const paths: (readonly string[])[] = [];
+ const runs: Array<
+ Omit<ReturnType<typeof entry>, "event"> & { event?: string }
+ > = [
+ ...mainIds.map((id) => entry(id, "schedule")),
+ ...Array.from({ length: prCount }, (_, index) => entry(100 + index)),
+ ];
+ const api: FailureReportApi = {
+ async getRun(_repository, id) {
+ const selected = runs.find((run) => run.id === Number(id))!;
+ return {
+ id,
+ run_attempt: selected.runAttempt,
+ status: "completed",
+ conclusion: "failure",
+ ...(selected.event ? { event: selected.event } : {}),
+ head_branch: selected.headBranch,
+ path: ".github/workflows/ci.yml@main",
+ head_sha: mainIds.includes(Number(id)) ? mainSha : prSha,
+ repository: { full_name: repository },
+ pull_requests:
+ selected.event === "pull_request"
+ ? [{ base: { sha: mainSha, repo: { full_name: repository } } }]
+ : [],
+ };
+ },
+ async getJobs(_repository, id, attempt) {
+ jobs.push(Number(id));
+ return [
+ {
+ name: "Static checks",
+ status: "completed",
+ conclusion: "failure",
+ run_id: Number(id),
+ run_attempt: attempt,
+ },
+ ];
+ },
+ async getStaticEvidence(_repository, run) {
+ artifacts.push(Number(run.id));
+ return evidence(run, [diagnostic]);
+ },
+ async getAncestry() {
+ return { status: "identical", mergeBaseSha: mainSha };
+ },
+ async getBlobIdentity(_repository, _ancestor, _descendant, requested) {
+ paths.push(requested);
+ return Object.fromEntries(requested.map((path) => [path, true]));
+ },
+ async verifyExecutionSource() {
+ return true;
+ },
+ };
+ return { api, runs, jobs, artifacts, paths };
+}
+
+async function report(input: ReturnType<typeof fixture>) {
+ return await reportCiFailures({
+ repository,
+ defaultBranch: "main",
+ batchManifest: { repository, runs: input.runs },
+ api: input.api,
+ outputDirectory: await createTestTempDirectory("rea-ci-budget-"),
+ });
+}
+
+it("selects PR failures before the cap even after a day of scheduled main failures", async () => {
+ const input = fixture(
+ Array.from({ length: 24 }, (_, index) => 20 + index),
+ 1,
+ );
+ input.runs.push({
+ id: 200,
+ runAttempt: 1,
+ workflow: "CI",
+ headBranch: "feature",
+ status: "collected",
+ conclusion: "failure",
+ });
+ const result = await report(input);
+ expect(result.failures).toMatchObject([
+ { runId: 100, classification: "matching-main-diagnostics" },
+ ]);
+ expect(result.boundedInput).toMatchObject({
+ candidateRuns: 1,
+ mainCandidateRuns: 5,
+ manifestTruncated: false,
+ });
+ expect(input.jobs).toEqual([100, 20]);
+});
+
+it("reuses five main job/artifact observations across twenty completed PR comparisons", async () => {
+ const input = fixture([20, 21, 22, 23, 24], 20);
+ const load = input.api.getStaticEvidence;
+ input.api.getStaticEvidence = async (...args) => {
+ const result = await load(...args);
+ return Number(args[1].id) < 24
+ ? evidence(args[1], [{ ...diagnostic, message: "different" }])
+ : result;
+ };
+ const result = await report(input);
+ expect(result.counts).toEqual({
+ "matching-main-diagnostics": 20,
+ unknown: 0,
+ });
+ expect(input.paths).toHaveLength(100);
+ expect(input.jobs).toHaveLength(25);
+ expect(input.artifacts).toHaveLength(25);
+ for (const id of [20, 21, 22, 23, 24]) {
+ expect(input.jobs.filter((item) => item === id)).toHaveLength(1);
+ expect(input.artifacts.filter((item) => item === id)).toHaveLength(1);
+ }
+});
+
+it.each(["missing", "artifact error", "job error", "wrong job attempt"])(
+ "reuses unavailable main evidence without retries: %s",
+ async (failure) => {
+ const input = fixture([20], 3);
+ const load = input.api.getStaticEvidence;
+ const jobs = input.api.getJobs;
+ input.api.getJobs = async (...args) => {
+ const result = await jobs(...args);
+ if (Number(args[1]) !== 20) return result;
+ if (failure === "job error") throw new Error("job_snapshot_unavailable");
+ return failure === "wrong job attempt"
+ ? result.map((job) => ({ ...job, run_attempt: 2 }))
+ : result;
+ };
+ input.api.getStaticEvidence = async (...args) => {
+ const result = await load(...args);
+ if (Number(args[1].id) !== 20) return result;
+ if (failure === "artifact error") throw new Error("artifact_unavailable");
+ return null;
+ };
+ const result = await report(input);
+ expect(result.counts.unknown).toBe(3);
+ expect(input.jobs.filter((id) => id === 20)).toHaveLength(1);
+ expect(input.artifacts.filter((id) => id === 20)).toHaveLength(
+ failure === "job error" || failure === "wrong job attempt" ? 0 : 1,
+ );
+ },
+);
+
+it.each([1, 2])(
+ "refreshes unavailable main evidence in the next report (attempt %s)",
+ async (attempt) => {
+ const input = fixture([20], 1);
+ expect((await report(input)).counts["matching-main-diagnostics"]).toBe(1);
+ input.runs[0]!.runAttempt = attempt;
+ const load = input.api.getStaticEvidence;
+ input.api.getStaticEvidence = async (...args) => {
+ const result = await load(...args);
+ return Number(args[1].id) === 20 ? null : result;
+ };
+ expect((await report(input)).counts.unknown).toBe(1);
+ expect(input.jobs.filter((id) => id === 20)).toHaveLength(2);
+ expect(input.artifacts.filter((id) => id === 20)).toHaveLength(2);
+ },
+);
+
+it.each([1, 51])(
+ "fetches only PR paths with %s PR diagnostics and noisy main evidence",
+ async (count) => {
+ const input = fixture([20], 1);
+ const load = input.api.getStaticEvidence;
+ const diagnostics = Array.from({ length: count }, (_, index) => ({
+ ...diagnostic,
+ path: `src/pr-${index}.ts`,
+ }));
+ input.api.getStaticEvidence = async (...args) => {
+ await load(...args);
+ const noise =
+ Number(args[1].id) === 20
+ ? Array.from({ length: 60 }, (_, index) => ({
+ ...diagnostic,
+ path: `src/main-${index}.ts`,
+ }))
+ : [];
+ return evidence(args[1], [...noise, ...diagnostics]);
+ };
+ const result = await report(input);
+ expect(input.paths).toEqual([
+ diagnostics.slice(0, 50).map((item) => item.path),
+ ]);
+ expect(result.failures[0]!.classification).toBe(
+ count === 1 ? "matching-main-diagnostics" : "unknown",
+ );
+ },
+);
+
+it.each(["jobs", "artifact", "comparison"])(
+ "continues after an isolated main %s failure and preserves its provenance",
+ async (failure) => {
+ const input = fixture([20, 21], 3);
+ const jobs = input.api.getJobs;
+ const load = input.api.getStaticEvidence;
+ input.api.getJobs = async (...args) => {
+ const result = await jobs(...args);
+ if (Number(args[1]) === 20 && failure === "jobs")
+ throw new Error("main_jobs_unavailable");
+ return result;
+ };
+ input.api.getStaticEvidence = async (...args) => {
+ const result = await load(...args);
+ if (Number(args[1].id) === 20 && failure === "artifact")
+ throw new Error("main_artifact_unavailable");
+ return Number(args[1].id) === 20 && failure === "comparison"
+ ? {
+ ...result!,
+ context: { ...result!.context, sourceSha: "d".repeat(40) },
+ }
+ : result;
+ };
+ input.api.verifyExecutionSource = async (_repository, run, context) => {
+ if (Number(run.id) === 20 && context.sourceSha !== run.head_sha)
+ throw new Error("main_comparison_unavailable");
+ return true;
+ };
+ const result = await report(input);
+ expect(result.counts).toEqual({
+ "matching-main-diagnostics": 3,
+ unknown: 0,
+ });
+ for (const item of result.failures) {
+ expect(item.candidateFailures).toEqual([
+ {
+ runId: 20,
+ runAttempt: 1,
+ reason: `main_${failure}_unavailable`,
+ },
+ ]);
+ }
+ expect(input.jobs.filter((id) => id === 20)).toHaveLength(1);
+ expect(input.jobs.filter((id) => id === 21)).toHaveLength(1);
+ expect(input.artifacts.filter((id) => id === 20)).toHaveLength(
+ failure === "jobs" ? 0 : 1,
+ );
+ expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1);
+ },
+);
diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts
new file mode 100644
index 000000000..c61cb0d92
--- /dev/null
+++ b/tests/boundary/filesystem/ciFeedback.test.ts
@@ -0,0 +1,313 @@
+import { execFile } from "node:child_process";
+import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { runInNewContext } from "node:vm";
+import { expect, it } from "vitest";
+import { parse } from "yaml";
+import { z } from "zod";
+
+import {
+ parseStaticTaskContext,
+ recordStaticTaskContext,
+} from "../../../scripts/ci/static-task-context.mjs";
+import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const execute = promisify(execFile);
+const scopeFlags = async () =>
+ z
+ .object({ CI_FLAGS: z.array(z.string()) })
+ .parse(
+ await import(
+ new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
+ ),
+ ).CI_FLAGS;
+const workflow = async () =>
+ z
+ .object({
+ jobs: z.record(
+ z.string(),
+ z
+ .object({
+ needs: z.union([z.string(), z.array(z.string())]).optional(),
+ if: z.string().optional(),
+ steps: z
+ .array(
+ z
+ .object({
+ run: z.string().optional(),
+ env: z.record(z.string(), z.string()).optional(),
+ })
+ .passthrough(),
+ )
+ .optional(),
+ })
+ .passthrough(),
+ ),
+ })
+ .parse(
+ parse(
+ await readFile(
+ new URL("../../../.github/workflows/ci.yml", import.meta.url),
+ "utf8",
+ ),
+ ),
+ ).jobs;
+
+it("starts source feedback before build and requires both groups without waiving missing jobs", async () => {
+ const jobs = await workflow();
+ expect(
+ jobs.static?.steps?.find(
+ (step) => step.run === "npm run ci:static-context",
+ )?.["continue-on-error"],
+ ).toBe(true);
+ expect(jobs["test-source"]?.needs).toBe("changes");
+ expect(jobs["test-shard"]?.needs).toEqual(["changes", "build"]);
+ expect(
+ jobs["test-source"]?.steps?.some(
+ (step) =>
+ step.run === "npm run test:fast -- --reporter=default --reporter=blob",
+ ),
+ ).toBe(true);
+ const required = fileURLToPath(
+ new URL("../../../scripts/ci/required.mjs", import.meta.url),
+ );
+ const flags = await scopeFlags();
+ for (const tier of ["fast", "full"]) {
+ for (const result of [
+ "success",
+ "failure",
+ "cancelled",
+ "skipped",
+ "missing",
+ ]) {
+ const needs = Object.fromEntries(
+ Object.keys(jobs)
+ .filter((name) => name !== "required")
+ .map((name) => [name, { result: "success" }]),
+ );
+ const outputs = {
+ ...Object.fromEntries(
+ flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
+ ),
+ tier,
+ code: "true",
+ };
+ const context: Record<
+ string,
+ { result: string; outputs?: Record<string, string> }
+ > = { ...needs, changes: { result: "success", outputs } };
+ if (result === "missing") delete context["test-source"];
+ else context["test-source"] = { result };
+ const run = execute(process.execPath, [required], {
+ env: {
+ ...process.env,
+ CI_JOBS: JSON.stringify(context),
+ CI_QUALIFICATION_DIRECTORY: "",
+ },
+ });
+ if (
+ result === "success" ||
+ (tier === "full" && ["skipped", "missing"].includes(result))
+ )
+ await expect(run).resolves.toMatchObject({ stderr: "" });
+ else
+ await expect(run).rejects.toMatchObject({
+ code: 1,
+ stderr: expect.stringContaining("test-source"),
+ });
+ }
+ }
+});
+
+it("records the actual static task graph with explicit source identity and no ambient environment", async () => {
+ const directory = await createTestTempDirectory("rea-ci-context-");
+ const context = await recordStaticTaskContext({ outputDirectory: directory });
+ const head = await execute("git", ["rev-parse", "HEAD"]);
+ expect(context.sourceSha).toBe(head.stdout.trim());
+ expect(context.taskHashes["lint:built"]).toMatch(/^[a-f0-9]{16}$/u);
+ expect(context.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u);
+ expect(context).not.toHaveProperty("environmentVariables");
+ expect(context).not.toHaveProperty("env");
+ expect(
+ JSON.parse(await readFile(`${directory}/static-task-context.json`, "utf8")),
+ ).toEqual(context);
+ for (const invalid of [
+ undefined,
+ {},
+ { ...context, sourceSha: "bad" },
+ { ...context, prHeadSha: "bad" },
+ { ...context, taskHashes: { jscpd: "stale" } },
+ ])
+ expect(() => parseStaticTaskContext(invalid)).toThrow();
+});
+
+it("rejects mutation methods at the shared GitHub transport before invoking the CLI", async () => {
+ const request = createGhApiRequest({ env: { PATH: "" } });
+ await expect(
+ request("POST", "repos/owner/repo/actions/runs/1/rerun", {
+ requests: 0,
+ startedAt: Date.now(),
+ limitMs: 1000,
+ }),
+ ).rejects.toThrow("read-only GET");
+});
+
+it("collects basic CI telemetry before optional dependency installation can fail", async () => {
+ const { jobs } = z
+ .object({
+ jobs: z.object({
+ report: z.object({
+ steps: z.array(
+ z.object({
+ id: z.string().optional(),
+ run: z.string().optional(),
+ if: z.string().optional(),
+ "continue-on-error": z.boolean().optional(),
+ }),
+ ),
+ }),
+ }),
+ })
+ .parse(
+ parse(
+ await readFile(
+ new URL("../../../.github/workflows/ci-timings.yml", import.meta.url),
+ "utf8",
+ ),
+ ),
+ );
+ const steps = jobs.report.steps;
+ const installation = steps.findIndex((step) => step.id === "failure-deps");
+ for (const script of ["pr-ci-health.mjs", "job-timings-batch.mjs"]) {
+ const collection = steps.findIndex(
+ (step) => step.run === `node scripts/ci/${script}`,
+ );
+ expect(collection).toBeGreaterThanOrEqual(0);
+ expect(installation).toBeGreaterThan(collection);
+ }
+ expect(steps[installation]).toMatchObject({
+ run: "npm ci",
+ "continue-on-error": true,
+ });
+ expect(
+ steps.find((step) => step.id === "npm-cache")?.["continue-on-error"],
+ ).toBe(true);
+ expect(
+ steps.find((step) => step.run === "node scripts/ci/failure-report.mjs")?.if,
+ ).toContain("steps.failure-deps.outcome == 'success'");
+});
+
+it("merges source and built reports exactly once using the workflow's shard matrix", async () => {
+ const jobs = await workflow();
+ const matrix = z
+ .object({
+ matrix: z.object({ shard: z.string().regex(/^\$\{\{.*\}\}$/u) }),
+ })
+ .parse(jobs["test-shard"]?.strategy).matrix.shard;
+ const shards = z
+ .array(z.string().regex(/^\d+\/\d+$/u))
+ .min(1)
+ .parse(
+ runInNewContext(
+ matrix.slice(3, -2),
+ {
+ fromJSON: JSON.parse,
+ needs: { changes: { outputs: { tier: "fast" } } },
+ },
+ { timeout: 1_000 },
+ ),
+ );
+ const labels = Object.fromEntries(
+ ["source", "built"].map((group) => [
+ group,
+ z
+ .string()
+ .min(1)
+ .parse(
+ jobs[group === "source" ? "test-source" : "test-shard"]?.steps?.find(
+ (step) => step.env?.VITEST_BLOB_LABEL,
+ )?.env?.VITEST_BLOB_LABEL,
+ ),
+ ]),
+ );
+ const root = await createTestTempDirectory("rea-ci-group-merge-");
+ const config = join(root, "vitest.config.mjs");
+ await writeFile(
+ config,
+ `export default { test: { maxWorkers: 1, projects: [${["source", "built"].map((name) => `{ test: { name: "${name}", globals: true, include: ["${name}*.test.js"] } }`).join(",")}] } };`,
+ );
+ const files = [
+ "source.test.js",
+ ...Array.from(
+ { length: shards.length },
+ (_, index) => `built-${index}.test.js`,
+ ),
+ ];
+ for (const file of files)
+ await writeFile(
+ join(root, file),
+ 'it("runs its selected group", () => expect(true).toBe(true));',
+ );
+ const vitest = fileURLToPath(
+ new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url),
+ );
+ const blobs = new Map<string, Buffer>();
+ const runs = [
+ { group: "source", shard: [] },
+ ...shards.map((shard) => ({ group: "built", shard: [`--shard=${shard}`] })),
+ ];
+ for (const { group, shard } of runs) {
+ await execute(
+ process.execPath,
+ [
+ vitest,
+ "run",
+ "--config",
+ config,
+ "--project",
+ group,
+ "--reporter=blob",
+ ...shard,
+ ],
+ {
+ cwd: root,
+ env: { ...process.env, VITEST_BLOB_LABEL: labels[group]! },
+ timeout: 20_000,
+ },
+ );
+ for (const filename of await readdir(join(root, ".vitest/blob"))) {
+ expect(blobs.has(filename)).toBe(false);
+ blobs.set(filename, await readFile(join(root, ".vitest/blob", filename)));
+ }
+ // Separate hosted jobs start with independent report directories.
+ await rm(join(root, ".vitest/blob"), { recursive: true });
+ }
+ expect(blobs.size).toBe(runs.length);
+ await mkdir(join(root, ".vitest/blob"), { recursive: true });
+ for (const [filename, bytes] of blobs)
+ await writeFile(join(root, ".vitest/blob", filename), bytes);
+ await execute(
+ process.execPath,
+ [
+ vitest,
+ "--config",
+ config,
+ "--merge-reports",
+ "--reporter=json",
+ "--outputFile.json=merged.json",
+ ],
+ { cwd: root, timeout: 20_000 },
+ );
+ const merged = z
+ .object({
+ success: z.literal(true),
+ testResults: z.array(z.object({ name: z.string() })),
+ })
+ .parse(JSON.parse(await readFile(join(root, "merged.json"), "utf8")));
+ expect(
+ merged.testResults.map(({ name }) => name.split("/").at(-1)).sort(),
+ ).toEqual(files.toSorted());
+});
diff --git a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts
index 6c2658c00..e0dcecc82 100644
--- a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts
+++ b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts
@@ -18,6 +18,8 @@ it("collects the latest attempt for recent completed CI and Release runs into a
workflow_runs: [
{
id: 42,
+ event: "push",
+ head_branch: "main",
run_attempt: 3,
status: "completed",
conclusion: "failure",
@@ -43,51 +45,9 @@ it("collects the latest attempt for recent completed CI and Release runs into a
},
],
},
- [
- {
- jobs: [
- {
- name: "Release lane",
- status: "completed",
- conclusion: "success",
- created_at: recent,
- started_at: recent,
- completed_at: new Date(Date.parse(recent) + 50_000).toISOString(),
- steps: [],
- },
- ],
- },
- ],
- [
- {
- jobs: [
- {
- name: "Failed lane",
- status: "completed",
- conclusion: "failure",
- created_at: recent,
- started_at: recent,
- completed_at: new Date(Date.parse(recent) + 55_000).toISOString(),
- steps: [],
- },
- ],
- },
- ],
- [
- {
- jobs: [
- {
- name: "Cancelled lane",
- status: "completed",
- conclusion: "cancelled",
- created_at: recent,
- started_at: recent,
- completed_at: new Date(Date.parse(recent) + 30_000).toISOString(),
- steps: [],
- },
- ],
- },
- ],
+ completedJobs("Release lane", "success", recent, 50),
+ completedJobs("Failed lane", "failure", recent, 55),
+ completedJobs("Cancelled lane", "cancelled", recent, 30),
]);
const result = await runBatch(fixture);
const manifest = JSON.parse(
@@ -116,8 +76,20 @@ it("collects the latest attempt for recent completed CI and Release runs into a
expect(manifest.errors).toEqual([]);
expect(manifest.runs).toMatchObject([
{ id: 43, runAttempt: 2, conclusion: "success" },
- { id: 42, runAttempt: 3, conclusion: "failure" },
- { id: 41, runAttempt: 1, conclusion: "cancelled" },
+ {
+ id: 42,
+ runAttempt: 3,
+ conclusion: "failure",
+ event: "push",
+ headBranch: "main",
+ },
+ {
+ id: 41,
+ runAttempt: 1,
+ conclusion: "cancelled",
+ event: null,
+ headBranch: null,
+ },
]);
expect(manifest.runs).toEqual(
expect.arrayContaining([
@@ -234,6 +206,31 @@ it("retains a partial manifest when its collection budget is exhausted", async (
expect(manifest.truncation.join(" ")).toContain("Time budget");
});
+function completedJobs(
+ name: string,
+ conclusion: string,
+ startedAt: string,
+ seconds: number,
+) {
+ return [
+ {
+ jobs: [
+ {
+ name,
+ status: "completed",
+ conclusion,
+ created_at: startedAt,
+ started_at: startedAt,
+ completed_at: new Date(
+ Date.parse(startedAt) + seconds * 1000,
+ ).toISOString(),
+ steps: [],
+ },
+ ],
+ },
+ ];
+}
+
async function createFixture(responses: unknown[]) {
const directory = await createTestTempDirectory("rea-ci-job-timings-batch-");
const fakeBin = join(directory, "bin");
diff --git a/tests/boundary/filesystem/ciPrHealth.test.ts b/tests/boundary/filesystem/ciPrHealth.test.ts
new file mode 100644
index 000000000..756fb4748
--- /dev/null
+++ b/tests/boundary/filesystem/ciPrHealth.test.ts
@@ -0,0 +1,617 @@
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import { delimiter, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { expect, it } from "vitest";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import {
+ classifyPrCi,
+ collectPrCiHealth,
+ matchRunToPullRequest,
+ writePrCiHealthReport,
+} from "../../../scripts/ci/pr-ci-health.mjs";
+import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs";
+import type {
+ PrCiPullRequest,
+ PrCiWorkflowRun,
+} from "../../../scripts/ci/pr-ci-health.mjs";
+import { executeWorkflowFixture } from "../../support/workflowGit.js";
+
+const reporter = fileURLToPath(
+ new URL("../../../scripts/ci/pr-ci-health.mjs", import.meta.url),
+);
+const observedAt = Date.parse("2026-10-14T00:00:00Z");
+const headSha = "a".repeat(40);
+const mergeSha = "b".repeat(40);
+
+it("classifies workflow states without confusing absent, pending, and completed CI", () => {
+ const pull = validPull();
+ const cases = [
+ [
+ "startup_failure",
+ run({ status: "completed", conclusion: "startup_failure" }),
+ ],
+ [
+ "queued_pending_executing",
+ run({ status: "in_progress", conclusion: null }),
+ ],
+ ["approval_required", run({ status: "waiting", conclusion: null })],
+ ["completed_failure", run({ status: "completed", conclusion: "failure" })],
+ [
+ "completed_canceled",
+ run({ status: "completed", conclusion: "cancelled" }),
+ ],
+ ["completed_success", run({ status: "completed", conclusion: "success" })],
+ ["completed_other", run({ status: "completed", conclusion: "neutral" })],
+ [
+ "completed_other",
+ run({ status: "completed", conclusion: "constructor" }),
+ ],
+ ] as const;
+ for (const [category, evidence] of cases) {
+ for (const runsComplete of [true, false]) {
+ const result = classifyPrCi({
+ pull,
+ runs: [
+ {
+ run: { ...evidence, run_attempt: 1 },
+ basis: "event_pr_number_and_head_sha",
+ },
+ ],
+ runsComplete,
+ pullRequestInventoryComplete: true,
+ ambiguousRuns: [],
+ });
+ expect(result).toMatchObject({
+ category: runsComplete ? category : "unknown",
+ ...(runsComplete
+ ? {}
+ : { reason: "workflow_run_inventory_incomplete" }),
+ runs: [
+ {
+ id: 77,
+ attempt: 1,
+ status: evidence.status,
+ conclusion: evidence.conclusion,
+ },
+ ],
+ });
+ }
+ }
+ for (const runsComplete of [true, false]) {
+ expect(
+ classifyPrCi({
+ pull: { ...pull, mergeable: false },
+ runs: [],
+ runsComplete,
+ pullRequestInventoryComplete: true,
+ ambiguousRuns: [],
+ }).category,
+ ).toBe("merge_conflict");
+ }
+});
+
+it("requires complete inventories, known mergeability, and immutable head identity to report missing", () => {
+ const pull = validPull({ mergeable: true });
+ const classify = (overrides = {}) =>
+ classifyPrCi({
+ pull,
+ runs: [],
+ runsComplete: true,
+ pullRequestInventoryComplete: true,
+ ambiguousRuns: [],
+ ...overrides,
+ });
+
+ expect(classify().category).toBe("missing");
+ expect(classify({ runsComplete: false })).toMatchObject({
+ category: "deferred",
+ reason: "lookup_inventory_incomplete",
+ });
+ expect(classify({ pullRequestInventoryComplete: false }).category).toBe(
+ "deferred",
+ );
+ expect(
+ classify({
+ ambiguousRuns: [{ run: run(), basis: "ambiguous_head_identity" }],
+ }).category,
+ ).toBe("unknown");
+ expect(classify({ pull: { ...pull, mergeable: undefined } }).category).toBe(
+ "unknown",
+ );
+ expect(classify({ pull: { ...pull, identityValid: false } }).category).toBe(
+ "unknown",
+ );
+});
+
+it("accepts event PR/head evidence for merge-SHA runs and refuses ambiguous branch identities", () => {
+ const pull = validPull();
+ const associatedMergeRun = run({
+ head_sha: mergeSha,
+ pull_requests: [
+ {
+ number: pull.number!,
+ head: { sha: headSha, repo: { id: pull.head.repositoryId } },
+ },
+ ],
+ });
+ expect(
+ matchRunToPullRequest({
+ run: associatedMergeRun,
+ pull,
+ identityCounts: new Map([["73\u0000feature", 1]]),
+ }),
+ ).toEqual({ kind: "match", basis: "event_pr_number_and_head_sha" });
+ expect(
+ matchRunToPullRequest({
+ run: run({
+ pull_requests: [
+ { number: pull.number!, head: { sha: "c".repeat(40) } },
+ ],
+ }),
+ pull,
+ identityCounts: new Map([["73\u0000feature", 1]]),
+ }).kind,
+ ).toBe("none");
+
+ const branchRun = run({ pull_requests: [] });
+ expect(
+ matchRunToPullRequest({
+ run: branchRun,
+ pull,
+ identityCounts: new Map([["73\u0000feature", 1]]),
+ inventoryComplete: false,
+ }).kind,
+ ).toBe("ambiguous");
+ expect(
+ matchRunToPullRequest({
+ run: branchRun,
+ pull,
+ identityCounts: new Map([["73\u0000feature", 1]]),
+ }),
+ ).toEqual({
+ kind: "match",
+ basis: "event_unique_head_ref_and_repository_id",
+ });
+ expect(
+ matchRunToPullRequest({
+ run: run({ head_sha: mergeSha, pull_requests: [] }),
+ pull,
+ identityCounts: new Map([["73\u0000feature", 1]]),
+ }).kind,
+ ).toBe("none");
+ expect(
+ matchRunToPullRequest({
+ run: run({
+ pull_requests: [{ number: pull.number!, head: { sha: headSha } }],
+ }),
+ pull,
+ identityCounts: new Map([["73\u0000feature", 1]]),
+ }).kind,
+ ).toBe("ambiguous");
+ expect(
+ matchRunToPullRequest({
+ run: branchRun,
+ pull,
+ identityCounts: new Map([["73\u0000feature", 2]]),
+ }).kind,
+ ).toBe("ambiguous");
+ expect(
+ matchRunToPullRequest({
+ run: run({ event: "workflow_dispatch" }),
+ pull,
+ identityCounts: new Map([["73\u0000feature", 1]]),
+ }).kind,
+ ).toBe("none");
+});
+
+it("scans target-branch PRs through public gh GET requests and writes JSON and Markdown", async () => {
+ const fixture = await createFixture();
+ const calls: string[] = [];
+ const apiRequest: NonNullable<
+ Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
+ > = async (method, endpoint) => {
+ calls.push(`${method} ${endpoint}`);
+ if (method !== "GET") throw new Error("fixture rejected a non-GET request");
+ if (endpoint.includes("/pulls?"))
+ return [
+ rawPull(42, { mergeable: true }),
+ rawPull(43, { mergeable: false }),
+ rawPull(44, {
+ base: { ref: "release", repo: { id: 11 } },
+ mergeable: true,
+ }),
+ rawPull(45, { draft: true, mergeable: true }),
+ ];
+ if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
+ return {
+ total_count: 1,
+ workflow_runs: [
+ run({
+ id: 9001,
+ event: "pull_request",
+ status: "completed",
+ conclusion: "failure",
+ head_sha: mergeSha,
+ pull_requests: [
+ {
+ number: 42,
+ head: { sha: headSha, repo: { id: 73 } },
+ },
+ ],
+ }),
+ ],
+ };
+ throw new Error(`unexpected GET endpoint: ${endpoint}`);
+ };
+ const report = await collectPrCiHealth({
+ repository: "owner/rea",
+ repositoryId: 11,
+ defaultBranch: "main",
+ now: observedAt,
+ apiRequest,
+ });
+ expect(calls).toHaveLength(2);
+ expect(calls.every((call) => call.startsWith("GET "))).toBe(true);
+ expect(calls[0]).toContain("state=open");
+ expect(calls[0]).not.toContain("base=");
+ expect(calls[1]).toContain("event=pull_request");
+ expect(report.pullRequests).toHaveLength(2);
+ expect(report.pullRequests.map(({ number }) => number)).toEqual([42, 43]);
+ expect(report.pullRequests.map(({ ci }) => ci.category)).toEqual([
+ "completed_failure",
+ "merge_conflict",
+ ]);
+ expect(report.scan).toMatchObject({
+ status: "complete",
+ pullRequestInventoryComplete: true,
+ workflowRunInventoryComplete: true,
+ apiRequests: 2,
+ });
+
+ const output = join(fixture.directory, ".cache", "ci-pr-health");
+ const paths = await writePrCiHealthReport(report, output);
+ const written = JSON.parse(await readFile(paths.jsonPath, "utf8"));
+ const markdown = await readFile(paths.markdownPath, "utf8");
+ expect(written.counts.completed_failure).toBe(1);
+ expect(markdown).toContain("# Pull request CI health snapshot");
+ expect(markdown).toContain("#42");
+ expect(markdown).toContain("completed_failure");
+});
+
+it("counts older open PRs when deciding whether a branch/ref fallback is unique", async () => {
+ const olderPull = rawPull(63, {
+ created_at: "2026-09-01T00:00:00Z",
+ head: { ...rawPull(63).head, ref: "shared-feature" },
+ });
+ const currentPull = rawPull(64, {
+ mergeable: true,
+ head: { ...olderPull.head },
+ });
+ const apiRequest: NonNullable<
+ Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
+ > = async (_method, endpoint) => {
+ if (endpoint.includes("/pulls?")) return [currentPull, olderPull];
+ if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
+ return {
+ workflow_runs: [
+ run({
+ head_branch: "shared-feature",
+ pull_requests: [],
+ }),
+ ],
+ };
+ throw new Error(`unexpected GET endpoint: ${endpoint}`);
+ };
+ const report = await collectPrCiHealth({
+ repository: "owner/rea",
+ repositoryId: 11,
+ defaultBranch: "main",
+ now: observedAt,
+ apiRequest,
+ });
+ expect(report.scan.pullRequestInventoryComplete).toBe(true);
+ expect(report.pullRequests.map(({ number }) => number)).toEqual([64]);
+ expect(report.pullRequests[0]?.ci).toMatchObject({
+ category: "unknown",
+ reason: "workflow_run_identity_ambiguous",
+ });
+});
+
+it.each([false, true])(
+ "keeps page-capped scans uncertain (observed success: %s)",
+ async (observedSuccess) => {
+ const pull = rawPull(62, { mergeable: true });
+ const apiRequest: NonNullable<
+ Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
+ > = async (_method, endpoint) => {
+ if (endpoint.includes("/pulls?")) return [pull];
+ const page = Number(
+ new URL(`https://fixture.invalid/${endpoint}`).searchParams.get("page"),
+ );
+ return {
+ workflow_runs: Array.from({ length: 100 }, (_item, index) =>
+ observedSuccess && page === 1 && index === 0
+ ? run({
+ run_attempt: 1,
+ pull_requests: [
+ { number: 62, head: { sha: headSha, repo: { id: 73 } } },
+ ],
+ })
+ : run({
+ id: page * 100 + index,
+ head_branch: "different-branch",
+ head_repository: { id: 999 },
+ pull_requests: [],
+ }),
+ ),
+ };
+ };
+ const report = await collectPrCiHealth({
+ repository: "owner/rea",
+ repositoryId: 11,
+ defaultBranch: "main",
+ now: observedAt,
+ apiRequest,
+ });
+ expect(report.scan.workflowRunPages).toBe(5);
+ expect(report.scan.workflowRunInventoryComplete).toBe(false);
+ const firstPull = report.pullRequests[0];
+ if (firstPull === undefined) throw new Error("fixture PR was not scanned");
+ expect(firstPull.ci).toMatchObject({
+ category: observedSuccess ? "unknown" : "deferred",
+ reason: observedSuccess
+ ? "workflow_run_inventory_incomplete"
+ : "lookup_inventory_incomplete",
+ });
+ expect(firstPull.ci.runs).toHaveLength(observedSuccess ? 1 : 0);
+ },
+);
+
+it("keeps the result unknown when the head changes between API snapshots", async () => {
+ const apiRequest: NonNullable<
+ Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
+ > = async (_method, endpoint) => {
+ if (endpoint.includes("/pulls?")) return [rawPull(63)];
+ if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
+ return { workflow_runs: [] };
+ return {
+ ...rawPull(63, { mergeable: true }),
+ head: {
+ ...rawPull(63).head,
+ sha: mergeSha,
+ },
+ };
+ };
+ const report = await collectPrCiHealth({
+ repository: "owner/rea",
+ repositoryId: 11,
+ defaultBranch: "main",
+ now: observedAt,
+ apiRequest,
+ });
+ expect(report.pullRequests[0]).toMatchObject({
+ identityChangedDuringScan: true,
+ ci: { category: "unknown", reason: "pull_request_head_identity_invalid" },
+ });
+ expect(report.scan).toMatchObject({
+ status: "partial",
+ errors: ["pull_request_identity_changed_during_scan"],
+ });
+});
+
+it("public CLI uses only GET and produces a report from fixture GitHub responses", async () => {
+ const fixture = await createFixture();
+ const ghCalls = join(fixture.directory, "gh-calls.jsonl");
+ const ghResponse = join(fixture.directory, "gh-response.json");
+ const fakeGh = join(fixture.fakeBin, "gh");
+ await writeFile(
+ ghResponse,
+ JSON.stringify([
+ [rawPull(51, { mergeable: true })],
+ { total_count: 0, workflow_runs: [] },
+ ]),
+ );
+ await writeFile(
+ fakeGh,
+ `#!/usr/bin/env node\nconst fs=require("node:fs");\nconst args=process.argv.slice(2);\nfs.appendFileSync(process.env.CI_GH_CALLS, JSON.stringify(args)+"\\n");\nif(args[0]!=="api"||args[1]!=="--method"||args[2]!=="GET")process.exit(41);\nconst page=Number(new URL("https://fixture.invalid/"+args.at(-1)).searchParams.get("page"));\nconst responses=JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSE,"utf8"));\nprocess.stdout.write(JSON.stringify(responses[page===1&&args.at(-1).includes("/pulls?")?0:1]));\n`,
+ );
+ await chmod(fakeGh, 0o755);
+ const outputDirectory = join(fixture.directory, ".cache", "ci-pr-health");
+ await executeWorkflowFixture(process.execPath, [reporter], {
+ cwd: fixture.directory,
+ env: {
+ ...process.env,
+ PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
+ CI_REPOSITORY: "owner/rea",
+ CI_REPOSITORY_ID: "11",
+ CI_DEFAULT_BRANCH: "main",
+ CI_PR_HEALTH_DIRECTORY: outputDirectory,
+ CI_GH_CALLS: ghCalls,
+ CI_GH_RESPONSE: ghResponse,
+ },
+ });
+ const calls = (await readFile(ghCalls, "utf8"))
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ expect(calls).toHaveLength(2);
+ expect(
+ calls.every(
+ (args: string[]) => args[1] === "--method" && args[2] === "GET",
+ ),
+ ).toBe(true);
+ const report = JSON.parse(
+ await readFile(join(outputDirectory, "report.json"), "utf8"),
+ );
+ expect(report.pullRequests[0].ci.category).toBe("missing");
+ await expect(
+ readFile(join(outputDirectory, "summary.md"), "utf8"),
+ ).resolves.toContain("missing");
+});
+
+it("rejects non-GET methods before launching the GitHub client", async () => {
+ const fixture = await createFixture();
+ const request = createGhApiRequest({
+ env: {
+ ...process.env,
+ PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
+ CI_GH_CALLS: join(fixture.directory, "should-not-exist"),
+ },
+ });
+ await expect(
+ request("POST", "repos/owner/rea/pulls", {
+ requests: 0,
+ startedAt: observedAt,
+ limitMs: 1_000,
+ }),
+ ).rejects.toThrow("read-only GET");
+ await expect(
+ readFile(join(fixture.directory, "should-not-exist")),
+ ).rejects.toMatchObject({ code: "ENOENT" });
+});
+
+function validPull(overrides: Partial<PrCiPullRequest> = {}): PrCiPullRequest {
+ return {
+ number: 42,
+ identityValid: true,
+ mergeable: true,
+ head: { repositoryId: 73, ref: "feature", sha: headSha },
+ ...overrides,
+ };
+}
+
+it.each([
+ ["newer run", { id: 78, created_at: "2026-10-13T01:00:00Z" }, "unknown"],
+ [
+ "older run",
+ { id: 78, created_at: "2026-10-12T23:00:00Z" },
+ "completed_success",
+ ],
+ ["equal timestamps", { id: 78 }, "unknown"],
+ [
+ "retry of an older run",
+ { id: 78, created_at: "2026-10-12T23:00:00Z", run_attempt: 2 },
+ "unknown",
+ ],
+ ["unknown timestamp", { id: 78, created_at: "invalid" }, "unknown"],
+ ["newer attempt", { run_attempt: 3 }, "unknown"],
+ ["older attempt", { run_attempt: 1 }, "completed_success"],
+] as const)(
+ "retains ambiguity evidence for a %s without reporting a stale completion",
+ (_label, overrides, category) => {
+ const currentAttempt = "id" in overrides ? 1 : 2;
+ const result = classifyPrCi({
+ pull: validPull(),
+ runs: [
+ {
+ run: run({ run_attempt: currentAttempt }),
+ basis: "event_pr_number_and_head_sha",
+ },
+ ],
+ ambiguousRuns: [
+ {
+ run: run({ run_attempt: currentAttempt, ...overrides }),
+ basis: "ambiguous_head_identity",
+ },
+ ],
+ runsComplete: true,
+ pullRequestInventoryComplete: true,
+ });
+ expect(result.category).toBe(category);
+ expect(result.runs).toHaveLength(2);
+ },
+);
+
+it("does not let an older proven run hide a newer run with an unverified repository ID", async () => {
+ const pull = rawPull(42);
+ const older = run({
+ id: 70,
+ created_at: "2026-10-13T00:00:00Z",
+ pull_requests: [{ number: 42, head: { sha: headSha, repo: { id: 73 } } }],
+ });
+ const newer = run({
+ id: 71,
+ created_at: "2026-10-13T01:00:00Z",
+ status: "in_progress",
+ conclusion: null,
+ pull_requests: [{ number: 42, head: { sha: headSha } }],
+ });
+ const report = await collectPrCiHealth({
+ repository: "owner/rea",
+ repositoryId: 11,
+ defaultBranch: "main",
+ now: observedAt,
+ apiRequest: async (_method, endpoint) => {
+ if (endpoint.includes("/pulls?")) return [pull];
+ if (endpoint.endsWith("/pulls/42")) return { ...pull, mergeable: true };
+ return { workflow_runs: [newer, older] };
+ },
+ });
+ expect(report.pullRequests[0]?.ci).toMatchObject({
+ category: "unknown",
+ reason: "workflow_run_identity_ambiguous",
+ });
+ expect(report.pullRequests[0]?.ci.runs).toHaveLength(2);
+});
+
+it("keeps a proven newest run even when two older matched runs have equal timestamps", () => {
+ const result = classifyPrCi({
+ pull: validPull(),
+ runs: [
+ run({ id: 70, run_attempt: 1 }),
+ run({ id: 71, run_attempt: 1, conclusion: "failure" }),
+ run({
+ id: 72,
+ run_attempt: 1,
+ created_at: "2026-10-13T01:00:00Z",
+ status: "in_progress",
+ conclusion: null,
+ }),
+ ].map((run) => ({ run, basis: "event_pr_number_and_head_sha" })),
+ ambiguousRuns: [],
+ runsComplete: true,
+ pullRequestInventoryComplete: true,
+ });
+ expect(result.category).toBe("queued_pending_executing");
+ expect(result.runs).toHaveLength(3);
+});
+
+function run(overrides: Partial<PrCiWorkflowRun> = {}): PrCiWorkflowRun {
+ return {
+ id: 77,
+ event: "pull_request",
+ head_branch: "feature",
+ head_repository: { id: 73 },
+ head_sha: headSha,
+ status: "completed",
+ conclusion: "success",
+ created_at: "2026-10-13T00:00:00Z",
+ pull_requests: [],
+ ...overrides,
+ };
+}
+
+function rawPull(number: number, overrides: Record<string, unknown> = {}) {
+ return {
+ number,
+ draft: false,
+ created_at: "2026-10-13T00:00:00Z",
+ html_url: `https://github.com/owner/rea/pull/${String(number)}`,
+ base: { ref: "main", repo: { id: 11 } },
+ head: {
+ ref: `feature-${String(number)}`,
+ sha: headSha,
+ repo: { id: 73, full_name: "contributor/rea" },
+ },
+ ...overrides,
+ };
+}
+
+async function createFixture() {
+ const directory = await createTestTempDirectory("rea-ci-pr-health-");
+ const fakeBin = join(directory, "bin");
+ await mkdir(fakeBin);
+ return { directory, fakeBin };
+}
diff --git a/tests/boundary/filesystem/ciQualification.test.ts b/tests/boundary/filesystem/ciQualification.test.ts
index b6a5a1e1a..673b09d34 100644
--- a/tests/boundary/filesystem/ciQualification.test.ts
+++ b/tests/boundary/filesystem/ciQualification.test.ts
@@ -44,7 +44,7 @@ function run(overrides: Record<string, unknown> = {}) {
id: 7,
status: "completed",
conclusion: "success",
- path: ".github/workflows/ci.yml",
+ path: ".github/workflows/ci.yml@main",
event: "schedule",
created_at: new Date(now - 120_000).toISOString(),
head_sha: validSha,
diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts
index d7ac1418e..d2191eb01 100644
--- a/tests/boundary/filesystem/ciRequiredLanes.test.ts
+++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts
@@ -121,7 +121,11 @@ it("exports every canonical scope with read authority for admission and keeps te
);
expect(timings.on.workflow_run).toBeUndefined();
expect(timings.on.schedule).toHaveLength(1);
- expect(timings.permissions).toEqual({ contents: "read", actions: "read" });
+ expect(timings.permissions).toEqual({
+ contents: "read",
+ actions: "read",
+ "pull-requests": "read",
+ });
const checkout = timings.jobs.report.steps.find((step) =>
step.uses?.startsWith("actions/checkout@"),
);
@@ -154,6 +158,7 @@ it("holds native and provider fan-out behind static validation while portable fe
"static",
"build",
"test-shard",
+ "test-source",
"test",
"docs",
"required",
@@ -369,7 +374,7 @@ it("keeps fast feedback separate from complete qualification and release proof",
// release their concurrency slot without waiting for another runner.
expect(ci.jobs.required?.if).toBe("${{ !cancelled() }}");
const shards = ci.jobs["test-shard"]?.steps ?? [];
- const fast = shards.find((step) => step.run?.includes("npm run test:run"));
+ const fast = shards.find((step) => step.run?.includes("npm run test:built"));
const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));
expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'");
expect(fast?.run).toContain("--reporter=blob");
diff --git a/tests/boundary/filesystem/ciStaticArtifacts.test.ts b/tests/boundary/filesystem/ciStaticArtifacts.test.ts
new file mode 100644
index 000000000..45915f450
--- /dev/null
+++ b/tests/boundary/filesystem/ciStaticArtifacts.test.ts
@@ -0,0 +1,99 @@
+import { execFile } from "node:child_process";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { expect, it } from "vitest";
+
+import { parseStaticArchive } from "../../../scripts/ci/artifacts.mjs";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const execute = promisify(execFile);
+const producer = fileURLToPath(
+ new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url),
+);
+
+it.each([3.5, 100, 0])(
+ "retains real lint failures when the duplication threshold is %s",
+ async (threshold) => {
+ const root = await createTestTempDirectory("rea-ci-static-archive-");
+ await mkdir(join(root, "src"));
+ await mkdir(join(root, "tests"));
+ await writeFile(
+ join(root, ".oxlintrc.json"),
+ JSON.stringify({ rules: { "no-debugger": "error" } }),
+ );
+ await writeFile(
+ join(root, ".jscpd.json"),
+ JSON.stringify({ threshold, minLines: 5, minTokens: 50 }),
+ );
+ await writeFile(
+ join(root, "src/example.ts"),
+ "debugger;\nexport const example = true;\n",
+ );
+ const duplicate =
+ "export function compute(value: number) {\n" +
+ Array.from({ length: 20 }, (_, index) => ` value += ${index};\n`).join(
+ "",
+ ) +
+ " return value;\n}\n";
+ for (const name of threshold === 3.5 ? [] : ["first", "second"])
+ await writeFile(join(root, `src/${name}.ts`), duplicate);
+ await expect(
+ execute(process.execPath, [producer, "lint", "src/example.ts"], {
+ cwd: root,
+ }),
+ ).rejects.toMatchObject({ code: 1 });
+ const duplication = execute(process.execPath, [producer, "duplicates"], {
+ cwd: root,
+ });
+ if (threshold === 0)
+ await expect(duplication).rejects.toMatchObject({ code: 1 });
+ else await duplication;
+ const reports = join(root, ".cache/static-checks");
+ expect(
+ JSON.parse(
+ await readFile(join(reports, "jscpd/jscpd-report.json"), "utf8"),
+ ),
+ ).toMatchObject({ reaCheck: { exitCode: threshold === 0 ? 1 : 0 } });
+ await writeFile(
+ join(reports, "static-task-context.json"),
+ JSON.stringify({
+ version: 1,
+ sourceSha: "a".repeat(40),
+ sourceClean: true,
+ nodeVersion: process.version,
+ npmVersion: "11.16.0",
+ turboVersion: "2.11.7",
+ platform: process.platform,
+ architecture: process.arch,
+ taskHashes: { "lint:built": "a".repeat(16), jscpd: "b".repeat(16) },
+ }),
+ );
+ const archive = join(root, "reports.zip");
+ await execute(
+ "zip",
+ [
+ "-q",
+ archive,
+ "oxlint.json",
+ "jscpd/jscpd-report.json",
+ "static-task-context.json",
+ ],
+ { cwd: reports },
+ );
+ const evidence = await parseStaticArchive(archive);
+ expect(evidence.diagnostics.filter(({ kind }) => kind === "lint")).toEqual([
+ expect.objectContaining({
+ kind: "lint",
+ path: "src/example.ts",
+ severity: "error",
+ }),
+ ]);
+ const duplicates = evidence.diagnostics.filter(
+ ({ kind }) => kind === "jscpd",
+ );
+ if (threshold === 0) expect(duplicates.length).toBeGreaterThan(0);
+ else expect(duplicates).toEqual([]);
+ },
+);
diff --git a/tests/boundary/filesystem/ciTestHistory.test.ts b/tests/boundary/filesystem/ciTestHistory.test.ts
index 411e9ece9..84950d49c 100644
--- a/tests/boundary/filesystem/ciTestHistory.test.ts
+++ b/tests/boundary/filesystem/ciTestHistory.test.ts
@@ -35,7 +35,7 @@ function run(id: number, overrides: Record<string, unknown> = {}) {
id,
status: "completed",
conclusion: "success",
- path: ".github/workflows/ci.yml",
+ path: ".github/workflows/ci.yml@main",
event: "schedule",
created_at: new Date(now - 120_000).toISOString(),
head_branch: "main",
diff --git a/tests/boundary/filesystem/ciTestProjects.test.ts b/tests/boundary/filesystem/ciTestProjects.test.ts
new file mode 100644
index 000000000..f0394c5bb
--- /dev/null
+++ b/tests/boundary/filesystem/ciTestProjects.test.ts
@@ -0,0 +1,75 @@
+import { execFileSync } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+import {
+ BUILT_TEST_PROJECTS,
+ SOURCE_TEST_PROJECTS,
+ projectForTest,
+} from "../../../scripts/lib/test-projects.mjs";
+import {
+ developmentTestPlan,
+ parseDevelopmentTestRequest,
+} from "../../../scripts/lib/development-tests.mjs";
+
+const repository = process.cwd();
+
+describe("canonical test-project ownership", () => {
+ it("assigns every tracked test file exactly once to source or built", () => {
+ const files = execFileSync("git", ["ls-files", "*.test.ts"], {
+ cwd: repository,
+ encoding: "utf8",
+ })
+ .trim()
+ .split("\n")
+ .filter(Boolean);
+ const grouped = new Map<string, string[]>();
+ for (const file of files) {
+ const project = projectForTest(file);
+ grouped.set(project.group, [...(grouped.get(project.group) ?? []), file]);
+ }
+
+ expect(grouped.get("source")).not.toBeUndefined();
+ expect(grouped.get("built")).not.toBeUndefined();
+ expect(
+ (grouped.get("source")?.length ?? 0) +
+ (grouped.get("built")?.length ?? 0),
+ ).toBe(files.length);
+ expect(
+ SOURCE_TEST_PROJECTS.reduce(
+ (count, project) => count + project.include.length,
+ 0,
+ ),
+ ).toBeGreaterThan(0);
+ expect(
+ BUILT_TEST_PROJECTS.every((project) => project.group === "built"),
+ ).toBe(true);
+ expect(() => projectForTest("tests/unowned/example.test.ts")).toThrow(
+ "No test project owns",
+ );
+ });
+
+ it("owns the Hopper real-provider conformance test as built and requires preparation", () => {
+ const testPath =
+ "tests/conformance/providers/hopper/realHopperSearch.test.ts";
+ expect(projectForTest(testPath)).toMatchObject({
+ name: "boundary",
+ group: "built",
+ });
+ expect(() => parseDevelopmentTestRequest("local", [testPath])).toThrow(
+ `Use test:focused for built test projects: ${testPath}`,
+ );
+ const request = parseDevelopmentTestRequest("focused", [testPath]);
+ expect(developmentTestPlan(request)).toMatchObject({ needsBuild: true });
+ });
+
+ it("uses canonical selectors in Vitest while retaining independent runner policy", async () => {
+ const config = await readFile(join(repository, "vitest.config.ts"), "utf8");
+ expect(config).toContain(
+ "[...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]",
+ );
+ expect(config).toContain("PROJECT_POLICIES");
+ expect(config).not.toContain('"src/{contracts,domain}/**/*.test.ts"');
+ });
+});
diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts
index 3e2aadcd4..6a5942448 100644
--- a/tests/boundary/filesystem/ciTestReports.test.ts
+++ b/tests/boundary/filesystem/ciTestReports.test.ts
@@ -3,6 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
+import { runInNewContext } from "node:vm";
import { expect, it } from "vitest";
import { parse } from "yaml";
@@ -48,9 +49,7 @@ const workflow = async () =>
it("merges available failed-shard diagnostics and skips unavailable shard reports", async () => {
const steps = (await workflow()).test!.steps;
const download = steps.find(
- (step) =>
- step.uses?.includes("actions/download-artifact") &&
- step.with?.pattern === "vitest-report-*",
+ (step) => step.with?.pattern === "vitest-report-*",
);
const merge = steps.findIndex((step) => step.run?.includes("test:ci:merge"));
const upload = steps.findIndex((step) => step.with?.name === "test-results");
@@ -59,8 +58,7 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
);
expect(merge).toBeGreaterThan(0);
expect(download?.if).toContain("!cancelled()");
- expect(download?.if).toContain("needs.test-shard.result == 'success'");
- expect(download?.if).toContain("needs.test-shard.result == 'failure'");
+
expect(upload).toBeGreaterThan(merge);
expect(gate).toBeGreaterThan(upload);
const merges = steps.filter((step) => step.run?.includes("--reporter=json"));
@@ -70,8 +68,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
"--reporter=./scripts/ci/test-timing-reporter.mjs",
);
expect(step.if).toContain("!cancelled()");
- expect(step.if).toContain("needs.test-shard.result == 'success'");
- expect(step.if).toContain("needs.test-shard.result == 'failure'");
}
expect(steps[upload]?.with?.path).toContain("test-timings.json");
expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
@@ -86,12 +82,23 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
);
const command = steps[gate]?.run;
if (command === undefined) throw new Error("Missing shard result gate");
- for (const result of ["failure", "cancelled", "skipped", ""]) {
- await expect(
- execute("bash", ["-c", command], {
- env: { ...process.env, CODE_REQUIRED: "true", SHARD_RESULT: result },
- }),
- ).rejects.toMatchObject({ code: 1 });
+ for (const tier of ["fast", "full"]) {
+ for (const shard of ["success", "failure", "cancelled", "skipped", ""]) {
+ for (const source of ["success", "failure", "cancelled", "skipped", ""]) {
+ const check = execute("bash", ["-c", command], {
+ env: {
+ ...process.env,
+ CODE_REQUIRED: "true",
+ SHARD_RESULT: shard,
+ SOURCE_REQUIRED: String(tier === "fast"),
+ SOURCE_RESULT: source,
+ },
+ });
+ if (shard === "success" && (tier !== "fast" || source === "success"))
+ await expect(check).resolves.toMatchObject({ stderr: "" });
+ else await expect(check).rejects.toMatchObject({ code: 1 });
+ }
+ }
}
await expect(
execute("bash", ["-c", command], {
@@ -100,6 +107,80 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
).resolves.toMatchObject({ stderr: "" });
});
+// These workflow conditions use the shared boolean/string expression subset.
+// Evaluate the checked-in expressions so failure-state coverage cannot drift
+// into a separate test implementation of the collection policy.
+function conditionValue(
+ condition: string | undefined,
+ needs: Record<string, unknown>,
+ canceled: boolean,
+) {
+ if (!condition) throw new Error("Missing report collection condition");
+ return runInNewContext(
+ condition
+ .replaceAll("needs.test-shard", 'needs["test-shard"]')
+ .replaceAll("needs.test-source", 'needs["test-source"]'),
+ { needs, cancelled: () => canceled },
+ { timeout: 1_000 },
+ );
+}
+
+it("collects either fast test group while keeping full reports tied to full shards", async () => {
+ const steps = (await workflow()).test.steps;
+ const download = steps.find(
+ (step) => step.with?.pattern === "vitest-report-*",
+ );
+ const fast = steps.find(
+ (step) => step.name === "Merge fast-suite JUnit and timing reports",
+ );
+ const full = steps.find(
+ (step) => step.name === "Merge coverage, JUnit, and timing reports",
+ );
+ for (const tier of ["fast", "full"]) {
+ for (const shard of ["success", "failure", "cancelled", "skipped", ""]) {
+ for (const source of ["success", "failure", "cancelled", "skipped", ""]) {
+ for (const [code, canceled] of [
+ ["true", false],
+ ["true", true],
+ ["false", false],
+ ["false", true],
+ ] as const) {
+ const needs = {
+ changes: { outputs: { code, tier } },
+ "test-shard": { result: shard },
+ "test-source": { result: source },
+ };
+ const builtAvailable = ["success", "failure"].includes(shard);
+ const sourceAvailable = ["success", "failure"].includes(source);
+ const enabled = code === "true" && !canceled;
+ const available =
+ builtAvailable || (tier === "fast" && sourceAvailable);
+ const expected = enabled
+ ? {
+ download: available,
+ fast: tier === "fast" && available,
+ full: tier === "full" && builtAvailable,
+ }
+ : { download: false, fast: false, full: false };
+ expect(conditionValue(download?.if, needs, canceled)).toBe(
+ expected.download,
+ );
+ expect(conditionValue(fast?.if, needs, canceled)).toBe(expected.fast);
+ expect(conditionValue(full?.if, needs, canceled)).toBe(expected.full);
+ }
+ }
+ }
+ }
+ const summary = steps.find((step) =>
+ step.run?.includes("scripts/ci/test-report.mjs"),
+ );
+ expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}");
+ expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}");
+ expect(summary?.env?.SOURCE_REQUIRED).toBe(
+ "${{ needs.changes.outputs.tier == 'fast' }}",
+ );
+});
+
it("keeps diagnosis available after merge failure and publishes only complete successful history", async () => {
const steps = (await workflow()).test!.steps;
const diagnosis = steps.find((step) => step.with?.name === "test-results");
@@ -220,7 +301,12 @@ it("exports portable timing history and removes it on a failed, partial, or inva
{ name: join(root, "tests/fast.test.ts"), durationMs: 10 },
],
};
- const run = async (value: unknown, shards = "success", merge = "success") => {
+ const run = async (
+ value: unknown,
+ shards = "success",
+ merge = "success",
+ source?: string,
+ ) => {
await writeFile(join(root, "test-timings.json"), JSON.stringify(value));
await writeFile(summary, "");
return execute(process.execPath, [fileURLToPath(script)], {
@@ -229,6 +315,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva
...process.env,
GITHUB_STEP_SUMMARY: summary,
SHARD_RESULT: shards,
+ SOURCE_REQUIRED: String(source !== undefined),
+ SOURCE_RESULT: source ?? "",
MERGE_RESULT: merge,
},
});
@@ -256,6 +344,24 @@ it("exports portable timing history and removes it on a failed, partial, or inva
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
expect(await readFile(summary, "utf8")).toContain("diagnostic only");
}
+ await run(report, "success", "success", "success");
+ expect(await readFile(history, "utf8")).toContain('"success":true');
+ for (const [shards, source] of [
+ ["skipped", "success"],
+ ["success", "skipped"],
+ ["success", "cancelled"],
+ ["success", "failure"],
+ ["success", ""],
+ ]) {
+ await run(report);
+ await run(report, shards, "success", source);
+ await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
+ const text = await readFile(summary, "utf8");
+ expect(text).toContain(`Source result: ${source}.`);
+ expect(text).toContain("diagnostic only");
+ if (shards === "skipped" || source !== "failure")
+ expect(text).toContain("Partial test diagnostics");
+ }
await run(report);
await expect(
run({
@@ -352,6 +458,21 @@ it("recovers complete module costs from real blob reports without executing test
expect(file.durationMs).toBeCloseTo(
Object.values(file.phases).reduce((sum, value) => sum + value, 0),
);
+ const partial = await execute(process.execPath, [fileURLToPath(script)], {
+ cwd: root,
+ env: {
+ ...process.env,
+ SHARD_RESULT: "skipped",
+ SOURCE_REQUIRED: "true",
+ SOURCE_RESULT: "success",
+ MERGE_RESULT: "success",
+ },
+ });
+ expect(partial.stdout).toContain("Partial test diagnostics");
+ expect(partial.stdout).toContain("sample.test.js");
+ await expect(
+ readFile(join(root, ".cache/ci-test-history/test-results.json")),
+ ).rejects.toMatchObject({ code: "ENOENT" });
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
@@ -420,13 +541,15 @@ it("records failed omissions from full reports without treating partial or unhan
state: index === 1 ? "failed" : "passed",
})),
};
- const run = async (report: unknown) => {
+ const run = async (report: unknown, source = "failure") => {
await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: {
...process.env,
SHARD_RESULT: "failure",
+ SOURCE_REQUIRED: "true",
+ SOURCE_RESULT: source,
MERGE_RESULT: "failure",
CI_VERIFY_TEST_INVENTORY: "true",
CI_TEST_SELECTION: JSON.stringify(
@@ -458,6 +581,8 @@ it("records failed omissions from full reports without treating partial or unhan
{ ...timing, reason: "interrupted" },
])
expect(await run(report)).toMatchObject({ complete: false });
+ for (const source of ["skipped", "cancelled", ""])
+ expect(await run(timing, source)).toMatchObject({ complete: false });
await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
code: 1,
});
diff --git a/tests/boundary/filesystem/ciWorkflowRun.test.ts b/tests/boundary/filesystem/ciWorkflowRun.test.ts
new file mode 100644
index 000000000..cbebb5c4d
--- /dev/null
+++ b/tests/boundary/filesystem/ciWorkflowRun.test.ts
@@ -0,0 +1,54 @@
+import { expect, it } from "vitest";
+import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs";
+
+it.each([
+ [".github/workflows/ci.yml", ".github/workflows/ci.yml", null],
+ [".github/workflows/ci.yml@main", ".github/workflows/ci.yml", "main"],
+ [
+ ".github/workflows/ci.yml@refs/heads/release/v1",
+ ".github/workflows/ci.yml",
+ "refs/heads/release/v1",
+ ],
+ [
+ ".github/workflows/ci.yml@feature@v1",
+ ".github/workflows/ci.yml",
+ "feature@v1",
+ ],
+ [
+ ".github/workflows/release.yaml@main",
+ ".github/workflows/release.yaml",
+ "main",
+ ],
+])(
+ "preserves API path %s while separating its file identity and ref",
+ (path, file, ref) => {
+ const raw = { id: 10, path, head_sha: "a".repeat(40) };
+ const parsed = normalizeWorkflowRun(raw);
+ expect(parsed).toEqual({ ...raw, workflowPath: { file, ref } });
+ expect(raw).not.toHaveProperty("workflowPath");
+ },
+);
+
+it.each([
+ undefined,
+ null,
+ 42,
+ "",
+ ".github/workflows/ci.yml@",
+ ".github/workflows/ci.yml@main\n",
+ ".github/workflows/ci.yml@main\0",
+ "../.github/workflows/ci.yml",
+ ".github/workflows/../ci.yml@main",
+ ".github/workflows/ci.yml-extra@main",
+ "https://example.test/ci.yml@main",
+])(
+ "keeps malformed workflow path %s unknown instead of inventing a file identity",
+ (path) => {
+ const parsed = normalizeWorkflowRun({
+ path,
+ workflowPath: { file: ".github/workflows/ci.yml", ref: "main" },
+ });
+ expect(parsed.path).toBe(path);
+ expect(parsed.workflowPath).toBeNull();
+ },
+);
diff --git a/turbo.json b/turbo.json
index ffee2a070..35e7ece04 100644
--- a/turbo.json
+++ b/turbo.json
@@ -63,9 +63,7 @@
"!docs/.vitepress/dist/**",
"!website/_site/**",
"!experiments/analysis-efficiency/results/**",
- "vitest.reverse-tools.config.ts",
- "vitest.reverse-tools-semantics.config.ts",
- "vitest.config.ts"
+ "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
]
},
"lint:built": {
@@ -86,9 +84,7 @@
"!docs/.vitepress/dist/**",
"!website/_site/**",
"!experiments/analysis-efficiency/results/**",
- "vitest.reverse-tools.config.ts",
- "vitest.reverse-tools-semantics.config.ts",
- "vitest.config.ts"
+ "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
]
},
"format:check": {},
@@ -104,8 +100,8 @@
"tests/**/*.ts",
"tsconfig.json",
"tsconfig.build.json",
- "vitest.config.ts",
- "docs/.vitepress/config.ts"
+ "docs/.vitepress/config.ts",
+ "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
]
},
"jscpd": {
diff --git a/vitest.config.ts b/vitest.config.ts
index a8bf6b045..2c034beb2 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -3,6 +3,10 @@ import { availableParallelism, tmpdir } from "node:os";
import { join } from "node:path";
import { defineConfig } from "vitest/config";
+import {
+ BUILT_TEST_PROJECTS,
+ SOURCE_TEST_PROJECTS,
+} from "./scripts/lib/test-projects.mjs";
import { CiTestSequencer } from "./tests/ciTestSequencer.js";
@@ -28,57 +32,32 @@ const COVERAGE_SHARD = process.argv.some((argument) =>
// Keep local and CI concurrency bounded by the runner rather than command locks.
const MAX_TEST_WORKERS = Math.min(2, availableParallelism());
-const TEST_PROJECTS = [
- {
- name: "domain",
- include: ["src/{contracts,domain}/**/*.test.ts"],
+const TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
+const PROJECT_POLICIES = {
+ domain: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Domain and contract modules are pure and own no process-global state, so
// per-file module isolation adds startup cost without protecting a seam.
isolate: false,
},
- {
- name: "services",
- include: ["src/application/**/*.test.ts"],
+ services: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Service tests use recording ports and own no process-global state. Share
// their module graph so file startup does not dominate the lane.
isolate: false,
},
- {
- name: "adapters",
- include: [
- "src/*.test.ts",
- "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
- ],
- pool: "forks" as const,
- maxWorkers: MAX_TEST_WORKERS,
- },
- {
- name: "composition",
- include: ["tests/composition/**/*.test.ts"],
+ adapters: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
+ composition: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Composition tests use provider-neutral sessions and test-scoped cleanup;
// sharing their immutable module graph avoids per-file startup work.
isolate: false,
},
- {
- name: "boundary",
- include: ["tests/boundary/**/*.test.ts"],
- exclude: [
- "tests/boundary/cli/**/*.test.ts",
- "tests/boundary/mcp/**/*.test.ts",
- "tests/boundary/process/**/*.test.ts",
- ],
- pool: "forks" as const,
- maxWorkers: MAX_TEST_WORKERS,
- },
- {
- name: "cli-boundary",
- include: ["tests/boundary/cli/**/*.test.ts"],
+ boundary: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
+ "cli-boundary": {
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
// CLI tests create fresh command instances or subprocesses and own their
@@ -88,54 +67,36 @@ const TEST_PROJECTS = [
// before their two heaps compete with the real CLI child processes.
execArgv: ["--max-old-space-size=384"],
},
- {
- // Process-tree observations share host sampling resources; keep this small
- // lane serial while the independent projects run together.
- name: "process-boundary",
- include: ["tests/boundary/process/**/*.test.ts"],
+ "process-boundary": {
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
+ // Process-tree observations share host sampling resources; keep this small
+ // lane serial while the independent projects run together.
fileParallelism: false,
sequence: { groupOrder: 1 },
},
- {
- name: "mcp-boundary",
- include: ["tests/boundary/mcp/**/*.test.ts"],
+ "mcp-boundary": {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Each file creates and closes an explicit in-memory MCP session. Reuse the
// immutable server graph so module startup does not outweigh transport work.
isolate: false,
},
- {
- name: "acceptance",
- include: ["tests/acceptance/**/*.test.ts"],
- pool: "forks" as const,
- maxWorkers: MAX_TEST_WORKERS,
- },
- {
- name: "process-global",
- include: ["tests/process-global/**/*.test.ts"],
- pool: "forks" as const,
- maxWorkers: MAX_TEST_WORKERS,
- },
- {
- name: "conformance",
- include: ["tests/conformance/**/*.test.ts"],
+ acceptance: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
+ "process-global": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
+ conformance: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Contract cases use recording providers and own no process-global state.
isolate: false,
},
- {
- name: "evaluation",
- include: ["tests/evaluation/**/*.test.ts"],
+ evaluation: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Evaluator tests inspect deterministic inputs without mutating shared state.
isolate: false,
},
-];
+};
const ZERO_COVERAGE_THRESHOLDS = {
statements: 0,
@@ -156,10 +117,21 @@ const ZERO_COVERAGE_THRESHOLDS = {
},
};
-const projects = TEST_PROJECTS.map((project) => ({
- extends: true as const,
- test: project,
-}));
+const projects = TEST_PROJECTS.map((project) => {
+ const policy =
+ PROJECT_POLICIES[project.name as keyof typeof PROJECT_POLICIES];
+ if (!policy)
+ throw new Error(`Missing test runner policy for ${project.name}`);
+ return {
+ extends: true as const,
+ test: {
+ name: project.name,
+ include: [...project.include],
+ ...(project.exclude ? { exclude: [...project.exclude] } : {}),
+ ...policy,
+ },
+ };
+});
export default defineConfig({
test: {