Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

ci: observe affected-owner test plans against full results

morlutofa0a9b2d328a
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b1af43269..28373f89d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -61,6 +61,7 @@ jobs:
nativeaot: ${{ steps.plan.outputs.nativeaot }}
website: ${{ steps.plan.outputs.website }}
matrix: ${{ steps.plan.outputs.matrix }}
+ test_selection: ${{ steps.plan.outputs.test_selection }}
steps:
- uses: actions/checkout@v4
with:
@@ -315,6 +316,8 @@ jobs:
SHARD_RESULT: ${{ needs.test-shard.result }}
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
CI_VERIFY_TEST_INVENTORY: "true"
+ CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
+ CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: node scripts/ci/test-report.mjs
- uses: actions/upload-artifact@v4
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
@@ -325,6 +328,8 @@ jobs:
test-results.json
test-timings.json
coverage/
+ .cache/ci-test-selection/report.json
+ include-hidden-files: true
if-no-files-found: error
retention-days: 30
- name: Publish successful timing history
diff --git a/docs/testing.md b/docs/testing.md
index ac1a01f84..03121e172 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -805,6 +805,27 @@ remain present and fail if classification fails. Every code change retains the
complete six-shard Linux suite; individual test files are not selected by
imports or filenames.
+Ordinary PRs also produce an **observational** test-selection plan. The candidate
+leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately
+from provider-job scopes. It proposes keeping affected owners plus every unowned
+test, including all CLI/MCP, acceptance, composition, conformance, and process
+tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall
+back to the complete inventory. Renames account for both paths. Main, scheduled,
+manual, and `ci:full` runs always propose the full suite.
+
+All six shards still execute the complete suite. The existing report merge
+compares the proposal with actual module outcomes and retains
+`.cache/ci-test-selection/report.json` in `test-results`, including selected and
+omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and
+executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate,
+skipped, interrupted, or unhandled results cannot establish a complete comparison;
+even a complete passing comparison is not proof of ownership completeness.
+Before enabling omissions, review representative full-run comparisons and
+cross-owner consumers, including failures, and measure completed CI feedback and
+runner demand. There is no automatic promotion or switch to skip tests. The
+initial proposal measures only leaf producer-test omissions; it does not promise
+the savings of a broader affected-test planner.
+
`scripts/ci/plan.mjs` selects lanes using the ownership table in
`scripts/ci/scopes.mjs`. Provider implementation changes run their real-provider
verifiers and the Linux x64 installed-package/Inspector baseline. The `apple`
diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs
index 3c711a1f9..86879216d 100644
--- a/scripts/ci/plan.mjs
+++ b/scripts/ci/plan.mjs
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { matchesGlob } from "node:path";
import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
+import { planTestSelection } from "./test-selection.mjs";
// Test files select the host/runtime lanes that execute their focused suites.
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
@@ -250,11 +251,17 @@ const matrix = {
]
: [linux],
};
-const plan = { ...selected, tier, matrix, files, reasons };
+const testSelection = planTestSelection(files, event, tier);
+const plan = { ...selected, tier, matrix, files, reasons, testSelection };
if (process.env.GITHUB_OUTPUT)
appendFileSync(
process.env.GITHUB_OUTPUT,
- Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) })
+ Object.entries({
+ ...selected,
+ tier,
+ matrix: JSON.stringify(matrix),
+ test_selection: JSON.stringify(testSelection),
+ })
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
);
diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs
index 86419ae9f..035a01990 100644
--- a/scripts/ci/test-report.mjs
+++ b/scripts/ci/test-report.mjs
@@ -4,25 +4,33 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path";
import { z } from "zod";
+import { compareTestSelection } from "./test-selection.mjs";
+
const root = process.cwd();
const historyDirectory = join(root, ".cache/ci-test-history");
const historyPath = join(historyDirectory, "test-results.json");
+const selectionDirectory = join(root, ".cache/ci-test-selection");
+const selectionPath = join(selectionDirectory, "report.json");
// Never let a failed or partial rerun leave a previous successful history file.
await rm(historyPath, { force: true });
+await rm(selectionPath, { force: true });
const report = z
.object({
version: z.literal(1),
success: z.boolean(),
+ reason: z.string().optional(),
+ unhandledErrors: z.number().int().nonnegative().optional(),
files: z.array(
z.object({
name: z.string().min(1),
durationMs: z.number().finite().nonnegative(),
+ state: z.string().optional(),
}),
),
})
.parse(JSON.parse(await readFile("test-timings.json", "utf8")));
-const files = report.files.map(({ name, durationMs }) => {
+const files = report.files.map(({ name, durationMs, state }) => {
const path = relative(root, resolve(root, name));
if (
path === "" ||
@@ -31,21 +39,47 @@ const files = report.files.map(({ name, durationMs }) => {
isAbsolute(path)
)
throw new Error(`Test report path is outside the checkout: ${name}`);
- return { name: path.split(sep).join("/"), durationMs };
+ return { name: path.split(sep).join("/"), durationMs, state };
});
const complete = process.env.SHARD_RESULT === "success";
const passed =
complete && process.env.MERGE_RESULT === "success" && report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
-if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
- const expected = execFileSync(
- "git",
- ["ls-files", "-z", "--", "src", "tests"],
- { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 },
- )
- .split("\0")
- .filter((path) => path.endsWith(".test.ts"));
+const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
+const expected =
+ verifyInventory || process.env.CI_TEST_SELECTION
+ ? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
+ encoding: "utf8",
+ maxBuffer: 8 * 1024 * 1024,
+ })
+ .split("\0")
+ .filter((path) => path.endsWith(".test.ts"))
+ : [];
+let selection;
+if (process.env.CI_TEST_SELECTION) {
+ const comparison = compareTestSelection(
+ JSON.parse(process.env.CI_TEST_SELECTION),
+ expected,
+ files,
+ );
+ selection = {
+ ...comparison,
+ complete:
+ comparison.complete &&
+ report.unhandledErrors === 0 &&
+ ["passed", "failed"].includes(report.reason) &&
+ ["success", "failure"].includes(process.env.SHARD_RESULT) &&
+ ["success", "failure"].includes(process.env.MERGE_RESULT),
+ executionSha: process.env.GITHUB_SHA ?? null,
+ prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
+ runId: process.env.GITHUB_RUN_ID ?? null,
+ runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
+ };
+ await mkdir(selectionDirectory, { recursive: true });
+ await writeFile(selectionPath, JSON.stringify(selection));
+}
+if (passed && verifyInventory) {
const actual = files.map(({ name }) => name);
const expectedSet = new Set(expected);
const actualSet = new Set(actual);
@@ -59,6 +93,16 @@ if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
const summary = [
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
"",
+ ...(selection
+ ? [
+ "### Observational PR test selection",
+ "",
+ `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
+ `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
+ "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
+ "",
+ ]
+ : []),
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
passed
? "Successful timing history is available."
@@ -85,6 +129,10 @@ if (passed) {
await mkdir(historyDirectory, { recursive: true });
await writeFile(
historyPath,
- JSON.stringify({ version: 1, success: true, files }),
+ JSON.stringify({
+ version: 1,
+ success: true,
+ files: files.map(({ name, durationMs }) => ({ name, durationMs })),
+ }),
);
}
diff --git a/scripts/ci/test-selection.d.mts b/scripts/ci/test-selection.d.mts
new file mode 100644
index 000000000..c19414398
--- /dev/null
+++ b/scripts/ci/test-selection.d.mts
@@ -0,0 +1,35 @@
+export interface TestSelectionPlan {
+ readonly version: 1;
+ readonly mode: "full" | "candidate";
+ readonly owners: readonly string[];
+ readonly changedTests: readonly string[];
+ readonly changedPaths: readonly string[];
+ readonly reason: string;
+}
+export function planTestSelection(
+ files: readonly string[],
+ event: string,
+ tier: string,
+): TestSelectionPlan;
+export function compareTestSelection(
+ plan: unknown,
+ inventory: readonly string[],
+ results: readonly {
+ readonly name: string;
+ readonly durationMs: number;
+ readonly state?: string;
+ }[],
+): {
+ readonly version: 1;
+ readonly observational: true;
+ readonly plan: TestSelectionPlan;
+ readonly selected: readonly string[];
+ readonly omitted: readonly string[];
+ readonly missing: readonly string[];
+ readonly unexpected: readonly string[];
+ readonly duplicateFiles: number;
+ readonly unobserved: readonly string[];
+ readonly missedFailures: readonly string[];
+ readonly complete: boolean;
+ readonly omittedDurationMs: number;
+};
diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs
new file mode 100644
index 000000000..8165e3658
--- /dev/null
+++ b/scripts/ci/test-selection.mjs
@@ -0,0 +1,151 @@
+import { matchesGlob } from "node:path";
+
+// Candidate ownership only: these leaf adapters and their producer tests can be
+// compared with full-suite outcomes. This table never changes test execution.
+// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
+const OWNERS = {
+ adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
+ apktool: ["src/apktool/**"],
+ dotnet: ["src/dotnet/**"],
+ evm: ["src/evm/**", "tests/boundary/evm/**"],
+ firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
+ flutter: ["src/flutter/**"],
+ ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
+ hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
+ ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
+ jeb: ["src/jeb/**"],
+ pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
+ sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
+ wasm: ["src/wasm/**"],
+};
+const ownersOf = (path) =>
+ Object.entries(OWNERS)
+ .filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
+ .map(([owner]) => owner);
+const validPath = (path) =>
+ typeof path === "string" &&
+ /^(src|tests)\//.test(path) &&
+ !/[\\\0\r\n]/.test(path) &&
+ !path.split("/").some((part) => ["", ".", ".."].includes(part));
+const full = (reason, changedPaths = []) => ({
+ version: 1,
+ mode: "full",
+ owners: [],
+ changedTests: [],
+ changedPaths,
+ reason,
+});
+
+/** Plan an observational PR subset from the classifier's existing Git diff. */
+export function planTestSelection(files, event, tier) {
+ if (event !== "pull_request" || tier !== "fast")
+ return full("Only ordinary PRs are candidates", files);
+ if (!files.length) return full("No changed paths available");
+ const owners = new Set();
+ const changedTests = [];
+ for (const path of files) {
+ if (!validPath(path))
+ return full(`Shared or unknown input: ${path}`, files);
+ const matches = ownersOf(path);
+ if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
+ for (const owner of matches) owners.add(owner);
+ if (path.endsWith(".test.ts")) changedTests.push(path);
+ }
+ return {
+ version: 1,
+ mode: "candidate",
+ owners: [...owners].sort(),
+ changedTests: [...new Set(changedTests)].sort(),
+ changedPaths: files,
+ reason:
+ "Observe affected leaf owners; retain integration and unowned tests",
+ };
+}
+
+/** Compare a proposal with the full run; missing observations never prove parity. */
+export function compareTestSelection(plan, inventory, results) {
+ if (
+ plan?.version !== 1 ||
+ !["full", "candidate"].includes(plan.mode) ||
+ !Array.isArray(plan.owners) ||
+ plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
+ !Array.isArray(plan.changedTests) ||
+ !Array.isArray(plan.changedPaths) ||
+ plan.changedPaths.some((path) => typeof path !== "string") ||
+ plan.changedTests.some(
+ (path) => !validPath(path) || !path.endsWith(".test.ts"),
+ ) ||
+ typeof plan.reason !== "string" ||
+ (plan.mode === "candidate" && !plan.owners.length) ||
+ (plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
+ )
+ throw new Error("Invalid observational test selection plan");
+ const expected = new Set(inventory);
+ if (!expected.size || expected.size !== inventory.length)
+ throw new Error("Test selection requires a nonempty unique inventory");
+ const ownersByFile = new Map(
+ plan.mode === "candidate"
+ ? inventory.map((path) => [path, ownersOf(path)])
+ : [],
+ );
+ // Removed/renamed tests have no current executable identity. Keep all tests.
+ const removed = plan.changedTests.filter((path) => !expected.has(path));
+ const missingOwners = plan.owners.filter(
+ (owner) =>
+ ![...ownersByFile.values()].some((owners) => owners.includes(owner)),
+ );
+ const effective = removed.length
+ ? full(
+ `Changed tests absent from current inventory: ${removed.join(", ")}`,
+ plan.changedPaths,
+ )
+ : missingOwners.length
+ ? full(
+ `Owners without executable tests: ${missingOwners.join(", ")}`,
+ plan.changedPaths,
+ )
+ : plan;
+ const selected = [];
+ const omitted = [];
+ for (const path of inventory.toSorted()) {
+ const owners = ownersByFile.get(path) ?? [];
+ const keep =
+ effective.mode === "full" ||
+ !owners.length ||
+ effective.changedTests.includes(path) ||
+ owners.some((owner) => effective.owners.includes(owner));
+ (keep ? selected : omitted).push(path);
+ }
+ const actual = new Set(results.map((file) => file.name));
+ const missing = inventory.filter((path) => !actual.has(path));
+ const unexpected = [...actual].filter((path) => !expected.has(path));
+ const duplicateFiles = results.length - actual.size;
+ const unobserved = results
+ .filter((file) => !["passed", "failed"].includes(file.state))
+ .map((file) => file.name);
+ const omittedSet = new Set(omitted);
+ const missedFailures = results
+ .filter((file) => omittedSet.has(file.name) && file.state === "failed")
+ .map((file) => file.name);
+ return {
+ version: 1,
+ observational: true,
+ plan: effective,
+ selected,
+ omitted,
+ missing,
+ unexpected,
+ duplicateFiles,
+ unobserved,
+ missedFailures,
+ complete:
+ !missing.length &&
+ !unexpected.length &&
+ !duplicateFiles &&
+ !unobserved.length,
+ // Phase totals are scheduling weights, not an estimate of wall-time savings.
+ omittedDurationMs: results
+ .filter((file) => omittedSet.has(file.name))
+ .reduce((sum, file) => sum + file.durationMs, 0),
+ };
+}
diff --git a/scripts/ci/test-timing-reporter.mjs b/scripts/ci/test-timing-reporter.mjs
index 29f0cc582..ce0b1c5dc 100644
--- a/scripts/ci/test-timing-reporter.mjs
+++ b/scripts/ci/test-timing-reporter.mjs
@@ -21,6 +21,7 @@ export default class TestTimingReporter {
};
return {
name: module.moduleId,
+ state: module.state(),
durationMs: Object.values(phases).reduce(
(total, value) => total + value,
0,
@@ -33,6 +34,8 @@ export default class TestTimingReporter {
JSON.stringify({
version: 1,
success: reason === "passed" && errors.length === 0,
+ reason,
+ unhandledErrors: errors.length,
files,
}),
);
diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts
index 714d2f980..fa4b71ebf 100644
--- a/tests/boundary/filesystem/ciChangeScope.test.ts
+++ b/tests/boundary/filesystem/ciChangeScope.test.ts
@@ -178,6 +178,23 @@ it("keeps unknown CI scripts on complete qualification", async () => {
for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true);
});
+it("emits an observational owner proposal without changing ordinary or full portable execution", async () => {
+ const { directory, base } = await fixture();
+ const head = await change(directory, ["src/evm/Example.ts"]);
+ expect(await classify(directory, base, head)).toMatchObject({
+ code: true,
+ testSelection: { mode: "candidate", owners: ["evm"] },
+ });
+ expect(
+ await classify(directory, base, head, "pull_request", true),
+ ).toMatchObject({
+ code: true,
+ testSelection: { mode: "full", owners: [] },
+ });
+ const output = await readFile(join(directory, "scope-output"), "utf8");
+ expect(output).toContain('test_selection={"version":1,"mode":"full"');
+});
+
it("does not charge a documentation PR for unrelated implementation updates on main", async () => {
const { directory, base } = await fixture();
await git(directory, ["switch", "-c", "docs/change"]);
diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts
index 333e93f08..06212ad07 100644
--- a/tests/boundary/filesystem/ciTestReports.test.ts
+++ b/tests/boundary/filesystem/ciTestReports.test.ts
@@ -8,6 +8,8 @@ import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
+import { planTestSelection } from "../../../scripts/ci/test-selection.mjs";
+
import { parseTestDurations } from "../../ciTestSequencer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { workflowGit } from "../../support/workflowGit.js";
@@ -61,6 +63,16 @@ it("collects failed-shard diagnostics before enforcing the shard result", async
"--reporter=./scripts/ci/test-timing-reporter.mjs",
);
expect(steps[upload]?.with?.path).toContain("test-timings.json");
+ expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
+ expect(steps[upload]?.with?.path).toContain(
+ ".cache/ci-test-selection/report.json",
+ );
+ const summary = steps.find((step) =>
+ step.run?.includes("scripts/ci/test-report.mjs"),
+ );
+ expect(summary?.env?.CI_TEST_SELECTION).toBe(
+ "${{ needs.changes.outputs.test_selection }}",
+ );
const command = steps[gate]?.run;
if (command === undefined) throw new Error("Missing shard result gate");
for (const result of ["failure", "cancelled", "skipped", ""]) {
@@ -304,6 +316,13 @@ it("recovers complete module costs from real blob reports without executing test
),
})
.parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")));
+ expect(
+ JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")),
+ ).toMatchObject({
+ reason: "passed",
+ unhandledErrors: 0,
+ files: [{ state: "passed" }],
+ });
expect(report.files).toHaveLength(1);
const file = report.files[0]!;
const json = z
@@ -349,7 +368,13 @@ it("recovers complete module costs from real blob reports without executing test
const failed = JSON.parse(
await readFile(join(root, "test-timings.json"), "utf8"),
) as unknown;
- expect(failed).toMatchObject({ version: 1, success: false });
+ expect(failed).toMatchObject({
+ version: 1,
+ success: false,
+ reason: "failed",
+ unhandledErrors: 0,
+ files: [{ state: "failed" }],
+ });
// Even a caller accidentally marking the merge successful cannot publish this report.
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
@@ -359,3 +384,76 @@ it("recovers complete module costs from real blob reports without executing test
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
});
+
+it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => {
+ const root = await createTestTempDirectory("rea-ci-selection-report-");
+ await workflowGit(root, ["init"]);
+ const paths = [
+ "src/sqlite/provider.test.ts",
+ "src/evm/provider.test.ts",
+ "tests/boundary/mcp/workflow.test.ts",
+ ];
+ for (const path of paths) {
+ await mkdir(join(root, path, ".."), { recursive: true });
+ await writeFile(join(root, path), "// fixture\n");
+ }
+ await workflowGit(root, ["add", "src", "tests"]);
+ const timing = {
+ version: 1,
+ success: false,
+ reason: "failed",
+ unhandledErrors: 0,
+ files: paths.map((name, index) => ({
+ name,
+ durationMs: 10,
+ state: index === 1 ? "failed" : "passed",
+ })),
+ };
+ const run = async (report: unknown) => {
+ await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
+ await execute(process.execPath, [fileURLToPath(script)], {
+ cwd: root,
+ env: {
+ ...process.env,
+ SHARD_RESULT: "failure",
+ MERGE_RESULT: "failure",
+ CI_VERIFY_TEST_INVENTORY: "true",
+ CI_TEST_SELECTION: JSON.stringify(
+ planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"),
+ ),
+ GITHUB_SHA: "merge-sha",
+ CI_PR_HEAD_SHA: "pr-head-sha",
+ },
+ });
+ return JSON.parse(
+ await readFile(
+ join(root, ".cache/ci-test-selection/report.json"),
+ "utf8",
+ ),
+ ) as unknown;
+ };
+ expect(await run(timing)).toMatchObject({
+ observational: true,
+ complete: true,
+ selected: [paths[0], paths[2]],
+ omitted: [paths[1]],
+ missedFailures: [paths[1]],
+ executionSha: "merge-sha",
+ prHeadSha: "pr-head-sha",
+ });
+ for (const report of [
+ { ...timing, files: timing.files.slice(0, 1) },
+ { ...timing, unhandledErrors: 1 },
+ { ...timing, reason: "interrupted" },
+ ])
+ expect(await run(report)).toMatchObject({ complete: false });
+ await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
+ code: 1,
+ });
+ await expect(
+ readFile(join(root, ".cache/ci-test-selection/report.json")),
+ ).rejects.toMatchObject({ code: "ENOENT" });
+ await expect(
+ readFile(join(root, ".cache/ci-test-history/test-results.json")),
+ ).rejects.toMatchObject({ code: "ENOENT" });
+});
diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts
new file mode 100644
index 000000000..1dff7f9e3
--- /dev/null
+++ b/tests/evaluation/ciTestSelection.test.ts
@@ -0,0 +1,240 @@
+import { expect, it } from "vitest";
+
+import {
+ compareTestSelection,
+ planTestSelection,
+} from "../../scripts/ci/test-selection.mjs";
+
+const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts";
+const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts";
+const adbSource = "src/adb/AdbProvider.test.ts";
+const adbBoundary =
+ "tests/boundary/providers/adb/adbBugreportPublication.test.ts";
+const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts";
+const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts";
+const processBoundary =
+ "tests/boundary/process/processCaptureLifecycle.test.ts";
+const acceptance = "tests/acceptance/tools/localToolContracts.test.ts";
+const composition = "tests/composition/directAnalysisCleanup.test.ts";
+const evaluation = "tests/evaluation/testDiscovery.test.ts";
+
+const passed = (name: string) => ({ name, durationMs: 10, state: "passed" });
+
+it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => {
+ const plan = planTestSelection(
+ ["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary],
+ "pull_request",
+ "fast",
+ );
+ const inventory = [
+ sqliteSource,
+ sqliteBoundary,
+ adbSource,
+ adbBoundary,
+ cliBoundary,
+ mcpBoundary,
+ processBoundary,
+ acceptance,
+ composition,
+ evaluation,
+ ];
+ const report = compareTestSelection(plan, inventory, inventory.map(passed));
+
+ expect(plan).toMatchObject({
+ mode: "candidate",
+ owners: ["adb", "sqlite"],
+ });
+ expect(report.selected).toEqual(
+ expect.arrayContaining([
+ sqliteSource,
+ sqliteBoundary,
+ adbSource,
+ adbBoundary,
+ cliBoundary,
+ mcpBoundary,
+ processBoundary,
+ acceptance,
+ composition,
+ evaluation,
+ ]),
+ );
+ expect(report.omitted).toEqual([]);
+});
+
+it("omits only unrelated owned tests while retaining dynamic boundary suites", () => {
+ const plan = planTestSelection(
+ ["src/sqlite/SqliteDatabaseProvider.ts"],
+ "pull_request",
+ "fast",
+ );
+ const inventory = [
+ sqliteSource,
+ sqliteBoundary,
+ adbSource,
+ adbBoundary,
+ cliBoundary,
+ mcpBoundary,
+ processBoundary,
+ acceptance,
+ composition,
+ evaluation,
+ ];
+ const report = compareTestSelection(plan, inventory, inventory.map(passed));
+
+ expect(report.selected).toEqual(
+ expect.arrayContaining([
+ sqliteSource,
+ sqliteBoundary,
+ cliBoundary,
+ mcpBoundary,
+ processBoundary,
+ acceptance,
+ composition,
+ evaluation,
+ ]),
+ );
+ expect(report.omitted).toEqual([adbSource, adbBoundary]);
+ expect(report.complete).toBe(true);
+});
+
+it.each([
+ ["unowned shared source", "src/application/Workflow.ts"],
+ ["fixture input", "tests/conformance/c/fixture.c"],
+ ["configuration", "vitest.config.ts"],
+ ["documentation", "README.md"],
+ ["CLI implementation", "src/cli/apktoolCommands.ts"],
+ ["MCP implementation", "src/server/registerSqliteTools.ts"],
+ ["process ownership", "src/process/ProcessOwnership.ts"],
+ ["acceptance test", acceptance],
+ ["composition test", composition],
+])("uses full fallback for %s changes", (_description, path) => {
+ expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({
+ mode: "full",
+ owners: [],
+ changedTests: [],
+ });
+});
+
+it("uses full fallback outside ordinary fast pull requests or without diff paths", () => {
+ expect(
+ planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast")
+ .mode,
+ ).toBe("full");
+ expect(
+ planTestSelection(
+ ["src/sqlite/SqliteDatabaseProvider.ts"],
+ "pull_request",
+ "full",
+ ).mode,
+ ).toBe("full");
+ expect(planTestSelection([], "pull_request", "fast").mode).toBe("full");
+});
+
+it("keeps new changed tests and falls back when a changed test was removed", () => {
+ const addedTest = "src/sqlite/NewDatabaseRule.test.ts";
+ const candidate = planTestSelection([addedTest], "pull_request", "fast");
+ expect(candidate.changedTests).toEqual([addedTest]);
+ const inventory = [addedTest, adbSource];
+ expect(
+ compareTestSelection(candidate, inventory, inventory.map(passed)).selected,
+ ).toContain(addedTest);
+
+ const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts";
+ const report = compareTestSelection(
+ planTestSelection([removedTest], "pull_request", "fast"),
+ [sqliteSource, adbSource],
+ [passed(sqliteSource), passed(adbSource)],
+ );
+ expect(report.plan.mode).toBe("full");
+ expect(report.selected).toEqual([adbSource, sqliteSource]);
+ expect(report.omitted).toEqual([]);
+});
+
+it("marks incomplete inventories and nonterminal outcomes as incomplete", () => {
+ const plan = planTestSelection(
+ ["src/sqlite/SqliteDatabaseProvider.ts"],
+ "pull_request",
+ "fast",
+ );
+ const inventory = [sqliteSource, adbSource];
+ const completeResults = inventory.map(passed);
+
+ expect(
+ compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete,
+ ).toBe(false);
+ expect(
+ compareTestSelection(plan, inventory, [
+ ...completeResults,
+ passed("tests/evaluation/unexpected.test.ts"),
+ ]),
+ ).toMatchObject({
+ complete: false,
+ unexpected: ["tests/evaluation/unexpected.test.ts"],
+ });
+ expect(
+ compareTestSelection(plan, inventory, [
+ ...completeResults,
+ passed(adbSource),
+ ]),
+ ).toMatchObject({ complete: false, duplicateFiles: 1 });
+ for (const state of ["pending", "skipped"])
+ expect(
+ compareTestSelection(plan, inventory, [
+ passed(sqliteSource),
+ { name: adbSource, durationMs: 0, state },
+ ]),
+ ).toMatchObject({ complete: false, unobserved: [adbSource] });
+});
+
+it("reports a failure from an omitted test, including collection or module failures", () => {
+ const plan = planTestSelection(
+ ["src/sqlite/SqliteDatabaseProvider.ts"],
+ "pull_request",
+ "fast",
+ );
+ const report = compareTestSelection(
+ plan,
+ [sqliteSource, adbSource],
+ [passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }],
+ );
+
+ expect(report.omitted).toContain(adbSource);
+ expect(report.missedFailures).toEqual([adbSource]);
+});
+
+it("rejects plans that name unowned owners or malformed changed tests", () => {
+ const plan = planTestSelection(
+ ["src/sqlite/SqliteDatabaseProvider.ts"],
+ "pull_request",
+ "fast",
+ );
+ expect(() =>
+ compareTestSelection(
+ { ...plan, owners: ["unknown"] },
+ [sqliteSource],
+ [passed(sqliteSource)],
+ ),
+ ).toThrow("Invalid observational test selection plan");
+ expect(() =>
+ compareTestSelection(
+ { ...plan, changedTests: ["src/sqlite/not-a-test.ts"] },
+ [sqliteSource],
+ [passed(sqliteSource)],
+ ),
+ ).toThrow("Invalid observational test selection plan");
+});
+
+it("uses full fallback when a changed owner has no executable tests", () => {
+ const report = compareTestSelection(
+ planTestSelection(
+ ["src/sqlite/SqliteDatabaseProvider.ts"],
+ "pull_request",
+ "fast",
+ ),
+ [adbSource],
+ [passed(adbSource)],
+ );
+
+ expect(report.plan.mode).toBe("full");
+ expect(report.omitted).toEqual([]);
+});