Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(ci): reject incomplete verification plans

morlutobd2c202dd930
diff --git a/docs/testing.md b/docs/testing.md
index 22b5c0c77..6dddda64b 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -874,7 +874,10 @@ summary. Representative Git-diff cases live in
`tests/boundary/filesystem/ciChangeScope.test.ts`.
`CI required` always runs and checks classification and every selected lane,
-rejecting failed, cancelled, or unexpectedly skipped work. Branch protection
+rejecting failed, cancelled, or unexpectedly skipped work. It rejects missing or
+unknown tiers and requires every canonical scope to be explicitly `true` or
+`false`; incomplete classification cannot silently omit verification lanes.
+Branch protection
should require this aggregate, `Static checks`, `Portable tests`, and
`Conventional PR title`. Title validation runs separately on PR opening,
reopening, commits, and edits; correcting a title refreshes only this lightweight
diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs
index afe52c2b8..01907de37 100644
--- a/scripts/ci/required.mjs
+++ b/scripts/ci/required.mjs
@@ -2,22 +2,31 @@ import { CI_FLAGS } from "./scopes.mjs";
import { recordQualification } from "./qualification.mjs";
const jobs = JSON.parse(process.env.CI_JOBS);
-const outputs = jobs.changes.outputs;
+const outputs = jobs?.changes?.outputs;
+if (!["fast", "full", "reused"].includes(outputs?.tier))
+ throw new Error(
+ "CI classification requires a valid tier: fast, full, or reused",
+ );
+const invalidFlags = CI_FLAGS.filter(
+ (flag) => outputs[flag] !== "true" && outputs[flag] !== "false",
+);
+if (invalidFlags.length)
+ throw new Error(
+ `CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`,
+ );
+const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true");
const reused = outputs.tier === "reused";
if (
reused &&
(process.env.GITHUB_EVENT_NAME !== "schedule" ||
!/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") ||
!Number.isSafeInteger(Number(outputs.qualification_run_id)) ||
- CI_FLAGS.some((flag) => outputs[flag] !== "false"))
+ selectedScopes.length !== 0)
)
throw new Error(
"Reused qualification requires a scheduled admission receipt and no selected lanes",
);
-if (
- outputs.tier === "full" &&
- CI_FLAGS.some((flag) => outputs[flag] !== "true")
-)
+if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
const required = reused ? ["changes"] : ["changes", "static", "test"];
const lanes = {
@@ -31,9 +40,7 @@ const lanes = {
docs: ["docs"],
workflows: ["static"],
};
-for (const [scope, value] of Object.entries(outputs)) {
- if (value === "true") required.push(...(lanes[scope] ?? [scope]));
-}
+for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope]));
const failed = [...new Set(required)].filter(
(name) => jobs[name]?.result !== "success",
);
diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts
index 8ed397e84..759f5c367 100644
--- a/tests/boundary/filesystem/ciChangeScope.test.ts
+++ b/tests/boundary/filesystem/ciChangeScope.test.ts
@@ -662,8 +662,16 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn
const command = fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
);
+ const { CI_FLAGS } = await loadScopes();
const jobs = {
- changes: { result: "success", outputs: { evm: "true" } },
+ changes: {
+ result: "success",
+ outputs: {
+ ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])),
+ tier: "fast",
+ evm: "true",
+ },
+ },
static: { result: "success" },
test: { result: "success" },
evm: { result: "success" },
diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts
index 1bac771a2..e460bb52b 100644
--- a/tests/boundary/filesystem/ciRequiredLanes.test.ts
+++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts
@@ -26,6 +26,15 @@ const workflow = async (filename = "ci.yml") =>
),
);
+const scopeFlags = async () =>
+ z
+ .object({ CI_FLAGS: z.array(z.string()) })
+ .parse(
+ await import(
+ new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
+ ),
+ ).CI_FLAGS;
+
it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => {
expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe(
"needs.changes.outputs.sqlite == 'true'",
@@ -57,13 +66,7 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and
it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => {
const { jobs } = await workflow();
- const { CI_FLAGS } = z
- .object({ CI_FLAGS: z.array(z.string()) })
- .parse(
- await import(
- new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
- ),
- );
+ const CI_FLAGS = await scopeFlags();
const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs);
expect(jobs.changes?.permissions).toEqual({
contents: "read",
@@ -294,7 +297,14 @@ it.each([
);
needs.changes = {
result: "success",
- outputs: { code: "true", [scope]: selected },
+ outputs: {
+ ...Object.fromEntries(
+ (await scopeFlags()).map((flag) => [flag, "false"]),
+ ),
+ tier: "fast",
+ code: "true",
+ [scope]: selected,
+ },
};
if (result === "missing") delete needs[lane];
else needs[lane] = { result };
@@ -319,3 +329,69 @@ it.each([
}
},
);
+
+it("rejects incomplete or malformed classification even when available jobs succeeded", async () => {
+ const flags = await scopeFlags();
+ const valid = {
+ ...Object.fromEntries(flags.map((flag) => [flag, "false"])),
+ tier: "fast",
+ };
+ const run = (outputs: unknown) =>
+ executeWorkflowFixture(
+ process.execPath,
+ [
+ fileURLToPath(
+ new URL("../../../scripts/ci/required.mjs", import.meta.url),
+ ),
+ ],
+ {
+ env: {
+ ...process.env,
+ CI_JOBS: JSON.stringify({
+ changes: { result: "success", outputs },
+ static: { result: "success" },
+ test: { result: "success" },
+ }),
+ CI_QUALIFICATION_DIRECTORY: "",
+ },
+ },
+ );
+ await expect(run(valid)).resolves.toMatchObject({ stderr: "" });
+ // Metadata is not a scope, even when its string happens to be "true".
+ await expect(
+ run({ ...valid, qualification_run_id: "true" }),
+ ).resolves.toMatchObject({ stderr: "" });
+ for (const outputs of [
+ undefined,
+ null,
+ {},
+ { ...valid, tier: undefined },
+ { ...valid, tier: "" },
+ { ...valid, tier: "unknown" },
+ ]) {
+ await expect(run(outputs)).rejects.toMatchObject({
+ code: 1,
+ stderr: expect.stringContaining("valid tier"),
+ });
+ }
+ for (const flag of flags) {
+ await expect(run({ ...valid, [flag]: undefined })).rejects.toMatchObject({
+ code: 1,
+ stderr: expect.stringContaining(`scope outputs: ${flag}`),
+ });
+ }
+ for (const value of ["", "TRUE", "yes", true, false, 0, null, [], {}]) {
+ await expect(run({ ...valid, sqlite: value })).rejects.toMatchObject({
+ code: 1,
+ stderr: expect.stringContaining("scope outputs: sqlite"),
+ });
+ }
+ for (const tier of ["full", "reused"]) {
+ await expect(
+ run({ ...valid, tier, sqlite: undefined }),
+ ).rejects.toMatchObject({
+ code: 1,
+ stderr: expect.stringContaining("scope outputs: sqlite"),
+ });
+ }
+});