Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(ci): preserve per-job findings and unavailable blob evidence

morluto22552525d54f
diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs
index 9a7cc1c21..250a15872 100644
--- a/scripts/ci/artifacts.mjs
+++ b/scripts/ci/artifacts.mjs
@@ -87,10 +87,13 @@ export function classifyFailure({
"unknown",
"main_run_does_not_contain_the_same_structured_diagnostic",
);
- if (!sameBlobEvidence(diagnostics, blobs))
+ const blobStatus = blobIdentityStatus(diagnostics, blobs);
+ if (blobStatus !== "identical")
return outcome(
"unknown",
- "diagnostic_source_content_not_identical_at_main_and_tested_base",
+ blobStatus === "different"
+ ? "diagnostic_source_content_not_identical_at_main_and_tested_base"
+ : "diagnostic_source_blob_identity_unavailable",
);
return outcome(
"matching-main-diagnostics",
@@ -366,9 +369,12 @@ function diagnosticKey(item) {
return JSON.stringify([item.kind, ...[...item.paths].sort()]);
return null;
}
-function sameBlobEvidence(diagnostics, blobs) {
+function blobIdentityStatus(diagnostics, blobs) {
const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];
- return paths.length > 0 && paths.every((path) => blobs?.[path] === true);
+ if (paths.some((path) => blobs?.[path] === false)) return "different";
+ return paths.length > 0 && paths.every((path) => blobs?.[path] === true)
+ ? "identical"
+ : "unavailable";
}
function sameTaskHashes(diagnostics, left, right) {
return diagnostics.every((diagnostic) => {
diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs
index 2ce6d813c..12364c59f 100644
--- a/scripts/ci/failure-report.mjs
+++ b/scripts/ci/failure-report.mjs
@@ -153,16 +153,21 @@ async function analyzeFailedRun({
];
const failures = [];
for (const job of failedJobs) {
- const evidence = await api.getStaticEvidence(repository, run, job);
- const classification = await attributeToMain({
- repository,
- defaultBranch,
- run,
- job,
- evidence,
- mainCandidates,
- api,
- });
+ let classification;
+ try {
+ const evidence = await api.getStaticEvidence(repository, run, job);
+ classification = await attributeToMain({
+ repository,
+ defaultBranch,
+ run,
+ job,
+ evidence,
+ mainCandidates,
+ api,
+ });
+ } catch (error) {
+ classification = outcome("unknown", errorCode(error));
+ }
failures.push({
runId: entry.id,
runAttempt: entry.runAttempt,
@@ -415,26 +420,25 @@ export function createGitHubApi({
for (const path of paths) {
if (!validRepoPath(path)) continue;
const [left, right] = await Promise.all([
- getContent(repository, ancestor, path, request),
- getContent(repository, descendant, path, request),
+ getContentBlobSha(repository, ancestor, path, request),
+ getContentBlobSha(repository, descendant, path, request),
]);
- result[path] = left !== null && left === right;
+ result[path] = left === right;
}
return result;
},
};
}
-async function getContent(repository, ref, path, request) {
- try {
- const encodedPath = path.split("/").map(encodeURIComponent).join("/");
- const value = await request(
- `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
- );
- return typeof value.sha === "string" ? value.sha : null;
- } catch {
- return null;
- }
+async function getContentBlobSha(repository, ref, path, request) {
+ const encodedPath = path.split("/").map(encodeURIComponent).join("/");
+ const value = await request(
+ `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
+ );
+ const sha = safeSha(value?.sha);
+ if (value?.type !== "file" || sha === null)
+ throw new Error("github_content_blob_identity_unavailable");
+ return sha;
}
function outcome(classification, reason) {
diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts
index 26feb3ee2..c2a282906 100644
--- a/tests/boundary/filesystem/ciFailureReport.test.ts
+++ b/tests/boundary/filesystem/ciFailureReport.test.ts
@@ -120,6 +120,22 @@ it.each([
expect(result.classification).toBe("unknown");
});
+it.each([
+ [{}, "diagnostic_source_blob_identity_unavailable"],
+ [
+ { "src/example.ts": false },
+ "diagnostic_source_content_not_identical_at_main_and_tested_base",
+ ],
+] as const)(
+ "distinguishes unavailable blob identity from verified differences (%s)",
+ (blobs, reason) => {
+ expect(classifyFailure({ ...comparison(), blobs })).toEqual({
+ classification: "unknown",
+ reason,
+ });
+ },
+);
+
it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => {
const directory = await createTestTempDirectory("rea-ci-failure-test-");
const calls: string[] = [];
diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.ts
index 92fe3e4ae..1caacde16 100644
--- a/tests/boundary/filesystem/ciFailureReportBudget.test.ts
+++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts
@@ -2,6 +2,7 @@ import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
reportCiFailures,
+ createGitHubApi,
type CiRun,
type FailureReportApi,
type StaticDiagnostic,
@@ -235,6 +236,10 @@ it.each([1, 51])(
expect(result.failures[0]!.classification).toBe(
count === 1 ? "matching-main-diagnostics" : "unknown",
);
+ if (count > 50)
+ expect(result.failures[0]!.reason).toBe(
+ "diagnostic_source_blob_identity_unavailable",
+ );
},
);
@@ -288,3 +293,115 @@ it.each(["jobs", "artifact", "comparison"])(
expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1);
},
);
+
+it.each([0, 1, 2])(
+ "preserves every failed job when artifact analysis fails at position %s",
+ async (position) => {
+ const input = fixture([20], 1);
+ const jobs = [
+ { name: "Python producer", conclusion: "failure" },
+ { name: "Test shard (1)", conclusion: "timed_out" },
+ ];
+ jobs.splice(position, 0, { name: "Static checks", conclusion: "failure" });
+ input.api.getJobs = async (_repository, id, attempt) =>
+ jobs.map((job) => ({
+ ...job,
+ status: "completed",
+ run_id: Number(id),
+ run_attempt: attempt,
+ }));
+ input.api.getStaticEvidence = async (_repository, _run, job) => {
+ if (job.name === "Static checks")
+ throw new Error("static_artifact_unavailable");
+ return null;
+ };
+ const result = await report(input);
+ expect(result.counts.unknown).toBe(3);
+ expect(result.failures).toEqual(
+ jobs.map((job) => ({
+ runId: 100,
+ runAttempt: 1,
+ runHead: prSha,
+ event: "pull_request",
+ job: job.name,
+ conclusion: job.conclusion,
+ classification: "unknown",
+ reason:
+ job.name === "Static checks"
+ ? "static_artifact_unavailable"
+ : "no_matching_verified_main_failure",
+ })),
+ );
+ },
+);
+
+it("keeps unavailable job inventories at run scope", async () => {
+ const input = fixture([], 1);
+ input.api.getJobs = async () => {
+ throw new Error("job_inventory_unavailable");
+ };
+ expect((await report(input)).failures).toEqual([
+ {
+ runId: 100,
+ runAttempt: 1,
+ classification: "unknown",
+ reason: "job_inventory_unavailable",
+ },
+ ]);
+});
+
+it.each([
+ ["request", "github_api_temporarily_unavailable"],
+ ["budget", "github_api_request_budget_exhausted"],
+ ["missing sha", "github_content_blob_identity_unavailable"],
+ ["invalid sha", "github_content_blob_identity_unavailable"],
+ ["directory", "github_content_blob_identity_unavailable"],
+] as const)(
+ "retains unavailable blob evidence as a comparison error: %s",
+ async (failure, reason) => {
+ const input = fixture([20], 1);
+ const contentApi = createGitHubApi({
+ token: "fixture-token",
+ apiRequest: async () => {
+ if (failure === "request") throw new Error(reason);
+ if (failure === "missing sha") return { type: "file" };
+ if (failure === "invalid sha")
+ return { type: "file", sha: "not-a-sha" };
+ if (failure === "directory") return [];
+ return { type: "file", sha: "e".repeat(40) };
+ },
+ });
+ if (failure === "budget") {
+ for (let id = 1; id <= 250; id++) await contentApi.getRun(repository, id);
+ }
+ input.api.getBlobIdentity = contentApi.getBlobIdentity;
+ const result = await report(input);
+ expect(result.failures).toMatchObject([
+ {
+ job: "Static checks",
+ classification: "unknown",
+ reason: "no_matching_verified_main_failure",
+ candidateFailures: [{ runId: 20, runAttempt: 1, reason }],
+ },
+ ]);
+ },
+);
+
+it.each([true, false])(
+ "reports blob equality only from verified file SHAs (equal: %s)",
+ async (equal) => {
+ const api = createGitHubApi({
+ token: "fixture-token",
+ apiRequest: async (_method, endpoint) => ({
+ type: "file",
+ sha:
+ equal || endpoint.includes(`ref=${mainSha}`)
+ ? "e".repeat(40)
+ : "f".repeat(40),
+ }),
+ });
+ expect(
+ await api.getBlobIdentity(repository, mainSha, prSha, ["src/example.ts"]),
+ ).toEqual({ "src/example.ts": equal });
+ },
+);