Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(ci): preserve per-job findings and unavailable blob evidence
morluto22552525d54f
diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjsindex 9a7cc1c21..250a15872 100644--- a/scripts/ci/artifacts.mjs+++ b/scripts/ci/artifacts.mjs@@ -87,10 +87,13 @@ export function classifyFailure({"unknown","main_run_does_not_contain_the_same_structured_diagnostic",);- if (!sameBlobEvidence(diagnostics, blobs))+ const blobStatus = blobIdentityStatus(diagnostics, blobs);+ if (blobStatus !== "identical")return outcome("unknown",- "diagnostic_source_content_not_identical_at_main_and_tested_base",+ blobStatus === "different"+ ? "diagnostic_source_content_not_identical_at_main_and_tested_base"+ : "diagnostic_source_blob_identity_unavailable",);return outcome("matching-main-diagnostics",@@ -366,9 +369,12 @@ function diagnosticKey(item) {return JSON.stringify([item.kind, ...[...item.paths].sort()]);return null;}-function sameBlobEvidence(diagnostics, blobs) {+function blobIdentityStatus(diagnostics, blobs) {const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];- return paths.length > 0 && paths.every((path) => blobs?.[path] === true);+ if (paths.some((path) => blobs?.[path] === false)) return "different";+ return paths.length > 0 && paths.every((path) => blobs?.[path] === true)+ ? "identical"+ : "unavailable";}function sameTaskHashes(diagnostics, left, right) {return diagnostics.every((diagnostic) => {diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjsindex 2ce6d813c..12364c59f 100644--- a/scripts/ci/failure-report.mjs+++ b/scripts/ci/failure-report.mjs@@ -153,16 +153,21 @@ async function analyzeFailedRun({];const failures = [];for (const job of failedJobs) {- const evidence = await api.getStaticEvidence(repository, run, job);- const classification = await attributeToMain({- repository,- defaultBranch,- run,- job,- evidence,- mainCandidates,- api,- });+ let classification;+ try {+ const evidence = await api.getStaticEvidence(repository, run, job);+ classification = await attributeToMain({+ repository,+ defaultBranch,+ run,+ job,+ evidence,+ mainCandidates,+ api,+ });+ } catch (error) {+ classification = outcome("unknown", errorCode(error));+ }failures.push({runId: entry.id,runAttempt: entry.runAttempt,@@ -415,26 +420,25 @@ export function createGitHubApi({for (const path of paths) {if (!validRepoPath(path)) continue;const [left, right] = await Promise.all([- getContent(repository, ancestor, path, request),- getContent(repository, descendant, path, request),+ getContentBlobSha(repository, ancestor, path, request),+ getContentBlobSha(repository, descendant, path, request),]);- result[path] = left !== null && left === right;+ result[path] = left === right;}return result;},};}-async function getContent(repository, ref, path, request) {- try {- const encodedPath = path.split("/").map(encodeURIComponent).join("/");- const value = await request(- `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,- );- return typeof value.sha === "string" ? value.sha : null;- } catch {- return null;- }+async function getContentBlobSha(repository, ref, path, request) {+ const encodedPath = path.split("/").map(encodeURIComponent).join("/");+ const value = await request(+ `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,+ );+ const sha = safeSha(value?.sha);+ if (value?.type !== "file" || sha === null)+ throw new Error("github_content_blob_identity_unavailable");+ return sha;}function outcome(classification, reason) {diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.tsindex 26feb3ee2..c2a282906 100644--- a/tests/boundary/filesystem/ciFailureReport.test.ts+++ b/tests/boundary/filesystem/ciFailureReport.test.ts@@ -120,6 +120,22 @@ it.each([expect(result.classification).toBe("unknown");});+it.each([+ [{}, "diagnostic_source_blob_identity_unavailable"],+ [+ { "src/example.ts": false },+ "diagnostic_source_content_not_identical_at_main_and_tested_base",+ ],+] as const)(+ "distinguishes unavailable blob identity from verified differences (%s)",+ (blobs, reason) => {+ expect(classifyFailure({ ...comparison(), blobs })).toEqual({+ classification: "unknown",+ reason,+ });+ },+);+it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => {const directory = await createTestTempDirectory("rea-ci-failure-test-");const calls: string[] = [];diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.tsindex 92fe3e4ae..1caacde16 100644--- a/tests/boundary/filesystem/ciFailureReportBudget.test.ts+++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts@@ -2,6 +2,7 @@ import { expect, it } from "vitest";import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";import {reportCiFailures,+ createGitHubApi,type CiRun,type FailureReportApi,type StaticDiagnostic,@@ -235,6 +236,10 @@ it.each([1, 51])(expect(result.failures[0]!.classification).toBe(count === 1 ? "matching-main-diagnostics" : "unknown",);+ if (count > 50)+ expect(result.failures[0]!.reason).toBe(+ "diagnostic_source_blob_identity_unavailable",+ );},);@@ -288,3 +293,115 @@ it.each(["jobs", "artifact", "comparison"])(expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1);},);++it.each([0, 1, 2])(+ "preserves every failed job when artifact analysis fails at position %s",+ async (position) => {+ const input = fixture([20], 1);+ const jobs = [+ { name: "Python producer", conclusion: "failure" },+ { name: "Test shard (1)", conclusion: "timed_out" },+ ];+ jobs.splice(position, 0, { name: "Static checks", conclusion: "failure" });+ input.api.getJobs = async (_repository, id, attempt) =>+ jobs.map((job) => ({+ ...job,+ status: "completed",+ run_id: Number(id),+ run_attempt: attempt,+ }));+ input.api.getStaticEvidence = async (_repository, _run, job) => {+ if (job.name === "Static checks")+ throw new Error("static_artifact_unavailable");+ return null;+ };+ const result = await report(input);+ expect(result.counts.unknown).toBe(3);+ expect(result.failures).toEqual(+ jobs.map((job) => ({+ runId: 100,+ runAttempt: 1,+ runHead: prSha,+ event: "pull_request",+ job: job.name,+ conclusion: job.conclusion,+ classification: "unknown",+ reason:+ job.name === "Static checks"+ ? "static_artifact_unavailable"+ : "no_matching_verified_main_failure",+ })),+ );+ },+);++it("keeps unavailable job inventories at run scope", async () => {+ const input = fixture([], 1);+ input.api.getJobs = async () => {+ throw new Error("job_inventory_unavailable");+ };+ expect((await report(input)).failures).toEqual([+ {+ runId: 100,+ runAttempt: 1,+ classification: "unknown",+ reason: "job_inventory_unavailable",+ },+ ]);+});++it.each([+ ["request", "github_api_temporarily_unavailable"],+ ["budget", "github_api_request_budget_exhausted"],+ ["missing sha", "github_content_blob_identity_unavailable"],+ ["invalid sha", "github_content_blob_identity_unavailable"],+ ["directory", "github_content_blob_identity_unavailable"],+] as const)(+ "retains unavailable blob evidence as a comparison error: %s",+ async (failure, reason) => {+ const input = fixture([20], 1);+ const contentApi = createGitHubApi({+ token: "fixture-token",+ apiRequest: async () => {+ if (failure === "request") throw new Error(reason);+ if (failure === "missing sha") return { type: "file" };+ if (failure === "invalid sha")+ return { type: "file", sha: "not-a-sha" };+ if (failure === "directory") return [];+ return { type: "file", sha: "e".repeat(40) };+ },+ });+ if (failure === "budget") {+ for (let id = 1; id <= 250; id++) await contentApi.getRun(repository, id);+ }+ input.api.getBlobIdentity = contentApi.getBlobIdentity;+ const result = await report(input);+ expect(result.failures).toMatchObject([+ {+ job: "Static checks",+ classification: "unknown",+ reason: "no_matching_verified_main_failure",+ candidateFailures: [{ runId: 20, runAttempt: 1, reason }],+ },+ ]);+ },+);++it.each([true, false])(+ "reports blob equality only from verified file SHAs (equal: %s)",+ async (equal) => {+ const api = createGitHubApi({+ token: "fixture-token",+ apiRequest: async (_method, endpoint) => ({+ type: "file",+ sha:+ equal || endpoint.includes(`ref=${mainSha}`)+ ? "e".repeat(40)+ : "f".repeat(40),+ }),+ });+ expect(+ await api.getBlobIdentity(repository, mainSha, prSha, ["src/example.ts"]),+ ).toEqual({ "src/example.ts": equal });+ },+);