Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
import assert from "node:assert/strict";
import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { isDeepStrictEqual, parseArgs } from "node:util";
import {
sazRequest,
verifySazRecords,
verifySazOracleIdentity,
sazEvidenceParameters,
} from "./saz-entry.mjs";
const { values } = parseArgs({
options: {
trial: { type: "string" },
oracle: { type: "string" },
"runtime-root": { type: "string" },
artifact: { type: "string" },
selection: { type: "string" },
},
});
if (
!values.trial ||
!values.oracle ||
!values["runtime-root"] ||
!["baseline", "focused"].includes(values.selection)
)
throw new Error(
"Use --trial DIR --oracle JSON --runtime-root DIR --artifact ID --selection baseline|focused",
);
const load = async (file) =>
JSON.parse(await readFile(resolve(values.trial, file), "utf8"));
const settings = await load("trial-settings.json"),
summary = await load("summary.json");
const oracle = JSON.parse(await readFile(values.oracle, "utf8"));
verifySazOracleIdentity(values.artifact, oracle);
const runtime = (file) =>
pathToFileURL(resolve(values["runtime-root"], file)).href;
const { parseEvidence } = await import(runtime("dist/domain/evidence.js"));
const { inspectWebNetworkCaptureInputSchema, webNetworkCaptureSchema } =
await import(runtime("dist/domain/webNetworkCapture.js"));
const { inspectAnalysisViewInputSchema, analysisViewResultSchema } =
await import(runtime("dist/domain/analysisView/analysisView.js"));
const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8"))
.split("\n")
.filter(Boolean)
.map(JSON.parse)
.filter(
(event) =>
event.type === "item.completed" && event.item?.type === "mcp_tool_call",
)
.map((event) => event.item);
const planned = inspectWebNetworkCaptureInputSchema.parse(
sazRequest(
values.artifact,
settings.selected_artifact_path,
values.selection,
),
);
const reasons = [],
workflowReasons = [],
authenticated = new Map(),
parents = new Set(),
conflicts = new Set();
let transportRecovery = false;
for (const call of calls) {
try {
assert.equal(call.server, "rea");
assert.equal(call.status, "completed");
assert.ok(!call.error, "RPC did not provide observations");
const text = call.result.content.filter((item) => item.type === "text");
assert.equal(text.length, 1, "Ambiguous Evidence text carrier");
let evidence = JSON.parse(text[0].text);
if (call.result.isError) {
const limits = evidence.error?.details?.reported_limits;
assert.equal(call.tool, "inspect_web_network_capture");
assert.equal(evidence.error?.code, "resource_constraint");
assert.equal(
evidence.error.details.operation,
"inspect_web_network_capture",
);
assert.equal(limits?.boundary, "mcp-response");
const input = inspectWebNetworkCaptureInputSchema.parse(call.arguments);
assert.deepEqual(input, planned);
const source = inspectAnalysisViewInputSchema.parse({
source: limits.evidence_reference,
view: { kind: "summary" },
}).source;
assert.equal(source.kind, "retained-evidence");
parents.add(source.evidence_id);
transportRecovery = true;
continue;
}
if (call.result.structuredContent !== undefined)
assert.deepEqual(evidence, call.result.structuredContent);
evidence = parseEvidence(evidence);
assert.equal(evidence.subject.local_path, settings.selected_artifact_path);
assert.equal(evidence.subject.digest.sha256, oracle.sha256);
let records;
if (call.tool === "inspect_web_network_capture") {
const { detail, ...arguments_ } = call.arguments;
const input = inspectWebNetworkCaptureInputSchema.parse(arguments_);
assert.equal(input.capture_path, settings.selected_artifact_path);
assert.equal(input.format, "saz");
if (!isDeepStrictEqual(input, planned))
workflowReasons.push(
"Inspection selection differs from planned arguments",
);
if (detail === "summary") {
workflowReasons.push(
"Summary delivery differs from the fixed complete-delivery plan",
);
assert.equal(evidence.operation, "inspect_analysis_view");
const report = analysisViewResultSchema.parse(
evidence.normalized_result,
);
assert.equal(report.kind, "summary");
assert.equal(report.parent_operation, "inspect_web_network_capture");
assert.equal(report.summary.format, "saz");
assert.equal(report.summary.total_records, oracle.total_records);
assert.deepEqual(evidence.parameters, {
parent_evidence_id: report.parent_evidence_id,
view: { kind: "summary" },
});
parents.add(report.parent_evidence_id);
authenticated.set(evidence.evidence_id, []);
continue;
}
assert.ok(detail === undefined || detail === "complete");
assert.equal(evidence.operation, "inspect_web_network_capture");
assert.deepEqual(
evidence.parameters,
sazEvidenceParameters(input, oracle.total_records),
);
const report = webNetworkCaptureSchema.parse(evidence.normalized_result);
assert.equal(report.total_records, oracle.total_records);
assert.equal(report.format, "saz");
records = report.records;
parents.add(evidence.evidence_id);
} else {
assert.equal(call.tool, "inspect_analysis_view");
const input = inspectAnalysisViewInputSchema.parse(call.arguments);
assert.equal(input.source.kind, "retained-evidence");
assert.ok(
parents.has(input.source.evidence_id),
"Unknown retained parent",
);
assert.equal(evidence.operation, "inspect_analysis_view");
assert.deepEqual(evidence.parameters, {
parent_evidence_id: input.source.evidence_id,
view: input.view,
});
const page = analysisViewResultSchema.parse(evidence.normalized_result);
assert.deepEqual(page.view, input.view);
assert.equal(page.parent_evidence_id, input.source.evidence_id);
assert.equal(page.parent_operation, "inspect_web_network_capture");
if (page.kind === "summary") {
assert.equal(page.summary.total_records, oracle.total_records);
assert.equal(page.summary.format, "saz");
records = [];
} else {
assert.equal(page.kind, "capture");
records = page.view.detail === "complete" ? page.items : [];
}
}
const prior = authenticated.get(evidence.evidence_id);
if (prior && !isDeepStrictEqual(prior, records)) {
conflicts.add(evidence.evidence_id);
authenticated.delete(evidence.evidence_id);
parents.delete(evidence.evidence_id);
throw new Error("Conflicting observations share an Evidence ID");
}
if (!conflicts.has(evidence.evidence_id))
authenticated.set(evidence.evidence_id, records);
} catch (cause) {
workflowReasons.push(String(cause));
}
}
if (
calls[0]?.tool !== "inspect_web_network_capture" ||
calls.filter((call) => call.tool === "inspect_web_network_capture").length !==
1 ||
(!transportRecovery && calls.length !== 1)
)
workflowReasons.push(
"Completed call sequence differs from the fixed initial call/recovery plan",
);
try {
const raw = (await readFile(resolve(values.trial, "answer.txt"), "utf8"))
.trim()
.replace(/^```(?:json)?\s*|\s*```$/g, "");
const answer = JSON.parse(raw);
assert.ok(
Array.isArray(answer.evidence_ids) && answer.evidence_ids.length > 0,
"Missing Evidence citations",
);
const observed = new Map();
for (const id of answer.evidence_ids) {
assert.ok(authenticated.has(id), "Unknown/failed cited Evidence");
for (const row of authenticated.get(id)) {
const prior = observed.get(row.ordinal);
assert.ok(
!prior || isDeepStrictEqual(prior, row),
"Conflicting cited record identities",
);
observed.set(row.ordinal, row);
}
}
const facts = verifySazRecords(
oracle.record_ordinals.map((ordinal) => observed.get(ordinal)),
oracle,
);
assert.equal(answer.records.length, facts.length, "Answer coverage differs");
assert.deepEqual(
[...answer.records].sort((a, b) => a.ordinal - b.ordinal),
[...facts].sort((a, b) => a.ordinal - b.ordinal),
"Answer facts differ from cited stored bytes",
);
if (
!isDeepStrictEqual(
answer.records.map((row) => row.ordinal),
oracle.record_ordinals,
)
)
workflowReasons.push("Final answer changes caller-selected order");
if (
!Array.isArray(answer.limitations) ||
answer.limitations.length === 0 ||
answer.limitations.some(
(value) => typeof value !== "string" || !value.trim(),
)
)
workflowReasons.push("Final answer omits limitations");
assert.equal(settings.selected_artifact_sha256_before, oracle.sha256);
assert.equal(summary.selected_artifact_sha256_after, oracle.sha256);
if (
summary.exit_code !== 0 ||
summary.timeout ||
summary.selected_artifact_bytes_unchanged !== true
)
workflowReasons.push(
"Trial did not complete with original bytes unchanged",
);
} catch (cause) {
reasons.push(String(cause));
}
const report = {
factual_correctness: reasons.length ? "failed" : "passed",
workflow_correctness:
reasons.length || workflowReasons.length ? "failed" : "passed",
reasons,
workflow_reasons: workflowReasons,
completed_calls: calls.length,
wall_seconds: summary.wall_seconds,
usage: summary.usage,
selected_artifact_sha256: oracle.sha256,
scope:
"Authenticated cited stored HTTP records and independent raw-byte checks; no live attribution, body decompression or certificate interpretation",
limitation_prose: "Presence checked; wording requires transcript review",
};
await writeFile(
resolve(values.trial, "saz-review.json"),
JSON.stringify(report, null, 2) + "\n",
);
console.log(JSON.stringify(report));
if (report.workflow_correctness !== "passed") process.exitCode = 1;