Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(hopper): verify real boundaries and prune redundant happy paths
morluto7005166f5878
diff --git a/docs/testing.md b/docs/testing.mdindex 3db5623b3..52e9f42e3 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -151,6 +151,17 @@ packaging/install behavior and fake-provider integration; use the correspondingreal-provider lanes for engine claims. Real Apple dispatch and InterfaceBuilder verifiers currently prove format integration through production readers.+`verify:hopper` exercises an installed Hopper through the production stdio MCP+server and CLI. It checks source-owned call chains, CFG edges, references and+complete large inventories, then probes unnamed bookmarks, annotation clearing,+batch validation before mutation, malformed addresses and regexes, segment-end+partial reads, and synthetic file-offset rejection. Advertised schemas are+validated in their JSON Schema dialect and successful replies are checked against+their advertised output schemas. Disposable binary copies prove that switching+and closing actually removes the native document, and that CLI byte results and+invalid-address diagnostics agree with MCP. No provider is mocked in this lane.+The Linux demo lane remains a separate `verify:hopper:linux` command.+Golden tests use immutable captured text inputs with producer/source provenanceunder `tests/fixtures/golden/`. Expected results are reviewed for the semanticclaim; capture commands do not automatically approve new expected outputs.diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjsnew file mode 100644index 000000000..0b8279e4f--- /dev/null+++ b/scripts/lib/real-hopper-boundaries.mjs@@ -0,0 +1,411 @@+import assert from "node:assert/strict";+import { execFile } from "node:child_process";+import { copyFile, mkdtemp, readFile, realpath, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { basename, join } from "node:path";+import { promisify } from "node:util";+import Ajv from "ajv/dist/2020.js";+import addFormats from "ajv-formats";+import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs";++/** Exercise real Hopper annotation and malformed-input boundaries over MCP. */+export async function verifyHopperBoundaryContracts(+ client,+ options,+ procedure,+ targetPath,+) {+ const catalog = await client.listTools();+ const ajv = new Ajv({ strict: false });+ addFormats(ajv);+ const validators = new Map();+ for (const tool of catalog.tools) {+ assert.ok(+ ajv.validateSchema(tool.inputSchema),+ `${tool.name} input schema`,+ );+ assert.ok(tool.outputSchema, `${tool.name} omitted its output schema`);+ assert.ok(+ ajv.validateSchema(tool.outputSchema),+ `${tool.name} output schema`,+ );+ validators.set(tool.name, ajv.compile(tool.outputSchema));+ }+ let successfulCalls = 0;+ let rejectedCalls = 0;+ const rejectedOperations = new Set();+ const call = async (name, args = {}) => {+ const reply = await client.callTool({ name, arguments: args }, options);+ const result = requireMcpResult(reply, name);+ const validate = validators.get(name);+ assert.ok(validate, `${name} is missing from tools/list`);+ assert.ok(+ validate(reply.structuredContent),+ `${name}: ${JSON.stringify(validate.errors)}`,+ );+ assert.deepEqual(reply.structuredContent, JSON.parse(mcpTextValue(reply)));+ successfulCalls += 1;+ return result;+ };+ const invalid = async (name, args, reason) => {+ const reply = await client.callTool({ name, arguments: args }, options);+ assert.equal(reply.isError, true, `${name} accepted invalid input`);+ const { error } = JSON.parse(mcpTextValue(reply));+ assert.equal(error.code, "invalid_request");+ assert.equal(error.category, "invalid_input");+ assert.equal(error.details.operation, name);+ assert.match(error.message, reason);+ rejectedCalls += 1;+ rejectedOperations.add(name);+ };+ const address = procedure.address;+ const originalName = await call("address_name", { address });+ const originalComment = await call("comment", { address });+ const originalInlineComment = await call("inline_comment", { address });+ const activeDocument = await call("current_document");+ const foreignDocument = (await call("list_documents")).find(+ (name) => name !== activeDocument,+ );+ try {+ if (foreignDocument !== undefined) {+ await invalid(+ "read_bytes",+ { address, length: 8, document: foreignDocument },+ /belongs to another target/u,+ );+ assert.equal(await call("current_document"), activeDocument);+ }+ for (const label of [undefined, "", "REA boundary bookmark 测试"]) {+ assert.equal(+ await call("set_bookmark", {+ address,+ ...(label === undefined ? {} : { name: label }),+ }),+ true,+ );+ const bookmarks = await call("list_bookmarks");+ const bookmark = bookmarks.find((item) => item.address === address);+ assert.ok(bookmark, "set_bookmark did not create a bookmark");+ assert.equal(bookmark.name, label || null);+ const context = await call("inspect_address_context", { address });+ assert.equal(context.bookmarks.state, "available");+ assert.deepEqual(context.bookmarks.value, [bookmark]);+ assert.equal(await call("unset_bookmark", { address }), true);+ assert.ok(+ !(await call("list_bookmarks")).some(+ (item) => item.address === address,+ ),+ );+ }++ for (const [setter, getter] of [+ ["set_comment", "comment"],+ ["set_inline_comment", "inline_comment"],+ ]) {+ const comment = "REA boundary comment 测试";+ assert.equal(await call(setter, { address, comment }), true);+ assert.equal(await call(getter, { address }), comment);+ assert.equal(await call(setter, { address, comment: "" }), true);+ assert.equal(await call(getter, { address }), null);+ }+ const inline = "REA inline\nsecond line";+ const exact = await call("set_inline_comment", {+ address,+ comment: inline,+ });+ const observed = await call("inline_comment", { address });+ assert.equal(+ exact,+ observed === inline,+ "inline normalization must remain visible",+ );++ await invalid(+ "set_addresses_names",+ {+ names: {+ [address]: "rea_boundary_should_not_be_applied",+ __rea_missing_batch_address__: "missing",+ },+ },+ /Unknown Hopper address or name: __rea_missing_batch_address__/u,+ );+ assert.equal(await call("address_name", { address }), originalName);+ assert.equal(+ (await call("procedure_info", { procedure: address })).name,+ originalName,+ );++ const renamed = "rea_boundary_renamed";+ assert.equal(+ await call("set_addresses_names", { names: { [address]: renamed } }).then(+ (value) => value[address],+ ),+ true,+ );+ assert.equal(await call("address_name", { address }), renamed);+ const matches = await call("search_procedures", {+ pattern: renamed,+ case_sensitive: true,+ });+ assert.ok(+ matches.some(+ (item) => item.address === address && item.value === renamed,+ ),+ );+ assert.equal(+ (await call("analyze_function", { procedure: address })).procedure.name,+ renamed,+ );+ assert.equal(+ await call("set_address_name", { address, name: "add" }),+ true,+ );+ assert.equal(+ await call("procedure_address", { procedure: "add" }),+ address,+ );+ assert.equal(+ (await call("procedure_info", { procedure: "add" })).entrypoint,+ address,+ );+ assert.equal(+ (await call("read_bytes", { address: "add", length: 8 })).address,+ address,+ );+ assert.equal(+ await call("procedure_address", { procedure: address.slice(2) }),+ address,+ );++ for (const query of ["-1", "0x10000000000000000", "0xffffffffffffffff"]) {+ for (const name of [+ "read_bytes",+ "address_to_file_offset",+ "resolve_containing_procedure",+ "procedure_info",+ ]) {+ await invalid(+ name,+ name === "procedure_info"+ ? { procedure: query }+ : {+ address: query,+ ...(name === "read_bytes" ? { length: 8 } : {}),+ },+ /usable unsigned 64-bit range/u,+ );+ }+ }+ await invalid(+ "read_bytes",+ { address: "0x0", length: 8 },+ /Address is outside every segment/u,+ );+ await invalid(+ "procedure_info",+ { procedure: address, document: "__rea_missing_document__" },+ /Unknown Hopper document/u,+ );+ await invalid(+ "search_procedures",+ { pattern: "[", mode: "regex" },+ /Invalid regex pattern/u,+ );+ // Invalid calls must leave the same authenticated bridge usable.+ assert.equal(+ (await call("read_bytes", { address, length: 8 })).returned_bytes,+ 8,+ );+ const mapping = await call("address_to_file_offset", { address });+ const bytes = await call("read_bytes", { address, length: 8 });+ const file = await readFile(targetPath);+ assert.equal(+ bytes.bytes_hex,+ file+ .subarray(mapping.file_offset, mapping.file_offset + 8)+ .toString("hex"),+ );+ const segments = await call("list_segments");+ const containing = segments.find(+ (segment) =>+ BigInt(segment.start) <= BigInt(address) &&+ BigInt(address) < BigInt(segment.end),+ );+ assert.ok(containing, "fixture procedure has no containing segment");+ const lastByte = `0x${(BigInt(containing.end) - 1n).toString(16)}`;+ const prefix = await call("read_bytes", { address: lastByte, length: 8 });+ assert.equal(prefix.returned_bytes, 1);+ assert.equal(prefix.bytes_hex.length, 2);+ assert.equal(prefix.complete, false);+ const synthetic = segments.find(+ (segment) => segment.name === "External Symbols",+ );+ assert.ok(synthetic, "fixture omitted synthetic external-symbol memory");+ await invalid(+ "address_to_file_offset",+ { address: synthetic.start },+ /no authoritative file-offset mapping/u,+ );+ const health = (await call("binary_session")).provider_operation_health;+ assert.equal(+ health.state,+ "idle",+ "a live GUI bridge must remain healthy after its launcher helper exits",+ );+ } finally {+ await call("set_address_name", { address, name: originalName ?? "" });+ await call("set_comment", { address, comment: originalComment ?? "" });+ await call("set_inline_comment", {+ address,+ comment: originalInlineComment ?? "",+ });+ await call("unset_bookmark", { address });+ }+ return {+ schemaCount: catalog.tools.length,+ successfulCalls,+ rejectedCalls,+ rejectedOperations: [...rejectedOperations],+ foreignDocumentVerified: foreignDocument !== undefined,+ };+}++/** Prove actual document disposal and shared CLI/MCP behavior on disposable copies. */+export async function verifyHopperLifecycleAndCli(client, options, targets) {+ const directory = await realpath(+ await mkdtemp(join(tmpdir(), "hopper-boundary-")),+ );+ const suffix = basename(directory);+ const primary = join(directory, `primary-${suffix}`);+ const secondary = join(directory, `secondary-${suffix}`);+ const call = async (name, args = {}) =>+ requireMcpResult(+ await client.callTool({ name, arguments: args }, options),+ name,+ );+ const cli = promisify(execFile);+ const runCli = (args) => {+ const pending = cli(process.execPath, args, {+ timeout: 180_000,+ maxBuffer: 1_048_576,+ });+ if (pending.child.pid !== undefined)+ targets.ownedProcessIds.add(pending.child.pid);+ return pending;+ };+ try {+ await copyFile(targets.primary, primary);+ await copyFile(targets.secondary, secondary);+ await call("open_binary", { path: primary });+ const firstDocument = await call("current_document");+ await call("open_binary", { path: secondary });+ const secondDocument = await call("current_document");+ assert.notEqual(firstDocument, secondDocument);+ assert.ok(+ !(await call("list_documents")).includes(firstDocument),+ "target switching retained the old native document",+ );+ const procedures = await call("list_procedures");+ await verifyBatchCancellation(client, options, procedures);+ const address = procedures[0].address;+ const expected = await call("read_bytes", {+ address,+ length: 8,+ document: secondDocument,+ });+ await call("close_binary");++ const dispatcher = new URL("../rea.mjs", import.meta.url).pathname;+ const common = [+ "--length",+ "8",+ "--provider",+ "hopper",+ "--format",+ "json",+ ];+ const { stdout } = await runCli([+ dispatcher,+ "read-bytes",+ secondary,+ address,+ ...common,+ ]);+ const evidence = JSON.parse(stdout);+ assert.equal(evidence.provider.id, "hopper");+ assert.equal(evidence.subject.local_path, secondary);+ assert.deepEqual(evidence.normalized_result, expected);+ let failure;+ try {+ await runCli([+ dispatcher,+ "read-bytes",+ secondary,+ "0x10000000000000000",+ ...common,+ ]);+ } catch (error) {+ assert.equal(error.code, 1);+ failure = JSON.parse(error.stdout);+ }+ assert.ok(failure, "CLI accepted an overflowing address");+ assert.equal(failure.code, "invalid_request");+ assert.equal(failure.category, "invalid_input");+ assert.match(failure.message, /usable unsigned 64-bit range/u);++ await call("open_binary", { path: primary });+ assert.ok(+ !(await call("list_documents")).includes(secondDocument),+ "MCP or CLI cleanup retained the closed native document",+ );+ await call("close_binary");+ return {+ targetSwitchDisposedDocument: true,+ cliByteParity: true,+ cliInvalidAddress: true,+ callerCancellationRecovered: true,+ closedDocumentAbsent: true,+ };+ } finally {+ await call("close_binary");+ await rm(directory, { recursive: true, force: true });+ }+}++const verifyBatchCancellation = async (client, options, procedures) => {+ const controller = new AbortController();+ let cancelled = false;+ await assert.rejects(+ client.callTool(+ {+ name: "batch_decompile",+ arguments: { addresses: procedures.map((item) => item.address) },+ },+ {+ ...options,+ signal: controller.signal,+ onprogress: (update) => {+ options.onprogress?.(update);+ // Cancel between real decompilations, after Hopper served one item.+ if (update.message?.includes("Hopper bridge completed request")) {+ cancelled = true;+ controller.abort();+ }+ },+ },+ ),+ /aborted/u,+ );+ assert.ok(cancelled, "cancellation did not follow a real Hopper response");+ const recovered = requireMcpResult(+ await client.callTool({ name: "list_procedures", arguments: {} }, options),+ "list_procedures after cancellation",+ );+ assert.deepEqual(recovered, procedures);+ const status = requireMcpResult(+ await client.callTool({ name: "binary_session", arguments: {} }, options),+ "binary_session after cancellation",+ );+ assert.equal(status.provider_operation_health.state, "idle");+};diff --git a/scripts/lib/real-hopper-semantic.mjs b/scripts/lib/real-hopper-semantic.mjsindex 8bd20c440..6cf855f2c 100644--- a/scripts/lib/real-hopper-semantic.mjs+++ b/scripts/lib/real-hopper-semantic.mjs@@ -57,7 +57,9 @@ export function requireBridgeDiagnostic(chunks) {(diagnostic) =>diagnostic.type === "bridge-diagnostic" &&diagnostic.category === "invalid_request" &&- diagnostic.message === "Invalid Hopper bridge request",+ diagnostic.message.includes(+ "Unknown Hopper address or name: __rea_missing_procedure__",+ ),))throw new Error("The MCP runtime omitted the correlated Hopper diagnostic");@@ -153,7 +155,13 @@ export async function verifyRealHopperFixture({}/** Reject malformed, unsafe, or unexpected failure diagnostics from MCP. */-export function requireSafeDiagnostics(chunks) {+export function requireSafeDiagnostics(+ chunks,+ expectedErrorTools = ["procedure_info"],+ expectedErrorTags = ["HopperRemoteError"],+) {+ const expected = new Set(expectedErrorTools);+ const tags = new Set(expectedErrorTags);const lines = chunks.join("").split("\n")@@ -173,10 +181,8 @@ export function requireSafeDiagnostics(chunks) {typeof diagnostic.level !== "number" ||diagnostic.level >= 50 ||(diagnostic.level >= 40 &&- diagnostic.errorTag !== "HopperRemoteError" &&- !(- diagnostic.tool === "procedure_info" && diagnostic.status === "error"- ))+ !tags.has(diagnostic.errorTag) &&+ !(expected.has(diagnostic.tool) && diagnostic.status === "error")))throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`);}diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjsindex 8877cc688..621cc2359 100644--- a/scripts/verify-real-hopper.mjs+++ b/scripts/verify-real-hopper.mjs@@ -1,5 +1,7 @@import { execFile } from "node:child_process";-import { readdir } from "node:fs/promises";+import { readFile, readdir } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";import { promisify } from "node:util";import { Client } from "@modelcontextprotocol/client";@@ -33,6 +35,10 @@ import { openAndVerifyLargeFixture } from "./lib/real-hopper-exhaustive-search.mimport { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";import { requireHopperSelection } from "./lib/real-hopper-selection.mjs";import { verifyHopperInventories } from "./lib/real-hopper-inventory.mjs";+import {+ verifyHopperBoundaryContracts,+ verifyHopperLifecycleAndCli,+} from "./lib/real-hopper-boundaries.mjs";import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs";const execFileAsync = promisify(execFile);const verifierRun = createVerifierRun();@@ -47,9 +53,9 @@ const parseServerArgs = (encoded) => {return parsed;};-const sessionsBefore = new Set(- (await readdir("/tmp")).filter((name) => name.startsWith("rea-")),-);+const runtimeParent = process.platform === "darwin" ? "/tmp" : tmpdir();+const sessionsBefore = new Set(await readdir(runtimeParent));+const ownedProcessIds = new Set();const textValue = mcpTextValue;const requireSuccessfulTool = requireMcpResult;@@ -250,6 +256,9 @@ let summary;try {await client.connect(transport);+ if (transport.pid === null)+ throw new Error("The verifier did not capture its MCP process identity");+ ownedProcessIds.add(transport.pid);const progressUpdates = [];const options = {timeout,@@ -358,6 +367,12 @@ try {oracle: fixtureTargets.oracle,normalizedResult: requireSuccessfulTool,});+ const boundaryContracts = await verifyHopperBoundaryContracts(+ client,+ options,+ fixtureAnalysis.procedures.entry,+ targetA,+ );const switched = await client.callTool({ name: "open_binary", arguments: { path: targetB } },options,@@ -396,15 +411,14 @@ try {throw new Error("binary_overview retained another document after target switch",);- const expectedDocumentsAfterTargetSwitch =- process.platform === "linux"- ? firstDocuments.length- : firstDocuments.length + 1;+ const expectedDocumentsAfterTargetSwitch = firstDocuments.length;if (!Array.isArray(documentsAfterTargetSwitch) ||documentsAfterTargetSwitch.length !== expectedDocumentsAfterTargetSwitch)- throw new Error("A distinct target did not receive one Hopper document");+ throw new Error(+ "Target switching did not replace the previous Hopper document",+ );const reopenedTarget = await client.callTool({ name: "open_binary", arguments: { path: targetB } },options,@@ -441,20 +455,32 @@ try {"Hopper's bundled MCP server was running during verification",);}- const diagnosticCount = requireSafeDiagnostics(stderrChunks);- requireBridgeDiagnostic(stderrChunks);-const closed = await client.callTool({ name: "close_binary", arguments: {} },options,);if (closed.isError === true) throw new Error(textValue(closed));+ const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, {+ primary: targetA,+ secondary: targetB,+ ownedProcessIds,+ });const closedSession = requireMcpResult(await fullSessionStatus(),"binary_session",);if (closedSession.open !== false)throw new Error("The real session remained open after close_binary");+ const diagnosticCount = requireSafeDiagnostics(+ stderrChunks,+ [+ "procedure_info",+ "batch_decompile",+ ...boundaryContracts.rejectedOperations,+ ],+ ["HopperRemoteError", "HopperCancelledError", "AnalysisCancelledError"],+ );+ requireBridgeDiagnostic(stderrChunks);summary = {toolCount: actualNames.length,@@ -463,6 +489,8 @@ try {segmentCount: firstOverview.segment_count,analyses: [firstAnalysis, secondAnalysis],fixtureAnalysis,+ boundaryContracts,+ lifecycleAndCli,inventoryCounts,largeInventory,fixtureManifest: fixtureTargets.manifestPath,@@ -491,29 +519,37 @@ try {}await new Promise((resolve) => setTimeout(resolve, 500));-const sessionsAfter = (await readdir("/tmp")).filter(- (name) => name.startsWith("rea-") && !sessionsBefore.has(name),-);-if (sessionsAfter.length > 0) {- throw new Error("The MCP runtime leaked a bridge session directory");-}-const afterCloseProcesses = await execFileAsync("ps", [- "-ax",- "-o",- "command=",-]);-if (- afterCloseProcesses.stdout- .split("\n")- .some((line) => line.includes("node dist/main.js"))-) {- throw new Error(- "The TypeScript MCP server remained alive after client close",- );+for (const entry of await readdir(runtimeParent, { withFileTypes: true })) {+ if (+ !entry.isDirectory() ||+ !entry.name.startsWith("rea-") ||+ sessionsBefore.has(entry.name)+ )+ continue;+ let ownership;+ try {+ ownership = JSON.parse(+ await readFile(join(runtimeParent, entry.name, "ownership.json"), "utf8"),+ );+ } catch (cause) {+ if (cause.code === "ENOENT") continue;+ throw cause;+ }+ if (ownedProcessIds.has(ownership.parent_pid))+ throw new Error(+ `An owned Hopper runtime leaked its directory: ${entry.name}`,+ );}if (summary === undefined)throw new Error("Real-Hopper verification did not produce a summary");const completedVerifierRun = await completeVerifierRun(verifierRun);+if (+ completedVerifierRun.process_lineage.status !== "verified" ||+ completedVerifierRun.process_lineage.descendants.length !== 0+)+ throw new Error(+ "The verifier's process lineage did not prove owned child cleanup",+ );await new Promise((resolve, reject) => {process.stdout.write(`${JSON.stringify({ verifier_run: completedVerifierRun, ...summary, unrelatedHopperSurvived, cleanShutdown: true }, null, 2)}\n`,diff --git a/tests/boundary/providers/hopper/hopperClient.health.test.ts b/tests/boundary/providers/hopper/hopperClient.health.test.tsindex 8e8bd1930..a21313c4c 100644--- a/tests/boundary/providers/hopper/hopperClient.health.test.ts+++ b/tests/boundary/providers/hopper/hopperClient.health.test.ts@@ -61,19 +61,6 @@ const start = async (owned: boolean) => {};describe("Hopper provider lifecycle health", () => {- it("keeps a live bridge healthy after a non-owning launcher helper exits", async () => {- const { client, launcher } = await start(false);- launcher.observed.exit(0);- expect(client.operationHealth().state).toBe("idle");- expect(await client.callTool("echo", { label: "GUI still alive" })).toEqual(- {- ok: true,- value: { label: "GUI still alive" },- },- );- expect(client.operationHealth().state).toBe("idle");- });-it("settles disconnected requests without attributing the helper exit to the GUI", async () => {const { client, launcher } = await start(false);launcher.observed.exit(0);diff --git a/tests/conformance/providers/hopper/realHopperSemantic.test.ts b/tests/conformance/providers/hopper/realHopperSemantic.test.tsindex da04a24ae..99b4f41bf 100644--- a/tests/conformance/providers/hopper/realHopperSemantic.test.ts+++ b/tests/conformance/providers/hopper/realHopperSemantic.test.ts@@ -27,22 +27,6 @@ describe("real Hopper semantic fixture resolution", () => {).rejects.toThrow(/current_document/u);});- it("resolves the public search value and preserves its reported name", async () => {- const procedure = await resolveFixtureProcedure(- clientWithItems([- {- address: "0x1000",- value: "_rea_entry",- },- ]),- {},- "rea_entry",- normalize,- );-- expect(procedure).toEqual({ address: "0x1000", name: "_rea_entry" });- });-it.each([["empty", []],[