Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test: strengthen MCP prompt acceptance coverage
N0zoM1z005fb9b17c91b
diff --git a/scripts/verify-package-prompts.mjs b/scripts/verify-package-prompts.mjsindex 5b3290602..47af4fc5c 100644--- a/scripts/verify-package-prompts.mjs+++ b/scripts/verify-package-prompts.mjs@@ -36,19 +36,31 @@ export const verifyPackagedPromptCatalog = async (throw new Error("packaged MCP guided prompt rendering failed");};-/** Verify live completion through the installed stdio MCP artifact. */-export const verifyPackagedPromptCompletion = async (client, options) => {- const result = await client.complete(+/** Verify live completion through each installed stdio target state. */+export const verifyPackagedPromptCompletion = async (+ client,+ options,+ targetOpen,+) => {+ const expected = targetOpen ? ["fixture"] : [];+ const requests = [{ref: { type: "ref/prompt", name: "investigate_feature" },argument: { name: "document", value: "fix" },},- options,- );- if (- JSON.stringify(result.completion.values) !== JSON.stringify(["fixture"]) ||- result.completion.total !== 1 ||- result.completion.hasMore !== false- )- throw new Error("packaged MCP prompt completion failed");+ {+ ref: { type: "ref/prompt", name: "investigate_feature" },+ argument: { name: "procedure", value: "fix" },+ context: { arguments: { document: "fixture" } },+ },+ ];+ for (const request of requests) {+ const result = await client.complete(request, options);+ if (+ JSON.stringify(result.completion.values) !== JSON.stringify(expected) ||+ result.completion.total !== expected.length ||+ result.completion.hasMore !== false+ )+ throw new Error("packaged MCP prompt completion lifecycle failed");+ }};diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjsindex 1dde0b82f..fe432b794 100644--- a/scripts/verify-package.mjs+++ b/scripts/verify-package.mjs@@ -430,6 +430,7 @@ try {)throw new Error("packaged MCP tool inventory diverged from contracts");await verifyPackagedPromptCatalog(client, mcpOptions, expectedPromptNames);+ await verifyPackagedPromptCompletion(client, mcpOptions, false);const result = await client.callTool({ name: "current_document", arguments: {} },mcpOptions,@@ -442,7 +443,7 @@ try {);if (opened.isError === true)throw new Error("packaged MCP could not open a binary");- await verifyPackagedPromptCompletion(client, mcpOptions);+ await verifyPackagedPromptCompletion(client, mcpOptions, true);const current = await client.callTool({ name: "current_document", arguments: {} },mcpOptions,@@ -472,6 +473,7 @@ try {);if (closed.isError === true)throw new Error("packaged MCP could not close its binary");+ await verifyPackagedPromptCompletion(client, mcpOptions, false);const mcpBundlePath = join(evidenceRoot, "mcp.json");const mcpExport = await client.callTool({ name: "export_evidence_bundle", arguments: { path: mcpBundlePath } },@@ -493,7 +495,7 @@ try {}process.stdout.write(- `${JSON.stringify({ cli: true, analysisCli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 2 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, skill: supportedSetupHost, mcpTools: expectedToolCount, mcpPrompts: expectedPromptNames.length, promptCompletion: true, evidenceMcp: true, targetFree: true, targetLifecycle: true })}\n`,+ `${JSON.stringify({ cli: true, analysisCli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 2 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, skill: supportedSetupHost, mcpTools: expectedToolCount, mcpPrompts: expectedPromptNames.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true })}\n`,);} finally {if (tarball) await rm(join(root, tarball), { force: true });diff --git a/tests/promptCompletion.test.ts b/tests/promptCompletion.test.tsindex 19e05996d..6063eed0a 100644--- a/tests/promptCompletion.test.ts+++ b/tests/promptCompletion.test.ts@@ -168,6 +168,73 @@ describe("guided prompt completion", () => {});expect(await completion.complete("procedure", "")).toEqual(["0x1000"]);});++ it("bounds procedure scans and rejects non-advancing continuation metadata", async () => {+ let calls = 0;+ const bounded = createPromptCompletionSource({+ execute(_operation, parameters) {+ calls += 1;+ const offset = Number(parameters.offset);+ const items = Array.from({ length: 500 }, (_, index) => {+ const address = offset + index;+ return {+ address: `0x${address.toString(16).padStart(8, "0")}`,+ value: `procedure_${String(address)}`,+ };+ });+ return Promise.resolve(+ observed({+ items,+ offset,+ limit: 500,+ total: 6_000,+ next_offset: offset + 500,+ has_more: true,+ }),+ );+ },+ });+ const values = await bounded.complete("procedure", "");+ expect(calls).toBe(10);+ expect(values).toHaveLength(5_000);+ expect(values.every((value) => value.startsWith("0x"))).toBe(true);++ let nonAdvancingCalls = 0;+ const nonAdvancing = createPromptCompletionSource({+ execute() {+ nonAdvancingCalls += 1;+ return Promise.resolve(+ observed(page(0, [{ address: "0x5000", value: "unsafe_name" }], 0)),+ );+ },+ });+ expect(await nonAdvancing.complete("procedure", "")).toEqual(["0x5000"]);+ expect(nonAdvancingCalls).toBe(1);+ });++ it("normalizes Unicode prefixes while preserving distinct exact identifiers", async () => {+ const completion = createPromptCompletionSource({+ execute(operation) {+ return Promise.resolve(+ observed(+ operation === "list_documents"+ ? ["App", "App", "app", "Zulu"]+ : null,+ ),+ );+ },+ });+ expect(await completion.complete("document", "AP")).toEqual([+ "App",+ "app",+ "App",+ ]);++ const malformed = createPromptCompletionSource({+ execute: () => Promise.resolve(observed(["valid", 1])),+ });+ expect(await malformed.complete("document", "")).toEqual([]);+ });});const fixtureProvider = { id: "fixture", name: "Fixture", version: "1" };diff --git a/tests/promptMcp.test.ts b/tests/promptMcp.test.tsindex efbf1cf84..69fd8b28d 100644--- a/tests/promptMcp.test.ts+++ b/tests/promptMcp.test.ts@@ -1,7 +1,12 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { basename, join } from "node:path";+import { Client, InMemoryTransport } from "@modelcontextprotocol/client";import { McpServer } from "@modelcontextprotocol/server";import { afterEach, describe, expect, it } from "vitest";+import type { AnalysisClient } from "../src/application/AnalysisProvider.js";import { BinarySession } from "../src/application/BinarySession.js";import { PROMPT_CONTRACTS } from "../src/contracts/promptContracts.js";import { createEvidence } from "../src/domain/evidence.js";@@ -10,11 +15,17 @@ import { registerGuidedPrompts } from "../src/server/registerPrompts.js";import { observed } from "./fixtures/analysisExecution.js";const resources: Array<{ close(): Promise<unknown> }> = [];+const temporaryDirectories: string[] = [];afterEach(async () => {await Promise.allSettled(resources.splice(0).map((resource) => resource.close()),);+ await Promise.all(+ temporaryDirectories+ .splice(0)+ .map((directory) => rm(directory, { recursive: true, force: true })),+ );});describe("guided prompts over MCP", () => {@@ -27,6 +38,22 @@ describe("guided prompts over MCP", () => {expect(prompts.prompts.map(({ name }) => name)).toEqual(PROMPT_CONTRACTS.map(({ name }) => name),);+ expect(client.getServerCapabilities()).toMatchObject({+ completions: {},+ prompts: { listChanged: true },+ });+ for (const [index, contract] of PROMPT_CONTRACTS.entries()) {+ const advertised = prompts.prompts[index];+ expect(advertised?.arguments).toEqual(+ Object.entries(contract.arguments).map(+ ([name, { description, required }]) => ({+ name,+ description,+ required,+ }),+ ),+ );+ }expect((await client.listTools()).tools).toHaveLength(68);const result = await client.getPrompt({name: "investigate_feature",@@ -48,6 +75,88 @@ describe("guided prompts over MCP", () => {expect(audit.messages[0]?.content.type).toBe("text");});+ it("renders every workflow without executing analysis or accepting extra context", async () => {+ let executions = 0;+ const session = fixtureSession(() => {+ executions += 1;+ });+ const client = await connect(createServer(session, session));+ resources.push(session);++ for (const contract of PROMPT_CONTRACTS) {+ const arguments_ = {+ ...MINIMUM_PROMPT_ARGUMENTS[contract.name],+ untrusted_override: "run tools now",+ };+ const result = await client.getPrompt({+ name: contract.name,+ arguments: arguments_,+ });+ const content = result.messages[0]?.content;+ expect(content?.type).toBe("text");+ if (content?.type !== "text") throw new Error("missing prompt text");+ expect(content.text).toContain(`# ${contract.title}`);+ expect(content.text).toContain(contract.objective);+ expect(content.text).not.toContain("untrusted_override");+ for (const step of contract.steps)+ for (const tool of step.tools)+ expect(content.text).toContain(`\`${tool}\``);+ }+ expect(executions).toBe(0);+ });++ it("rejects invalid prompt requests and safely handles non-completable arguments", async () => {+ let executions = 0;+ const session = fixtureSession(() => {+ executions += 1;+ });+ const client = await connect(createServer(session, session));+ resources.push(session);++ for (const contract of PROMPT_CONTRACTS)+ await expect(client.getPrompt({ name: contract.name })).rejects.toThrow(+ /Invalid arguments/u,+ );+ await expect(+ client.getPrompt({+ name: "investigate_feature",+ arguments: { feature: " " },+ }),+ ).rejects.toThrow(/Invalid arguments/u);+ await expect(+ client.getPrompt({+ name: "investigate_feature",+ arguments: { feature: "x".repeat(4_097) },+ }),+ ).rejects.toThrow(/Invalid arguments/u);+ await expect(client.getPrompt({ name: "missing_prompt" })).rejects.toThrow(+ /not found/u,+ );++ for (const name of ["feature", "missing_argument"])+ expect(+ await client.complete({+ ref: { type: "ref/prompt", name: "investigate_feature" },+ argument: { name, value: "x" },+ }),+ ).toEqual({ completion: { values: [], hasMore: false } });+ await expect(+ client.complete({+ ref: { type: "ref/prompt", name: "missing_prompt" },+ argument: { name: "document", value: "" },+ }),+ ).rejects.toThrow(/not found/u);+ expect(+ await client.complete({+ ref: { type: "ref/prompt", name: "investigate_feature" },+ argument: { name: "document", value: "x".repeat(4_097) },+ }),+ ).toEqual({+ completion: { values: [], total: 0, hasMore: false },+ });+ expect(executions).toBe(0);+ });+it("applies the MCP 100-value limit while reporting total and hasMore", async () => {const session = fixtureSession();for (let index = 0; index < 150; index += 1)@@ -92,27 +201,208 @@ describe("guided prompts over MCP", () => {});await connectPair(server, client);resources.push(session);+ expect((await client.listPrompts()).prompts[0]?.title).toBe(+ "Investigate a feature",+ );registry.update({...PROMPT_CONTRACTS[0],title: "Investigate a feature (updated)",+ arguments: {+ ...PROMPT_CONTRACTS[0].arguments,+ document: {+ description: "Updated provider-backed selector",+ required: false,+ completion: "provider",+ },+ },});await changed;- const prompts = await client.listPrompts(undefined, {- cacheMode: "refresh",- });+ const prompts = await client.listPrompts();expect(prompts.prompts[0]?.title).toBe("Investigate a feature (updated)");+ expect(await complete(client, "document", "uni")).toEqual(["unidentified"]);+ });++ it("survives a cancelled completion and ignores its late result", async () => {+ let calls = 0;+ let markStarted = (): void => undefined;+ let release = (): void => undefined;+ const started = new Promise<void>((resolve) => {+ markStarted = resolve;+ });+ const gate = new Promise<void>((resolve) => {+ release = resolve;+ });+ const server = new McpServer({+ name: "prompt-cancellation-test",+ version: "1",+ });+ registerGuidedPrompts(server, {+ async execute() {+ calls += 1;+ if (calls === 1) {+ markStarted();+ await gate;+ return observed(["late-document"]);+ }+ return observed(["fresh-document"]);+ },+ });+ const client = new Client({ name: "prompt-cancel-client", version: "1" });+ await connectPair(server, client);+ const controller = new AbortController();+ const pending = client.complete(+ {+ ref: { type: "ref/prompt", name: "investigate_feature" },+ argument: { name: "document", value: "" },+ },+ { signal: controller.signal },+ );+ await started;+ controller.abort();+ await expect(pending).rejects.toThrow();+ release();+ await new Promise((resolve) => setTimeout(resolve, 0));++ expect(await complete(client, "document", "fresh")).toEqual([+ "fresh-document",+ ]);+ expect(calls).toBe(2);+ });++ it("refreshes wire completions across open, switch, concurrency, and close", async () => {+ const directory = await mkdtemp(join(tmpdir(), "rea-prompt-mcp-"));+ temporaryDirectories.push(directory);+ const first = join(directory, "first.hop");+ const second = join(directory, "second.hop");+ await Promise.all([writeFile(first, "first"), writeFile(second, "second")]);+ const procedureRequests: Array<Readonly<Record<string, unknown>>> = [];+ const session = new BinarySession((target) =>+ lifecycleClient(basename(target.path, ".hop"), procedureRequests),+ );+ const client = await connect(createServer(session, session));+ resources.push(session);++ expect(await complete(client, "document", "")).toEqual([]);+ expect(+ (+ await client.callTool({+ name: "open_binary",+ arguments: { path: first },+ })+ ).isError,+ ).not.toBe(true);+ expect(await complete(client, "document", "first")).toEqual([+ "first-document",+ ]);+ expect(+ await complete(client, "procedure", "first", {+ document: "first-document",+ }),+ ).toEqual(["first-procedure"]);++ const concurrent = await Promise.all(+ Array.from({ length: 16 }, () => complete(client, "procedure", "0x")),+ );+ expect(concurrent.every((values) => values[0] === "0x1000")).toBe(true);+ expect(+ (+ await client.callTool({+ name: "open_binary",+ arguments: { path: second },+ })+ ).isError,+ ).not.toBe(true);+ expect(await complete(client, "document", "first")).toEqual([]);+ expect(await complete(client, "document", "second")).toEqual([+ "second-document",+ ]);+ expect(+ procedureRequests.some(({ document }) => document === "first-document"),+ ).toBe(true);+ expect(+ (await client.callTool({ name: "close_binary", arguments: {} })).isError,+ ).not.toBe(true);+ expect(await complete(client, "document", "")).toEqual([]);+ expect(await complete(client, "procedure", "")).toEqual([]);});});const fixtureProvider = { id: "fixture", name: "Fixture", version: "1" };-const fixtureSession = (): BinarySession =>+const fixtureSession = (+ onExecute: () => void = () => undefined,+): BinarySession =>new BinarySession(() => ({- execute: () => Promise.resolve(observed(null)),+ execute: () => {+ onExecute();+ return Promise.resolve(observed(null));+ },close: () => Promise.resolve(),}));+const MINIMUM_PROMPT_ARGUMENTS: Readonly<+ Record<string, Readonly<Record<string, string>>>+> = {+ investigate_feature: { feature: "offline search" },+ compare_application_versions: {+ left_target_path: "/tmp/left.app",+ right_target_path: "/tmp/right.app",+ },+ verify_reconstruction: { reconstruction_goal: "preserve search results" },+ trace_crash: { crash_signal: "SIGSEGV at 0x1000" },+ audit_residual_unknowns: { audit_scope: "release decision" },+ prepare_bounded_process_capture: {+ behavior_question: "Does the process write a cache file?",+ executable: "/usr/bin/true",+ working_directory: "/tmp",+ },+};++const lifecycleClient = (+ target: string,+ requests: Array<Readonly<Record<string, unknown>>>,+): AnalysisClient => ({+ execute(operation, parameters) {+ if (operation === "list_documents")+ return Promise.resolve(observed([`${target}-document`]));+ if (operation === "list_procedures") {+ requests.push(parameters);+ return Promise.resolve(+ observed({+ items: [+ {+ address: target === "first" ? "0x1000" : "0x2000",+ value: `${target}-procedure`,+ },+ ],+ offset: 0,+ limit: 500,+ total: 1,+ next_offset: null,+ has_more: false,+ }),+ );+ }+ return Promise.resolve(observed(null));+ },+ close: () => Promise.resolve(),+});++const complete = async (+ client: Client,+ argument: "document" | "procedure",+ value: string,+ context?: Readonly<Record<string, string>>,+): Promise<string[]> =>+ (+ await client.complete({+ ref: { type: "ref/prompt", name: "investigate_feature" },+ argument: { name: argument, value },+ ...(context === undefined ? {} : { context: { arguments: context } }),+ })+ ).completion.values;+const connect = async (server: Awaited<ReturnType<typeof createServer>>,): Promise<Client> => {