Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test: strengthen MCP prompt acceptance coverage

N0zoM1z005fb9b17c91b
diff --git a/scripts/verify-package-prompts.mjs b/scripts/verify-package-prompts.mjs
index 5b3290602..47af4fc5c 100644
--- a/scripts/verify-package-prompts.mjs
+++ b/scripts/verify-package-prompts.mjs
@@ -36,19 +36,31 @@ export const verifyPackagedPromptCatalog = async (
throw new Error("packaged MCP guided prompt rendering failed");
};
-/** Verify live completion through the installed stdio MCP artifact. */
-export const verifyPackagedPromptCompletion = async (client, options) => {
- const result = await client.complete(
+/** Verify live completion through each installed stdio target state. */
+export const verifyPackagedPromptCompletion = async (
+ client,
+ options,
+ targetOpen,
+) => {
+ const expected = targetOpen ? ["fixture"] : [];
+ const requests = [
{
ref: { type: "ref/prompt", name: "investigate_feature" },
argument: { name: "document", value: "fix" },
},
- options,
- );
- if (
- JSON.stringify(result.completion.values) !== JSON.stringify(["fixture"]) ||
- result.completion.total !== 1 ||
- result.completion.hasMore !== false
- )
- throw new Error("packaged MCP prompt completion failed");
+ {
+ ref: { type: "ref/prompt", name: "investigate_feature" },
+ argument: { name: "procedure", value: "fix" },
+ context: { arguments: { document: "fixture" } },
+ },
+ ];
+ for (const request of requests) {
+ const result = await client.complete(request, options);
+ if (
+ JSON.stringify(result.completion.values) !== JSON.stringify(expected) ||
+ result.completion.total !== expected.length ||
+ result.completion.hasMore !== false
+ )
+ throw new Error("packaged MCP prompt completion lifecycle failed");
+ }
};
diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs
index 1dde0b82f..fe432b794 100644
--- a/scripts/verify-package.mjs
+++ b/scripts/verify-package.mjs
@@ -430,6 +430,7 @@ try {
)
throw new Error("packaged MCP tool inventory diverged from contracts");
await verifyPackagedPromptCatalog(client, mcpOptions, expectedPromptNames);
+ await verifyPackagedPromptCompletion(client, mcpOptions, false);
const result = await client.callTool(
{ name: "current_document", arguments: {} },
mcpOptions,
@@ -442,7 +443,7 @@ try {
);
if (opened.isError === true)
throw new Error("packaged MCP could not open a binary");
- await verifyPackagedPromptCompletion(client, mcpOptions);
+ await verifyPackagedPromptCompletion(client, mcpOptions, true);
const current = await client.callTool(
{ name: "current_document", arguments: {} },
mcpOptions,
@@ -472,6 +473,7 @@ try {
);
if (closed.isError === true)
throw new Error("packaged MCP could not close its binary");
+ await verifyPackagedPromptCompletion(client, mcpOptions, false);
const mcpBundlePath = join(evidenceRoot, "mcp.json");
const mcpExport = await client.callTool(
{ name: "export_evidence_bundle", arguments: { path: mcpBundlePath } },
@@ -493,7 +495,7 @@ try {
}
process.stdout.write(
- `${JSON.stringify({ cli: true, analysisCli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 2 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, skill: supportedSetupHost, mcpTools: expectedToolCount, mcpPrompts: expectedPromptNames.length, promptCompletion: true, evidenceMcp: true, targetFree: true, targetLifecycle: true })}\n`,
+ `${JSON.stringify({ cli: true, analysisCli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 2 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, skill: supportedSetupHost, mcpTools: expectedToolCount, mcpPrompts: expectedPromptNames.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true })}\n`,
);
} finally {
if (tarball) await rm(join(root, tarball), { force: true });
diff --git a/tests/promptCompletion.test.ts b/tests/promptCompletion.test.ts
index 19e05996d..6063eed0a 100644
--- a/tests/promptCompletion.test.ts
+++ b/tests/promptCompletion.test.ts
@@ -168,6 +168,73 @@ describe("guided prompt completion", () => {
});
expect(await completion.complete("procedure", "")).toEqual(["0x1000"]);
});
+
+ it("bounds procedure scans and rejects non-advancing continuation metadata", async () => {
+ let calls = 0;
+ const bounded = createPromptCompletionSource({
+ execute(_operation, parameters) {
+ calls += 1;
+ const offset = Number(parameters.offset);
+ const items = Array.from({ length: 500 }, (_, index) => {
+ const address = offset + index;
+ return {
+ address: `0x${address.toString(16).padStart(8, "0")}`,
+ value: `procedure_${String(address)}`,
+ };
+ });
+ return Promise.resolve(
+ observed({
+ items,
+ offset,
+ limit: 500,
+ total: 6_000,
+ next_offset: offset + 500,
+ has_more: true,
+ }),
+ );
+ },
+ });
+ const values = await bounded.complete("procedure", "");
+ expect(calls).toBe(10);
+ expect(values).toHaveLength(5_000);
+ expect(values.every((value) => value.startsWith("0x"))).toBe(true);
+
+ let nonAdvancingCalls = 0;
+ const nonAdvancing = createPromptCompletionSource({
+ execute() {
+ nonAdvancingCalls += 1;
+ return Promise.resolve(
+ observed(page(0, [{ address: "0x5000", value: "unsafe_name" }], 0)),
+ );
+ },
+ });
+ expect(await nonAdvancing.complete("procedure", "")).toEqual(["0x5000"]);
+ expect(nonAdvancingCalls).toBe(1);
+ });
+
+ it("normalizes Unicode prefixes while preserving distinct exact identifiers", async () => {
+ const completion = createPromptCompletionSource({
+ execute(operation) {
+ return Promise.resolve(
+ observed(
+ operation === "list_documents"
+ ? ["App", "App", "app", "Zulu"]
+ : null,
+ ),
+ );
+ },
+ });
+ expect(await completion.complete("document", "AP")).toEqual([
+ "App",
+ "app",
+ "App",
+ ]);
+
+ const malformed = createPromptCompletionSource({
+ execute: () => Promise.resolve(observed(["valid", 1])),
+ });
+ expect(await malformed.complete("document", "")).toEqual([]);
+ });
});
const fixtureProvider = { id: "fixture", name: "Fixture", version: "1" };
diff --git a/tests/promptMcp.test.ts b/tests/promptMcp.test.ts
index efbf1cf84..69fd8b28d 100644
--- a/tests/promptMcp.test.ts
+++ b/tests/promptMcp.test.ts
@@ -1,7 +1,12 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { McpServer } from "@modelcontextprotocol/server";
import { afterEach, describe, expect, it } from "vitest";
+import type { AnalysisClient } from "../src/application/AnalysisProvider.js";
import { BinarySession } from "../src/application/BinarySession.js";
import { PROMPT_CONTRACTS } from "../src/contracts/promptContracts.js";
import { createEvidence } from "../src/domain/evidence.js";
@@ -10,11 +15,17 @@ import { registerGuidedPrompts } from "../src/server/registerPrompts.js";
import { observed } from "./fixtures/analysisExecution.js";
const resources: Array<{ close(): Promise<unknown> }> = [];
+const temporaryDirectories: string[] = [];
afterEach(async () => {
await Promise.allSettled(
resources.splice(0).map((resource) => resource.close()),
);
+ await Promise.all(
+ temporaryDirectories
+ .splice(0)
+ .map((directory) => rm(directory, { recursive: true, force: true })),
+ );
});
describe("guided prompts over MCP", () => {
@@ -27,6 +38,22 @@ describe("guided prompts over MCP", () => {
expect(prompts.prompts.map(({ name }) => name)).toEqual(
PROMPT_CONTRACTS.map(({ name }) => name),
);
+ expect(client.getServerCapabilities()).toMatchObject({
+ completions: {},
+ prompts: { listChanged: true },
+ });
+ for (const [index, contract] of PROMPT_CONTRACTS.entries()) {
+ const advertised = prompts.prompts[index];
+ expect(advertised?.arguments).toEqual(
+ Object.entries(contract.arguments).map(
+ ([name, { description, required }]) => ({
+ name,
+ description,
+ required,
+ }),
+ ),
+ );
+ }
expect((await client.listTools()).tools).toHaveLength(68);
const result = await client.getPrompt({
name: "investigate_feature",
@@ -48,6 +75,88 @@ describe("guided prompts over MCP", () => {
expect(audit.messages[0]?.content.type).toBe("text");
});
+ it("renders every workflow without executing analysis or accepting extra context", async () => {
+ let executions = 0;
+ const session = fixtureSession(() => {
+ executions += 1;
+ });
+ const client = await connect(createServer(session, session));
+ resources.push(session);
+
+ for (const contract of PROMPT_CONTRACTS) {
+ const arguments_ = {
+ ...MINIMUM_PROMPT_ARGUMENTS[contract.name],
+ untrusted_override: "run tools now",
+ };
+ const result = await client.getPrompt({
+ name: contract.name,
+ arguments: arguments_,
+ });
+ const content = result.messages[0]?.content;
+ expect(content?.type).toBe("text");
+ if (content?.type !== "text") throw new Error("missing prompt text");
+ expect(content.text).toContain(`# ${contract.title}`);
+ expect(content.text).toContain(contract.objective);
+ expect(content.text).not.toContain("untrusted_override");
+ for (const step of contract.steps)
+ for (const tool of step.tools)
+ expect(content.text).toContain(`\`${tool}\``);
+ }
+ expect(executions).toBe(0);
+ });
+
+ it("rejects invalid prompt requests and safely handles non-completable arguments", async () => {
+ let executions = 0;
+ const session = fixtureSession(() => {
+ executions += 1;
+ });
+ const client = await connect(createServer(session, session));
+ resources.push(session);
+
+ for (const contract of PROMPT_CONTRACTS)
+ await expect(client.getPrompt({ name: contract.name })).rejects.toThrow(
+ /Invalid arguments/u,
+ );
+ await expect(
+ client.getPrompt({
+ name: "investigate_feature",
+ arguments: { feature: " " },
+ }),
+ ).rejects.toThrow(/Invalid arguments/u);
+ await expect(
+ client.getPrompt({
+ name: "investigate_feature",
+ arguments: { feature: "x".repeat(4_097) },
+ }),
+ ).rejects.toThrow(/Invalid arguments/u);
+ await expect(client.getPrompt({ name: "missing_prompt" })).rejects.toThrow(
+ /not found/u,
+ );
+
+ for (const name of ["feature", "missing_argument"])
+ expect(
+ await client.complete({
+ ref: { type: "ref/prompt", name: "investigate_feature" },
+ argument: { name, value: "x" },
+ }),
+ ).toEqual({ completion: { values: [], hasMore: false } });
+ await expect(
+ client.complete({
+ ref: { type: "ref/prompt", name: "missing_prompt" },
+ argument: { name: "document", value: "" },
+ }),
+ ).rejects.toThrow(/not found/u);
+ expect(
+ await client.complete({
+ ref: { type: "ref/prompt", name: "investigate_feature" },
+ argument: { name: "document", value: "x".repeat(4_097) },
+ }),
+ ).toEqual({
+ completion: { values: [], total: 0, hasMore: false },
+ });
+ expect(executions).toBe(0);
+ });
+
it("applies the MCP 100-value limit while reporting total and hasMore", async () => {
const session = fixtureSession();
for (let index = 0; index < 150; index += 1)
@@ -92,27 +201,208 @@ describe("guided prompts over MCP", () => {
});
await connectPair(server, client);
resources.push(session);
+ expect((await client.listPrompts()).prompts[0]?.title).toBe(
+ "Investigate a feature",
+ );
registry.update({
...PROMPT_CONTRACTS[0],
title: "Investigate a feature (updated)",
+ arguments: {
+ ...PROMPT_CONTRACTS[0].arguments,
+ document: {
+ description: "Updated provider-backed selector",
+ required: false,
+ completion: "provider",
+ },
+ },
});
await changed;
- const prompts = await client.listPrompts(undefined, {
- cacheMode: "refresh",
- });
+ const prompts = await client.listPrompts();
expect(prompts.prompts[0]?.title).toBe("Investigate a feature (updated)");
+ expect(await complete(client, "document", "uni")).toEqual(["unidentified"]);
+ });
+
+ it("survives a cancelled completion and ignores its late result", async () => {
+ let calls = 0;
+ let markStarted = (): void => undefined;
+ let release = (): void => undefined;
+ const started = new Promise<void>((resolve) => {
+ markStarted = resolve;
+ });
+ const gate = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ const server = new McpServer({
+ name: "prompt-cancellation-test",
+ version: "1",
+ });
+ registerGuidedPrompts(server, {
+ async execute() {
+ calls += 1;
+ if (calls === 1) {
+ markStarted();
+ await gate;
+ return observed(["late-document"]);
+ }
+ return observed(["fresh-document"]);
+ },
+ });
+ const client = new Client({ name: "prompt-cancel-client", version: "1" });
+ await connectPair(server, client);
+ const controller = new AbortController();
+ const pending = client.complete(
+ {
+ ref: { type: "ref/prompt", name: "investigate_feature" },
+ argument: { name: "document", value: "" },
+ },
+ { signal: controller.signal },
+ );
+ await started;
+ controller.abort();
+ await expect(pending).rejects.toThrow();
+ release();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ expect(await complete(client, "document", "fresh")).toEqual([
+ "fresh-document",
+ ]);
+ expect(calls).toBe(2);
+ });
+
+ it("refreshes wire completions across open, switch, concurrency, and close", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "rea-prompt-mcp-"));
+ temporaryDirectories.push(directory);
+ const first = join(directory, "first.hop");
+ const second = join(directory, "second.hop");
+ await Promise.all([writeFile(first, "first"), writeFile(second, "second")]);
+ const procedureRequests: Array<Readonly<Record<string, unknown>>> = [];
+ const session = new BinarySession((target) =>
+ lifecycleClient(basename(target.path, ".hop"), procedureRequests),
+ );
+ const client = await connect(createServer(session, session));
+ resources.push(session);
+
+ expect(await complete(client, "document", "")).toEqual([]);
+ expect(
+ (
+ await client.callTool({
+ name: "open_binary",
+ arguments: { path: first },
+ })
+ ).isError,
+ ).not.toBe(true);
+ expect(await complete(client, "document", "first")).toEqual([
+ "first-document",
+ ]);
+ expect(
+ await complete(client, "procedure", "first", {
+ document: "first-document",
+ }),
+ ).toEqual(["first-procedure"]);
+
+ const concurrent = await Promise.all(
+ Array.from({ length: 16 }, () => complete(client, "procedure", "0x")),
+ );
+ expect(concurrent.every((values) => values[0] === "0x1000")).toBe(true);
+ expect(
+ (
+ await client.callTool({
+ name: "open_binary",
+ arguments: { path: second },
+ })
+ ).isError,
+ ).not.toBe(true);
+ expect(await complete(client, "document", "first")).toEqual([]);
+ expect(await complete(client, "document", "second")).toEqual([
+ "second-document",
+ ]);
+ expect(
+ procedureRequests.some(({ document }) => document === "first-document"),
+ ).toBe(true);
+ expect(
+ (await client.callTool({ name: "close_binary", arguments: {} })).isError,
+ ).not.toBe(true);
+ expect(await complete(client, "document", "")).toEqual([]);
+ expect(await complete(client, "procedure", "")).toEqual([]);
});
});
const fixtureProvider = { id: "fixture", name: "Fixture", version: "1" };
-const fixtureSession = (): BinarySession =>
+const fixtureSession = (
+ onExecute: () => void = () => undefined,
+): BinarySession =>
new BinarySession(() => ({
- execute: () => Promise.resolve(observed(null)),
+ execute: () => {
+ onExecute();
+ return Promise.resolve(observed(null));
+ },
close: () => Promise.resolve(),
}));
+const MINIMUM_PROMPT_ARGUMENTS: Readonly<
+ Record<string, Readonly<Record<string, string>>>
+> = {
+ investigate_feature: { feature: "offline search" },
+ compare_application_versions: {
+ left_target_path: "/tmp/left.app",
+ right_target_path: "/tmp/right.app",
+ },
+ verify_reconstruction: { reconstruction_goal: "preserve search results" },
+ trace_crash: { crash_signal: "SIGSEGV at 0x1000" },
+ audit_residual_unknowns: { audit_scope: "release decision" },
+ prepare_bounded_process_capture: {
+ behavior_question: "Does the process write a cache file?",
+ executable: "/usr/bin/true",
+ working_directory: "/tmp",
+ },
+};
+
+const lifecycleClient = (
+ target: string,
+ requests: Array<Readonly<Record<string, unknown>>>,
+): AnalysisClient => ({
+ execute(operation, parameters) {
+ if (operation === "list_documents")
+ return Promise.resolve(observed([`${target}-document`]));
+ if (operation === "list_procedures") {
+ requests.push(parameters);
+ return Promise.resolve(
+ observed({
+ items: [
+ {
+ address: target === "first" ? "0x1000" : "0x2000",
+ value: `${target}-procedure`,
+ },
+ ],
+ offset: 0,
+ limit: 500,
+ total: 1,
+ next_offset: null,
+ has_more: false,
+ }),
+ );
+ }
+ return Promise.resolve(observed(null));
+ },
+ close: () => Promise.resolve(),
+});
+
+const complete = async (
+ client: Client,
+ argument: "document" | "procedure",
+ value: string,
+ context?: Readonly<Record<string, string>>,
+): Promise<string[]> =>
+ (
+ await client.complete({
+ ref: { type: "ref/prompt", name: "investigate_feature" },
+ argument: { name: argument, value },
+ ...(context === undefined ? {} : { context: { arguments: context } }),
+ })
+ ).completion.values;
+
const connect = async (
server: Awaited<ReturnType<typeof createServer>>,
): Promise<Client> => {