Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(ci): validate artifact MCP replies against named schemas (#6021)
gensokyo-devf86206535f8e
diff --git a/docs/testing.md b/docs/testing.mdindex e19db750b..9376578c4 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -256,6 +256,22 @@ pass that definition through the SDK's `toolDefinition` call option. This keepsthe selected output-schema validation without eagerly compiling unrelatedcatalog schemas. Re-read definitions if the server advertises a changed list.+Artifact verification uses `artifactMcpCall` for both lifecycle and analysis+calls. A fresh SDK client has no held tool listing, so a bare `callTool` can+skip output-schema validation. The helper discovers actual advertised+definitions, rejects missing or ambiguous schemas, and validates the requested+tool without compiling unrelated catalog schemas. REA registers process-static+tool contracts; bindings share one discovery per negotiated connection and are+discarded on reconnect or failed discovery. Keep raw artifact calls on this+shared path and retain explicit request options such as timeouts.+Caller-supplied tool definitions are rejected: the helper owns selection from+the advertised catalog so a stale or permissive schema cannot bypass it.++The regression fixture changes only `canonical_path` in a real stdio response+to a number. The old helper accepts that response; the named schema must reject+it. This client-boundary fault injection does not establish native Apple+behavior, which remains covered by the macOS verification lane.+Report unavailable authority as a named skip. Await owned cleanup before averifier reports success. Native process inspectors are released at Vitestworker teardown; fork termination does not run Node exit hooks, and per-filediff --git a/scripts/lib/artifact-e2e.mjs b/scripts/lib/artifact-e2e.mjsindex 444639319..a2ab021fe 100644--- a/scripts/lib/artifact-e2e.mjs+++ b/scripts/lib/artifact-e2e.mjs@@ -4,6 +4,67 @@ import { promisify } from "node:util";import { fileURLToPath } from "node:url";import { Client } from "@modelcontextprotocol/client";import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";+import { closeVerifierResources } from "./verifier-cleanup.mjs";++// REA registers process-static tool contracts. Bind their parsed definitions to+// the SDK's negotiated connection identity, including when a Client reconnects.+const toolBindings = new WeakMap();++const advertisedTool = async (client, name) => {+ const connection = client.getServerVersion();+ assert.ok(connection, "MCP must be initialized before binding tool schemas");+ let binding = toolBindings.get(client);+ if (binding?.connection !== connection) {+ binding = {+ connection,+ definitions: client.listTools().then(({ tools }) => {+ const definitions = new Map();+ for (const tool of tools) {+ assert.ok(+ !definitions.has(tool.name),+ `Duplicate MCP tool: ${tool.name}`,+ );+ definitions.set(tool.name, tool);+ }+ return definitions;+ }),+ };+ toolBindings.set(client, binding);+ }+ let definitions;+ try {+ definitions = await binding.definitions;+ assert.equal(+ client.getServerVersion(),+ connection,+ "MCP connection changed while binding tool schemas",+ );+ } catch (cause) {+ if (toolBindings.get(client) === binding) toolBindings.delete(client);+ throw cause;+ }+ const tool = definitions.get(name);+ assert.ok(tool, `MCP catalog did not advertise ${name}`);+ return tool;+};++/** Call an advertised tool with SDK output validation of its actual schema. */+export async function artifactMcpCall(client, parameters, options = {}) {+ options.signal?.throwIfAborted();+ assert.equal(+ options.toolDefinition,+ undefined,+ "Artifact verification owns the advertised schema; pass request options only",+ );+ const toolDefinition = await advertisedTool(client, parameters.name);+ assert.notEqual(+ toolDefinition.outputSchema,+ undefined,+ `MCP tool omitted its output schema: ${parameters.name}`,+ );+ options.signal?.throwIfAborted();+ return client.callTool(parameters, { ...options, toolDefinition });+}const environment = () => {const compileCache =@@ -66,9 +127,10 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {stderr: "pipe",});const client = new Client({ name: "artifact-real-e2e", version: "1" });+ let failure;try {await client.connect(transport);- const opened = await client.callTool({+ const opened = await artifactMcpCall(client, {name: openTool,arguments: { path: target },});@@ -76,9 +138,15 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {if (openTool === "open_bundle_resources")assert.equal(opened.structuredContent?.result?.caller_path, target);await verify(client);+ } catch (cause) {+ failure = cause;+ throw cause;} finally {- await client.close();- await transport.close();+ toolBindings.delete(client);+ await closeVerifierResources(+ [() => client.close(), () => transport.close()],+ failure,+ );}};@@ -94,7 +162,10 @@ export async function withBundleResourcesMcp(target, verify) {/** Require a successful structured MCP tool result rather than its text rendering. */export async function artifactMcpResult(client, name, arguments_ = {}) {- const result = await client.callTool({ name, arguments: arguments_ });+ const result = await artifactMcpCall(client, {+ name,+ arguments: arguments_,+ });assert.notEqual(result.isError, true, JSON.stringify(result));assert.ok(result.structuredContent?.normalized_result,diff --git a/scripts/verify-native-calls.mjs b/scripts/verify-native-calls.mjsindex 7a2262075..2d43a0457 100644--- a/scripts/verify-native-calls.mjs+++ b/scripts/verify-native-calls.mjs@@ -10,6 +10,7 @@ import { promisify } from "node:util";import {artifactCli,artifactMcpResult,+ artifactMcpCall,withArtifactMcp,} from "./lib/artifact-e2e.mjs";@@ -225,7 +226,8 @@ const checkHardenedRuntime = async (directory, binary) => {assert.equal(denied.code, "capability_unavailable");assert.match(denied.details.reason, /^debugger-attach-denied: /u);await withArtifactMcp(blocked, async (client) => {- const deniedMcp = await client.callTool(+ const deniedMcp = await artifactMcpCall(+ client,{name: "observe_native_calls",arguments: INPUT,diff --git a/scripts/verify/apple/asset-catalog.mjs b/scripts/verify/apple/asset-catalog.mjsindex 4de295f06..a194d9211 100644--- a/scripts/verify/apple/asset-catalog.mjs+++ b/scripts/verify/apple/asset-catalog.mjs@@ -9,6 +9,7 @@ import { promisify } from "node:util";import {artifactCli,artifactMcpResult,+ artifactMcpCall,withArtifactMcp,withBundleResourcesMcp,} from "../../lib/artifact-e2e.mjs";@@ -97,7 +98,7 @@ try {offset: first.next_offset,});assert.deepEqual([...first.records, ...next.records], cli.records);- const invalid = await client.callTool({+ const invalid = await artifactMcpCall(client, {name: "inspect_asset_catalog",arguments: { offset: -1 },});diff --git a/scripts/verify/apple/keyed-archive.mjs b/scripts/verify/apple/keyed-archive.mjsindex 471b76ef6..e44929d7e 100644--- a/scripts/verify/apple/keyed-archive.mjs+++ b/scripts/verify/apple/keyed-archive.mjs@@ -9,6 +9,7 @@ import { createHash } from "node:crypto";import {artifactCli,artifactMcpResult,+ artifactMcpCall,withArtifactMcp,} from "../../lib/artifact-e2e.mjs";import {@@ -53,7 +54,7 @@ try {});assert.deepEqual(page.objects, [graph.objects[1]]);assert.equal(page.next_offset, 2);- const invalid = await client.callTool({+ const invalid = await artifactMcpCall(client, {name: "inspect_keyed_archive",arguments: { path: "../other.plist" },});diff --git a/scripts/verify/apple/macho-occurrences.mjs b/scripts/verify/apple/macho-occurrences.mjsindex 9a7da4f4e..d1719af43 100644--- a/scripts/verify/apple/macho-occurrences.mjs+++ b/scripts/verify/apple/macho-occurrences.mjs@@ -5,6 +5,7 @@ import { relative, sep } from "node:path";import { parseArtifactInventoryEvidence } from "../../../dist/domain/artifactInventoryEvidence.js";import {artifactCliEvidence,+ artifactMcpCall,withArtifactMcp,} from "../../lib/artifact-e2e.mjs";@@ -53,7 +54,7 @@ export async function verifyMachOOccurrences(app, executable) {assert.equal(node.sha256, direct.manifest.root_sha256);assert.equal(node.format, root.artifact_format);await withArtifactMcp(app, async (client) => {- const reply = await client.callTool({+ const reply = await artifactMcpCall(client, {name: "inspect_artifact",arguments: {},});diff --git a/scripts/verify/apple/macos-dylib-e2e.mjs b/scripts/verify/apple/macos-dylib-e2e.mjsindex d9538c4cc..59a55bb7c 100644--- a/scripts/verify/apple/macos-dylib-e2e.mjs+++ b/scripts/verify/apple/macos-dylib-e2e.mjs@@ -7,6 +7,7 @@ import { promisify } from "node:util";import {artifactCliEvidence,artifactMcpResult,+ artifactMcpCall,withArtifactMcp,} from "../../lib/artifact-e2e.mjs";@@ -180,7 +181,7 @@ export async function verifyDylibResolution(app) {await artifactMcpResult(client, "trace_dylib_resolution"),trace,);- const invalid = await client.callTool({+ const invalid = await artifactMcpCall(client, {name: "trace_dylib_resolution",arguments: { roots: ["../outside"] },});diff --git a/scripts/verify/apple/native-provenance.mjs b/scripts/verify/apple/native-provenance.mjsindex 63ae2183b..a225fafb9 100644--- a/scripts/verify/apple/native-provenance.mjs+++ b/scripts/verify/apple/native-provenance.mjs@@ -5,6 +5,7 @@ import { readFile } from "node:fs/promises";import { toolContract } from "../../../dist/contracts/toolContracts.js";import {artifactCliEvidence,+ artifactMcpCall,withArtifactMcp,withBundleResourcesMcp,} from "../../lib/artifact-e2e.mjs";@@ -27,7 +28,7 @@ export async function verifyNativeProvenance(app, executable) {assert.equal(resourcePlistCli.subject, null);assert.equal(resourcePlistCli.raw_result, null);await withBundleResourcesMcp(app, async (client) => {- const response = await client.callTool({+ const response = await artifactMcpCall(client, {name: "inspect_plist",arguments: {},});@@ -58,13 +59,10 @@ export async function verifyNativeProvenance(app, executable) {assert.equal(binaryPlistCli.raw_result, null);assert.equal(binaryPlistCli.normalized_result.resource_root, undefined);await withArtifactMcp(app, async (client) => {- const definitions = new Map(- (await client.listTools()).tools.map((tool) => [tool.name, tool]),- );- const plistResponse = await client.callTool(- { name: "inspect_plist", arguments: {} },- { toolDefinition: definitions.get("inspect_plist") },- );+ const plistResponse = await artifactMcpCall(client, {+ name: "inspect_plist",+ arguments: {},+ });assert.notEqual(plistResponse.isError, true, JSON.stringify(plistResponse));const binaryPlistMcp = toolContract("inspect_plist").outputSchema.parse(plistResponse.structuredContent,@@ -98,13 +96,10 @@ export async function verifyNativeProvenance(app, executable) {arguments_,),);- const response = await client.callTool(- {- name: operation,- arguments: operation === "demangle_swift" ? { symbols } : {},- },- { toolDefinition: definitions.get(operation) },- );+ const response = await artifactMcpCall(client, {+ name: operation,+ arguments: operation === "demangle_swift" ? { symbols } : {},+ });assert.notEqual(response.isError, true, JSON.stringify(response));const mcp = toolContract(operation).outputSchema.parse(response.structuredContent,diff --git a/tests/boundary/process/artifactMcpVerifierOutput.test.ts b/tests/boundary/process/artifactMcpVerifierOutput.test.tsnew file mode 100644index 000000000..91da42a92--- /dev/null+++ b/tests/boundary/process/artifactMcpVerifierOutput.test.ts@@ -0,0 +1,126 @@+import { execFile } from "node:child_process";+import { mkdir, writeFile } from "node:fs/promises";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import { promisify } from "node:util";+import { expect, it } from "vitest";+import { z } from "zod";++import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++const exec = promisify(execFile);+const observationSchema = z.strictObject({+ accepted: z.boolean(),+ catalogRequests: z.number().int().nonnegative(),+ corruptedReplies: z.number().int().nonnegative(),+ toolCalls: z.number().int().nonnegative(),+ toolTimeouts: z.array(z.number().nullable()),+ error: z+ .strictObject({+ name: z.string(),+ message: z.string(),+ code: z.union([z.number(), z.string()]).nullable(),+ })+ .nullable(),+});++async function probe(mode: string, name = "Fixture.app") {+ const root = await createTestTempDirectory("rea-artifact-mcp-schema-");+ const target = join(root, name);+ await mkdir(join(target, "Contents", "Resources"), { recursive: true });+ await writeFile(+ join(target, "Contents", "Info.plist"),+ '<?xml version="1.0"?><plist version="1.0"><dict><key>CFBundleIdentifier</key><string>dev.rea.schema-probe</string></dict></plist>',+ );+ const { stdout } = await exec(+ process.execPath,+ [+ fileURLToPath(+ new URL("../../fixtures/artifactMcpVerifierProbe.mjs", import.meta.url),+ ),+ mode,+ target,+ ],+ // Retain cleanup time within the existing 30s test budget.+ { timeout: 25_000 },+ );+ return observationSchema.parse(JSON.parse(stdout));+}++it("validates the real resource admission reply against its advertised schema", async () => {+ expect(await probe("healthy", "Fixture.app")).toEqual({+ accepted: true,+ catalogRequests: 1,+ corruptedReplies: 0,+ toolCalls: 1,+ toolTimeouts: [null],+ error: null,+ });+});++it.each(["Fixture.app", "Fixture with spaces 雨.app"])(+ "rejects a corrupted canonical path in a real admission reply: %s",+ async (name) => {+ const result = await probe("corrupt", name);+ expect(result.accepted).toBe(false);+ expect(result.catalogRequests).toBe(1);+ expect(result.corruptedReplies).toBe(1);+ expect(result.error).toMatchObject({+ name: "ProtocolError",+ code: -32602,+ message: expect.stringMatching(/output schema.*canonical_path/u),+ });+ },+);++it.each(["reuse", "concurrent"])(+ "shares one schema discovery within one MCP connection: %s",+ async (mode) => {+ const result = await probe(mode);+ expect(result.accepted).toBe(true);+ expect(result.catalogRequests).toBe(1);+ expect(result.toolCalls).toBe(mode === "reuse" ? 3 : 2);+ },+);++it("binds the new advertised schema after reconnecting the same client", async () => {+ const result = await probe("reconnect");+ expect(result.accepted).toBe(false);+ expect(result.catalogRequests).toBe(2);+ expect(result.toolCalls).toBe(2);+ expect(result.error).toMatchObject({+ code: -32602,+ message: expect.stringContaining("__rea_second_connection"),+ });+});++it("rejects an omitted output schema before calling the advertised tool", async () => {+ const result = await probe("missing-schema");+ expect(result.accepted).toBe(false);+ expect(result.toolCalls).toBe(0);+ expect(result.error?.message).toMatch(/omitted its output schema/u);+});++it("rejects a supplied schema override before dispatching that call", async () => {+ const result = await probe("override");+ expect(result.accepted).toBe(false);+ expect(result.toolCalls).toBe(1);+ expect(result.error?.message).toMatch(/owns the advertised schema/u);+});++it("preserves explicit request timeouts on schema-bound calls", async () => {+ const result = await probe("request-options");+ expect(result.accepted).toBe(true);+ expect(result.catalogRequests).toBe(1);+ expect(result.toolTimeouts).toEqual([null, 12_345]);+});++it.each(["duplicate-first", "duplicate-last"])(+ "rejects ambiguous named schemas independently of catalog order: %s",+ async (mode) => {+ const result = await probe(mode);+ expect(result.accepted).toBe(false);+ expect(result.toolCalls).toBe(0);+ expect(result.error?.message).toMatch(/Duplicate MCP tool/u);+ },+);diff --git a/tests/fixtures/artifactMcpVerifierProbe.mjs b/tests/fixtures/artifactMcpVerifierProbe.mjsnew file mode 100644index 000000000..731d39881--- /dev/null+++ b/tests/fixtures/artifactMcpVerifierProbe.mjs@@ -0,0 +1,209 @@+import assert from "node:assert/strict";+import { fileURLToPath } from "node:url";+import { Client } from "@modelcontextprotocol/client";+import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";++import * as artifact from "../../scripts/lib/artifact-e2e.mjs";+import { closeVerifierResources } from "../../scripts/lib/verifier-cleanup.mjs";++const [mode, target] = process.argv.slice(2);+assert.ok(+ [+ "healthy",+ "corrupt",+ "reuse",+ "concurrent",+ "reconnect",+ "missing-schema",+ "duplicate-first",+ "duplicate-last",+ "override",+ "request-options",+ ].includes(mode),+);+assert.equal(typeof target, "string");++// Isolate the fault to this child client. The server, resource admission and+// advertised schema are real; alter only the reply before tool validation.+const request = Client.prototype.request;+let catalogRequests = 0;+let corruptedReplies = 0;+let toolCalls = 0;+let connection = 0;+const toolTimeouts = [];+Client.prototype.request = async function (input, options) {+ if (input.method === "tools/list") catalogRequests++;+ if (input.method === "tools/call") {+ toolCalls++;+ toolTimeouts.push(options?.timeout ?? null);+ }+ const result = await request.call(this, input, options);+ if (+ mode === "override" &&+ input.method === "tools/call" &&+ input.params.name === "binary_session"+ ) {+ corruptedReplies++;+ return { ...result, structuredContent: { result: 777 } };+ }+ if (input.method === "tools/list") {+ if (mode === "missing-schema")+ return {+ ...result,+ tools: result.tools.map((tool) => {+ if (tool.name !== "open_bundle_resources") return tool;+ const { outputSchema: _schema, ...withoutSchema } = tool;+ return withoutSchema;+ }),+ };+ if (mode === "duplicate-first" || mode === "duplicate-last") {+ const listed = result.tools.find(+ (tool) => tool.name === "open_bundle_resources",+ );+ assert.ok(listed);+ const duplicate = {+ ...listed,+ outputSchema: { type: "object", required: ["__rea_wrong_schema"] },+ };+ return {+ ...result,+ tools:+ mode === "duplicate-first"+ ? [duplicate, ...result.tools]+ : [...result.tools, duplicate],+ };+ }+ if (mode === "reconnect" && connection === 2)+ return {+ ...result,+ tools: result.tools.map((tool) =>+ tool.name === "binary_session"+ ? {+ ...tool,+ outputSchema: {+ type: "object",+ required: ["__rea_second_connection"],+ },+ }+ : tool,+ ),+ };+ }+ if (+ mode !== "corrupt" ||+ input.method !== "tools/call" ||+ input.params.name !== "open_bundle_resources"+ )+ return result;+ assert.equal(+ typeof result.structuredContent?.result?.canonical_path,+ "string",+ "The real reply omitted the field selected for fault injection",+ );+ corruptedReplies++;+ return {+ ...result,+ structuredContent: {+ ...result.structuredContent,+ result: { ...result.structuredContent.result, canonical_path: 777 },+ },+ };+};++const status = async (client) => {+ const result = await artifact.artifactMcpCall(client, {+ name: "binary_session",+ arguments: {},+ });+ assert.notEqual(result.isError, true, JSON.stringify(result));+};++async function standaloneConnections() {+ const client = new Client({ name: "artifact-binding-probe", version: "1" });+ let firstVersion;+ for (+ connection = 1;+ connection <= (mode === "reconnect" ? 2 : 1);+ connection+++ ) {+ const transport = new StdioClientTransport({+ command: process.execPath,+ args: [+ fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)),+ "mcp",+ ],+ env: {+ PATH: process.env.PATH ?? "/usr/bin:/bin",+ REA_LOG_LEVEL: "silent",+ },+ stderr: "pipe",+ });+ let failure;+ try {+ await client.connect(transport);+ const version = client.getServerVersion();+ assert.ok(version);+ if (firstVersion !== undefined) {+ assert.notEqual(version, firstVersion);+ assert.equal(version.name, firstVersion.name);+ assert.equal(version.version, firstVersion.version);+ }+ firstVersion = version;+ if (mode === "concurrent")+ await Promise.all([status(client), status(client)]);+ else await status(client);+ } catch (cause) {+ failure = cause;+ throw cause;+ } finally {+ await closeVerifierResources(+ [() => client.close(), () => transport.close()],+ failure,+ );+ }+ }+}++let error;+try {+ if (mode === "concurrent" || mode === "reconnect")+ await standaloneConnections();+ else+ await artifact.withBundleResourcesMcp(target, async (client) => {+ if (mode === "reuse") await Promise.all([status(client), status(client)]);+ if (mode === "override")+ await artifact.artifactMcpCall(+ client,+ { name: "binary_session", arguments: {} },+ {+ toolDefinition: {+ name: "binary_session",+ inputSchema: { type: "object" },+ outputSchema: { type: "object" },+ },+ },+ );+ if (mode === "request-options")+ await artifact.artifactMcpCall(+ client,+ { name: "binary_session", arguments: {} },+ { timeout: 12_345 },+ );+ });+} catch (cause) {+ error = {+ name: cause instanceof Error ? cause.name : "unknown",+ message: cause instanceof Error ? cause.message : String(cause),+ code: cause?.code ?? null,+ };+}+process.stdout.write(+ JSON.stringify({+ accepted: error === undefined,+ catalogRequests,+ corruptedReplies,+ toolCalls,+ toolTimeouts,+ error: error ?? null,+ }) + "\n",+);