Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(clients): verify native Claude Code integration (#1549)

Ysopwnable8c7569aca793
diff --git a/docs/testing.md b/docs/testing.md
index 52271a140..26da7f15c 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -132,6 +132,20 @@ context and synthetic usage. Skills use an explicit isolated `skills.paths`
directory, so default OS-home discovery remains unverified. Live model APIs and
native Windows are also unverified.
+`verify:claude-client` requires installed Claude Code (verified with
+`@anthropic-ai/[email protected]`); select it with `REA_VERIFY_CLAUDE_COMMAND`.
+The optional POSIX lane uses native `CLAUDE_CONFIG_DIR` discovery, an independent
+Git workspace, user settings and default built-in tools. It preserves caller
+preferences and backups, checks idempotent setup, waits through the native
+`WaitForMcpServers` tool when discovery is pending, and validates the complete
+catalog and forwarded input schemas. Call mode loads the personal skill through
+native `Skill`, checks its full body and verifies named-schema JavaScript Evidence
+for a Unicode path. Use `-- chat` for ordinary chat with REA enabled, or
+`REA_VERIFY_RUNTIME_ROOT` for a production-only installed package. The local
+Anthropic Messages/SSE fixture preserves native resource-hint envelopes in its
+request artifacts. Usage is synthetic; live Anthropic API, native Windows and
+bare-mode personal-skill activation remain unverified by this lane.
+
`verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`;
verified with `@deepseek-ai/[email protected]`) and Git. Set
`REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane
diff --git a/package.json b/package.json
index 080f901f6..75cd2eb1c 100644
--- a/package.json
+++ b/package.json
@@ -182,6 +182,7 @@
"verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",
"verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs",
"verify:opencode-client": "npm run build:cached && node scripts/verify-opencode-client.mjs",
+ "verify:claude-client": "npm run build:cached && node scripts/verify-claude-client.mjs",
"verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi",
"verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes",
"verify:deepseek-client": "npm run build:cached && node scripts/verify-deepseek-client.mjs",
diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs
new file mode 100644
index 000000000..d1b7aa933
--- /dev/null
+++ b/scripts/verify-claude-client.mjs
@@ -0,0 +1,368 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import Ajv2020 from "ajv/dist/2020.js";
+import {
+ TOOL_CONTRACTS,
+ toolContract,
+} from "../dist/contracts/toolContracts.js";
+import { exec } from "./lib/verify-package-core.mjs";
+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
+import { createAnthropicModelFixture } from "./verify/clients/anthropic-model-fixture.mjs";
+
+assert(
+ process.platform !== "win32",
+ "Native Claude Code verifier requires POSIX; Windows is unverified",
+);
+const repo = fileURLToPath(new URL("..", import.meta.url));
+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
+const command = process.env.REA_VERIFY_CLAUDE_COMMAND ?? "claude";
+const mode = process.argv[2] ?? "call";
+assert(
+ ["chat", "call"].includes(mode),
+ "Usage: verify-claude-client.mjs [chat|call]",
+);
+const verifier = createVerifierRun();
+const lab = await mkdtemp(join(tmpdir(), "rea-claude-client-"));
+const profile = join(lab, "selected profile");
+const workspace = join(lab, "workspace");
+const configPath = join(profile, ".claude.json");
+const skillDirectory = join(profile, "skills", "reverse-engineer-anything");
+await Promise.all([mkdir(profile), mkdir(workspace)]);
+await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
+const environment = {
+ ...process.env,
+ CLAUDE_CONFIG_DIR: profile,
+ DISABLE_AUTOUPDATER: "1",
+ CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
+ REA_PROCESS_RUN_ID: verifier.run_id,
+};
+let clientVersion;
+try {
+ clientVersion = (
+ await exec(command, ["--version"], { env: environment, timeout: 30_000 })
+ ).stdout.trim();
+} catch (cause) {
+ throw new Error(
+ `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND. ${String(cause)}`,
+ );
+}
+const target = join(lab, "fixture α测试");
+await mkdir(target);
+await writeFile(
+ join(target, "app.js"),
+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
+);
+let skillRequested = false;
+let skillLoaded = false;
+let expectedSkillBody;
+let evidenceSeen = false;
+let finalSeen = false;
+let catalogSize = 0;
+let schemasChecked = 0;
+let pendingServerWaits = 0;
+let artifactDigest;
+const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
+const containsText = (value, text) => {
+ if (typeof value === "string") return value.includes(text);
+ if (value && typeof value === "object")
+ return Object.values(value).some((nested) => containsText(nested, text));
+ return false;
+};
+const extractEvidence = (value) => {
+ if (typeof value === "string") {
+ try {
+ // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts.
+ const json = value.replace(
+ /\n\n<system-reminder>\n<total_tokens>\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u,
+ "",
+ );
+ return extractEvidence(JSON.parse(json));
+ } catch {
+ return undefined;
+ }
+ }
+ if (value && typeof value === "object") {
+ if (
+ typeof value.evidence_id === "string" &&
+ value.normalized_result?.graph?.nodes?.length > 0
+ )
+ return value;
+ for (const nested of Object.values(value)) {
+ const evidence = extractEvidence(nested);
+ if (evidence) return evidence;
+ }
+ }
+ return undefined;
+};
+const fixture = await createAnthropicModelFixture({
+ directory: lab,
+ onRequest: async ({ body, tools, requestIndex }) => {
+ if (tools.length === 0)
+ return [{ type: "text", text: "REA fixture title" }];
+ assert(requestIndex <= 8, "Local model workflow exceeded its turn budget");
+ const call = (name, input) => [
+ { type: "tool_use", id: `toolu_rea_${requestIndex}`, name, input },
+ ];
+ const analysis = tools.find((t) =>
+ t.name.endsWith("analyze_javascript_application"),
+ );
+ if (!analysis) {
+ // Native automatic discovery can begin the conversation while MCP is still connecting.
+ const wait = tools.find((t) => t.name === "WaitForMcpServers");
+ assert(
+ wait,
+ "Claude must expose REA tools or its native pending-server wait",
+ );
+ pendingServerWaits++;
+ return call(wait.name, { servers: ["rea"] });
+ }
+ const prefix = analysis.name.slice(
+ 0,
+ -"analyze_javascript_application".length,
+ );
+ const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort();
+ const reaTools = tools.filter((t) => t.name.startsWith(prefix));
+ assert.deepEqual(
+ reaTools.map((t) => t.name).sort(),
+ expected,
+ "Claude must forward the complete REA catalog after connection",
+ );
+ catalogSize = expected.length;
+ if (schemasChecked === 0) {
+ for (const tool of reaTools) {
+ assert(
+ tool.input_schema,
+ `${tool.name}: missing advertised input schema`,
+ );
+ assert(
+ schemaValidator.validateSchema(tool.input_schema),
+ `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`,
+ );
+ schemasChecked++;
+ }
+ }
+ skillLoaded ||=
+ expectedSkillBody !== undefined &&
+ containsText(body.messages, expectedSkillBody) &&
+ containsText(body.messages, skillDirectory);
+ const observed = extractEvidence(body.messages);
+ if (observed) {
+ const evidence = toolContract(
+ "analyze_javascript_application",
+ ).outputSchema.parse(observed);
+ assert.equal(evidence.subject.local_path, target);
+ const shape = evidence.normalized_result.graph.nodes
+ .flatMap((n) => n.observations)
+ .map((o) => o.properties)
+ .find(
+ (p) =>
+ p.semantic_role === "export-return-shapes" &&
+ p.exported_name === "result",
+ );
+ assert(shape?.return_shape_coverage?.projection_complete);
+ const fields = shape.static_return_shapes.flatMap((s) => s.fields);
+ assert(
+ fields.some(
+ (f) =>
+ f.path === "/clientCompatibility" &&
+ f.state === "literal" &&
+ f.value === "REA_CLIENT_FIXTURE",
+ ),
+ );
+ assert(
+ fields.some(
+ (f) => f.path === "/count" && f.state === "literal" && f.value === 7,
+ ),
+ );
+ const bytes = JSON.stringify(evidence);
+ artifactDigest = createHash("sha256").update(bytes).digest("hex");
+ await writeFile(join(lab, "claude-evidence.json"), bytes);
+ evidenceSeen = true;
+ }
+ if (mode === "chat" || evidenceSeen) {
+ finalSeen = true;
+ return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }];
+ }
+ if (!skillRequested) {
+ const skill = tools.find((t) => t.name === "Skill");
+ assert(skill, "Claude must expose its native Skill tool");
+ skillRequested = true;
+ return call(skill.name, { skill: "reverse-engineer-anything" });
+ }
+ assert(
+ skillLoaded,
+ "Claude must load the complete installed personal REA skill",
+ );
+ return call(analysis.name, { input_path: target, format: "directory" });
+ },
+});
+const original = JSON.stringify({ theme: "light", mcpServers: {} }, null, 2);
+await writeFile(configPath, original);
+let failure;
+try {
+ for (const args of [
+ ["setup", "--client", "claude_code", "--dry-run"],
+ ["setup", "--client", "claude_code", "--yes"],
+ ["doctor", "--client", "claude_code", "--skill"],
+ ]) {
+ const result = await exec(
+ process.execPath,
+ [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
+ { env: environment, cwd: workspace, timeout: 60_000 },
+ );
+ await writeFile(
+ join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
+ result.stdout,
+ );
+ const parsed = JSON.parse(result.stdout);
+ if (args[0] === "doctor") assert.equal(parsed.healthy, true);
+ else {
+ assert.equal(
+ parsed.status,
+ args.includes("--dry-run") ? "planned" : "ready",
+ );
+ if (args.includes("--dry-run")) {
+ assert(parsed.plannedActions.some((a) => a.target === configPath));
+ assert(parsed.plannedActions.some((a) => a.target === skillDirectory));
+ assert(
+ parsed.plannedActions.every(
+ (a) =>
+ a.target === configPath || a.target.startsWith(skillDirectory),
+ ),
+ );
+ }
+ }
+ }
+ assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+ const configured = await readFile(configPath, "utf8");
+ assert.equal(JSON.parse(configured).theme, "light");
+ const repeated = await exec(
+ process.execPath,
+ [
+ join(runtimeRoot, "scripts/rea.mjs"),
+ "setup",
+ "--client",
+ "claude_code",
+ "--yes",
+ "--json",
+ ],
+ { env: environment, cwd: workspace, timeout: 60_000 },
+ );
+ assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
+ assert.equal(await readFile(configPath, "utf8"), configured);
+ assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+ const installedSkill = await readFile(
+ join(skillDirectory, "SKILL.md"),
+ "utf8",
+ );
+ assert(installedSkill.includes("name: reverse-engineer-anything"));
+ const lines = installedSkill.split("\n");
+ const headerEnd = lines.indexOf("---", 1);
+ assert(
+ lines[0] === "---" && headerEnd > 0,
+ "Installed skill must have its authored frontmatter",
+ );
+ expectedSkillBody = lines
+ .slice(headerEnd + 1)
+ .join("\n")
+ .trim();
+ assert(expectedSkillBody.length > 0);
+ const execution = exec(
+ command,
+ [
+ "--setting-sources",
+ "user",
+ "--print",
+ "--output-format",
+ "json",
+ "--no-session-persistence",
+ "--model",
+ "claude-sonnet-4-6",
+ "--tools",
+ "default",
+ "--permission-mode",
+ "dontAsk",
+ "--settings",
+ JSON.stringify({
+ permissions: {
+ allow: [
+ "Skill",
+ "WaitForMcpServers",
+ "mcp__rea__analyze_javascript_application",
+ ],
+ },
+ }),
+ mode === "chat"
+ ? "Say hello briefly."
+ : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`,
+ ],
+ {
+ env: {
+ ...environment,
+ ANTHROPIC_API_KEY: "local-fixture-key",
+ ANTHROPIC_AUTH_TOKEN: "",
+ ANTHROPIC_BASE_URL: fixture.baseUrl,
+ },
+ cwd: workspace,
+ timeout: 120_000,
+ maxBuffer: 10 * 1024 * 1024,
+ },
+ );
+ execution.child.stdin.end();
+ const response = await execution;
+ await writeFile(join(lab, `${mode}.stdout`), response.stdout);
+ await writeFile(join(lab, `${mode}.stderr`), response.stderr);
+ assert.equal(fixture.failure, undefined);
+ const parsed = JSON.parse(response.stdout);
+ assert.equal(parsed.is_error, false);
+ assert.equal(parsed.subtype, "success");
+ assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK");
+ assert.deepEqual(parsed.permission_denials, []);
+ assert(finalSeen);
+ if (mode === "call") assert(evidenceSeen && skillLoaded);
+} catch (cause) {
+ failure = cause;
+ if (typeof cause?.stdout === "string")
+ await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
+ if (typeof cause?.stderr === "string")
+ await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
+} finally {
+ await fixture.close();
+ const receipt = {
+ client: "Claude Code",
+ clientVersion,
+ mode,
+ passed: failure === undefined,
+ artifacts: lab,
+ runtimeRoot,
+ host: {
+ platform: process.platform,
+ architecture: process.arch,
+ node: process.versions.node,
+ },
+ modelFixture:
+ "loopback Anthropic Messages/SSE; synthetic usage and native resource hints; no live API claim",
+ discovery:
+ "native CLAUDE_CONFIG_DIR user configuration and skills; separate Git root, user settings only, default built-in tools",
+ bareMode:
+ "not used; native bare print mode did not load the personal skill",
+ catalogSize,
+ schemasChecked,
+ pendingServerWaits,
+ evidenceSeen,
+ skillLoaded,
+ artifactDigest,
+ finalSeen,
+ requests: fixture.requests,
+ probes: fixture.probes,
+ fixtureFailure: fixture.failure,
+ verifier: await completeVerifierRun(verifier),
+ };
+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
+ console.log(JSON.stringify(receipt));
+}
+if (failure !== undefined) throw failure;
diff --git a/scripts/verify/clients/anthropic-model-fixture.mjs b/scripts/verify/clients/anthropic-model-fixture.mjs
new file mode 100644
index 000000000..8f8e448c9
--- /dev/null
+++ b/scripts/verify/clients/anthropic-model-fixture.mjs
@@ -0,0 +1,158 @@
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+/** Exercise native Anthropic adapters locally, without live-provider claims. */
+export const createAnthropicModelFixture = async ({ directory, onRequest }) => {
+ const requests = [];
+ const probes = [];
+ let incoming = 0;
+ let failure;
+ const server = createServer(async (request, response) => {
+ try {
+ const chunks = [];
+ let bytes = 0;
+ for await (const chunk of request) {
+ bytes += chunk.length;
+ assert(
+ bytes <= 10 * 1024 * 1024,
+ "Model fixture request exceeds 10 MiB budget",
+ );
+ chunks.push(chunk);
+ }
+ const raw = Buffer.concat(chunks).toString("utf8");
+ const body = raw.length === 0 ? null : JSON.parse(raw);
+ const path = new URL(request.url, "http://localhost").pathname;
+ // Retain producer bodies and URLs, without transport authentication headers.
+ await writeFile(
+ join(directory, `claude-incoming-${++incoming}.json`),
+ JSON.stringify({ method: request.method, path: request.url, body }),
+ );
+ if (path === "/v1/messages/count_tokens") {
+ probes.push({ path, kind: "synthetic-token-count" });
+ response.setHeader("Content-Type", "application/json");
+ response.end(JSON.stringify({ input_tokens: 100 }));
+ return;
+ }
+ if (path !== "/v1/messages") {
+ probes.push({ path, status: 404 });
+ response.statusCode = 404;
+ response.end(
+ JSON.stringify({
+ error: { message: "Unsupported fixture endpoint" },
+ }),
+ );
+ return;
+ }
+ assert.equal(request.method, "POST");
+ assert(body && typeof body === "object" && !Array.isArray(body));
+ assert(Array.isArray(body.messages));
+ const tools = body.tools ?? [];
+ requests.push({
+ path: request.url,
+ stream: body.stream,
+ tools: tools.map((t) => t.name),
+ messageRoles: body.messages.map((m) => m.role),
+ });
+ const content = await onRequest({
+ body,
+ tools,
+ requestIndex: requests.length,
+ });
+ const stopReason = content.some((c) => c.type === "tool_use")
+ ? "tool_use"
+ : "end_turn";
+ const usage = {
+ input_tokens: 100,
+ output_tokens: 20,
+ cache_creation_input_tokens: 0,
+ cache_read_input_tokens: 0,
+ };
+ const result = {
+ id: `msg_rea_${requests.length}`,
+ type: "message",
+ role: "assistant",
+ model: body.model,
+ content,
+ stop_reason: stopReason,
+ stop_sequence: null,
+ usage,
+ };
+ if (body.stream) {
+ response.setHeader("Content-Type", "text/event-stream");
+ const emit = (event, data) =>
+ response.write(
+ `event: ${event}\ndata: ${JSON.stringify({ type: event, ...data })}\n\n`,
+ );
+ emit("message_start", {
+ message: {
+ ...result,
+ content: [],
+ stop_reason: null,
+ usage: { ...usage, output_tokens: 0 },
+ },
+ });
+ for (const [index, block] of content.entries()) {
+ emit("content_block_start", {
+ index,
+ content_block:
+ block.type === "tool_use"
+ ? { ...block, input: {} }
+ : { type: "text", text: "" },
+ });
+ if (block.type === "tool_use") {
+ const json = JSON.stringify(block.input);
+ const split = Math.floor(json.length / 2);
+ for (const partialJson of [
+ json.slice(0, split),
+ json.slice(split),
+ ]) {
+ emit("content_block_delta", {
+ index,
+ delta: { type: "input_json_delta", partial_json: partialJson },
+ });
+ }
+ } else {
+ emit("content_block_delta", {
+ index,
+ delta: { type: "text_delta", text: block.text },
+ });
+ }
+ emit("content_block_stop", { index });
+ }
+ emit("message_delta", {
+ delta: { stop_reason: stopReason, stop_sequence: null },
+ usage: { output_tokens: 20 },
+ });
+ emit("message_stop", {});
+ response.end();
+ } else {
+ response.setHeader("Content-Type", "application/json");
+ response.end(JSON.stringify(result));
+ }
+ } catch (cause) {
+ failure = String(cause);
+ response.statusCode = 400;
+ response.setHeader("Content-Type", "application/json");
+ response.end(
+ JSON.stringify({
+ type: "error",
+ error: { type: "invalid_request_error", message: failure },
+ }),
+ );
+ }
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const address = server.address();
+ assert(address !== null && typeof address !== "string");
+ return {
+ baseUrl: `http://127.0.0.1:${address.port}`,
+ requests,
+ probes,
+ get failure() {
+ return failure;
+ },
+ close: () => new Promise((resolve) => server.close(resolve)),
+ };
+};