Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(clients): verify native Qwen Pi and Hermes workflows (#1511)

Ysopwnableb25922e4e8fe
diff --git a/docs/testing.md b/docs/testing.md
index 9730f027b..82b529929 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -88,6 +88,21 @@ entrypoint and include its verifier/fixture files in their cache inputs.
## End-to-end, integration and golden evidence
+The optional `verify:qwen-client`, `verify:pi-client`, and `verify:hermes-client`
+lanes require an installed native client. Select its executable with
+`REA_VERIFY_QWEN_COMMAND`, `REA_VERIFY_PI_COMMAND`, or `REA_VERIFY_HERMES_COMMAND`.
+They configure disposable profiles through the public REA CLI, preserve caller
+settings and backups, and exercise real stdio MCP calls and skill loading. The
+`call` mode checks complete catalog discovery and actual JavaScript Evidence;
+pass `-- chat` to check plain chat with REA enabled. Qwen also reads the full
+result that its client offloads, and Pi exercises default codemode execution.
+Use `REA_VERIFY_RUNTIME_ROOT` to select a production-only installed REA package.
+Receipts retain client versions, result digests, host coverage and owned-process
+cleanup. These POSIX lanes use a deterministic loopback model, so they do not
+prove live model-provider or native Windows compatibility. Qwen and Pi use
+caller-configured skill directories to isolate the fixture account from the OS
+home; this does not prove their default home-directory discovery.
+
Full E2E tests invoke the production command dispatcher and real providers,
without fake launchers, runners or responses. `verify:keyed-archive` writes an
actual Foundation binary archive, runs the CLI and a separate stdio MCP
diff --git a/package.json b/package.json
index 439746c70..0c47e5901 100644
--- a/package.json
+++ b/package.json
@@ -179,6 +179,9 @@
"verify:readiness": "npm run build:cached && npm run verify:fixtures && node scripts/verify-reconstruction-readiness.mjs",
"verify:package": "npm run build:cached && node scripts/verify-package.mjs",
"verify:package:windows": "npm run build:cached && node scripts/verify-windows-package.mjs",
+ "verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",
+ "verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi",
+ "verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes",
"probe:windows-capabilities": "npm run build:cached && node scripts/windows-capabilities.mjs",
"verify:published": "node scripts/verify-published-package.mjs",
"config:print": "node scripts/print-mcp-config.mjs",
diff --git a/scripts/verify-native-agent-client.mjs b/scripts/verify-native-agent-client.mjs
new file mode 100644
index 000000000..f561e353e
--- /dev/null
+++ b/scripts/verify-native-agent-client.mjs
@@ -0,0 +1,456 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+ TOOL_CONTRACTS,
+ toolContract,
+} from "../dist/contracts/toolContracts.js";
+import { exec } from "./lib/verify-package-core.mjs";
+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
+import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
+
+const client = process.argv[2];
+const mode = process.argv[3] ?? "call";
+assert(["pi", "hermes"].includes(client), "Select pi or hermes.");
+assert(["chat", "call"].includes(mode), "Select chat or call.");
+assert(
+ process.platform !== "win32",
+ "This real-client lane requires POSIX; native Windows is unverified.",
+);
+const repo = fileURLToPath(new URL("..", import.meta.url));
+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
+const command =
+ process.env[`REA_VERIFY_${client.toUpperCase()}_COMMAND`] ?? client;
+const lab = await mkdtemp(join(tmpdir(), `rea-${client}-client-`));
+const account = join(lab, "account");
+const profile = join(lab, "selected profile");
+const workspace = join(lab, "workspace");
+const target = join(workspace, "fixture α测试");
+const artifactPath = join(workspace, "evidence.json");
+const skillPath = join(
+ client === "pi" ? join(account, ".agents") : profile,
+ "skills",
+ "reverse-engineer-anything",
+ "SKILL.md",
+);
+const verifier = createVerifierRun();
+await Promise.all([
+ mkdir(account),
+ mkdir(profile),
+ mkdir(target, { recursive: true }),
+]);
+await writeFile(
+ join(target, "app.js"),
+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
+);
+const environment = {
+ PATH: process.env.PATH ?? "",
+ USERPROFILE: account,
+ [client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile,
+ PI_OFFLINE: "1",
+ REA_PROCESS_RUN_ID: verifier.run_id,
+};
+const configPath = join(profile, client === "pi" ? "mcp.json" : "config.yaml");
+const original =
+ client === "pi"
+ ? JSON.stringify({
+ mcpServers: { other: { command: "unrelated-server", enabled: false } },
+ })
+ : "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n";
+await writeFile(configPath, original);
+if (client === "pi") {
+ // The CLI fixture account and native os.homedir() differ; use Pi's caller-owned skill setting.
+ await writeFile(
+ join(profile, "settings.json"),
+ JSON.stringify({ skills: [join(account, ".agents", "skills")] }),
+ );
+}
+const clientVersion = (
+ await exec(command, ["--version"], { env: environment, timeout: 30_000 })
+).stdout.trim();
+for (const args of [
+ ["setup", "--client", client, "--dry-run"],
+ ["setup", "--client", client, "--yes"],
+ ["doctor", "--client", client, "--skill"],
+]) {
+ const result = await exec(
+ process.execPath,
+ [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
+ { env: environment, cwd: workspace, timeout: 60_000 },
+ );
+ const parsed = JSON.parse(result.stdout);
+ if (args[0] === "doctor") assert.equal(parsed.healthy, true);
+ else
+ assert.equal(
+ parsed.status,
+ args.includes("--dry-run") ? "planned" : "ready",
+ );
+ await writeFile(
+ join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
+ result.stdout,
+ );
+}
+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+const installedConfiguration = await readFile(configPath, "utf8");
+assert(installedConfiguration.includes("unrelated-server"));
+if (client === "hermes")
+ assert(installedConfiguration.includes("# Keep caller settings."));
+const repeated = await exec(
+ process.execPath,
+ [
+ join(runtimeRoot, "scripts/rea.mjs"),
+ "setup",
+ "--client",
+ client,
+ "--yes",
+ "--json",
+ ],
+ { env: environment, cwd: workspace, timeout: 60_000 },
+);
+assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
+assert.equal(await readFile(configPath, "utf8"), installedConfiguration);
+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+assert(
+ (await readFile(skillPath, "utf8")).includes(
+ "name: reverse-engineer-anything",
+ ),
+);
+let evidenceSeen = false;
+let skillLoaded = false;
+let skillRequested = false;
+let callRequested = false;
+let searchRequested = false;
+let describeRequested = false;
+let finalSeen = false;
+let catalog;
+let artifactDigest;
+const extractEvidence = (value) => {
+ if (typeof value === "string") {
+ if (
+ value.startsWith("<untrusted_tool_result ") &&
+ value.endsWith("</untrusted_tool_result>")
+ ) {
+ // Hermes wraps external MCP content with its own instruction banner.
+ // Decode only that observed envelope, retaining the original request artifact.
+ const start = value.indexOf("\n\n");
+ if (start >= 0)
+ return extractEvidence(
+ value
+ .slice(start + 2, value.lastIndexOf("</untrusted_tool_result>"))
+ .trim(),
+ );
+ }
+ try {
+ return extractEvidence(JSON.parse(value));
+ } catch {
+ return undefined;
+ }
+ }
+ if (value && typeof value === "object") {
+ if (
+ typeof value.evidence_id === "string" &&
+ value.normalized_result?.graph?.nodes
+ )
+ return value;
+ for (const child of Object.values(value)) {
+ const found = extractEvidence(child);
+ if (found) return found;
+ }
+ }
+};
+const validateEvidence = async (value) => {
+ const evidence = toolContract(
+ "analyze_javascript_application",
+ ).outputSchema.parse(value);
+ assert.equal(evidence.subject.local_path, target);
+ const shape = evidence.normalized_result.graph.nodes
+ .flatMap((n) => n.observations)
+ .map((o) => o.properties)
+ .find(
+ (p) =>
+ p.semantic_role === "export-return-shapes" &&
+ p.exported_name === "result",
+ );
+ assert(shape?.return_shape_coverage?.projection_complete);
+ const fields = shape.static_return_shapes.flatMap((s) => s.fields);
+ assert(
+ fields.some(
+ (f) =>
+ f.path === "/clientCompatibility" &&
+ f.state === "literal" &&
+ f.value === "REA_CLIENT_FIXTURE",
+ ),
+ );
+ assert(
+ fields.some(
+ (f) => f.path === "/count" && f.state === "literal" && f.value === 7,
+ ),
+ );
+ const serialized = JSON.stringify(evidence);
+ await writeFile(join(lab, "evidence.json"), serialized);
+ artifactDigest = createHash("sha256").update(serialized).digest("hex");
+ evidenceSeen = true;
+};
+const fixture = await createOpenAiModelFixture({
+ directory: lab,
+ prefix: `${client}-${mode}`,
+ onRequest: async ({ body, tools, requestIndex }) => {
+ assert(
+ requestIndex <= 6,
+ "Client did not complete the bounded fixture workflow.",
+ );
+ const messages = body.messages ?? [];
+ const toolMessages = messages.filter((m) => m.role === "tool");
+ const serializedTools = JSON.stringify(toolMessages);
+ const call = (name, args) => ({
+ role: "assistant",
+ content: null,
+ tool_calls: [
+ {
+ id: `call_rea_${requestIndex}`,
+ type: "function",
+ function: { name, arguments: JSON.stringify(args) },
+ },
+ ],
+ });
+ if (
+ client === "pi" &&
+ callRequested &&
+ serializedTools.includes("REA_CLIENT_FIXTURE")
+ ) {
+ const fact = toolMessages
+ .flatMap((m) => {
+ const text =
+ typeof m.content === "string"
+ ? m.content
+ : JSON.stringify(m.content);
+ const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true');
+ if (marker < 0) return [];
+ const end = text.indexOf("}\n", marker);
+ try {
+ return [
+ JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)),
+ ];
+ } catch {
+ return [];
+ }
+ })
+ .find((v) => Array.isArray(v.catalog));
+ assert(
+ fact,
+ "Pi codemode must return its actual catalog and skill-read receipt.",
+ );
+ catalog = fact.catalog;
+ skillLoaded = fact.skill_loaded === true;
+ await validateEvidence(JSON.parse(await readFile(artifactPath, "utf8")));
+ } else if (client === "hermes") {
+ const search = tools.find((t) => t.name === "tool_search");
+ const exposed = tools
+ .filter((t) => t.name.startsWith("mcp__rea__"))
+ .map((t) => t.name.slice(10));
+ const deferred = [
+ ...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu),
+ ].map((m) => m[1]);
+ const native = [...new Set([...exposed, ...deferred])];
+ // Hermes advertises MCP prompt utilities beside server tools.
+ if (native.length)
+ catalog = native.filter(
+ (name) => !["get_prompt", "list_prompts"].includes(name),
+ );
+ skillLoaded ||=
+ serializedTools.includes(skillPath) &&
+ serializedTools.includes("reverse-engineer-anything");
+ for (const message of toolMessages) {
+ const evidence = extractEvidence(message.content);
+ if (evidence) await validateEvidence(evidence);
+ }
+ }
+ if (mode === "chat" || evidenceSeen) {
+ finalSeen = true;
+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
+ }
+ if (client === "pi") {
+ const tool = tools.find((t) => t.name === "codemode");
+ assert(
+ tool,
+ `Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`,
+ );
+ assert(
+ JSON.stringify(messages).includes(skillPath),
+ "Pi must advertise the installed REA skill.",
+ );
+ callRequested = true;
+ return call(tool.name, {
+ code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`,
+ });
+ }
+ if (!skillRequested) {
+ const tool = tools.find((t) => t.name === "skill_view");
+ assert(tool, "Hermes skill_view must be available.");
+ skillRequested = true;
+ return call(tool.name, { name: "reverse-engineer-anything" });
+ }
+ if (!searchRequested) {
+ const tool = tools.find((t) => t.name === "tool_search");
+ assert(tool, "Hermes native deferred tool_search must be available.");
+ searchRequested = true;
+ return call(tool.name, { queries: ["analyze_javascript_application"] });
+ }
+ assert(
+ serializedTools.includes("mcp__rea__analyze_javascript_application"),
+ "Hermes must discover the analysis tool.",
+ );
+ if (!describeRequested) {
+ const tool = tools.find((t) => t.name === "tool_describe");
+ assert(tool, "Hermes must load the actual parameter schema.");
+ describeRequested = true;
+ return call(tool.name, {
+ names: ["mcp__rea__analyze_javascript_application"],
+ });
+ }
+ const tool = tools.find((t) => t.name === "tool_call");
+ assert(tool, "Hermes native deferred tool_call must be available.");
+ callRequested = true;
+ return call(tool.name, {
+ calls: [
+ {
+ name: "mcp__rea__analyze_javascript_application",
+ arguments: { input_path: target, format: "directory" },
+ },
+ ],
+ });
+ },
+});
+if (client === "pi") {
+ await writeFile(
+ join(profile, "models.json"),
+ JSON.stringify({
+ providers: {
+ "rea-fixture": {
+ baseUrl: fixture.baseUrl,
+ api: "openai-completions",
+ apiKey: "local-fixture-key",
+ models: [
+ {
+ id: "rea-client-fixture",
+ reasoning: false,
+ contextWindow: 200000,
+ maxTokens: 4096,
+ },
+ ],
+ },
+ },
+ }),
+ );
+} else {
+ const configured = await readFile(configPath, "utf8");
+ await writeFile(
+ configPath,
+ configured +
+ `\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`,
+ );
+}
+const args =
+ client === "pi"
+ ? [
+ "--offline",
+ "--provider",
+ "rea-fixture",
+ "--model",
+ "rea-client-fixture",
+ "--no-session",
+ "--no-context-files",
+ "--mode",
+ "json",
+ "-p",
+ mode === "chat"
+ ? "Say hello briefly."
+ : `Use REA to analyze ${target}.`,
+ ]
+ : [
+ "--oneshot",
+ mode === "chat"
+ ? "Say hello briefly."
+ : `Use REA to analyze ${target}.`,
+ "--provider",
+ "custom",
+ "--model",
+ "rea-client-fixture",
+ "--reasoning",
+ "none",
+ ];
+let failure;
+try {
+ const execution = exec(command, args, {
+ env: {
+ ...environment,
+ OPENAI_API_KEY: "local-fixture-key",
+ OPENAI_BASE_URL: fixture.baseUrl,
+ },
+ cwd: workspace,
+ timeout: 120_000,
+ maxBuffer: 10 * 1024 * 1024,
+ });
+ // Pi consumes piped input before its first prompt; an unused open pipe is not EOF.
+ execution.child.stdin.end();
+ const result = await execution;
+ await writeFile(join(lab, "client.stdout"), result.stdout);
+ await writeFile(join(lab, "client.stderr"), result.stderr);
+ assert.equal(fixture.failure, undefined);
+ assert(finalSeen);
+ assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
+ if (mode === "call") {
+ assert(evidenceSeen);
+ assert(
+ skillLoaded,
+ "Native client must actually read the installed skill.",
+ );
+ assert.deepEqual(
+ catalog?.sort(),
+ TOOL_CONTRACTS.map((t) => t.name).sort(),
+ "The complete REA catalog must survive native client discovery.",
+ );
+ }
+} catch (cause) {
+ failure = cause;
+ if (typeof cause?.stdout === "string")
+ await writeFile(join(lab, "client.stdout"), cause.stdout);
+ if (typeof cause?.stderr === "string")
+ await writeFile(join(lab, "client.stderr"), cause.stderr);
+} finally {
+ await fixture.close();
+ const receipt = {
+ client,
+ clientVersion,
+ mode,
+ passed: failure === undefined,
+ artifacts: lab,
+ runtimeRoot,
+ host: {
+ platform: process.platform,
+ architecture: process.arch,
+ node: process.versions.node,
+ },
+ modelFixture:
+ "loopback deterministic OpenAI-compatible endpoint; no live provider claim",
+ skillDiscovery:
+ client === "pi"
+ ? "caller-configured isolated shared directory; default OS home discovery unverified"
+ : "native HERMES_HOME personal directory",
+ catalogSize: catalog?.length,
+ evidenceSeen,
+ skillLoaded,
+ artifactDigest,
+ finalSeen,
+ requests: fixture.requests,
+ probes: fixture.probes,
+ fixtureFailure: fixture.failure,
+ verifier: await completeVerifierRun(verifier),
+ };
+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
+ console.log(JSON.stringify(receipt));
+}
+if (failure !== undefined) throw failure;
diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs
new file mode 100644
index 000000000..4b85470d6
--- /dev/null
+++ b/scripts/verify-qwen-client.mjs
@@ -0,0 +1,403 @@
+import assert from "node:assert/strict";
+import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
+import {
+ mkdir,
+ writeFile,
+ readFile,
+ mkdtemp,
+ realpath,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { createHash } from "node:crypto";
+import { exec } from "./lib/verify-package-core.mjs";
+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
+import {
+ TOOL_CONTRACTS,
+ toolContract,
+} from "../dist/contracts/toolContracts.js";
+
+assert(
+ process.platform !== "win32",
+ "The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",
+);
+const repo = fileURLToPath(new URL("..", import.meta.url));
+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
+const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
+const client = "qwen";
+const mode = process.argv[2] ?? "call";
+assert(
+ ["chat", "call"].includes(mode),
+ "Usage: node scripts/verify-qwen-client.mjs [chat|call]",
+);
+const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
+const verifier = createVerifierRun();
+const profile = join(lab, "selected profile");
+const account = join(lab, "account");
+const runtime = join(profile, "runtime");
+const workspace = join(lab, "workspace");
+const sharedSkills = join(account, ".agents", "skills");
+await Promise.all([
+ mkdir(profile, { recursive: true }),
+ mkdir(account),
+ mkdir(workspace),
+]);
+const baseEnvironment = {
+ PATH: process.env.PATH ?? "",
+ USERPROFILE: account,
+ QWEN_HOME: profile,
+ QWEN_RUNTIME_DIR: runtime,
+ REA_PROCESS_RUN_ID: verifier.run_id,
+};
+let clientVersion;
+try {
+ clientVersion = (
+ await exec(command, ["--version"], {
+ env: baseEnvironment,
+ timeout: 30_000,
+ })
+ ).stdout.trim();
+} catch (cause) {
+ throw new Error(
+ `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,
+ );
+}
+const original = JSON.stringify(
+ {
+ security: {
+ auth: { selectedType: "openai" },
+ folderTrust: { enabled: false },
+ },
+ telemetry: { enabled: false },
+ model: { name: "rea-client-fixture" },
+ // Qwen uses os.homedir(), while REA's isolated CLI account uses USERPROFILE.
+ // This caller-owned setting lets the real client load the REA-installed bundle.
+ skills: { directories: [sharedSkills] },
+ mcpServers: { other: { command: "unrelated-server", disabled: true } },
+ },
+ null,
+ 2,
+);
+await writeFile(join(profile, "settings.json"), original);
+for (const step of [
+ ["setup", "--client", "qwen_code", "--dry-run"],
+ ["setup", "--client", "qwen_code", "--yes"],
+ ["doctor", "--client", "qwen_code", "--skill"],
+]) {
+ const response = await exec(
+ process.execPath,
+ [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"],
+ { env: baseEnvironment, cwd: repo, timeout: 60_000 },
+ );
+ const result = JSON.parse(response.stdout);
+ if (step.includes("--dry-run"))
+ assert(
+ result.plannedActions.every(
+ (a) =>
+ a.target.startsWith(profile + "/") ||
+ a.target.startsWith(sharedSkills + "/"),
+ ),
+ );
+ await writeFile(
+ join(lab, `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`),
+ response.stdout,
+ );
+}
+assert.equal(
+ await readFile(join(profile, "settings.json.rea.backup"), "utf8"),
+ original,
+);
+const configured = JSON.parse(
+ await readFile(join(profile, "settings.json"), "utf8"),
+);
+assert.deepEqual(configured.model, { name: "rea-client-fixture" });
+assert.deepEqual(configured.skills, { directories: [sharedSkills] });
+assert.deepEqual(configured.mcpServers.other, {
+ command: "unrelated-server",
+ disabled: true,
+});
+const target = join(lab, "fixture α测试");
+await mkdir(target, { recursive: true });
+await writeFile(
+ `${target}/app.js`,
+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
+);
+let evidenceSeen = false;
+let skillSeen = false;
+let finalSeen = false;
+let skillRequested = false;
+let searchRequested = false;
+let artifactReadByAgent = false;
+let artifactPath;
+let selectedSkillLoaded = false;
+let artifactDigest;
+const extractEvidence = (value) => {
+ if (typeof value === "string") {
+ try {
+ return extractEvidence(JSON.parse(value));
+ } catch {
+ return null;
+ }
+ }
+ if (Array.isArray(value)) {
+ for (const item of value) {
+ const result = extractEvidence(item);
+ if (result) return result;
+ }
+ } else if (value && typeof value === "object") {
+ if (
+ typeof value.evidence_id === "string" &&
+ value.normalized_result?.graph?.nodes?.length > 0
+ )
+ return value;
+ for (const nested of Object.values(value)) {
+ const result = extractEvidence(nested);
+ if (result) return result;
+ }
+ }
+ return null;
+};
+const fixture = await createOpenAiModelFixture({
+ directory: lab,
+ prefix: `${client}-${mode}`,
+ onRequest: async ({ body, tools, requestIndex }) => {
+ skillSeen ||= JSON.stringify(body.messages ?? []).includes(
+ "reverse-engineer-anything",
+ );
+ selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(
+ join(sharedSkills, "reverse-engineer-anything"),
+ );
+ const toolText = JSON.stringify(
+ (body.messages ?? []).filter((m) => m.role === "tool"),
+ );
+ if (
+ toolText.includes("REA_CLIENT_FIXTURE") &&
+ /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText)
+ ) {
+ artifactReadByAgent = true;
+ assert(artifactPath);
+ const artifact = await readFile(artifactPath);
+ artifactDigest = createHash("sha256").update(artifact).digest("hex");
+ const evidence = toolContract(
+ "analyze_javascript_application",
+ ).outputSchema.parse(JSON.parse(artifact.toString("utf8")));
+ assert(extractEvidence(evidence));
+ assert.equal(evidence.subject.local_path, target);
+ const shape = evidence.normalized_result.graph.nodes
+ .flatMap((n) => n.observations)
+ .map((o) => o.properties)
+ .find(
+ (p) =>
+ p.semantic_role === "export-return-shapes" &&
+ p.exported_name === "result",
+ );
+ assert(shape?.return_shape_coverage?.projection_complete);
+ const fields = shape.static_return_shapes.flatMap((s) => s.fields);
+ assert(
+ fields.some(
+ (f) =>
+ f.path === "/clientCompatibility" &&
+ f.state === "literal" &&
+ f.value === "REA_CLIENT_FIXTURE",
+ ),
+ );
+ assert(
+ fields.some(
+ (f) => f.path === "/count" && f.state === "literal" && f.value === 7,
+ ),
+ );
+ evidenceSeen = true;
+ await writeFile(
+ `${lab}/${client}-${mode}-evidence.json`,
+ JSON.stringify(evidence),
+ );
+ }
+ for (const message of body.messages ?? []) {
+ if (message.role !== "tool") continue;
+ const evidence = extractEvidence(message.content);
+ if (evidence) {
+ assert(
+ JSON.stringify(evidence.normalized_result).includes(
+ "REA_CLIENT_FIXTURE",
+ ),
+ );
+ evidenceSeen = true;
+ await writeFile(
+ `${lab}/${client}-${mode}-evidence.json`,
+ JSON.stringify(evidence),
+ );
+ }
+ }
+ const selected = tools.find((t) =>
+ t.name.includes("analyze_javascript_application"),
+ );
+ const reviewedNames = [
+ ...JSON.stringify(
+ (body.messages ?? []).filter((m) => m.role === "tool"),
+ ).matchAll(
+ /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g,
+ ),
+ ]
+ .map((m) => m[1])
+ .sort((a, b) => b.length - a.length);
+ const search = tools.find((t) => t.name === "tool_search");
+ const invoke = tools.find((t) => t.name === "tool_call");
+ const skill = tools.find((t) => t.name === "skill");
+ const shell = tools.find((t) => t.name === "run_shell_command");
+ const persisted = toolText.match(
+ /Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,
+ );
+ const call = (name, args) => ({
+ role: "assistant",
+ content: null,
+ tool_calls: [
+ {
+ id: `call_rea_fixture_${requestIndex}`,
+ type: "function",
+ function: { name, arguments: JSON.stringify(args) },
+ },
+ ],
+ });
+ let message;
+ if (mode === "chat" || evidenceSeen) {
+ message = { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
+ finalSeen = true;
+ } else if (skill && !skillRequested) {
+ skillRequested = true;
+ message = call(skill.name, { skill: "reverse-engineer-anything" });
+ } else if (persisted && shell) {
+ artifactPath = persisted[1];
+ assert(
+ (await realpath(artifactPath)).startsWith(
+ (await realpath(runtime)) + "/",
+ ),
+ );
+ const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
+ const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
+ message = call(shell.name, {
+ command: `node -e ${quote(script)}`,
+ description:
+ "Read the complete saved REA Evidence and verify the fixture facts.",
+ });
+ } else if (selected)
+ message = call(selected.name, {
+ input_path: target,
+ format: "directory",
+ });
+ else if (reviewedNames.length && invoke)
+ message = call(invoke.name, {
+ name: reviewedNames[0],
+ arguments: { input_path: target, format: "directory" },
+ });
+ else if (search && !searchRequested) {
+ searchRequested = true;
+ message = call(search.name, { query: "analyze_javascript_application" });
+ } else
+ throw new Error(
+ `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`,
+ );
+ return message;
+ },
+});
+const endpoint = fixture.baseUrl;
+const prompt =
+ mode === "chat"
+ ? "Say hello briefly."
+ : `Use REA to analyze the JavaScript directory ${target}. Report the result.`;
+const env = {
+ ...baseEnvironment,
+ OPENAI_API_KEY: "local-fixture-key",
+ OPENAI_BASE_URL: endpoint,
+};
+const args = [
+ "--auth-type",
+ "openai",
+ "--openai-base-url",
+ endpoint,
+ "--model",
+ "rea-client-fixture",
+ "--approval-mode",
+ mode === "chat" ? "default" : "yolo",
+ "--allowed-mcp-server-names",
+ "rea",
+ "--output-format",
+ "stream-json",
+ "--max-session-turns",
+ "8",
+ "-p",
+ prompt,
+];
+let response;
+let failure;
+try {
+ response = await exec(command, args, {
+ env,
+ cwd: workspace,
+ timeout: 120_000,
+ maxBuffer: 10 * 1024 * 1024,
+ });
+ await writeFile(join(lab, `${mode}.stdout`), response.stdout);
+ await writeFile(join(lab, `${mode}.stderr`), response.stderr);
+ const initialized = response.stdout
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => JSON.parse(line))
+ .find((v) => v.type === "system" && v.subtype === "init");
+ assert(initialized);
+ const observed = initialized.tools
+ .filter((n) => n.startsWith("mcp__rea__"))
+ .sort();
+ assert.deepEqual(
+ observed,
+ TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(),
+ "Qwen must discover the complete REA catalog",
+ );
+ assert.equal(fixture.failure, undefined);
+ assert(finalSeen);
+ if (mode === "call") {
+ assert(evidenceSeen);
+ assert(selectedSkillLoaded);
+ assert(artifactReadByAgent);
+ }
+} catch (cause) {
+ failure = cause;
+ if (typeof cause?.stdout === "string")
+ await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
+ if (typeof cause?.stderr === "string")
+ await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
+} finally {
+ await fixture.close();
+ const lineage = await completeVerifierRun(verifier);
+ const receipt = {
+ client: "Qwen Code",
+ clientVersion,
+ mode,
+ passed: failure === undefined,
+ artifacts: lab,
+ runtimeRoot,
+ host: {
+ platform: process.platform,
+ architecture: process.arch,
+ node: process.versions.node,
+ },
+ modelFixture:
+ "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim",
+ skillDiscovery:
+ "caller-configured isolated shared directory; default OS home discovery unverified",
+ catalogSize: TOOL_CONTRACTS.length,
+ requests: fixture.requests,
+ probes: fixture.probes,
+ evidenceSeen,
+ selectedSkillLoaded,
+ artifactReadByAgent,
+ artifactPath,
+ artifactDigest,
+ finalSeen,
+ fixtureFailure: fixture.failure,
+ verifier: lineage,
+ };
+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
+ console.log(JSON.stringify(receipt));
+}
+if (failure !== undefined) throw failure;
diff --git a/scripts/verify/clients/model-fixture.mjs b/scripts/verify/clients/model-fixture.mjs
new file mode 100644
index 000000000..cdc99a31c
--- /dev/null
+++ b/scripts/verify/clients/model-fixture.mjs
@@ -0,0 +1,171 @@
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+/** A loopback model fixture exercises real clients without claiming live-provider compatibility. */
+export const createOpenAiModelFixture = async ({
+ directory,
+ prefix,
+ onRequest,
+}) => {
+ const requests = [];
+ const probes = [];
+ let incoming = 0;
+ let failure;
+ const server = createServer(async (request, response) => {
+ try {
+ if (request.method === "GET" && request.url?.endsWith("/models")) {
+ response.setHeader("Content-Type", "application/json");
+ response.end(
+ JSON.stringify({
+ object: "list",
+ data: [
+ {
+ id: "rea-client-fixture",
+ object: "model",
+ owned_by: "local-fixture",
+ },
+ ],
+ }),
+ );
+ return;
+ }
+ const chunks = [];
+ let bytes = 0;
+ for await (const chunk of request) {
+ bytes += chunk.length;
+ assert(
+ bytes <= 10 * 1024 * 1024,
+ "Local model fixture exceeded its 10 MiB request resource budget",
+ );
+ chunks.push(chunk);
+ }
+ const rawBody = Buffer.concat(chunks).toString("utf8");
+ const body = rawBody.length === 0 ? null : JSON.parse(rawBody);
+ await writeFile(
+ join(directory, `${prefix}-incoming-${++incoming}.json`),
+ JSON.stringify({ method: request.method, path: request.url, body }),
+ );
+ if (request.url !== "/v1/chat/completions") {
+ // Hermes probes custom endpoints for Ollama/llama.cpp metadata.
+ // Unsupported HTTP endpoints return 404, as on an OpenAI-only server.
+ probes.push({
+ method: request.method,
+ path: request.url,
+ body,
+ status: 404,
+ });
+ response.statusCode = 404;
+ response.setHeader("Content-Type", "application/json");
+ response.end(
+ JSON.stringify({
+ error:
+ "This OpenAI fixture exposes models and chat completions only.",
+ }),
+ );
+ return;
+ }
+ assert.equal(request.method, "POST");
+ assert(body !== null && typeof body === "object" && !Array.isArray(body));
+ assert(Array.isArray(body.messages));
+ const tools = (body.tools ?? [])
+ .filter((t) => t.type === "function")
+ .map((t) => t.function ?? t);
+ requests.push({
+ path: request.url,
+ tools: tools.map((t) => t.name),
+ stream: body.stream,
+ messageRoles: body.messages.map((m) => m.role),
+ });
+ await writeFile(
+ join(directory, `${prefix}-request-${requests.length}.json`),
+ JSON.stringify(body),
+ );
+ const message = await onRequest({
+ body,
+ tools,
+ requestIndex: requests.length,
+ });
+ const base = {
+ id: `chatcmpl-rea-${requests.length}`,
+ object: "chat.completion",
+ created: Math.floor(Date.now() / 1000),
+ model: body.model,
+ };
+ const finish = message.tool_calls ? "tool_calls" : "stop";
+ const usage = {
+ prompt_tokens: 100,
+ completion_tokens: 20,
+ total_tokens: 120,
+ };
+ if (body.stream) {
+ response.setHeader("Content-Type", "text/event-stream");
+ const emit = (delta, finishReason = null) =>
+ response.write(
+ `data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta, finish_reason: finishReason }] })}\n\n`,
+ );
+ if (message.tool_calls) {
+ const fragments = message.tool_calls.map((call, index) => {
+ const split = Math.floor(call.function.arguments.length / 2);
+ return {
+ first: {
+ index,
+ ...call,
+ function: {
+ name: call.function.name,
+ arguments: call.function.arguments.slice(0, split),
+ },
+ },
+ second: {
+ index,
+ function: { arguments: call.function.arguments.slice(split) },
+ },
+ };
+ });
+ emit({
+ role: "assistant",
+ content: null,
+ tool_calls: fragments.map((f) => f.first),
+ });
+ emit({ tool_calls: fragments.map((f) => f.second) });
+ } else emit(message);
+ emit({}, finish);
+ response.write(
+ `data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [], usage })}\n\n`,
+ );
+ response.end("data: [DONE]\n\n");
+ } else {
+ response.setHeader("Content-Type", "application/json");
+ response.end(
+ JSON.stringify({
+ ...base,
+ choices: [{ index: 0, message, finish_reason: finish }],
+ usage,
+ }),
+ );
+ }
+ } catch (cause) {
+ failure = String(cause);
+ response.statusCode = 400;
+ response.setHeader("Content-Type", "application/json");
+ response.end(
+ JSON.stringify({
+ error: { message: failure, type: "invalid_request_error" },
+ }),
+ );
+ }
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const address = server.address();
+ assert(address !== null && typeof address !== "string");
+ return {
+ baseUrl: `http://127.0.0.1:${address.port}/v1`,
+ requests,
+ probes,
+ get failure() {
+ return failure;
+ },
+ close: () => new Promise((resolve) => server.close(resolve)),
+ };
+};
diff --git a/tests/boundary/cli/hermesClientSetup.test.ts b/tests/acceptance/setup/hermesClientSetup.test.ts
similarity index 100%
rename from tests/boundary/cli/hermesClientSetup.test.ts
rename to tests/acceptance/setup/hermesClientSetup.test.ts
diff --git a/tests/boundary/cli/piClientSetup.test.ts b/tests/acceptance/setup/piClientSetup.test.ts
similarity index 100%
rename from tests/boundary/cli/piClientSetup.test.ts
rename to tests/acceptance/setup/piClientSetup.test.ts
diff --git a/tests/acceptance/setup/qwenProfileSetup.test.ts b/tests/acceptance/setup/qwenProfileSetup.test.ts
new file mode 100644
index 000000000..9f52ba08f
--- /dev/null
+++ b/tests/acceptance/setup/qwenProfileSetup.test.ts
@@ -0,0 +1,96 @@
+import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { parse } from "jsonc-parser";
+import { expect } from "vitest";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+cliTest(
+ "keeps Qwen profile registration and shared skill consistent through setup, doctor and uninstall",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-qwen-profile-");
+ const home = join(root, "account");
+ const profile = join(root, "selected profile");
+ const settings = join(profile, "settings.json");
+ const skill = join(
+ home,
+ ".agents",
+ "skills",
+ "reverse-engineer-anything",
+ "SKILL.md",
+ );
+ const environment = { USERPROFILE: home, QWEN_HOME: profile };
+ const original =
+ '{\n // Keep my configuration.\n "model": {"name": "existing"},\n "mcpServers": {"other": {"command": "other-server"}}\n}\n';
+ await mkdir(profile, { recursive: true });
+ await writeFile(settings, original);
+ const run = (arguments_: readonly string[]) =>
+ cli.run({ arguments: [...arguments_, "--json"], environment });
+
+ const plan = await run(["setup", "--client", "qwen_code", "--dry-run"]);
+ expect(plan.exitCode, plan.stderr).toBe(0);
+ expect(plan.json).toMatchObject({
+ status: "planned",
+ plannedActions: [
+ { kind: "configure_client", target: settings },
+ {
+ kind: "install_skill",
+ target: join(home, ".agents", "skills", "reverse-engineer-anything"),
+ },
+ ],
+ });
+ expect(await readFile(settings, "utf8")).toBe(original);
+ await expect(access(skill)).rejects.toMatchObject({ code: "ENOENT" });
+
+ const installed = await run(["setup", "--client", "qwen_code", "--yes"]);
+ expect(installed.exitCode, installed.stderr).toBe(0);
+ expect(installed.json).toMatchObject({ status: "ready" });
+ expect(await readFile(`${settings}.rea.backup`, "utf8")).toBe(original);
+ const configured = await readFile(settings, "utf8");
+ expect(configured).toContain("// Keep my configuration.");
+ expect(parse(configured)).toMatchObject({
+ model: { name: "existing" },
+ mcpServers: {
+ other: { command: "other-server" },
+ rea: {
+ command:
+ process.platform === "win32"
+ ? process.execPath
+ : resolve("scripts/rea.mjs"),
+ args:
+ process.platform === "win32"
+ ? [resolve("scripts/rea.mjs"), "mcp"]
+ : ["mcp"],
+ },
+ },
+ });
+ expect(await readFile(skill, "utf8")).toContain(
+ "name: reverse-engineer-anything",
+ );
+
+ const doctor = await run(["doctor", "--client", "qwen_code", "--skill"]);
+ expect(doctor.exitCode, doctor.stderr).toBe(0);
+ expect(doctor.json).toMatchObject({ healthy: true });
+ const repeated = await run(["setup", "--client", "qwen_code", "--yes"]);
+ expect(repeated.exitCode, repeated.stderr).toBe(0);
+ expect(repeated.json).toMatchObject({
+ status: "ready",
+ appliedActions: [],
+ });
+ expect(await readFile(settings, "utf8")).toBe(configured);
+
+ const removed = await run(["uninstall"]);
+ expect(removed.exitCode, removed.stderr).toBe(0);
+ expect(removed.json).toMatchObject({ status: "complete" });
+ expect(parse(await readFile(settings, "utf8"))).toEqual({
+ model: { name: "existing" },
+ mcpServers: { other: { command: "other-server" } },
+ });
+ await expect(access(skill)).rejects.toMatchObject({ code: "ENOENT" });
+ await expect(access(join(home, ".qwen"))).rejects.toMatchObject({
+ code: "ENOENT",
+ });
+ },
+ 120_000,
+);