Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(clients): verify native Qwen Pi and Hermes workflows (#1511)
Ysopwnableb25922e4e8fe
diff --git a/docs/testing.md b/docs/testing.mdindex 9730f027b..82b529929 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -88,6 +88,21 @@ entrypoint and include its verifier/fixture files in their cache inputs.## End-to-end, integration and golden evidence+The optional `verify:qwen-client`, `verify:pi-client`, and `verify:hermes-client`+lanes require an installed native client. Select its executable with+`REA_VERIFY_QWEN_COMMAND`, `REA_VERIFY_PI_COMMAND`, or `REA_VERIFY_HERMES_COMMAND`.+They configure disposable profiles through the public REA CLI, preserve caller+settings and backups, and exercise real stdio MCP calls and skill loading. The+`call` mode checks complete catalog discovery and actual JavaScript Evidence;+pass `-- chat` to check plain chat with REA enabled. Qwen also reads the full+result that its client offloads, and Pi exercises default codemode execution.+Use `REA_VERIFY_RUNTIME_ROOT` to select a production-only installed REA package.+Receipts retain client versions, result digests, host coverage and owned-process+cleanup. These POSIX lanes use a deterministic loopback model, so they do not+prove live model-provider or native Windows compatibility. Qwen and Pi use+caller-configured skill directories to isolate the fixture account from the OS+home; this does not prove their default home-directory discovery.+Full E2E tests invoke the production command dispatcher and real providers,without fake launchers, runners or responses. `verify:keyed-archive` writes anactual Foundation binary archive, runs the CLI and a separate stdio MCPdiff --git a/package.json b/package.jsonindex 439746c70..0c47e5901 100644--- a/package.json+++ b/package.json@@ -179,6 +179,9 @@"verify:readiness": "npm run build:cached && npm run verify:fixtures && node scripts/verify-reconstruction-readiness.mjs","verify:package": "npm run build:cached && node scripts/verify-package.mjs","verify:package:windows": "npm run build:cached && node scripts/verify-windows-package.mjs",+ "verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",+ "verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi",+ "verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes","probe:windows-capabilities": "npm run build:cached && node scripts/windows-capabilities.mjs","verify:published": "node scripts/verify-published-package.mjs","config:print": "node scripts/print-mcp-config.mjs",diff --git a/scripts/verify-native-agent-client.mjs b/scripts/verify-native-agent-client.mjsnew file mode 100644index 000000000..f561e353e--- /dev/null+++ b/scripts/verify-native-agent-client.mjs@@ -0,0 +1,456 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import {+ TOOL_CONTRACTS,+ toolContract,+} from "../dist/contracts/toolContracts.js";+import { exec } from "./lib/verify-package-core.mjs";+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";+import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";++const client = process.argv[2];+const mode = process.argv[3] ?? "call";+assert(["pi", "hermes"].includes(client), "Select pi or hermes.");+assert(["chat", "call"].includes(mode), "Select chat or call.");+assert(+ process.platform !== "win32",+ "This real-client lane requires POSIX; native Windows is unverified.",+);+const repo = fileURLToPath(new URL("..", import.meta.url));+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;+const command =+ process.env[`REA_VERIFY_${client.toUpperCase()}_COMMAND`] ?? client;+const lab = await mkdtemp(join(tmpdir(), `rea-${client}-client-`));+const account = join(lab, "account");+const profile = join(lab, "selected profile");+const workspace = join(lab, "workspace");+const target = join(workspace, "fixture α测试");+const artifactPath = join(workspace, "evidence.json");+const skillPath = join(+ client === "pi" ? join(account, ".agents") : profile,+ "skills",+ "reverse-engineer-anything",+ "SKILL.md",+);+const verifier = createVerifierRun();+await Promise.all([+ mkdir(account),+ mkdir(profile),+ mkdir(target, { recursive: true }),+]);+await writeFile(+ join(target, "app.js"),+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',+);+const environment = {+ PATH: process.env.PATH ?? "",+ USERPROFILE: account,+ [client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile,+ PI_OFFLINE: "1",+ REA_PROCESS_RUN_ID: verifier.run_id,+};+const configPath = join(profile, client === "pi" ? "mcp.json" : "config.yaml");+const original =+ client === "pi"+ ? JSON.stringify({+ mcpServers: { other: { command: "unrelated-server", enabled: false } },+ })+ : "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n";+await writeFile(configPath, original);+if (client === "pi") {+ // The CLI fixture account and native os.homedir() differ; use Pi's caller-owned skill setting.+ await writeFile(+ join(profile, "settings.json"),+ JSON.stringify({ skills: [join(account, ".agents", "skills")] }),+ );+}+const clientVersion = (+ await exec(command, ["--version"], { env: environment, timeout: 30_000 })+).stdout.trim();+for (const args of [+ ["setup", "--client", client, "--dry-run"],+ ["setup", "--client", client, "--yes"],+ ["doctor", "--client", client, "--skill"],+]) {+ const result = await exec(+ process.execPath,+ [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],+ { env: environment, cwd: workspace, timeout: 60_000 },+ );+ const parsed = JSON.parse(result.stdout);+ if (args[0] === "doctor") assert.equal(parsed.healthy, true);+ else+ assert.equal(+ parsed.status,+ args.includes("--dry-run") ? "planned" : "ready",+ );+ await writeFile(+ join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),+ result.stdout,+ );+}+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);+const installedConfiguration = await readFile(configPath, "utf8");+assert(installedConfiguration.includes("unrelated-server"));+if (client === "hermes")+ assert(installedConfiguration.includes("# Keep caller settings."));+const repeated = await exec(+ process.execPath,+ [+ join(runtimeRoot, "scripts/rea.mjs"),+ "setup",+ "--client",+ client,+ "--yes",+ "--json",+ ],+ { env: environment, cwd: workspace, timeout: 60_000 },+);+assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);+assert.equal(await readFile(configPath, "utf8"), installedConfiguration);+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);+assert(+ (await readFile(skillPath, "utf8")).includes(+ "name: reverse-engineer-anything",+ ),+);+let evidenceSeen = false;+let skillLoaded = false;+let skillRequested = false;+let callRequested = false;+let searchRequested = false;+let describeRequested = false;+let finalSeen = false;+let catalog;+let artifactDigest;+const extractEvidence = (value) => {+ if (typeof value === "string") {+ if (+ value.startsWith("<untrusted_tool_result ") &&+ value.endsWith("</untrusted_tool_result>")+ ) {+ // Hermes wraps external MCP content with its own instruction banner.+ // Decode only that observed envelope, retaining the original request artifact.+ const start = value.indexOf("\n\n");+ if (start >= 0)+ return extractEvidence(+ value+ .slice(start + 2, value.lastIndexOf("</untrusted_tool_result>"))+ .trim(),+ );+ }+ try {+ return extractEvidence(JSON.parse(value));+ } catch {+ return undefined;+ }+ }+ if (value && typeof value === "object") {+ if (+ typeof value.evidence_id === "string" &&+ value.normalized_result?.graph?.nodes+ )+ return value;+ for (const child of Object.values(value)) {+ const found = extractEvidence(child);+ if (found) return found;+ }+ }+};+const validateEvidence = async (value) => {+ const evidence = toolContract(+ "analyze_javascript_application",+ ).outputSchema.parse(value);+ assert.equal(evidence.subject.local_path, target);+ const shape = evidence.normalized_result.graph.nodes+ .flatMap((n) => n.observations)+ .map((o) => o.properties)+ .find(+ (p) =>+ p.semantic_role === "export-return-shapes" &&+ p.exported_name === "result",+ );+ assert(shape?.return_shape_coverage?.projection_complete);+ const fields = shape.static_return_shapes.flatMap((s) => s.fields);+ assert(+ fields.some(+ (f) =>+ f.path === "/clientCompatibility" &&+ f.state === "literal" &&+ f.value === "REA_CLIENT_FIXTURE",+ ),+ );+ assert(+ fields.some(+ (f) => f.path === "/count" && f.state === "literal" && f.value === 7,+ ),+ );+ const serialized = JSON.stringify(evidence);+ await writeFile(join(lab, "evidence.json"), serialized);+ artifactDigest = createHash("sha256").update(serialized).digest("hex");+ evidenceSeen = true;+};+const fixture = await createOpenAiModelFixture({+ directory: lab,+ prefix: `${client}-${mode}`,+ onRequest: async ({ body, tools, requestIndex }) => {+ assert(+ requestIndex <= 6,+ "Client did not complete the bounded fixture workflow.",+ );+ const messages = body.messages ?? [];+ const toolMessages = messages.filter((m) => m.role === "tool");+ const serializedTools = JSON.stringify(toolMessages);+ const call = (name, args) => ({+ role: "assistant",+ content: null,+ tool_calls: [+ {+ id: `call_rea_${requestIndex}`,+ type: "function",+ function: { name, arguments: JSON.stringify(args) },+ },+ ],+ });+ if (+ client === "pi" &&+ callRequested &&+ serializedTools.includes("REA_CLIENT_FIXTURE")+ ) {+ const fact = toolMessages+ .flatMap((m) => {+ const text =+ typeof m.content === "string"+ ? m.content+ : JSON.stringify(m.content);+ const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true');+ if (marker < 0) return [];+ const end = text.indexOf("}\n", marker);+ try {+ return [+ JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)),+ ];+ } catch {+ return [];+ }+ })+ .find((v) => Array.isArray(v.catalog));+ assert(+ fact,+ "Pi codemode must return its actual catalog and skill-read receipt.",+ );+ catalog = fact.catalog;+ skillLoaded = fact.skill_loaded === true;+ await validateEvidence(JSON.parse(await readFile(artifactPath, "utf8")));+ } else if (client === "hermes") {+ const search = tools.find((t) => t.name === "tool_search");+ const exposed = tools+ .filter((t) => t.name.startsWith("mcp__rea__"))+ .map((t) => t.name.slice(10));+ const deferred = [+ ...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu),+ ].map((m) => m[1]);+ const native = [...new Set([...exposed, ...deferred])];+ // Hermes advertises MCP prompt utilities beside server tools.+ if (native.length)+ catalog = native.filter(+ (name) => !["get_prompt", "list_prompts"].includes(name),+ );+ skillLoaded ||=+ serializedTools.includes(skillPath) &&+ serializedTools.includes("reverse-engineer-anything");+ for (const message of toolMessages) {+ const evidence = extractEvidence(message.content);+ if (evidence) await validateEvidence(evidence);+ }+ }+ if (mode === "chat" || evidenceSeen) {+ finalSeen = true;+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };+ }+ if (client === "pi") {+ const tool = tools.find((t) => t.name === "codemode");+ assert(+ tool,+ `Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`,+ );+ assert(+ JSON.stringify(messages).includes(skillPath),+ "Pi must advertise the installed REA skill.",+ );+ callRequested = true;+ return call(tool.name, {+ code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`,+ });+ }+ if (!skillRequested) {+ const tool = tools.find((t) => t.name === "skill_view");+ assert(tool, "Hermes skill_view must be available.");+ skillRequested = true;+ return call(tool.name, { name: "reverse-engineer-anything" });+ }+ if (!searchRequested) {+ const tool = tools.find((t) => t.name === "tool_search");+ assert(tool, "Hermes native deferred tool_search must be available.");+ searchRequested = true;+ return call(tool.name, { queries: ["analyze_javascript_application"] });+ }+ assert(+ serializedTools.includes("mcp__rea__analyze_javascript_application"),+ "Hermes must discover the analysis tool.",+ );+ if (!describeRequested) {+ const tool = tools.find((t) => t.name === "tool_describe");+ assert(tool, "Hermes must load the actual parameter schema.");+ describeRequested = true;+ return call(tool.name, {+ names: ["mcp__rea__analyze_javascript_application"],+ });+ }+ const tool = tools.find((t) => t.name === "tool_call");+ assert(tool, "Hermes native deferred tool_call must be available.");+ callRequested = true;+ return call(tool.name, {+ calls: [+ {+ name: "mcp__rea__analyze_javascript_application",+ arguments: { input_path: target, format: "directory" },+ },+ ],+ });+ },+});+if (client === "pi") {+ await writeFile(+ join(profile, "models.json"),+ JSON.stringify({+ providers: {+ "rea-fixture": {+ baseUrl: fixture.baseUrl,+ api: "openai-completions",+ apiKey: "local-fixture-key",+ models: [+ {+ id: "rea-client-fixture",+ reasoning: false,+ contextWindow: 200000,+ maxTokens: 4096,+ },+ ],+ },+ },+ }),+ );+} else {+ const configured = await readFile(configPath, "utf8");+ await writeFile(+ configPath,+ configured ++ `\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`,+ );+}+const args =+ client === "pi"+ ? [+ "--offline",+ "--provider",+ "rea-fixture",+ "--model",+ "rea-client-fixture",+ "--no-session",+ "--no-context-files",+ "--mode",+ "json",+ "-p",+ mode === "chat"+ ? "Say hello briefly."+ : `Use REA to analyze ${target}.`,+ ]+ : [+ "--oneshot",+ mode === "chat"+ ? "Say hello briefly."+ : `Use REA to analyze ${target}.`,+ "--provider",+ "custom",+ "--model",+ "rea-client-fixture",+ "--reasoning",+ "none",+ ];+let failure;+try {+ const execution = exec(command, args, {+ env: {+ ...environment,+ OPENAI_API_KEY: "local-fixture-key",+ OPENAI_BASE_URL: fixture.baseUrl,+ },+ cwd: workspace,+ timeout: 120_000,+ maxBuffer: 10 * 1024 * 1024,+ });+ // Pi consumes piped input before its first prompt; an unused open pipe is not EOF.+ execution.child.stdin.end();+ const result = await execution;+ await writeFile(join(lab, "client.stdout"), result.stdout);+ await writeFile(join(lab, "client.stderr"), result.stderr);+ assert.equal(fixture.failure, undefined);+ assert(finalSeen);+ assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));+ if (mode === "call") {+ assert(evidenceSeen);+ assert(+ skillLoaded,+ "Native client must actually read the installed skill.",+ );+ assert.deepEqual(+ catalog?.sort(),+ TOOL_CONTRACTS.map((t) => t.name).sort(),+ "The complete REA catalog must survive native client discovery.",+ );+ }+} catch (cause) {+ failure = cause;+ if (typeof cause?.stdout === "string")+ await writeFile(join(lab, "client.stdout"), cause.stdout);+ if (typeof cause?.stderr === "string")+ await writeFile(join(lab, "client.stderr"), cause.stderr);+} finally {+ await fixture.close();+ const receipt = {+ client,+ clientVersion,+ mode,+ passed: failure === undefined,+ artifacts: lab,+ runtimeRoot,+ host: {+ platform: process.platform,+ architecture: process.arch,+ node: process.versions.node,+ },+ modelFixture:+ "loopback deterministic OpenAI-compatible endpoint; no live provider claim",+ skillDiscovery:+ client === "pi"+ ? "caller-configured isolated shared directory; default OS home discovery unverified"+ : "native HERMES_HOME personal directory",+ catalogSize: catalog?.length,+ evidenceSeen,+ skillLoaded,+ artifactDigest,+ finalSeen,+ requests: fixture.requests,+ probes: fixture.probes,+ fixtureFailure: fixture.failure,+ verifier: await completeVerifierRun(verifier),+ };+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));+ console.log(JSON.stringify(receipt));+}+if (failure !== undefined) throw failure;diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjsnew file mode 100644index 000000000..4b85470d6--- /dev/null+++ b/scripts/verify-qwen-client.mjs@@ -0,0 +1,403 @@+import assert from "node:assert/strict";+import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";+import {+ mkdir,+ writeFile,+ readFile,+ mkdtemp,+ realpath,+} from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import { createHash } from "node:crypto";+import { exec } from "./lib/verify-package-core.mjs";+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";+import {+ TOOL_CONTRACTS,+ toolContract,+} from "../dist/contracts/toolContracts.js";++assert(+ process.platform !== "win32",+ "The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",+);+const repo = fileURLToPath(new URL("..", import.meta.url));+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;+const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));+const client = "qwen";+const mode = process.argv[2] ?? "call";+assert(+ ["chat", "call"].includes(mode),+ "Usage: node scripts/verify-qwen-client.mjs [chat|call]",+);+const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";+const verifier = createVerifierRun();+const profile = join(lab, "selected profile");+const account = join(lab, "account");+const runtime = join(profile, "runtime");+const workspace = join(lab, "workspace");+const sharedSkills = join(account, ".agents", "skills");+await Promise.all([+ mkdir(profile, { recursive: true }),+ mkdir(account),+ mkdir(workspace),+]);+const baseEnvironment = {+ PATH: process.env.PATH ?? "",+ USERPROFILE: account,+ QWEN_HOME: profile,+ QWEN_RUNTIME_DIR: runtime,+ REA_PROCESS_RUN_ID: verifier.run_id,+};+let clientVersion;+try {+ clientVersion = (+ await exec(command, ["--version"], {+ env: baseEnvironment,+ timeout: 30_000,+ })+ ).stdout.trim();+} catch (cause) {+ throw new Error(+ `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,+ );+}+const original = JSON.stringify(+ {+ security: {+ auth: { selectedType: "openai" },+ folderTrust: { enabled: false },+ },+ telemetry: { enabled: false },+ model: { name: "rea-client-fixture" },+ // Qwen uses os.homedir(), while REA's isolated CLI account uses USERPROFILE.+ // This caller-owned setting lets the real client load the REA-installed bundle.+ skills: { directories: [sharedSkills] },+ mcpServers: { other: { command: "unrelated-server", disabled: true } },+ },+ null,+ 2,+);+await writeFile(join(profile, "settings.json"), original);+for (const step of [+ ["setup", "--client", "qwen_code", "--dry-run"],+ ["setup", "--client", "qwen_code", "--yes"],+ ["doctor", "--client", "qwen_code", "--skill"],+]) {+ const response = await exec(+ process.execPath,+ [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"],+ { env: baseEnvironment, cwd: repo, timeout: 60_000 },+ );+ const result = JSON.parse(response.stdout);+ if (step.includes("--dry-run"))+ assert(+ result.plannedActions.every(+ (a) =>+ a.target.startsWith(profile + "/") ||+ a.target.startsWith(sharedSkills + "/"),+ ),+ );+ await writeFile(+ join(lab, `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`),+ response.stdout,+ );+}+assert.equal(+ await readFile(join(profile, "settings.json.rea.backup"), "utf8"),+ original,+);+const configured = JSON.parse(+ await readFile(join(profile, "settings.json"), "utf8"),+);+assert.deepEqual(configured.model, { name: "rea-client-fixture" });+assert.deepEqual(configured.skills, { directories: [sharedSkills] });+assert.deepEqual(configured.mcpServers.other, {+ command: "unrelated-server",+ disabled: true,+});+const target = join(lab, "fixture α测试");+await mkdir(target, { recursive: true });+await writeFile(+ `${target}/app.js`,+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',+);+let evidenceSeen = false;+let skillSeen = false;+let finalSeen = false;+let skillRequested = false;+let searchRequested = false;+let artifactReadByAgent = false;+let artifactPath;+let selectedSkillLoaded = false;+let artifactDigest;+const extractEvidence = (value) => {+ if (typeof value === "string") {+ try {+ return extractEvidence(JSON.parse(value));+ } catch {+ return null;+ }+ }+ if (Array.isArray(value)) {+ for (const item of value) {+ const result = extractEvidence(item);+ if (result) return result;+ }+ } else if (value && typeof value === "object") {+ if (+ typeof value.evidence_id === "string" &&+ value.normalized_result?.graph?.nodes?.length > 0+ )+ return value;+ for (const nested of Object.values(value)) {+ const result = extractEvidence(nested);+ if (result) return result;+ }+ }+ return null;+};+const fixture = await createOpenAiModelFixture({+ directory: lab,+ prefix: `${client}-${mode}`,+ onRequest: async ({ body, tools, requestIndex }) => {+ skillSeen ||= JSON.stringify(body.messages ?? []).includes(+ "reverse-engineer-anything",+ );+ selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(+ join(sharedSkills, "reverse-engineer-anything"),+ );+ const toolText = JSON.stringify(+ (body.messages ?? []).filter((m) => m.role === "tool"),+ );+ if (+ toolText.includes("REA_CLIENT_FIXTURE") &&+ /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText)+ ) {+ artifactReadByAgent = true;+ assert(artifactPath);+ const artifact = await readFile(artifactPath);+ artifactDigest = createHash("sha256").update(artifact).digest("hex");+ const evidence = toolContract(+ "analyze_javascript_application",+ ).outputSchema.parse(JSON.parse(artifact.toString("utf8")));+ assert(extractEvidence(evidence));+ assert.equal(evidence.subject.local_path, target);+ const shape = evidence.normalized_result.graph.nodes+ .flatMap((n) => n.observations)+ .map((o) => o.properties)+ .find(+ (p) =>+ p.semantic_role === "export-return-shapes" &&+ p.exported_name === "result",+ );+ assert(shape?.return_shape_coverage?.projection_complete);+ const fields = shape.static_return_shapes.flatMap((s) => s.fields);+ assert(+ fields.some(+ (f) =>+ f.path === "/clientCompatibility" &&+ f.state === "literal" &&+ f.value === "REA_CLIENT_FIXTURE",+ ),+ );+ assert(+ fields.some(+ (f) => f.path === "/count" && f.state === "literal" && f.value === 7,+ ),+ );+ evidenceSeen = true;+ await writeFile(+ `${lab}/${client}-${mode}-evidence.json`,+ JSON.stringify(evidence),+ );+ }+ for (const message of body.messages ?? []) {+ if (message.role !== "tool") continue;+ const evidence = extractEvidence(message.content);+ if (evidence) {+ assert(+ JSON.stringify(evidence.normalized_result).includes(+ "REA_CLIENT_FIXTURE",+ ),+ );+ evidenceSeen = true;+ await writeFile(+ `${lab}/${client}-${mode}-evidence.json`,+ JSON.stringify(evidence),+ );+ }+ }+ const selected = tools.find((t) =>+ t.name.includes("analyze_javascript_application"),+ );+ const reviewedNames = [+ ...JSON.stringify(+ (body.messages ?? []).filter((m) => m.role === "tool"),+ ).matchAll(+ /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g,+ ),+ ]+ .map((m) => m[1])+ .sort((a, b) => b.length - a.length);+ const search = tools.find((t) => t.name === "tool_search");+ const invoke = tools.find((t) => t.name === "tool_call");+ const skill = tools.find((t) => t.name === "skill");+ const shell = tools.find((t) => t.name === "run_shell_command");+ const persisted = toolText.match(+ /Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,+ );+ const call = (name, args) => ({+ role: "assistant",+ content: null,+ tool_calls: [+ {+ id: `call_rea_fixture_${requestIndex}`,+ type: "function",+ function: { name, arguments: JSON.stringify(args) },+ },+ ],+ });+ let message;+ if (mode === "chat" || evidenceSeen) {+ message = { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };+ finalSeen = true;+ } else if (skill && !skillRequested) {+ skillRequested = true;+ message = call(skill.name, { skill: "reverse-engineer-anything" });+ } else if (persisted && shell) {+ artifactPath = persisted[1];+ assert(+ (await realpath(artifactPath)).startsWith(+ (await realpath(runtime)) + "/",+ ),+ );+ const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;+ const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";+ message = call(shell.name, {+ command: `node -e ${quote(script)}`,+ description:+ "Read the complete saved REA Evidence and verify the fixture facts.",+ });+ } else if (selected)+ message = call(selected.name, {+ input_path: target,+ format: "directory",+ });+ else if (reviewedNames.length && invoke)+ message = call(invoke.name, {+ name: reviewedNames[0],+ arguments: { input_path: target, format: "directory" },+ });+ else if (search && !searchRequested) {+ searchRequested = true;+ message = call(search.name, { query: "analyze_javascript_application" });+ } else+ throw new Error(+ `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`,+ );+ return message;+ },+});+const endpoint = fixture.baseUrl;+const prompt =+ mode === "chat"+ ? "Say hello briefly."+ : `Use REA to analyze the JavaScript directory ${target}. Report the result.`;+const env = {+ ...baseEnvironment,+ OPENAI_API_KEY: "local-fixture-key",+ OPENAI_BASE_URL: endpoint,+};+const args = [+ "--auth-type",+ "openai",+ "--openai-base-url",+ endpoint,+ "--model",+ "rea-client-fixture",+ "--approval-mode",+ mode === "chat" ? "default" : "yolo",+ "--allowed-mcp-server-names",+ "rea",+ "--output-format",+ "stream-json",+ "--max-session-turns",+ "8",+ "-p",+ prompt,+];+let response;+let failure;+try {+ response = await exec(command, args, {+ env,+ cwd: workspace,+ timeout: 120_000,+ maxBuffer: 10 * 1024 * 1024,+ });+ await writeFile(join(lab, `${mode}.stdout`), response.stdout);+ await writeFile(join(lab, `${mode}.stderr`), response.stderr);+ const initialized = response.stdout+ .split("\n")+ .filter(Boolean)+ .map((line) => JSON.parse(line))+ .find((v) => v.type === "system" && v.subtype === "init");+ assert(initialized);+ const observed = initialized.tools+ .filter((n) => n.startsWith("mcp__rea__"))+ .sort();+ assert.deepEqual(+ observed,+ TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(),+ "Qwen must discover the complete REA catalog",+ );+ assert.equal(fixture.failure, undefined);+ assert(finalSeen);+ if (mode === "call") {+ assert(evidenceSeen);+ assert(selectedSkillLoaded);+ assert(artifactReadByAgent);+ }+} catch (cause) {+ failure = cause;+ if (typeof cause?.stdout === "string")+ await writeFile(join(lab, `${mode}.stdout`), cause.stdout);+ if (typeof cause?.stderr === "string")+ await writeFile(join(lab, `${mode}.stderr`), cause.stderr);+} finally {+ await fixture.close();+ const lineage = await completeVerifierRun(verifier);+ const receipt = {+ client: "Qwen Code",+ clientVersion,+ mode,+ passed: failure === undefined,+ artifacts: lab,+ runtimeRoot,+ host: {+ platform: process.platform,+ architecture: process.arch,+ node: process.versions.node,+ },+ modelFixture:+ "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim",+ skillDiscovery:+ "caller-configured isolated shared directory; default OS home discovery unverified",+ catalogSize: TOOL_CONTRACTS.length,+ requests: fixture.requests,+ probes: fixture.probes,+ evidenceSeen,+ selectedSkillLoaded,+ artifactReadByAgent,+ artifactPath,+ artifactDigest,+ finalSeen,+ fixtureFailure: fixture.failure,+ verifier: lineage,+ };+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));+ console.log(JSON.stringify(receipt));+}+if (failure !== undefined) throw failure;diff --git a/scripts/verify/clients/model-fixture.mjs b/scripts/verify/clients/model-fixture.mjsnew file mode 100644index 000000000..cdc99a31c--- /dev/null+++ b/scripts/verify/clients/model-fixture.mjs@@ -0,0 +1,171 @@+import assert from "node:assert/strict";+import { createServer } from "node:http";+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++/** A loopback model fixture exercises real clients without claiming live-provider compatibility. */+export const createOpenAiModelFixture = async ({+ directory,+ prefix,+ onRequest,+}) => {+ const requests = [];+ const probes = [];+ let incoming = 0;+ let failure;+ const server = createServer(async (request, response) => {+ try {+ if (request.method === "GET" && request.url?.endsWith("/models")) {+ response.setHeader("Content-Type", "application/json");+ response.end(+ JSON.stringify({+ object: "list",+ data: [+ {+ id: "rea-client-fixture",+ object: "model",+ owned_by: "local-fixture",+ },+ ],+ }),+ );+ return;+ }+ const chunks = [];+ let bytes = 0;+ for await (const chunk of request) {+ bytes += chunk.length;+ assert(+ bytes <= 10 * 1024 * 1024,+ "Local model fixture exceeded its 10 MiB request resource budget",+ );+ chunks.push(chunk);+ }+ const rawBody = Buffer.concat(chunks).toString("utf8");+ const body = rawBody.length === 0 ? null : JSON.parse(rawBody);+ await writeFile(+ join(directory, `${prefix}-incoming-${++incoming}.json`),+ JSON.stringify({ method: request.method, path: request.url, body }),+ );+ if (request.url !== "/v1/chat/completions") {+ // Hermes probes custom endpoints for Ollama/llama.cpp metadata.+ // Unsupported HTTP endpoints return 404, as on an OpenAI-only server.+ probes.push({+ method: request.method,+ path: request.url,+ body,+ status: 404,+ });+ response.statusCode = 404;+ response.setHeader("Content-Type", "application/json");+ response.end(+ JSON.stringify({+ error:+ "This OpenAI fixture exposes models and chat completions only.",+ }),+ );+ return;+ }+ assert.equal(request.method, "POST");+ assert(body !== null && typeof body === "object" && !Array.isArray(body));+ assert(Array.isArray(body.messages));+ const tools = (body.tools ?? [])+ .filter((t) => t.type === "function")+ .map((t) => t.function ?? t);+ requests.push({+ path: request.url,+ tools: tools.map((t) => t.name),+ stream: body.stream,+ messageRoles: body.messages.map((m) => m.role),+ });+ await writeFile(+ join(directory, `${prefix}-request-${requests.length}.json`),+ JSON.stringify(body),+ );+ const message = await onRequest({+ body,+ tools,+ requestIndex: requests.length,+ });+ const base = {+ id: `chatcmpl-rea-${requests.length}`,+ object: "chat.completion",+ created: Math.floor(Date.now() / 1000),+ model: body.model,+ };+ const finish = message.tool_calls ? "tool_calls" : "stop";+ const usage = {+ prompt_tokens: 100,+ completion_tokens: 20,+ total_tokens: 120,+ };+ if (body.stream) {+ response.setHeader("Content-Type", "text/event-stream");+ const emit = (delta, finishReason = null) =>+ response.write(+ `data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta, finish_reason: finishReason }] })}\n\n`,+ );+ if (message.tool_calls) {+ const fragments = message.tool_calls.map((call, index) => {+ const split = Math.floor(call.function.arguments.length / 2);+ return {+ first: {+ index,+ ...call,+ function: {+ name: call.function.name,+ arguments: call.function.arguments.slice(0, split),+ },+ },+ second: {+ index,+ function: { arguments: call.function.arguments.slice(split) },+ },+ };+ });+ emit({+ role: "assistant",+ content: null,+ tool_calls: fragments.map((f) => f.first),+ });+ emit({ tool_calls: fragments.map((f) => f.second) });+ } else emit(message);+ emit({}, finish);+ response.write(+ `data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [], usage })}\n\n`,+ );+ response.end("data: [DONE]\n\n");+ } else {+ response.setHeader("Content-Type", "application/json");+ response.end(+ JSON.stringify({+ ...base,+ choices: [{ index: 0, message, finish_reason: finish }],+ usage,+ }),+ );+ }+ } catch (cause) {+ failure = String(cause);+ response.statusCode = 400;+ response.setHeader("Content-Type", "application/json");+ response.end(+ JSON.stringify({+ error: { message: failure, type: "invalid_request_error" },+ }),+ );+ }+ });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));+ const address = server.address();+ assert(address !== null && typeof address !== "string");+ return {+ baseUrl: `http://127.0.0.1:${address.port}/v1`,+ requests,+ probes,+ get failure() {+ return failure;+ },+ close: () => new Promise((resolve) => server.close(resolve)),+ };+};diff --git a/tests/boundary/cli/hermesClientSetup.test.ts b/tests/acceptance/setup/hermesClientSetup.test.tssimilarity index 100%rename from tests/boundary/cli/hermesClientSetup.test.tsrename to tests/acceptance/setup/hermesClientSetup.test.tsdiff --git a/tests/boundary/cli/piClientSetup.test.ts b/tests/acceptance/setup/piClientSetup.test.tssimilarity index 100%rename from tests/boundary/cli/piClientSetup.test.tsrename to tests/acceptance/setup/piClientSetup.test.tsdiff --git a/tests/acceptance/setup/qwenProfileSetup.test.ts b/tests/acceptance/setup/qwenProfileSetup.test.tsnew file mode 100644index 000000000..9f52ba08f--- /dev/null+++ b/tests/acceptance/setup/qwenProfileSetup.test.ts@@ -0,0 +1,96 @@+import { access, mkdir, readFile, writeFile } from "node:fs/promises";+import { join, resolve } from "node:path";+import { parse } from "jsonc-parser";+import { expect } from "vitest";++import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++cliTest(+ "keeps Qwen profile registration and shared skill consistent through setup, doctor and uninstall",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-qwen-profile-");+ const home = join(root, "account");+ const profile = join(root, "selected profile");+ const settings = join(profile, "settings.json");+ const skill = join(+ home,+ ".agents",+ "skills",+ "reverse-engineer-anything",+ "SKILL.md",+ );+ const environment = { USERPROFILE: home, QWEN_HOME: profile };+ const original =+ '{\n // Keep my configuration.\n "model": {"name": "existing"},\n "mcpServers": {"other": {"command": "other-server"}}\n}\n';+ await mkdir(profile, { recursive: true });+ await writeFile(settings, original);+ const run = (arguments_: readonly string[]) =>+ cli.run({ arguments: [...arguments_, "--json"], environment });++ const plan = await run(["setup", "--client", "qwen_code", "--dry-run"]);+ expect(plan.exitCode, plan.stderr).toBe(0);+ expect(plan.json).toMatchObject({+ status: "planned",+ plannedActions: [+ { kind: "configure_client", target: settings },+ {+ kind: "install_skill",+ target: join(home, ".agents", "skills", "reverse-engineer-anything"),+ },+ ],+ });+ expect(await readFile(settings, "utf8")).toBe(original);+ await expect(access(skill)).rejects.toMatchObject({ code: "ENOENT" });++ const installed = await run(["setup", "--client", "qwen_code", "--yes"]);+ expect(installed.exitCode, installed.stderr).toBe(0);+ expect(installed.json).toMatchObject({ status: "ready" });+ expect(await readFile(`${settings}.rea.backup`, "utf8")).toBe(original);+ const configured = await readFile(settings, "utf8");+ expect(configured).toContain("// Keep my configuration.");+ expect(parse(configured)).toMatchObject({+ model: { name: "existing" },+ mcpServers: {+ other: { command: "other-server" },+ rea: {+ command:+ process.platform === "win32"+ ? process.execPath+ : resolve("scripts/rea.mjs"),+ args:+ process.platform === "win32"+ ? [resolve("scripts/rea.mjs"), "mcp"]+ : ["mcp"],+ },+ },+ });+ expect(await readFile(skill, "utf8")).toContain(+ "name: reverse-engineer-anything",+ );++ const doctor = await run(["doctor", "--client", "qwen_code", "--skill"]);+ expect(doctor.exitCode, doctor.stderr).toBe(0);+ expect(doctor.json).toMatchObject({ healthy: true });+ const repeated = await run(["setup", "--client", "qwen_code", "--yes"]);+ expect(repeated.exitCode, repeated.stderr).toBe(0);+ expect(repeated.json).toMatchObject({+ status: "ready",+ appliedActions: [],+ });+ expect(await readFile(settings, "utf8")).toBe(configured);++ const removed = await run(["uninstall"]);+ expect(removed.exitCode, removed.stderr).toBe(0);+ expect(removed.json).toMatchObject({ status: "complete" });+ expect(parse(await readFile(settings, "utf8"))).toEqual({+ model: { name: "existing" },+ mcpServers: { other: { command: "other-server" } },+ });+ await expect(access(skill)).rejects.toMatchObject({ code: "ENOENT" });+ await expect(access(join(home, ".qwen"))).rejects.toMatchObject({+ code: "ENOENT",+ });+ },+ 120_000,+);