Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(clients): verify native Grok Build workflows (#1571)
Ysopwnableefd81f977604
diff --git a/docs/testing.md b/docs/testing.mdindex 22e409afa..61440dfb1 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -167,6 +167,26 @@ fix for that client/model limit. It preserves the complete catalog and schemas.Live model APIs, native Windows and default OS-home skill discovery remainunverified.+`verify:grok-client` requires installed Grok Build (verified with the official+Linux x64 1.0.50 binary); select it with `REA_VERIFY_GROK_COMMAND`. This optional+POSIX lane isolates `GROK_HOME`, an independent Git workspace and additional+skill roots, disables foreign configuration discovery, guards setup writes,+preserves unrelated registrations/backups and checks idempotence. Call mode+loads the full skill through native `read_file`, queries all REA names and input+schemas through native `search_tool`, and calls analysis through `use_tool`.+If discovery reports `partial`, the native agent diagnoses REA registration and+retries discovery. No startup-timeout override or fixed readiness delay is used.+A larger Unicode-path fixture exercises native result offloading: the full+retained Evidence is validated against its named schema, then native terminal+queries recover the selected export facts, subject and full artifact digest in+the next model request. This verifies useful artifact recovery without claiming+that every offloaded graph fact enters model context. Native line-number and+truncation envelopes remain in the request artifacts. Use `-- chat` for ordinary+chat, or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed package.+The loopback OpenAI completions/SSE custom model declares a one-million-token+context and synthetic usage. Live xAI APIs, native Windows and default OS-home+skill discovery remain unverified.+`verify:omp-client` requires installed OMP (verified with the official Linux x6418.8.7 binary); select it with `REA_VERIFY_OMP_COMMAND`. This optional POSIX laneisolates the default agent, global config and XDG roots, guards setup targets,diff --git a/package.json b/package.jsonindex 9e59d0c6e..ed36813a8 100644--- a/package.json+++ b/package.json@@ -182,6 +182,7 @@"verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs","verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs","verify:opencode-client": "npm run build:cached && node scripts/verify-opencode-client.mjs",+ "verify:grok-client": "npm run build:cached && node scripts/verify-grok-client.mjs","verify:omp-client": "npm run build:cached && node scripts/verify-omp-client.mjs","verify:copilot-client": "npm run build:cached && node scripts/verify-copilot-client.mjs","verify:claude-client": "npm run build:cached && node scripts/verify-claude-client.mjs",diff --git a/scripts/verify-grok-client.mjs b/scripts/verify-grok-client.mjsnew file mode 100644index 000000000..e1c066770--- /dev/null+++ b/scripts/verify-grok-client.mjs@@ -0,0 +1,539 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";+import { homedir, tmpdir } from "node:os";+import { isAbsolute, join, relative } from "node:path";+import { fileURLToPath } from "node:url";+import Ajv2020 from "ajv/dist/2020.js";+import {+ TOOL_CONTRACTS,+ toolContract,+} from "../dist/contracts/toolContracts.js";+import { exec } from "./lib/verify-package-core.mjs";+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";+import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";++assert(+ process.platform !== "win32",+ "Native Grok verifier requires POSIX; Windows is unverified",+);+const repo = fileURLToPath(new URL("..", import.meta.url));+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;+const command = process.env.REA_VERIFY_GROK_COMMAND ?? "grok";+const mode = process.argv[2] ?? "call";+assert(+ ["chat", "call"].includes(mode),+ "Usage: verify-grok-client.mjs [chat|call]",+);+const verifier = createVerifierRun();+const lab = await mkdtemp(join(tmpdir(), "rea-grok-client-"));+const account = join(lab, "account");+const profile = join(lab, "selected profile");+const workspace = join(lab, "workspace");+const configPath = join(profile, "config.toml");+const skillDirectory = join(+ account,+ ".agents",+ "skills",+ "reverse-engineer-anything",+);+await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]);+await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });+// Native compatibility discovery is independent of GROK_HOME. Disable those+// sources and ignore the real shared skill root instead of changing HOME.+const environment = {+ PATH: process.env.PATH,+ NODE_OPTIONS: process.env.NODE_OPTIONS,+ USERPROFILE: account,+ GROK_HOME: profile,+ GROK_DISABLE_AUTOUPDATER: "1",+ ...Object.fromEntries(+ ["CLAUDE", "CURSOR"].flatMap((vendor) =>+ ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map((cell) => [+ `GROK_${vendor}_${cell}_ENABLED`,+ "false",+ ]),+ ),+ ),+ XDG_DATA_HOME: join(lab, "data"),+ XDG_CONFIG_HOME: join(lab, "config"),+ XDG_CACHE_HOME: join(lab, "cache"),+ REA_PROCESS_RUN_ID: verifier.run_id,+};+const target = join(lab, "fixture α测试");+await mkdir(target);+await writeFile(+ join(target, "app.js"),+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' ++ // A real analysis result large enough to exercise native MCP offloading.+ Array.from(+ { length: 60 },+ (_, index) =>+ `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`,+ ).join("\n"),+);+const expectedNames = TOOL_CONTRACTS.map((tool) => "rea__" + tool.name).sort();+assert(+ expectedNames.length <= 255,+ "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools",+);+const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });+const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'";+const containsText = (value, text) => {+ if (typeof value === "string") return value.includes(text);+ return (+ value !== null &&+ typeof value === "object" &&+ Object.values(value).some((item) => containsText(item, text))+ );+};+let clientVersion;+let expectedSkill;+let turn = 0;+let skillRequested = false;+let skillLoaded = false;+let diagnosticsRequested = false;+let diagnosticsVerified = false;+let lastTool;+let catalogChecked = false;+let schemasChecked = 0;+let analysisRequested = false;+let evidenceSeen = false;+let spillQueryRequested = false;+let projectionSeen = false;+let artifactDigest;+let artifactBytes;+let retainedArtifact;+let expectedProjection;+let finalSeen = false;+let failure;+const call = (name, args) => {+ lastTool = name;+ return {+ role: "assistant",+ content: null,+ tool_calls: [+ {+ id: `call_grok_${turn}`,+ type: "function",+ function: { name, arguments: JSON.stringify(args) },+ },+ ],+ };+};+const verifyEvidence = (bytes) => {+ const evidence = toolContract(+ "analyze_javascript_application",+ ).outputSchema.parse(JSON.parse(bytes.toString("utf8")));+ assert.equal(evidence.subject.local_path, target);+ const shapes = evidence.normalized_result.graph.nodes+ .flatMap((node) => node.observations)+ .map((observation) => observation.properties)+ .filter(+ (properties) =>+ properties.semantic_role === "export-return-shapes" &&+ properties.exported_name === "result",+ );+ assert(+ shapes.length > 0 &&+ shapes.every((shape) => shape.return_shape_coverage?.projection_complete),+ );+ const fields = shapes.flatMap((shape) =>+ shape.static_return_shapes.flatMap((item) => item.fields),+ );+ assert(+ fields.some(+ (field) =>+ field.path === "/clientCompatibility" &&+ field.state === "literal" &&+ field.value === "REA_CLIENT_FIXTURE",+ ),+ );+ assert(+ fields.some(+ (field) =>+ field.path === "/count" &&+ field.state === "literal" &&+ field.value === 7,+ ),+ );+ artifactDigest = createHash("sha256").update(bytes).digest("hex");+ artifactBytes = bytes.length;+ evidenceSeen = true;+ return {+ evidence_id: evidence.evidence_id,+ subject: evidence.subject,+ operation: evidence.operation,+ artifact_sha256: artifactDigest,+ byte_length: artifactBytes,+ shapes,+ };+};+const diagnosticFile = join(lab, "native-doctor.json");+const diagnosticCode =+ "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));";+// Query the saved result through a native agent command. Only the facts needed+// for this analyst question enter context; the complete named-schema Evidence+// and its digest remain available in the native artifact.+const recoveryCode =+ "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));";+const fixture = await createOpenAiModelFixture({+ directory: lab,+ prefix: "grok",+ onRequest: async ({ body, tools }) => {+ // Native title/summary requests cannot execute the analyst workflow.+ if (!tools.some((tool) => tool.name === "search_tool"))+ return { role: "assistant", content: "REA fixture auxiliary response" };+ turn++;+ assert(+ containsText(body.messages, skillDirectory),+ "Grok must discover the isolated installed skill",+ );+ assert(+ !containsText(body.messages, join(homedir(), ".agents", "skills")),+ "Do not inherit real personal skills",+ );+ if (mode === "chat") {+ finalSeen = true;+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };+ }+ const replies = body.messages.filter((message) => message.role === "tool");+ if (!skillRequested) {+ skillRequested = true;+ return call("read_file", {+ target_file: join(skillDirectory, "SKILL.md"),+ });+ }+ assert(+ expectedSkill !== undefined && containsText(replies, expectedSkill),+ "Native read_file must return the full skill with its documented line-number rendering",+ );+ skillLoaded = true;+ const last = replies.at(-1)?.content;+ assert.equal(typeof last, "string");+ if (!catalogChecked) {+ const discovery =+ lastTool === "search_tool" ? JSON.parse(last) : undefined;+ if (discovery?.status === "partial" && !diagnosticsRequested) {+ diagnosticsRequested = true;+ const diagnostic = [+ process.execPath,+ join(runtimeRoot, "scripts/rea.mjs"),+ "doctor",+ "--client",+ "grok_build",+ "--skill",+ "--json",+ ]+ .map(shellQuote)+ .join(" ");+ const projection = [+ process.execPath,+ "-e",+ diagnosticCode,+ diagnosticFile,+ ]+ .map(shellQuote)+ .join(" ");+ return call("run_terminal_command", {+ command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`,+ description:+ "Diagnose REA registration while native MCP discovery is pending",+ });+ }+ if (lastTool === "run_terminal_command") {+ const line = last+ .split("\n")+ .find((item) => item.startsWith('{"rea_client_diagnostic":'));+ assert(+ line,+ "Native REA diagnostics failed or did not return the selected status",+ );+ const diagnostic = JSON.parse(line).rea_client_diagnostic;+ assert.equal(diagnostic.healthy, true);+ assert.equal(diagnostic.skill, "aligned");+ assert(+ diagnostic.registrations.some(+ (registration) =>+ registration.state === "aligned" &&+ registration.config_path === configPath,+ ),+ );+ diagnosticsVerified = true;+ }+ if (discovery?.status !== "ready")+ return call("search_tool", { query: "rea", limit: 255 });+ const inventory = discovery.results+ .filter((result) => result.server === "rea")+ .flatMap((result) => result.tools);+ assert.deepEqual(+ inventory.map((tool) => tool.tool_name).sort(),+ expectedNames,+ "Native search_tool must discover the complete REA catalog",+ );+ for (const tool of inventory) {+ assert(tool.input_schema, `${tool.tool_name}: missing input schema`);+ assert(+ schemaValidator.validateSchema(tool.input_schema),+ `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`,+ );+ schemasChecked++;+ }+ await writeFile(join(lab, "native-discovery.json"), last);+ catalogChecked = true;+ analysisRequested = true;+ return call("use_tool", {+ tool_name: "rea__analyze_javascript_application",+ tool_input: { input_path: target, format: "directory" },+ });+ }+ if (!spillQueryRequested) {+ const marker = "Full output written to: ";+ const start = last.indexOf(marker);+ if (start < 0) {+ verifyEvidence(Buffer.from(last, "utf8"));+ finalSeen = true;+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };+ }+ // The native hint explicitly calls for a query rather than read_file on+ // the saved JSON's very long line. Preserve the original hint in requests.+ const file = last+ .slice(start + marker.length)+ .split(". The full output ")[0];+ assert(isAbsolute(file));+ const childPath = relative(join(profile, "sessions"), file);+ assert(+ childPath !== "" &&+ !childPath.startsWith("..") &&+ !isAbsolute(childPath) &&+ file.endsWith(".json"),+ "Unexpected native MCP spill path",+ );+ retainedArtifact = file;+ expectedProjection = verifyEvidence(await readFile(file));+ spillQueryRequested = true;+ return call("run_terminal_command", {+ command: [process.execPath, "-e", recoveryCode, file]+ .map(shellQuote)+ .join(" "),+ description:+ "Query full saved REA Evidence while preserving artifact identity",+ });+ }+ const line = last+ .split("\n")+ .find((item) => item.startsWith('{"rea_recovered":'));+ assert(+ line,+ "The next model request must contain native artifact query output",+ );+ const projection = JSON.parse(line).rea_recovered;+ assert.deepEqual(+ projection,+ expectedProjection,+ "Native query must recover the observed facts and complete artifact digest",+ );+ projectionSeen = true;+ finalSeen = true;+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };+ },+});+const q = JSON.stringify;+const original = `# Keep unrelated settings and server state.+disabled_mcp_servers = ["rea", "other"]+[cli]+auto_update = false+[features]+telemetry = "off"+[skills]+paths = [${q(join(account, ".agents", "skills"))}]+ignore = [${q(join(homedir(), ".agents", "skills"))}]+[model.rea-client-fixture]+model = "rea-client-fixture"+base_url = ${q(fixture.baseUrl)}+api_key = "local-fixture-key"+api_backend = "chat_completions"+context_window = 1000000+max_completion_tokens = 4096+[mcp_servers.other]+command = "unrelated-server"+enabled = false+`;+await writeFile(configPath, original);+const rea = async (args) =>+ exec(+ process.execPath,+ [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],+ { env: environment, cwd: workspace, timeout: 60_000 },+ );+try {+ clientVersion = (+ await exec(command, ["--version"], { env: environment, timeout: 30_000 })+ ).stdout.trim();+ for (const args of [+ ["setup", "--client", "grok_build", "--dry-run"],+ ["setup", "--client", "grok_build", "--yes"],+ ["doctor", "--client", "grok_build", "--skill"],+ ]) {+ const result = await rea(args);+ await writeFile(+ join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),+ result.stdout,+ );+ const parsed = JSON.parse(result.stdout);+ if (args[0] === "doctor") assert.equal(parsed.healthy, true);+ else {+ assert.equal(+ parsed.status,+ args.includes("--dry-run") ? "planned" : "ready",+ );+ if (args.includes("--dry-run")) {+ assert(+ parsed.plannedActions.some((action) => action.target === configPath),+ );+ assert(+ parsed.plannedActions.some(+ (action) => action.target === skillDirectory,+ ),+ );+ assert(+ parsed.plannedActions.every(+ (action) =>+ action.target === configPath ||+ action.target === skillDirectory ||+ action.target.startsWith(skillDirectory + "/"),+ ),+ "Refuse writes outside the owned profile/skill",+ );+ }+ }+ }+ assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);+ const configured = await readFile(configPath, "utf8");+ assert(+ configured.includes("unrelated-server") &&+ configured.includes("# Keep unrelated settings"),+ );+ const repeated = JSON.parse(+ (await rea(["setup", "--client", "grok_build", "--yes"])).stdout,+ );+ assert.deepEqual(repeated.appliedActions, []);+ assert.equal(await readFile(configPath, "utf8"), configured);+ assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);+ const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8");+ expectedSkill = skill+ .trimEnd()+ .split("\n")+ .map((line, index) =>+ index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line,+ )+ .join("\n");+ const execution = exec(+ command,+ [+ "-p",+ mode === "chat"+ ? "Say hello briefly."+ : `Use the REA skill to analyze ${target}.`,+ "-m",+ "rea-client-fixture",+ "--output-format",+ "streaming-messages-json",+ "--always-approve",+ "--no-subagents",+ ],+ {+ env: environment,+ cwd: workspace,+ timeout: 120_000,+ maxBuffer: 10 * 1024 * 1024,+ },+ );+ execution.child.stdin.end();+ const response = await execution;+ await writeFile(join(lab, `${mode}.stdout`), response.stdout);+ await writeFile(join(lab, `${mode}.stderr`), response.stderr);+ assert.equal(fixture.failure, undefined);+ const events = response.stdout+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line));+ const init = events.find(+ (event) => event.type === "system" && event.subtype === "init",+ );+ assert(init?.mcp_servers?.some((server) => server.name === "rea"));+ assert(+ init.mcp_servers.every((server) => ["rea", "other"].includes(server.name)),+ "Do not inherit foreign MCP configuration",+ );+ assert(+ events.some(+ (event) =>+ event.type === "result" &&+ event.subtype === "success" &&+ event.is_error === false &&+ event.result === "REA_CLIENT_COMPATIBILITY_OK",+ ),+ );+ assert(finalSeen);+ if (mode === "call")+ assert(+ skillLoaded &&+ catalogChecked &&+ analysisRequested &&+ evidenceSeen &&+ (!spillQueryRequested || projectionSeen) &&+ (!diagnosticsRequested || diagnosticsVerified),+ );+} catch (cause) {+ failure = cause;+ if (typeof cause?.stdout === "string")+ await writeFile(join(lab, `${mode}.stdout`), cause.stdout);+ if (typeof cause?.stderr === "string")+ await writeFile(join(lab, `${mode}.stderr`), cause.stderr);+} finally {+ await fixture.close();+ const lineage = await completeVerifierRun(verifier);+ const receipt = {+ client: "Grok Build",+ clientVersion,+ mode,+ passed: failure === undefined,+ artifacts: lab,+ runtimeRoot,+ host: {+ platform: process.platform,+ architecture: process.arch,+ node: process.versions.node,+ },+ modelFixture:+ "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim",+ discovery:+ "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified",+ interface:+ "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal",+ catalogChecked,+ catalogSize: catalogChecked ? expectedNames.length : 0,+ schemasChecked,+ skillLoaded,+ diagnosticsRequested,+ diagnosticsVerified,+ analysisRequested,+ evidenceSeen,+ spillQueryRequested,+ projectionSeen,+ retainedArtifact,+ artifactBytes,+ artifactDigest,+ finalSeen,+ evidenceScope:+ "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact",+ requests: fixture.requests,+ probes: fixture.probes,+ fixtureFailure: fixture.failure,+ verifier: lineage,+ };+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));+ console.log(JSON.stringify(receipt));+}+if (failure !== undefined) throw failure;