Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(clients): verify native Grok Build workflows (#1571)

Ysopwnableefd81f977604
diff --git a/docs/testing.md b/docs/testing.md
index 22e409afa..61440dfb1 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -167,6 +167,26 @@ fix for that client/model limit. It preserves the complete catalog and schemas.
Live model APIs, native Windows and default OS-home skill discovery remain
unverified.
+`verify:grok-client` requires installed Grok Build (verified with the official
+Linux x64 1.0.50 binary); select it with `REA_VERIFY_GROK_COMMAND`. This optional
+POSIX lane isolates `GROK_HOME`, an independent Git workspace and additional
+skill roots, disables foreign configuration discovery, guards setup writes,
+preserves unrelated registrations/backups and checks idempotence. Call mode
+loads the full skill through native `read_file`, queries all REA names and input
+schemas through native `search_tool`, and calls analysis through `use_tool`.
+If discovery reports `partial`, the native agent diagnoses REA registration and
+retries discovery. No startup-timeout override or fixed readiness delay is used.
+A larger Unicode-path fixture exercises native result offloading: the full
+retained Evidence is validated against its named schema, then native terminal
+queries recover the selected export facts, subject and full artifact digest in
+the next model request. This verifies useful artifact recovery without claiming
+that every offloaded graph fact enters model context. Native line-number and
+truncation envelopes remain in the request artifacts. Use `-- chat` for ordinary
+chat, or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed package.
+The loopback OpenAI completions/SSE custom model declares a one-million-token
+context and synthetic usage. Live xAI APIs, native Windows and default OS-home
+skill discovery remain unverified.
+
`verify:omp-client` requires installed OMP (verified with the official Linux x64
18.8.7 binary); select it with `REA_VERIFY_OMP_COMMAND`. This optional POSIX lane
isolates the default agent, global config and XDG roots, guards setup targets,
diff --git a/package.json b/package.json
index 9e59d0c6e..ed36813a8 100644
--- a/package.json
+++ b/package.json
@@ -182,6 +182,7 @@
"verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",
"verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs",
"verify:opencode-client": "npm run build:cached && node scripts/verify-opencode-client.mjs",
+ "verify:grok-client": "npm run build:cached && node scripts/verify-grok-client.mjs",
"verify:omp-client": "npm run build:cached && node scripts/verify-omp-client.mjs",
"verify:copilot-client": "npm run build:cached && node scripts/verify-copilot-client.mjs",
"verify:claude-client": "npm run build:cached && node scripts/verify-claude-client.mjs",
diff --git a/scripts/verify-grok-client.mjs b/scripts/verify-grok-client.mjs
new file mode 100644
index 000000000..e1c066770
--- /dev/null
+++ b/scripts/verify-grok-client.mjs
@@ -0,0 +1,539 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+import { isAbsolute, join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
+import Ajv2020 from "ajv/dist/2020.js";
+import {
+ TOOL_CONTRACTS,
+ toolContract,
+} from "../dist/contracts/toolContracts.js";
+import { exec } from "./lib/verify-package-core.mjs";
+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
+import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
+
+assert(
+ process.platform !== "win32",
+ "Native Grok verifier requires POSIX; Windows is unverified",
+);
+const repo = fileURLToPath(new URL("..", import.meta.url));
+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
+const command = process.env.REA_VERIFY_GROK_COMMAND ?? "grok";
+const mode = process.argv[2] ?? "call";
+assert(
+ ["chat", "call"].includes(mode),
+ "Usage: verify-grok-client.mjs [chat|call]",
+);
+const verifier = createVerifierRun();
+const lab = await mkdtemp(join(tmpdir(), "rea-grok-client-"));
+const account = join(lab, "account");
+const profile = join(lab, "selected profile");
+const workspace = join(lab, "workspace");
+const configPath = join(profile, "config.toml");
+const skillDirectory = join(
+ account,
+ ".agents",
+ "skills",
+ "reverse-engineer-anything",
+);
+await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]);
+await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
+// Native compatibility discovery is independent of GROK_HOME. Disable those
+// sources and ignore the real shared skill root instead of changing HOME.
+const environment = {
+ PATH: process.env.PATH,
+ NODE_OPTIONS: process.env.NODE_OPTIONS,
+ USERPROFILE: account,
+ GROK_HOME: profile,
+ GROK_DISABLE_AUTOUPDATER: "1",
+ ...Object.fromEntries(
+ ["CLAUDE", "CURSOR"].flatMap((vendor) =>
+ ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map((cell) => [
+ `GROK_${vendor}_${cell}_ENABLED`,
+ "false",
+ ]),
+ ),
+ ),
+ XDG_DATA_HOME: join(lab, "data"),
+ XDG_CONFIG_HOME: join(lab, "config"),
+ XDG_CACHE_HOME: join(lab, "cache"),
+ REA_PROCESS_RUN_ID: verifier.run_id,
+};
+const target = join(lab, "fixture α测试");
+await mkdir(target);
+await writeFile(
+ join(target, "app.js"),
+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' +
+ // A real analysis result large enough to exercise native MCP offloading.
+ Array.from(
+ { length: 60 },
+ (_, index) =>
+ `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`,
+ ).join("\n"),
+);
+const expectedNames = TOOL_CONTRACTS.map((tool) => "rea__" + tool.name).sort();
+assert(
+ expectedNames.length <= 255,
+ "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools",
+);
+const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
+const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'";
+const containsText = (value, text) => {
+ if (typeof value === "string") return value.includes(text);
+ return (
+ value !== null &&
+ typeof value === "object" &&
+ Object.values(value).some((item) => containsText(item, text))
+ );
+};
+let clientVersion;
+let expectedSkill;
+let turn = 0;
+let skillRequested = false;
+let skillLoaded = false;
+let diagnosticsRequested = false;
+let diagnosticsVerified = false;
+let lastTool;
+let catalogChecked = false;
+let schemasChecked = 0;
+let analysisRequested = false;
+let evidenceSeen = false;
+let spillQueryRequested = false;
+let projectionSeen = false;
+let artifactDigest;
+let artifactBytes;
+let retainedArtifact;
+let expectedProjection;
+let finalSeen = false;
+let failure;
+const call = (name, args) => {
+ lastTool = name;
+ return {
+ role: "assistant",
+ content: null,
+ tool_calls: [
+ {
+ id: `call_grok_${turn}`,
+ type: "function",
+ function: { name, arguments: JSON.stringify(args) },
+ },
+ ],
+ };
+};
+const verifyEvidence = (bytes) => {
+ const evidence = toolContract(
+ "analyze_javascript_application",
+ ).outputSchema.parse(JSON.parse(bytes.toString("utf8")));
+ assert.equal(evidence.subject.local_path, target);
+ const shapes = evidence.normalized_result.graph.nodes
+ .flatMap((node) => node.observations)
+ .map((observation) => observation.properties)
+ .filter(
+ (properties) =>
+ properties.semantic_role === "export-return-shapes" &&
+ properties.exported_name === "result",
+ );
+ assert(
+ shapes.length > 0 &&
+ shapes.every((shape) => shape.return_shape_coverage?.projection_complete),
+ );
+ const fields = shapes.flatMap((shape) =>
+ shape.static_return_shapes.flatMap((item) => item.fields),
+ );
+ assert(
+ fields.some(
+ (field) =>
+ field.path === "/clientCompatibility" &&
+ field.state === "literal" &&
+ field.value === "REA_CLIENT_FIXTURE",
+ ),
+ );
+ assert(
+ fields.some(
+ (field) =>
+ field.path === "/count" &&
+ field.state === "literal" &&
+ field.value === 7,
+ ),
+ );
+ artifactDigest = createHash("sha256").update(bytes).digest("hex");
+ artifactBytes = bytes.length;
+ evidenceSeen = true;
+ return {
+ evidence_id: evidence.evidence_id,
+ subject: evidence.subject,
+ operation: evidence.operation,
+ artifact_sha256: artifactDigest,
+ byte_length: artifactBytes,
+ shapes,
+ };
+};
+const diagnosticFile = join(lab, "native-doctor.json");
+const diagnosticCode =
+ "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));";
+// Query the saved result through a native agent command. Only the facts needed
+// for this analyst question enter context; the complete named-schema Evidence
+// and its digest remain available in the native artifact.
+const recoveryCode =
+ "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));";
+const fixture = await createOpenAiModelFixture({
+ directory: lab,
+ prefix: "grok",
+ onRequest: async ({ body, tools }) => {
+ // Native title/summary requests cannot execute the analyst workflow.
+ if (!tools.some((tool) => tool.name === "search_tool"))
+ return { role: "assistant", content: "REA fixture auxiliary response" };
+ turn++;
+ assert(
+ containsText(body.messages, skillDirectory),
+ "Grok must discover the isolated installed skill",
+ );
+ assert(
+ !containsText(body.messages, join(homedir(), ".agents", "skills")),
+ "Do not inherit real personal skills",
+ );
+ if (mode === "chat") {
+ finalSeen = true;
+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
+ }
+ const replies = body.messages.filter((message) => message.role === "tool");
+ if (!skillRequested) {
+ skillRequested = true;
+ return call("read_file", {
+ target_file: join(skillDirectory, "SKILL.md"),
+ });
+ }
+ assert(
+ expectedSkill !== undefined && containsText(replies, expectedSkill),
+ "Native read_file must return the full skill with its documented line-number rendering",
+ );
+ skillLoaded = true;
+ const last = replies.at(-1)?.content;
+ assert.equal(typeof last, "string");
+ if (!catalogChecked) {
+ const discovery =
+ lastTool === "search_tool" ? JSON.parse(last) : undefined;
+ if (discovery?.status === "partial" && !diagnosticsRequested) {
+ diagnosticsRequested = true;
+ const diagnostic = [
+ process.execPath,
+ join(runtimeRoot, "scripts/rea.mjs"),
+ "doctor",
+ "--client",
+ "grok_build",
+ "--skill",
+ "--json",
+ ]
+ .map(shellQuote)
+ .join(" ");
+ const projection = [
+ process.execPath,
+ "-e",
+ diagnosticCode,
+ diagnosticFile,
+ ]
+ .map(shellQuote)
+ .join(" ");
+ return call("run_terminal_command", {
+ command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`,
+ description:
+ "Diagnose REA registration while native MCP discovery is pending",
+ });
+ }
+ if (lastTool === "run_terminal_command") {
+ const line = last
+ .split("\n")
+ .find((item) => item.startsWith('{"rea_client_diagnostic":'));
+ assert(
+ line,
+ "Native REA diagnostics failed or did not return the selected status",
+ );
+ const diagnostic = JSON.parse(line).rea_client_diagnostic;
+ assert.equal(diagnostic.healthy, true);
+ assert.equal(diagnostic.skill, "aligned");
+ assert(
+ diagnostic.registrations.some(
+ (registration) =>
+ registration.state === "aligned" &&
+ registration.config_path === configPath,
+ ),
+ );
+ diagnosticsVerified = true;
+ }
+ if (discovery?.status !== "ready")
+ return call("search_tool", { query: "rea", limit: 255 });
+ const inventory = discovery.results
+ .filter((result) => result.server === "rea")
+ .flatMap((result) => result.tools);
+ assert.deepEqual(
+ inventory.map((tool) => tool.tool_name).sort(),
+ expectedNames,
+ "Native search_tool must discover the complete REA catalog",
+ );
+ for (const tool of inventory) {
+ assert(tool.input_schema, `${tool.tool_name}: missing input schema`);
+ assert(
+ schemaValidator.validateSchema(tool.input_schema),
+ `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`,
+ );
+ schemasChecked++;
+ }
+ await writeFile(join(lab, "native-discovery.json"), last);
+ catalogChecked = true;
+ analysisRequested = true;
+ return call("use_tool", {
+ tool_name: "rea__analyze_javascript_application",
+ tool_input: { input_path: target, format: "directory" },
+ });
+ }
+ if (!spillQueryRequested) {
+ const marker = "Full output written to: ";
+ const start = last.indexOf(marker);
+ if (start < 0) {
+ verifyEvidence(Buffer.from(last, "utf8"));
+ finalSeen = true;
+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
+ }
+ // The native hint explicitly calls for a query rather than read_file on
+ // the saved JSON's very long line. Preserve the original hint in requests.
+ const file = last
+ .slice(start + marker.length)
+ .split(". The full output ")[0];
+ assert(isAbsolute(file));
+ const childPath = relative(join(profile, "sessions"), file);
+ assert(
+ childPath !== "" &&
+ !childPath.startsWith("..") &&
+ !isAbsolute(childPath) &&
+ file.endsWith(".json"),
+ "Unexpected native MCP spill path",
+ );
+ retainedArtifact = file;
+ expectedProjection = verifyEvidence(await readFile(file));
+ spillQueryRequested = true;
+ return call("run_terminal_command", {
+ command: [process.execPath, "-e", recoveryCode, file]
+ .map(shellQuote)
+ .join(" "),
+ description:
+ "Query full saved REA Evidence while preserving artifact identity",
+ });
+ }
+ const line = last
+ .split("\n")
+ .find((item) => item.startsWith('{"rea_recovered":'));
+ assert(
+ line,
+ "The next model request must contain native artifact query output",
+ );
+ const projection = JSON.parse(line).rea_recovered;
+ assert.deepEqual(
+ projection,
+ expectedProjection,
+ "Native query must recover the observed facts and complete artifact digest",
+ );
+ projectionSeen = true;
+ finalSeen = true;
+ return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
+ },
+});
+const q = JSON.stringify;
+const original = `# Keep unrelated settings and server state.
+disabled_mcp_servers = ["rea", "other"]
+[cli]
+auto_update = false
+[features]
+telemetry = "off"
+[skills]
+paths = [${q(join(account, ".agents", "skills"))}]
+ignore = [${q(join(homedir(), ".agents", "skills"))}]
+[model.rea-client-fixture]
+model = "rea-client-fixture"
+base_url = ${q(fixture.baseUrl)}
+api_key = "local-fixture-key"
+api_backend = "chat_completions"
+context_window = 1000000
+max_completion_tokens = 4096
+[mcp_servers.other]
+command = "unrelated-server"
+enabled = false
+`;
+await writeFile(configPath, original);
+const rea = async (args) =>
+ exec(
+ process.execPath,
+ [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
+ { env: environment, cwd: workspace, timeout: 60_000 },
+ );
+try {
+ clientVersion = (
+ await exec(command, ["--version"], { env: environment, timeout: 30_000 })
+ ).stdout.trim();
+ for (const args of [
+ ["setup", "--client", "grok_build", "--dry-run"],
+ ["setup", "--client", "grok_build", "--yes"],
+ ["doctor", "--client", "grok_build", "--skill"],
+ ]) {
+ const result = await rea(args);
+ await writeFile(
+ join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
+ result.stdout,
+ );
+ const parsed = JSON.parse(result.stdout);
+ if (args[0] === "doctor") assert.equal(parsed.healthy, true);
+ else {
+ assert.equal(
+ parsed.status,
+ args.includes("--dry-run") ? "planned" : "ready",
+ );
+ if (args.includes("--dry-run")) {
+ assert(
+ parsed.plannedActions.some((action) => action.target === configPath),
+ );
+ assert(
+ parsed.plannedActions.some(
+ (action) => action.target === skillDirectory,
+ ),
+ );
+ assert(
+ parsed.plannedActions.every(
+ (action) =>
+ action.target === configPath ||
+ action.target === skillDirectory ||
+ action.target.startsWith(skillDirectory + "/"),
+ ),
+ "Refuse writes outside the owned profile/skill",
+ );
+ }
+ }
+ }
+ assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+ const configured = await readFile(configPath, "utf8");
+ assert(
+ configured.includes("unrelated-server") &&
+ configured.includes("# Keep unrelated settings"),
+ );
+ const repeated = JSON.parse(
+ (await rea(["setup", "--client", "grok_build", "--yes"])).stdout,
+ );
+ assert.deepEqual(repeated.appliedActions, []);
+ assert.equal(await readFile(configPath, "utf8"), configured);
+ assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
+ const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8");
+ expectedSkill = skill
+ .trimEnd()
+ .split("\n")
+ .map((line, index) =>
+ index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line,
+ )
+ .join("\n");
+ const execution = exec(
+ command,
+ [
+ "-p",
+ mode === "chat"
+ ? "Say hello briefly."
+ : `Use the REA skill to analyze ${target}.`,
+ "-m",
+ "rea-client-fixture",
+ "--output-format",
+ "streaming-messages-json",
+ "--always-approve",
+ "--no-subagents",
+ ],
+ {
+ env: environment,
+ cwd: workspace,
+ timeout: 120_000,
+ maxBuffer: 10 * 1024 * 1024,
+ },
+ );
+ execution.child.stdin.end();
+ const response = await execution;
+ await writeFile(join(lab, `${mode}.stdout`), response.stdout);
+ await writeFile(join(lab, `${mode}.stderr`), response.stderr);
+ assert.equal(fixture.failure, undefined);
+ const events = response.stdout
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ const init = events.find(
+ (event) => event.type === "system" && event.subtype === "init",
+ );
+ assert(init?.mcp_servers?.some((server) => server.name === "rea"));
+ assert(
+ init.mcp_servers.every((server) => ["rea", "other"].includes(server.name)),
+ "Do not inherit foreign MCP configuration",
+ );
+ assert(
+ events.some(
+ (event) =>
+ event.type === "result" &&
+ event.subtype === "success" &&
+ event.is_error === false &&
+ event.result === "REA_CLIENT_COMPATIBILITY_OK",
+ ),
+ );
+ assert(finalSeen);
+ if (mode === "call")
+ assert(
+ skillLoaded &&
+ catalogChecked &&
+ analysisRequested &&
+ evidenceSeen &&
+ (!spillQueryRequested || projectionSeen) &&
+ (!diagnosticsRequested || diagnosticsVerified),
+ );
+} catch (cause) {
+ failure = cause;
+ if (typeof cause?.stdout === "string")
+ await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
+ if (typeof cause?.stderr === "string")
+ await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
+} finally {
+ await fixture.close();
+ const lineage = await completeVerifierRun(verifier);
+ const receipt = {
+ client: "Grok Build",
+ clientVersion,
+ mode,
+ passed: failure === undefined,
+ artifacts: lab,
+ runtimeRoot,
+ host: {
+ platform: process.platform,
+ architecture: process.arch,
+ node: process.versions.node,
+ },
+ modelFixture:
+ "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim",
+ discovery:
+ "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified",
+ interface:
+ "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal",
+ catalogChecked,
+ catalogSize: catalogChecked ? expectedNames.length : 0,
+ schemasChecked,
+ skillLoaded,
+ diagnosticsRequested,
+ diagnosticsVerified,
+ analysisRequested,
+ evidenceSeen,
+ spillQueryRequested,
+ projectionSeen,
+ retainedArtifact,
+ artifactBytes,
+ artifactDigest,
+ finalSeen,
+ evidenceScope:
+ "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact",
+ requests: fixture.requests,
+ probes: fixture.probes,
+ fixtureFailure: fixture.failure,
+ verifier: lineage,
+ };
+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
+ console.log(JSON.stringify(receipt));
+}
+if (failure !== undefined) throw failure;