Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(clients): verify native Gemini CLI integration (#1541)
Ysopwnableb9c439831d70
diff --git a/docs/testing.md b/docs/testing.mdindex 031baa824..e90436112 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -105,6 +105,19 @@ home; this does not prove their default home-directory discovery.Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection ofa named sticky profile with `HERMES_HOME` still pointing to its root.+`verify:gemini-client` requires an installed Gemini CLI (verified with+`@google/[email protected]`); select it with `REA_VERIFY_GEMINI_COMMAND`.+The optional POSIX lane uses native `GEMINI_CLI_HOME` discovery in an isolated+Git project, checks setup plans, backups and idempotence, activates the installed+personal skill, validates all forwarded input JSON Schemas against their declared+dialect, forwards the complete REA catalog and checks full JavaScript+Evidence for a Unicode path. Use `-- chat` for ordinary chat with REA enabled,+or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed REA package.+The local fixture exercises the real Gemini API adapter, including its+`untrusted_context` tool-result envelope. Token counts are synthetic; live+Google API and native Windows compatibility remain unverified. The lane disables+the client's memory-based relaunch to preserve the caller's Node heap budget.+`verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`;verified with `@deepseek-ai/[email protected]`) and Git. Set`REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lanediff --git a/package.json b/package.jsonindex 663055d64..cf60ba460 100644--- a/package.json+++ b/package.json@@ -180,6 +180,7 @@"verify:package": "npm run build:cached && node scripts/verify-package.mjs","verify:package:windows": "npm run build:cached && node scripts/verify-windows-package.mjs","verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",+ "verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs","verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi","verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes","verify:deepseek-client": "npm run build:cached && node scripts/verify-deepseek-client.mjs",diff --git a/scripts/verify-gemini-client.mjs b/scripts/verify-gemini-client.mjsnew file mode 100644index 000000000..c2d940077--- /dev/null+++ b/scripts/verify-gemini-client.mjs@@ -0,0 +1,378 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import Ajv2020 from "ajv/dist/2020.js";+import {+ TOOL_CONTRACTS,+ toolContract,+} from "../dist/contracts/toolContracts.js";+import { exec } from "./lib/verify-package-core.mjs";+import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";+import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs";++assert(+ process.platform !== "win32",+ "Native Gemini verifier requires POSIX; Windows is unverified",+);+const repo = fileURLToPath(new URL("..", import.meta.url));+const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;+const command = process.env.REA_VERIFY_GEMINI_COMMAND ?? "gemini";+const mode = process.argv[2] ?? "call";+assert(+ ["chat", "call"].includes(mode),+ "Usage: verify-gemini-client.mjs [chat|call]",+);+const verifier = createVerifierRun();+const lab = await mkdtemp(join(tmpdir(), "rea-gemini-client-"));+const account = join(lab, "account");+const profile = join(lab, "selected profile");+const workspace = join(lab, "workspace");+const configPath = join(profile, ".gemini", "settings.json");+const skillPath = join(+ profile,+ ".agents",+ "skills",+ "reverse-engineer-anything",+ "SKILL.md",+);+await Promise.all([+ mkdir(account),+ mkdir(workspace),+ mkdir(join(profile, ".gemini"), { recursive: true }),+]);+await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });+await writeFile(join(lab, "system-settings.json"), "{}\n");+await writeFile(join(lab, "system-defaults.json"), "{}\n");+const environment = {+ PATH: process.env.PATH ?? "",+ USERPROFILE: account,+ GEMINI_CLI_HOME: profile,+ GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"),+ GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"),+ // Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing.+ GEMINI_CLI_NO_RELAUNCH: "1",+ GEMINI_CLI_TRUST_WORKSPACE: "true",+ REA_PROCESS_RUN_ID: verifier.run_id,+};+let clientVersion;+try {+ clientVersion = (+ await exec(command, ["--version"], { env: environment, timeout: 30_000 })+ ).stdout.trim();+} catch (cause) {+ throw new Error(+ `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND. ${String(cause)}`,+ );+}+const original = JSON.stringify(+ {+ security: {+ auth: { selectedType: "gemini-api-key" },+ folderTrust: { enabled: false },+ },+ telemetry: { enabled: false },+ advanced: { autoConfigureMemory: false },+ model: { name: "gemini-2.5-flash", maxSessionTurns: 8 },+ mcpServers: { other: { command: "unrelated-server" } },+ },+ null,+ 2,+);+await writeFile(configPath, original);+for (const args of [+ ["setup", "--client", "gemini_cli", "--dry-run"],+ ["setup", "--client", "gemini_cli", "--yes"],+ ["doctor", "--client", "gemini_cli", "--skill"],+]) {+ const result = await exec(+ process.execPath,+ [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],+ { env: environment, cwd: workspace, timeout: 60_000 },+ );+ const parsed = JSON.parse(result.stdout);+ if (args[0] === "doctor") assert.equal(parsed.healthy, true);+ else {+ assert.equal(+ parsed.status,+ args.includes("--dry-run") ? "planned" : "ready",+ );+ if (args.includes("--dry-run")) {+ assert(parsed.plannedActions.some((a) => a.target === configPath));+ assert(+ parsed.plannedActions.some((a) =>+ a.target.startsWith(join(profile, ".agents", "skills") + "/"),+ ),+ );+ assert(+ parsed.plannedActions.every((a) => a.target.startsWith(profile + "/")),+ );+ }+ }+ await writeFile(+ join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),+ result.stdout,+ );+}+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);+const configured = await readFile(configPath, "utf8");+assert.deepEqual(JSON.parse(configured).mcpServers.other, {+ command: "unrelated-server",+});+const repeated = await exec(+ process.execPath,+ [+ join(runtimeRoot, "scripts/rea.mjs"),+ "setup",+ "--client",+ "gemini_cli",+ "--yes",+ "--json",+ ],+ { env: environment, cwd: workspace, timeout: 60_000 },+);+assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);+assert.equal(await readFile(configPath, "utf8"), configured);+assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);+assert(+ (await readFile(skillPath, "utf8")).includes(+ "name: reverse-engineer-anything",+ ),+);+const target = join(lab, "fixture α测试");+await mkdir(target);+await writeFile(+ join(target, "app.js"),+ 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',+);+let evidenceSeen = false;+let skillLoaded = false;+let skillRequested = false;+let finalSeen = false;+let catalogSize = 0;+let schemasChecked = 0;+let artifactDigest;+const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });+const extractEvidence = (value) => {+ if (typeof value === "string") {+ if (+ value.startsWith("<untrusted_context>\n") &&+ value.endsWith("\n</untrusted_context>")+ ) {+ return extractEvidence(+ value.slice(+ "<untrusted_context>\n".length,+ -"\n</untrusted_context>".length,+ ),+ );+ }+ try {+ return extractEvidence(JSON.parse(value));+ } catch {+ return undefined;+ }+ }+ if (value && typeof value === "object") {+ if (+ typeof value.evidence_id === "string" &&+ value.normalized_result?.graph?.nodes?.length > 0+ )+ return value;+ for (const nested of Object.values(value)) {+ const evidence = extractEvidence(nested);+ if (evidence) return evidence;+ }+ }+ return undefined;+};+const fixture = await createGeminiModelFixture({+ directory: lab,+ onRequest: async ({ body, tools }) => {+ const analysis = tools.find((t) =>+ t.name.endsWith("analyze_javascript_application"),+ );+ assert(analysis, "Gemini must expose the REA analysis tool");+ const prefix = analysis.name.slice(+ 0,+ -"analyze_javascript_application".length,+ );+ const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort();+ assert.deepEqual(+ tools+ .filter((t) => expected.includes(t.name))+ .map((t) => t.name)+ .sort(),+ expected,+ "Gemini must forward the complete REA tool catalog",+ );+ catalogSize = expected.length;+ if (schemasChecked === 0) {+ for (const tool of tools.filter((t) => expected.includes(t.name))) {+ assert(+ tool.parametersJsonSchema,+ `${tool.name}: missing advertised JSON Schema`,+ );+ assert(+ schemaValidator.validateSchema(tool.parametersJsonSchema),+ `${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`,+ );+ schemasChecked++;+ }+ }+ const responses = body.contents+ .flatMap((c) => c.parts ?? [])+ .filter((p) => p.functionResponse)+ .map((p) => p.functionResponse);+ skillLoaded ||= responses.some(+ (r) =>+ r.name === "activate_skill" &&+ r.response?.output?.includes(+ `<activated_skill name="reverse-engineer-anything">`,+ ) &&+ r.response.output.includes(+ join(profile, ".agents", "skills", "reverse-engineer-anything"),+ ),+ );+ const observed = extractEvidence(responses);+ if (observed) {+ const evidence = toolContract(+ "analyze_javascript_application",+ ).outputSchema.parse(observed);+ assert.equal(evidence.subject.local_path, target);+ const shape = evidence.normalized_result.graph.nodes+ .flatMap((n) => n.observations)+ .map((o) => o.properties)+ .find(+ (p) =>+ p.semantic_role === "export-return-shapes" &&+ p.exported_name === "result",+ );+ assert(shape?.return_shape_coverage?.projection_complete);+ const fields = shape.static_return_shapes.flatMap((s) => s.fields);+ assert(+ fields.some(+ (f) =>+ f.path === "/clientCompatibility" &&+ f.state === "literal" &&+ f.value === "REA_CLIENT_FIXTURE",+ ),+ );+ assert(+ fields.some(+ (f) => f.path === "/count" && f.state === "literal" && f.value === 7,+ ),+ );+ const bytes = JSON.stringify(evidence);+ artifactDigest = createHash("sha256").update(bytes).digest("hex");+ await writeFile(join(lab, "gemini-evidence.json"), bytes);+ evidenceSeen = true;+ }+ if (mode === "chat" || evidenceSeen) {+ finalSeen = true;+ return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }];+ }+ if (!skillRequested) {+ const skill = tools.find((t) => t.name === "activate_skill");+ assert(skill, "Gemini must expose its native skill activation tool");+ skillRequested = true;+ return [+ {+ functionCall: {+ name: skill.name,+ args: { name: "reverse-engineer-anything" },+ },+ },+ ];+ }+ assert(skillLoaded, "Gemini must load the installed personal REA skill");+ return [+ {+ functionCall: {+ name: analysis.name,+ args: { input_path: target, format: "directory" },+ },+ },+ ];+ },+});+let failure;+try {+ const response = await exec(+ command,+ [+ "--model",+ "gemini-2.5-flash",+ "--approval-mode",+ mode === "chat" ? "default" : "yolo",+ "--allowed-mcp-server-names",+ "rea",+ "--output-format",+ "stream-json",+ "-p",+ mode === "chat"+ ? "Say hello briefly."+ : `Use REA to analyze the JavaScript directory ${target}. Report the result.`,+ ],+ {+ env: {+ ...environment,+ GEMINI_API_KEY: "local-fixture-key",+ GOOGLE_GEMINI_BASE_URL: fixture.baseUrl,+ },+ cwd: workspace,+ timeout: 120_000,+ maxBuffer: 10 * 1024 * 1024,+ },+ );+ await writeFile(join(lab, `${mode}.stdout`), response.stdout);+ await writeFile(join(lab, `${mode}.stderr`), response.stderr);+ const events = response.stdout+ .split("\n")+ .filter(Boolean)+ .map((line) => JSON.parse(line));+ assert.equal(events.findLast((e) => e.type === "result")?.status, "success");+ assert.equal(fixture.failure, undefined);+ assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));+ if (mode === "call") assert(evidenceSeen && skillLoaded);+} catch (cause) {+ failure = cause;+ if (typeof cause?.stdout === "string")+ await writeFile(join(lab, `${mode}.stdout`), cause.stdout);+ if (typeof cause?.stderr === "string")+ await writeFile(join(lab, `${mode}.stderr`), cause.stderr);+} finally {+ await fixture.close();+ const lineage = await completeVerifierRun(verifier);+ const receipt = {+ client: "Gemini CLI",+ clientVersion,+ mode,+ passed: failure === undefined,+ artifacts: lab,+ runtimeRoot,+ host: {+ platform: process.platform,+ architecture: process.arch,+ node: process.versions.node,+ },+ modelFixture:+ "loopback native Gemini API adapter; token counts synthetic; no live Google API claim",+ skillDiscovery:+ "native GEMINI_CLI_HOME personal shared directory; isolated Git project",+ catalogSize,+ schemasChecked,+ evidenceSeen,+ skillLoaded,+ artifactDigest,+ finalSeen,+ requests: fixture.requests,+ probes: fixture.probes,+ fixtureFailure: fixture.failure,+ verifier: lineage,+ };+ await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));+ console.log(JSON.stringify(receipt));+}+if (failure !== undefined) throw failure;diff --git a/scripts/verify/clients/gemini-model-fixture.mjs b/scripts/verify/clients/gemini-model-fixture.mjsnew file mode 100644index 000000000..58b21302d--- /dev/null+++ b/scripts/verify/clients/gemini-model-fixture.mjs@@ -0,0 +1,104 @@+import assert from "node:assert/strict";+import { createServer } from "node:http";+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++/** Exercise the native Gemini API adapter against a local model, without cloud claims. */+export const createGeminiModelFixture = async ({ directory, onRequest }) => {+ const requests = [];+ const probes = [];+ let incoming = 0;+ let failure;+ const server = createServer(async (request, response) => {+ try {+ const chunks = [];+ let bytes = 0;+ for await (const chunk of request) {+ bytes += chunk.length;+ assert(+ bytes <= 10 * 1024 * 1024,+ "Model fixture request exceeds 10 MiB budget",+ );+ chunks.push(chunk);+ }+ const raw = Buffer.concat(chunks).toString("utf8");+ const body = raw.length === 0 ? null : JSON.parse(raw);+ await writeFile(+ join(directory, `gemini-incoming-${++incoming}.json`),+ JSON.stringify({ method: request.method, path: request.url, body }),+ );+ const path = new URL(request.url, "http://localhost").pathname;+ if (path.endsWith(":countTokens")) {+ probes.push({ path, kind: "synthetic-token-count" });+ response.setHeader("Content-Type", "application/json");+ response.end(JSON.stringify({ totalTokens: 100 }));+ return;+ }+ if (+ !/\/models\/[^/]+:(?:streamGenerateContent|generateContent)$/u.test(+ path,+ )+ ) {+ probes.push({ path, status: 404 });+ response.statusCode = 404;+ response.end(+ JSON.stringify({+ error: { message: "Unsupported fixture endpoint" },+ }),+ );+ return;+ }+ assert.equal(request.method, "POST");+ assert(body && typeof body === "object" && !Array.isArray(body));+ assert(Array.isArray(body.contents));+ const tools = (body.tools ?? []).flatMap(+ (t) => t.functionDeclarations ?? [],+ );+ requests.push({+ path,+ tools: tools.map((t) => t.name),+ contentRoles: body.contents.map((c) => c.role),+ });+ const parts = await onRequest({+ body,+ tools,+ requestIndex: requests.length,+ });+ const result = {+ candidates: [+ { index: 0, content: { role: "model", parts }, finishReason: "STOP" },+ ],+ usageMetadata: {+ promptTokenCount: 100,+ candidatesTokenCount: 20,+ totalTokenCount: 120,+ },+ modelVersion: "local-rea-fixture",+ };+ if (path.endsWith(":streamGenerateContent")) {+ response.setHeader("Content-Type", "text/event-stream");+ response.end(`data: ${JSON.stringify(result)}\n\n`);+ } else {+ response.setHeader("Content-Type", "application/json");+ response.end(JSON.stringify(result));+ }+ } catch (cause) {+ failure = String(cause);+ response.statusCode = 400;+ response.setHeader("Content-Type", "application/json");+ response.end(JSON.stringify({ error: { message: failure, code: 400 } }));+ }+ });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));+ const address = server.address();+ assert(address !== null && typeof address !== "string");+ return {+ baseUrl: `http://127.0.0.1:${address.port}`,+ requests,+ probes,+ get failure() {+ return failure;+ },+ close: () => new Promise((resolve) => server.close(resolve)),+ };+};