π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
test(metaharness): GCP-secret Γ scaffold Γ lifecycle Γ OpenRouter e2e (iter 26)
ruvf1331e6db469
diff --git a/plugins/ruflo-metaharness/scripts/smoke.sh b/plugins/ruflo-metaharness/scripts/smoke.shindex 40e2d00ec0..2400fed640 100755--- a/plugins/ruflo-metaharness/scripts/smoke.sh+++ b/plugins/ruflo-metaharness/scripts/smoke.sh@@ -170,6 +170,27 @@ if (!od.metaharness) { console.error('missing metaharness in optionalDependencieif (j.dependencies && j.dependencies.metaharness) { console.error('metaharness leaked into dependencies'); process.exit(1); }" 2>/dev/null && ok || bad "ruflo wrapper missing metaharness optionalDep"+step "17q. test-with-openrouter β GCP-secret Γ scaffold Γ lifecycle e2e (iter 26)"+F="$ROOT/scripts/test-with-openrouter.mjs"+miss=""+[[ -x "$F" ]] || miss="$miss not-executable"+node --check "$F" 2>/dev/null || miss="$miss syntax-error"+# Pulls the secret from GCP Secret Manager (not from env file)+grep -q "gcloud secrets versions access" "$F" || miss="$miss no-gcp-fetch"+grep -q "OPENROUTER_API_KEY" "$F" || miss="$miss no-secret-name"+# Verifies the key against OpenRouter (live HTTP)+grep -q "openrouter.ai/api/v1" "$F" || miss="$miss no-openrouter-http"+# Scaffold + lifecycle commands+grep -q "metaharness@latest.*new\|metaharness new\|'test-harness'" "$F" || miss="$miss no-scaffold-call"+grep -q "harness.*doctor\|harness', 'doctor\|\\['doctor'" "$F" || miss="$miss no-doctor-call"+grep -q "harness.*score\|'score'" "$F" || miss="$miss no-score-call"+# Anti-regression: scaffold MUST cd into a temp dir (--target is ignored+# by [email protected]+ which writes to \$CWD/<name>; iter 26 fix)+grep -q "cwd: fixture\|cwd: opts.cwd" "$F" || miss="$miss no-cwd-fix"+# Never echo the raw key+grep -q "apiKey.slice(0, 7)" "$F" || miss="$miss key-may-leak"+[[ -z "$miss" ]] && ok || bad "$miss"+step "17p. bench-recordpair-overhead β measures + gates iter-12 default-path cost (iter 24/25)"F="$ROOT/scripts/bench-recordpair-overhead.mjs"miss=""diff --git a/plugins/ruflo-metaharness/scripts/test-with-openrouter.mjs b/plugins/ruflo-metaharness/scripts/test-with-openrouter.mjsnew file mode 100755index 0000000000..dc0d1fafb5--- /dev/null+++ b/plugins/ruflo-metaharness/scripts/test-with-openrouter.mjs@@ -0,0 +1,229 @@+#!/usr/bin/env node+// test-with-openrouter.mjs β runtime test that exercises metaharness+// scaffolding + lifecycle commands using OPENROUTER_API_KEY fetched+// from GCP Secret Manager.+//+// WHAT IT DOES (end-to-end)+// 1. Fetch OPENROUTER_API_KEY from GCP Secret Manager via+// `gcloud secrets versions access latest --secret=OPENROUTER_API_KEY`+// 2. Verify the secret authenticates against OpenRouter by listing+// models (1 HTTP call, ~$0)+// 3. Scaffold a fresh harness into a temp dir via+// `metaharness new --name test-h --template vertical:coding+// --host claude-code --yes`+// 4. Run lifecycle commands against the scaffold:+// - harness doctor (smoke health check)+// - harness validate (full validation; --skip-gcp to avoid+// nested GCP fetches in this test)+// - harness score (5-dim scorecard of the scaffolded+// harness itself)+// - harness genome (7-section report)+// 5. (Optional) make one real LLM call through OpenRouter to a+// cheap model (e.g. openrouter/auto with a 1-token prompt) to+// prove the token works for actual inference+// 6. Clean up the temp dir+//+// COST: ~$0 (1 model-list call + optional <$0.0001 inference call)+//+// USAGE+// node scripts/test-with-openrouter.mjs # full e2e+// node scripts/test-with-openrouter.mjs --skip-inference # no LLM call+// node scripts/test-with-openrouter.mjs --keep-fixtures # leave scaffold for inspection+//+// EXIT CODES+// 0 all checks passed+// 1 at least one assertion failed+// 2 setup error (gcloud not authed, OPENROUTER_API_KEY missing in GCP, etc.)+// 3 cost/safety guard tripped++import { spawnSync, execSync } from 'node:child_process';+import { mkdtempSync, rmSync, mkdirSync, existsSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';++const ARGS = (() => {+ const a = { skipInference: false, keep: false, format: 'table' };+ for (let i = 2; i < process.argv.length; i++) {+ const v = process.argv[i];+ if (v === '--skip-inference') a.skipInference = true;+ else if (v === '--keep-fixtures') a.keep = true;+ else if (v === '--format') a.format = process.argv[++i];+ }+ return a;+})();++let passed = 0, failed = 0;+const failures = [];+function assert(cond, label) {+ if (cond) { console.log(` β ${label}`); passed++; }+ else { console.log(` β ${label}`); failures.push(label); failed++; }+}++function fetchSecretFromGcp(secretName) {+ try {+ const out = execSync(+ `gcloud secrets versions access latest --secret=${secretName}`,+ { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 15_000 },+ );+ return out.trim();+ } catch (e) {+ console.error(` β gcloud secret fetch failed: ${(e.message || '').slice(0, 120)}`);+ return null;+ }+}++async function listOpenRouterModels(apiKey) {+ const resp = await fetch('https://openrouter.ai/api/v1/models', {+ headers: { Authorization: `Bearer ${apiKey}` },+ });+ if (!resp.ok) {+ return { ok: false, status: resp.status, body: (await resp.text()).slice(0, 200) };+ }+ const data = await resp.json();+ return { ok: true, modelCount: data.data?.length ?? 0 };+}++async function cheapInferenceCall(apiKey) {+ // openrouter/auto picks the cheapest viable model; 5-token prompt;+ // max_tokens=1 to bound cost at ~$0.00005.+ const resp = await fetch('https://openrouter.ai/api/v1/chat/completions', {+ method: 'POST',+ headers: {+ Authorization: `Bearer ${apiKey}`,+ 'Content-Type': 'application/json',+ 'HTTP-Referer': 'https://github.com/ruvnet/ruflo',+ 'X-Title': 'ruflo-metaharness-test',+ },+ body: JSON.stringify({+ model: 'openrouter/auto',+ messages: [{ role: 'user', content: 'OK' }],+ max_tokens: 1,+ }),+ });+ if (!resp.ok) return { ok: false, status: resp.status, body: (await resp.text()).slice(0, 200) };+ const data = await resp.json();+ return { ok: true, model: data.model, usage: data.usage };+}++function runHarness(args, opts = {}) {+ const r = spawnSync('npx', ['-y', '-p', 'metaharness@latest', 'harness', ...args], {+ stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8',+ timeout: opts.timeoutMs ?? 60_000,+ env: { ...process.env, ...opts.env },+ });+ return { exitCode: r.status ?? 1, stdout: r.stdout || '', stderr: r.stderr || '' };+}++function runMetaharness(args, opts = {}) {+ const r = spawnSync('npx', ['-y', 'metaharness@latest', ...args], {+ stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8',+ timeout: opts.timeoutMs ?? 120_000,+ cwd: opts.cwd, // metaharness new writes to cwd/<name>; --target is ignored+ env: { ...process.env, ...opts.env },+ });+ return { exitCode: r.status ?? 1, stdout: r.stdout || '', stderr: r.stderr || '' };+}++async function main() {+ console.log('# test-with-openrouter β metaharness Γ GCP Γ OpenRouter e2e\n');++ // ββ 0. Preflight ββββββββββββββββββββββββββββββββββββββββββββββββββ+ console.log('Phase 0 β preflight');+ const gcloudWho = execSync('gcloud config get-value account 2>&1', { encoding: 'utf-8' }).trim();+ assert(!!gcloudWho && !/None/.test(gcloudWho), `gcloud authenticated (${gcloudWho || 'none'})`);++ // ββ 1. Fetch OPENROUTER_API_KEY βββββββββββββββββββββββββββββββββββ+ console.log('\nPhase 1 β fetch OPENROUTER_API_KEY from GCP Secret Manager');+ const apiKey = fetchSecretFromGcp('OPENROUTER_API_KEY');+ assert(typeof apiKey === 'string' && apiKey.length > 20, 'OPENROUTER_API_KEY fetched (length OK)');+ if (!apiKey) {+ console.error('Setup error: cannot proceed without OPENROUTER_API_KEY. Skipping further phases.');+ process.exit(2);+ }+ // Echo only length+prefix, never the raw key+ console.log(` key length: ${apiKey.length}, prefix: ${apiKey.slice(0, 7)}β¦`);++ // ββ 2. Verify the key authenticates against OpenRouter ββββββββββββ+ console.log('\nPhase 2 β verify OpenRouter authentication');+ const modelsResp = await listOpenRouterModels(apiKey);+ assert(modelsResp.ok, `OpenRouter /api/v1/models returns 2xx (got ${modelsResp.ok ? 'ok' : modelsResp.status})`);+ if (modelsResp.ok) {+ assert(modelsResp.modelCount > 10, `model list non-empty (${modelsResp.modelCount} models)`);+ }++ // ββ 3. Scaffold a fresh harness βββββββββββββββββββββββββββββββββββ+ // CRITICAL: `metaharness new <name>` writes to $CWD/<name> β the+ // --target flag is ignored by the CLI (verified 2026-06-16 against+ // [email protected]). Run from inside a fresh temp dir so the+ // scaffold lands there, not in the ruflo project root.+ console.log('\nPhase 3 β scaffold a fresh harness');+ const fixture = mkdtempSync(join(tmpdir(), 'ruflo-mh-openrouter-'));+ const target = join(fixture, 'test-harness');+ console.log(` fixture cwd: ${fixture}`);+ console.log(` expected target: ${target}`);+ const scaffold = runMetaharness(+ ['test-harness', '--template', 'vertical:coding', '--host', 'claude-code'],+ { timeoutMs: 180_000, cwd: fixture },+ );+ assert(scaffold.exitCode === 0, `metaharness new exit 0 (got ${scaffold.exitCode})`);+ assert(existsSync(target), `target dir created at ${target}`);++ // ββ 4. Run lifecycle commands against the scaffold ββββββββββββββββ+ console.log('\nPhase 4 β lifecycle commands on the scaffold');+ // harness doctor β quick smoke+ const doc = runHarness(['doctor', target]);+ assert(doc.exitCode === 0, `harness doctor exit 0 (got ${doc.exitCode})`);++ // harness score on the scaffold itself+ const score = runHarness(['score', target, '--json']);+ assert(score.exitCode === 0, `harness score exit 0 (got ${score.exitCode})`);+ const scoreJson = (() => {+ const m = /\{[\s\S]*\}/.exec(score.stdout);+ try { return m ? JSON.parse(m[0]) : null; } catch { return null; }+ })();+ assert(scoreJson && typeof scoreJson.score === 'number', 'score.json has numeric score');++ // harness genome β exit 0 (ready) or 1 (needs-work) both acceptable.+ // Only exit 2 (blocked / scan-error) is a real failure.+ const gen = runHarness(['genome', target, '--json']);+ assert(gen.exitCode === 0 || gen.exitCode === 1, `harness genome exit 0 or 1 (got ${gen.exitCode})`);++ // harness mcp-scan+ const scan = runHarness(['mcp-scan', target]);+ // mcp-scan can exit 1 on findings; either 0 or 1 is acceptable here.+ assert(scan.exitCode === 0 || scan.exitCode === 1, `harness mcp-scan exit 0 or 1 (got ${scan.exitCode})`);++ // ββ 5. (Optional) one real OpenRouter inference call ββββββββββββββ+ if (!ARGS.skipInference) {+ console.log('\nPhase 5 β single OpenRouter inference call (cheapest auto-route, max_tokens=1)');+ const inf = await cheapInferenceCall(apiKey);+ assert(inf.ok, `OpenRouter inference 2xx (got ${inf.ok ? 'ok' : inf.status})`);+ if (inf.ok) {+ console.log(` model used: ${inf.model}`);+ console.log(` usage: prompt=${inf.usage?.prompt_tokens} completion=${inf.usage?.completion_tokens}`);+ }+ } else {+ console.log('\nPhase 5 β SKIPPED (--skip-inference)');+ }++ // ββ 6. Cleanup ββββββββββββββββββββββββββββββββββββββββββββββββββββ+ if (!ARGS.keep) {+ rmSync(fixture, { recursive: true, force: true });+ console.log(`\nFixture cleaned: ${fixture}`);+ } else {+ console.log(`\nFixture kept at: ${fixture}`);+ }++ console.log(`\n${passed} passed, ${failed} failed`);+ if (failed > 0) {+ console.log('\nFailures:');+ for (const f of failures) console.log(` - ${f}`);+ process.exit(1);+ }+ console.log('\nβ All harness Γ OpenRouter integration checks passed.');+}++main().catch((e) => {+ console.error('test-with-openrouter crashed:', e.message || e);+ process.exit(2);+});