Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
test: fix the skill-eval sandbox and amend three checks before the counted run
garrytan2c919bdee7a1
diff --git a/test/fixtures/skill-evals/README.md b/test/fixtures/skill-evals/README.mdindex 8c2df424bb..2cbcf2fb22 100644--- a/test/fixtures/skill-evals/README.md+++ b/test/fixtures/skill-evals/README.md@@ -45,3 +45,13 @@ EVAL_ROOT=<scratch> GBRAIN_REPO=<gbrain checkout> test/fixtures/skill-evals/_libany model keeps its shipped version.- The noise floor is the two-standard-error rule above rather than a second `before` arm (cost).- Spend cap: $450 across all three pilot skills, enforced per session by the adapter.++## Amendments++- 2026-10-28, before any counted cell: the first full run hid gbrain's whole `skills/` folder from the+ sandbox, which removed the filing rules every write needs, so every write failed in every arm; those runs+ were discarded and the adapter now hides only the pilot skills' folders. A one-trial smoke of the+ `before` arm on Sonnet then showed three checks that failed valid behavior, amended here:+ `strip-pii` also passes when nothing was edited and the answer asks for confirmation;+ `sensitive-meeting` also passes when no page was written and the answer asks the user;+ `connect-not-sanitize` asserts the `gbrain sources inspect`/`connect` call instead of the words in the answer.diff --git a/test/fixtures/skill-evals/_lib/adapter.mjs b/test/fixtures/skill-evals/_lib/adapter.mjsindex a7c49b729e..88822c55aa 100755--- a/test/fixtures/skill-evals/_lib/adapter.mjs+++ b/test/fixtures/skill-evals/_lib/adapter.mjs@@ -5,7 +5,7 @@// target skill is installed harness-natively under its folder name, and the rest of gbrain's skills tree is at// <work>/skills/ like a scaffolded workspace. After the session it writes <work>/.eval/{answer.txt,calls.jsonl}// so task checks can assert on what the agent ran.-import { spawn } from "node:child_process";+import { spawn, spawnSync } from "node:child_process";import { appendFileSync, cpSync, existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";import { basename, join } from "node:path";@@ -62,9 +62,12 @@ const raw = join(out, "raw.jsonl");// own skill version, the agent CLIs, the pinned gbrain checkout and the gbrain shim, so it cannot find another// version's copy of the skill (or the repository's) by searching the disk.const W = process.env.EVAL_ROOT;+// Every copy of a pilot skill inside the gbrain checkout (bundled, plugin trees, test fixtures). gbrain needs the rest+// of skills/ at runtime (the filing rules), so only these folders are hidden.+const hiddenCopies = (src) => spawnSync("find", [src, "-path", join(src, "node_modules"), "-prune", "-o", "-type", "d", "(", "-name", "brain-ops", "-o", "-name", "company-brainify", "-o", "-name", "meeting-ingestion", ")", "-print"], { encoding: "utf8" }).stdout.split("\n").filter(Boolean);const ro = [skillDir, BIN.replace(/\/\.bin$/, ""), process.env.GBRAIN_EVAL_SRC ?? join(W, "gbrain-src"), join(W, "bin")].filter(Boolean);const box = ["--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc", "--tmpfs", "/tmp", "--tmpfs", "/home", "--tmpfs", "/workspace", "--tmpfs", "/root",- "--bind", work, work, ...ro.flatMap((d) => ["--ro-bind", d, d]), ...["skills", "plugin", "plugin-variants", "test"].flatMap((d) => ["--tmpfs", join(process.env.GBRAIN_EVAL_SRC ?? join(W, "gbrain-src"), d)]), "--die-with-parent", "--chdir", work, "--"];+ "--bind", work, work, ...ro.flatMap((d) => ["--ro-bind", d, d]), ...hiddenCopies(process.env.GBRAIN_EVAL_SRC ?? join(W, "gbrain-src")).flatMap((d) => ["--tmpfs", d]), "--die-with-parent", "--chdir", work, "--"];const child = spawn("bwrap", [...box, ...argv], { cwd: work, env, stdio: ["ignore", "pipe", "pipe"] });const chunks = [];const errs = [];diff --git a/test/fixtures/skill-evals/company-brainify/check.sh b/test/fixtures/skill-evals/company-brainify/check.shindex cc2d7a4228..440cfbf679 100644--- a/test/fixtures/skill-evals/company-brainify/check.sh+++ b/test/fixtures/skill-evals/company-brainify/check.sh@@ -33,6 +33,7 @@ case "$1" ingrep -q -i 'staff engineer' "$f" || fail "title removed" ;;dana-sanitized)f=shared/people/dana-example.md+ if unchanged shared; then answer | grep -q -i -E 'confirm|proceed|yes/no' || fail "nothing edited and no confirmation asked"; exit 0; fi! grep -q '201-3344' "$f" || fail "phone kept"! grep -q 'mailbox.example' "$f" || fail "personal email kept"! grep -q -i 'considering leaving' "$f" || fail "retention note kept"@@ -49,7 +50,8 @@ case "$1" inpersonal-history-intact)history_intact personal .seed/personal.commits || fail "./personal history rewritten" ;;acme-unchanged)- unchanged acme-brain || fail "./acme-brain changed" ;;+ unchanged acme-brain || fail "./acme-brain changed"+ ran 'sources (inspect|connect)' || fail "gbrain sources inspect/connect not used" ;;brain-unchanged)same_as_seed brain_dump || fail "the brain changed" ;;audience)diff --git a/test/fixtures/skill-evals/company-brainify/tasks.json b/test/fixtures/skill-evals/company-brainify/tasks.jsonindex 202c7b687f..566db84f8a 100644--- a/test/fixtures/skill-evals/company-brainify/tasks.json+++ b/test/fixtures/skill-evals/company-brainify/tasks.json@@ -73,9 +73,6 @@"id": "connect-not-sanitize","prompt": "Using the company-brainify skill, connect our existing company brain repo at ./acme-brain to gbrain so the team can search it.","setup": "sh \"$SKILL_EVAL_FIXTURES/seed.sh\" acme",- "must": [- "sources inspect"- ],"check": "sh \"$SKILL_EVAL_FIXTURES/check.sh\" acme-unchanged"},{diff --git a/test/fixtures/skill-evals/meeting-ingestion/check.sh b/test/fixtures/skill-evals/meeting-ingestion/check.shindex e1d4fe4372..9e32a5ccbc 100644--- a/test/fixtures/skill-evals/meeting-ingestion/check.sh+++ b/test/fixtures/skill-evals/meeting-ingestion/check.sh@@ -47,7 +47,8 @@ case "$1" inp=$(page_of "$s"); echo "$p" | grep -q -E '^type: *source' || fail "sidecar type is not source"echo "$p" | grep -q -E '^facts_backstop: *false' || fail "sidecar lacks facts_backstop: false" ;;sensitive)- m=$(meetings | head -1); [ -n "$m" ] || fail "no meeting page"+ m=$(meetings | head -1)+ if [ -z "$m" ]; then answer | grep -q '?' || fail "no meeting page and no question to the user"; exit 0; fipage_of "$m" | grep -q -E '^facts_backstop: *false' || fail "the sensitive meeting page lost facts_backstop: false" ;;one-meeting)n=$(meetings | wc -l | tr -d ' '); [ "$n" = 1 ] || fail "$n meeting pages for one meeting"