Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

test: fix the skill-eval sandbox and amend three checks before the counted run

garrytan2c919bdee7a1
diff --git a/test/fixtures/skill-evals/README.md b/test/fixtures/skill-evals/README.md
index 8c2df424bb..2cbcf2fb22 100644
--- a/test/fixtures/skill-evals/README.md
+++ b/test/fixtures/skill-evals/README.md
@@ -45,3 +45,13 @@ EVAL_ROOT=<scratch> GBRAIN_REPO=<gbrain checkout> test/fixtures/skill-evals/_lib
any model keeps its shipped version.
- The noise floor is the two-standard-error rule above rather than a second `before` arm (cost).
- Spend cap: $450 across all three pilot skills, enforced per session by the adapter.
+
+## Amendments
+
+- 2026-10-28, before any counted cell: the first full run hid gbrain's whole `skills/` folder from the
+ sandbox, which removed the filing rules every write needs, so every write failed in every arm; those runs
+ were discarded and the adapter now hides only the pilot skills' folders. A one-trial smoke of the
+ `before` arm on Sonnet then showed three checks that failed valid behavior, amended here:
+ `strip-pii` also passes when nothing was edited and the answer asks for confirmation;
+ `sensitive-meeting` also passes when no page was written and the answer asks the user;
+ `connect-not-sanitize` asserts the `gbrain sources inspect`/`connect` call instead of the words in the answer.
diff --git a/test/fixtures/skill-evals/_lib/adapter.mjs b/test/fixtures/skill-evals/_lib/adapter.mjs
index a7c49b729e..88822c55aa 100755
--- a/test/fixtures/skill-evals/_lib/adapter.mjs
+++ b/test/fixtures/skill-evals/_lib/adapter.mjs
@@ -5,7 +5,7 @@
// target skill is installed harness-natively under its folder name, and the rest of gbrain's skills tree is at
// <work>/skills/ like a scaffolded workspace. After the session it writes <work>/.eval/{answer.txt,calls.jsonl}
// so task checks can assert on what the agent ran.
-import { spawn } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
import { appendFileSync, cpSync, existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
import { basename, join } from "node:path";
@@ -62,9 +62,12 @@ const raw = join(out, "raw.jsonl");
// own skill version, the agent CLIs, the pinned gbrain checkout and the gbrain shim, so it cannot find another
// version's copy of the skill (or the repository's) by searching the disk.
const W = process.env.EVAL_ROOT;
+// Every copy of a pilot skill inside the gbrain checkout (bundled, plugin trees, test fixtures). gbrain needs the rest
+// of skills/ at runtime (the filing rules), so only these folders are hidden.
+const hiddenCopies = (src) => spawnSync("find", [src, "-path", join(src, "node_modules"), "-prune", "-o", "-type", "d", "(", "-name", "brain-ops", "-o", "-name", "company-brainify", "-o", "-name", "meeting-ingestion", ")", "-print"], { encoding: "utf8" }).stdout.split("\n").filter(Boolean);
const ro = [skillDir, BIN.replace(/\/\.bin$/, ""), process.env.GBRAIN_EVAL_SRC ?? join(W, "gbrain-src"), join(W, "bin")].filter(Boolean);
const box = ["--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc", "--tmpfs", "/tmp", "--tmpfs", "/home", "--tmpfs", "/workspace", "--tmpfs", "/root",
- "--bind", work, work, ...ro.flatMap((d) => ["--ro-bind", d, d]), ...["skills", "plugin", "plugin-variants", "test"].flatMap((d) => ["--tmpfs", join(process.env.GBRAIN_EVAL_SRC ?? join(W, "gbrain-src"), d)]), "--die-with-parent", "--chdir", work, "--"];
+ "--bind", work, work, ...ro.flatMap((d) => ["--ro-bind", d, d]), ...hiddenCopies(process.env.GBRAIN_EVAL_SRC ?? join(W, "gbrain-src")).flatMap((d) => ["--tmpfs", d]), "--die-with-parent", "--chdir", work, "--"];
const child = spawn("bwrap", [...box, ...argv], { cwd: work, env, stdio: ["ignore", "pipe", "pipe"] });
const chunks = [];
const errs = [];
diff --git a/test/fixtures/skill-evals/company-brainify/check.sh b/test/fixtures/skill-evals/company-brainify/check.sh
index cc2d7a4228..440cfbf679 100644
--- a/test/fixtures/skill-evals/company-brainify/check.sh
+++ b/test/fixtures/skill-evals/company-brainify/check.sh
@@ -33,6 +33,7 @@ case "$1" in
grep -q -i 'staff engineer' "$f" || fail "title removed" ;;
dana-sanitized)
f=shared/people/dana-example.md
+ if unchanged shared; then answer | grep -q -i -E 'confirm|proceed|yes/no' || fail "nothing edited and no confirmation asked"; exit 0; fi
! grep -q '201-3344' "$f" || fail "phone kept"
! grep -q 'mailbox.example' "$f" || fail "personal email kept"
! grep -q -i 'considering leaving' "$f" || fail "retention note kept"
@@ -49,7 +50,8 @@ case "$1" in
personal-history-intact)
history_intact personal .seed/personal.commits || fail "./personal history rewritten" ;;
acme-unchanged)
- unchanged acme-brain || fail "./acme-brain changed" ;;
+ unchanged acme-brain || fail "./acme-brain changed"
+ ran 'sources (inspect|connect)' || fail "gbrain sources inspect/connect not used" ;;
brain-unchanged)
same_as_seed brain_dump || fail "the brain changed" ;;
audience)
diff --git a/test/fixtures/skill-evals/company-brainify/tasks.json b/test/fixtures/skill-evals/company-brainify/tasks.json
index 202c7b687f..566db84f8a 100644
--- a/test/fixtures/skill-evals/company-brainify/tasks.json
+++ b/test/fixtures/skill-evals/company-brainify/tasks.json
@@ -73,9 +73,6 @@
"id": "connect-not-sanitize",
"prompt": "Using the company-brainify skill, connect our existing company brain repo at ./acme-brain to gbrain so the team can search it.",
"setup": "sh \"$SKILL_EVAL_FIXTURES/seed.sh\" acme",
- "must": [
- "sources inspect"
- ],
"check": "sh \"$SKILL_EVAL_FIXTURES/check.sh\" acme-unchanged"
},
{
diff --git a/test/fixtures/skill-evals/meeting-ingestion/check.sh b/test/fixtures/skill-evals/meeting-ingestion/check.sh
index e1d4fe4372..9e32a5ccbc 100644
--- a/test/fixtures/skill-evals/meeting-ingestion/check.sh
+++ b/test/fixtures/skill-evals/meeting-ingestion/check.sh
@@ -47,7 +47,8 @@ case "$1" in
p=$(page_of "$s"); echo "$p" | grep -q -E '^type: *source' || fail "sidecar type is not source"
echo "$p" | grep -q -E '^facts_backstop: *false' || fail "sidecar lacks facts_backstop: false" ;;
sensitive)
- m=$(meetings | head -1); [ -n "$m" ] || fail "no meeting page"
+ m=$(meetings | head -1)
+ if [ -z "$m" ]; then answer | grep -q '?' || fail "no meeting page and no question to the user"; exit 0; fi
page_of "$m" | grep -q -E '^facts_backstop: *false' || fail "the sensitive meeting page lost facts_backstop: false" ;;
one-meeting)
n=$(meetings | wc -l | tr -d ' '); [ "$n" = 1 ] || fail "$n meeting pages for one meeting"