Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(gan): require Playwright capability in harness

wellkilo6f9282af4191
diff --git a/scripts/gan-harness.sh b/scripts/gan-harness.sh
index 79dd5038f..fbab88742 100755
--- a/scripts/gan-harness.sh
+++ b/scripts/gan-harness.sh
@@ -19,6 +19,7 @@
# GAN_PROJECT_DIR — Working directory (default: current dir)
# GAN_SKIP_PLANNER — Set to "true" to skip planner phase
# GAN_EVAL_MODE — playwright, screenshot, or code-only (default: playwright)
+# playwright requires a connected MCP server named "playwright"
set -euo pipefail
@@ -96,6 +97,38 @@ score_passes() {
awk -v s="$score" -v t="$threshold" 'BEGIN { exit !(s >= t) }'
}
+playwright_mcp_is_connected() {
+ local status
+ local status_value
+ local check_mark
+ local heavy_check_mark
+ status=$(NO_COLOR=1 claude mcp get playwright 2>/dev/null) || return 1
+ status_value=$(printf '%s\n' "$status" | awk '
+ /^[[:space:]]*Status:[[:space:]]*/ {
+ sub(/^[[:space:]]*Status:[[:space:]]*/, "")
+ sub(/[[:space:]]*$/, "")
+ print
+ exit
+ }
+ ')
+ check_mark=$(printf '\342\234\223')
+ heavy_check_mark=$(printf '\342\234\224')
+
+ [ "$status_value" = "$check_mark Connected" ] || \
+ [ "$status_value" = "$heavy_check_mark Connected" ]
+}
+
+evaluator_tools_for_mode() {
+ local base_tools="Read,Write,Bash,Grep,Glob"
+ local playwright_tools="mcp__playwright__browser_navigate,mcp__playwright__browser_click,mcp__playwright__browser_take_screenshot,mcp__playwright__browser_snapshot,mcp__playwright__browser_type,mcp__playwright__browser_fill_form"
+
+ if [ "$1" = "playwright" ]; then
+ printf '%s,%s\n' "$base_tools" "$playwright_tools"
+ else
+ printf '%s\n' "$base_tools"
+ fi
+}
+
elapsed() {
local now=$(date +%s)
local diff=$((now - START_TIME))
@@ -104,6 +137,24 @@ elapsed() {
# ─── Setup ───────────────────────────────────────────────────────────────────
+case "$EVAL_MODE" in
+ playwright)
+ if ! playwright_mcp_is_connected; then
+ fail "GAN_EVAL_MODE=playwright requires a connected MCP server named 'playwright'."
+ fail "Run 'claude mcp get playwright' to inspect its status, or choose GAN_EVAL_MODE=screenshot or code-only."
+ exit 1
+ fi
+ ;;
+ screenshot|code-only)
+ ;;
+ *)
+ fail "Unsupported GAN_EVAL_MODE. Expected playwright, screenshot, or code-only."
+ exit 1
+ ;;
+esac
+
+EVALUATOR_TOOLS=$(evaluator_tools_for_mode "$EVAL_MODE")
+
phase "GAN-STYLE HARNESS — Setup"
log "Brief: ${CYAN}${BRIEF}${NC}"
@@ -205,8 +256,14 @@ Update gan-harness/generator-state.md." \
# ── EVALUATE ──
echo -e "${RED}>> EVALUATOR (iteration $i)${NC}"
+ if [ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected; then
+ fail "The Playwright MCP server disconnected before evaluator iteration $i."
+ fail "Run 'claude mcp get playwright' to inspect its status, then retry the harness."
+ exit 1
+ fi
+
claude -p --model "$EVALUATOR_MODEL" \
- --allowedTools "Read,Write,Bash,Grep,Glob" \
+ --allowedTools "$EVALUATOR_TOOLS" \
"You are the Evaluator in a GAN-style harness. Read agents/gan-evaluator.md for full instructions.
Iteration: $i
diff --git a/tests/gan-harness.test.js b/tests/gan-harness.test.js
index 36c7255ce..6a5041f92 100644
--- a/tests/gan-harness.test.js
+++ b/tests/gan-harness.test.js
@@ -12,7 +12,9 @@ const { spawnSync } = require('child_process');
const repoRoot = path.resolve(__dirname, '..');
const harnessPath = path.join(repoRoot, 'scripts', 'gan-harness.sh');
+const evaluatorPath = path.join(repoRoot, 'agents', 'gan-evaluator.md');
const harnessSource = fs.readFileSync(harnessPath, 'utf8');
+const evaluatorSource = fs.readFileSync(evaluatorPath, 'utf8');
if (process.platform === 'win32') {
console.log('\n=== GAN harness helpers ===\n');
@@ -34,15 +36,22 @@ function test(name, fn) {
}
}
-function runHarnessScript(script, args = []) {
+function runHarnessScript(script, args = [], env = {}) {
const bashExecutable = process.platform === 'win32' ? 'bash' : '/bin/bash';
const result = spawnSync(bashExecutable, ['-c', script, 'gan-harness-test', ...args], {
encoding: 'utf8',
+ env: { ...process.env, ...env },
});
assert.strictEqual(result.status, 0, result.stderr || 'GAN harness script failed');
return result.stdout.trim();
}
+function extractShellFunction(name) {
+ const functionMatch = harnessSource.match(new RegExp(`${name}\\(\\) \\{[\\s\\S]*?\\n\\}`));
+ assert.ok(functionMatch, `expected scripts/gan-harness.sh to define ${name}`);
+ return functionMatch[0];
+}
+
function extractScore(feedback) {
const functionMatch = harnessSource.match(/extract_score\(\) \{[\s\S]*?\n\}/);
assert.ok(functionMatch, 'expected scripts/gan-harness.sh to define extract_score');
@@ -58,6 +67,39 @@ function extractScore(feedback) {
}
}
+function probePlaywright(statusLine, commandStatus = 0) {
+ const script = [
+ 'claude() {',
+ ' [ "$#" -eq 3 ] && [ "$1" = mcp ] && [ "$2" = get ] && [ "$3" = playwright ] || return 64',
+ ' [ "$NO_COLOR" = 1 ] || return 65',
+ " printf '%s\\n' \"$GAN_TEST_MCP_STATUS\"",
+ ' return "$GAN_TEST_MCP_EXIT"',
+ '}',
+ extractShellFunction('playwright_mcp_is_connected'),
+ 'if playwright_mcp_is_connected; then printf connected; else printf unavailable; fi',
+ ].join('\n');
+
+ return runHarnessScript(script, [], {
+ GAN_TEST_MCP_STATUS: statusLine,
+ GAN_TEST_MCP_EXIT: String(commandStatus),
+ });
+}
+
+function evaluatorToolsForMode(mode) {
+ return runHarnessScript(
+ `${extractShellFunction('evaluator_tools_for_mode')}\nevaluator_tools_for_mode "$1"`,
+ [mode]
+ ).split(',');
+}
+
+function declaredEvaluatorTools() {
+ const frontmatter = evaluatorSource.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+ assert.ok(frontmatter, 'expected agents/gan-evaluator.md to have frontmatter');
+ const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m);
+ assert.ok(toolsLine, 'expected agents/gan-evaluator.md to declare tools');
+ return toolsLine[1].split(',').map(tool => tool.trim());
+}
+
console.log('\n=== GAN harness helpers ===\n');
const results = Object.freeze([
@@ -106,6 +148,48 @@ const results = Object.freeze([
assert.strictEqual(result, '0.0');
}),
+ test('Playwright preflight accepts only an explicitly connected server', () => {
+ assert.strictEqual(probePlaywright('Status: \u2713 Connected'), 'connected');
+ assert.strictEqual(probePlaywright('Status: \u2714 Connected'), 'connected');
+ for (const unavailableStatus of [
+ 'Status: ! Connected \u00b7 tools fetch failed',
+ 'Status: ! Needs authentication',
+ 'Status: \u2718 Failed to connect',
+ 'Status: \u23f8 Pending approval',
+ 'Status: \u2298 Disabled for this project',
+ '',
+ ]) {
+ assert.strictEqual(probePlaywright(unavailableStatus), 'unavailable');
+ }
+ assert.strictEqual(probePlaywright('Status: \u2713 Connected', 1), 'unavailable');
+ assert.strictEqual(
+ probePlaywright('Status: \u2718 Failed to connect\nStatus: \u2713 Connected'),
+ 'unavailable'
+ );
+ }),
+
+ test('evaluator tools follow mode and reuse the approved agent contract', () => {
+ assert.deepStrictEqual(evaluatorToolsForMode('playwright'), declaredEvaluatorTools());
+ for (const mode of ['screenshot', 'code-only']) {
+ assert.deepStrictEqual(
+ evaluatorToolsForMode(mode),
+ ['Read', 'Write', 'Bash', 'Grep', 'Glob']
+ );
+ }
+ }),
+
+ test('Playwright is checked before setup and again before evaluator launch', () => {
+ const preflightCall = harnessSource.indexOf('if ! playwright_mcp_is_connected');
+ const setupMutation = harnessSource.indexOf('mkdir -p "$FEEDBACK_DIR"');
+ const runtimeCheck = harnessSource.indexOf('[ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected');
+ const evaluatorLaunch = harnessSource.indexOf('claude -p --model "$EVALUATOR_MODEL"');
+
+ assert.ok(preflightCall >= 0 && preflightCall < setupMutation);
+ assert.ok(runtimeCheck >= 0 && runtimeCheck < evaluatorLaunch);
+ assert.match(harnessSource, /--allowedTools "\$EVALUATOR_TOOLS"/);
+ assert.match(harnessSource, /Unsupported GAN_EVAL_MODE/);
+ }),
+
test('final score lookup is compatible with the macOS Bash 3.2 runtime', () => {
const finalScoreBlock = harnessSource.match(
/NUM_ITERATIONS=\$\{#SCORES\[@\]\}\nif \[ "\$NUM_ITERATIONS"[\s\S]*?\nfi/