The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(gan): require Playwright capability in harness
wellkilo6f9282af4191
diff --git a/scripts/gan-harness.sh b/scripts/gan-harness.shindex 79dd5038f..fbab88742 100755--- a/scripts/gan-harness.sh+++ b/scripts/gan-harness.sh@@ -19,6 +19,7 @@# GAN_PROJECT_DIR — Working directory (default: current dir)# GAN_SKIP_PLANNER — Set to "true" to skip planner phase# GAN_EVAL_MODE — playwright, screenshot, or code-only (default: playwright)+# playwright requires a connected MCP server named "playwright"set -euo pipefail@@ -96,6 +97,38 @@ score_passes() {awk -v s="$score" -v t="$threshold" 'BEGIN { exit !(s >= t) }'}+playwright_mcp_is_connected() {+ local status+ local status_value+ local check_mark+ local heavy_check_mark+ status=$(NO_COLOR=1 claude mcp get playwright 2>/dev/null) || return 1+ status_value=$(printf '%s\n' "$status" | awk '+ /^[[:space:]]*Status:[[:space:]]*/ {+ sub(/^[[:space:]]*Status:[[:space:]]*/, "")+ sub(/[[:space:]]*$/, "")+ exit+ }+ ')+ check_mark=$(printf '\342\234\223')+ heavy_check_mark=$(printf '\342\234\224')++ [ "$status_value" = "$check_mark Connected" ] || \+ [ "$status_value" = "$heavy_check_mark Connected" ]+}++evaluator_tools_for_mode() {+ local base_tools="Read,Write,Bash,Grep,Glob"+ local playwright_tools="mcp__playwright__browser_navigate,mcp__playwright__browser_click,mcp__playwright__browser_take_screenshot,mcp__playwright__browser_snapshot,mcp__playwright__browser_type,mcp__playwright__browser_fill_form"++ if [ "$1" = "playwright" ]; then+ printf '%s,%s\n' "$base_tools" "$playwright_tools"+ else+ printf '%s\n' "$base_tools"+ fi+}+elapsed() {local now=$(date +%s)local diff=$((now - START_TIME))@@ -104,6 +137,24 @@ elapsed() {# ─── Setup ───────────────────────────────────────────────────────────────────+case "$EVAL_MODE" in+ playwright)+ if ! playwright_mcp_is_connected; then+ fail "GAN_EVAL_MODE=playwright requires a connected MCP server named 'playwright'."+ fail "Run 'claude mcp get playwright' to inspect its status, or choose GAN_EVAL_MODE=screenshot or code-only."+ exit 1+ fi+ ;;+ screenshot|code-only)+ ;;+ *)+ fail "Unsupported GAN_EVAL_MODE. Expected playwright, screenshot, or code-only."+ exit 1+ ;;+esac++EVALUATOR_TOOLS=$(evaluator_tools_for_mode "$EVAL_MODE")+phase "GAN-STYLE HARNESS — Setup"log "Brief: ${CYAN}${BRIEF}${NC}"@@ -205,8 +256,14 @@ Update gan-harness/generator-state.md." \# ── EVALUATE ──echo -e "${RED}>> EVALUATOR (iteration $i)${NC}"+ if [ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected; then+ fail "The Playwright MCP server disconnected before evaluator iteration $i."+ fail "Run 'claude mcp get playwright' to inspect its status, then retry the harness."+ exit 1+ fi+claude -p --model "$EVALUATOR_MODEL" \- --allowedTools "Read,Write,Bash,Grep,Glob" \+ --allowedTools "$EVALUATOR_TOOLS" \"You are the Evaluator in a GAN-style harness. Read agents/gan-evaluator.md for full instructions.Iteration: $idiff --git a/tests/gan-harness.test.js b/tests/gan-harness.test.jsindex 36c7255ce..6a5041f92 100644--- a/tests/gan-harness.test.js+++ b/tests/gan-harness.test.js@@ -12,7 +12,9 @@ const { spawnSync } = require('child_process');const repoRoot = path.resolve(__dirname, '..');const harnessPath = path.join(repoRoot, 'scripts', 'gan-harness.sh');+const evaluatorPath = path.join(repoRoot, 'agents', 'gan-evaluator.md');const harnessSource = fs.readFileSync(harnessPath, 'utf8');+const evaluatorSource = fs.readFileSync(evaluatorPath, 'utf8');if (process.platform === 'win32') {console.log('\n=== GAN harness helpers ===\n');@@ -34,15 +36,22 @@ function test(name, fn) {}}-function runHarnessScript(script, args = []) {+function runHarnessScript(script, args = [], env = {}) {const bashExecutable = process.platform === 'win32' ? 'bash' : '/bin/bash';const result = spawnSync(bashExecutable, ['-c', script, 'gan-harness-test', ...args], {encoding: 'utf8',+ env: { ...process.env, ...env },});assert.strictEqual(result.status, 0, result.stderr || 'GAN harness script failed');return result.stdout.trim();}+function extractShellFunction(name) {+ const functionMatch = harnessSource.match(new RegExp(`${name}\\(\\) \\{[\\s\\S]*?\\n\\}`));+ assert.ok(functionMatch, `expected scripts/gan-harness.sh to define ${name}`);+ return functionMatch[0];+}+function extractScore(feedback) {const functionMatch = harnessSource.match(/extract_score\(\) \{[\s\S]*?\n\}/);assert.ok(functionMatch, 'expected scripts/gan-harness.sh to define extract_score');@@ -58,6 +67,39 @@ function extractScore(feedback) {}}+function probePlaywright(statusLine, commandStatus = 0) {+ const script = [+ 'claude() {',+ ' [ "$#" -eq 3 ] && [ "$1" = mcp ] && [ "$2" = get ] && [ "$3" = playwright ] || return 64',+ ' [ "$NO_COLOR" = 1 ] || return 65',+ " printf '%s\\n' \"$GAN_TEST_MCP_STATUS\"",+ ' return "$GAN_TEST_MCP_EXIT"',+ '}',+ extractShellFunction('playwright_mcp_is_connected'),+ 'if playwright_mcp_is_connected; then printf connected; else printf unavailable; fi',+ ].join('\n');++ return runHarnessScript(script, [], {+ GAN_TEST_MCP_STATUS: statusLine,+ GAN_TEST_MCP_EXIT: String(commandStatus),+ });+}++function evaluatorToolsForMode(mode) {+ return runHarnessScript(+ `${extractShellFunction('evaluator_tools_for_mode')}\nevaluator_tools_for_mode "$1"`,+ [mode]+ ).split(',');+}++function declaredEvaluatorTools() {+ const frontmatter = evaluatorSource.match(/^---\r?\n([\s\S]*?)\r?\n---/);+ assert.ok(frontmatter, 'expected agents/gan-evaluator.md to have frontmatter');+ const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m);+ assert.ok(toolsLine, 'expected agents/gan-evaluator.md to declare tools');+ return toolsLine[1].split(',').map(tool => tool.trim());+}+console.log('\n=== GAN harness helpers ===\n');const results = Object.freeze([@@ -106,6 +148,48 @@ const results = Object.freeze([assert.strictEqual(result, '0.0');}),+ test('Playwright preflight accepts only an explicitly connected server', () => {+ assert.strictEqual(probePlaywright('Status: \u2713 Connected'), 'connected');+ assert.strictEqual(probePlaywright('Status: \u2714 Connected'), 'connected');+ for (const unavailableStatus of [+ 'Status: ! Connected \u00b7 tools fetch failed',+ 'Status: ! Needs authentication',+ 'Status: \u2718 Failed to connect',+ 'Status: \u23f8 Pending approval',+ 'Status: \u2298 Disabled for this project',+ '',+ ]) {+ assert.strictEqual(probePlaywright(unavailableStatus), 'unavailable');+ }+ assert.strictEqual(probePlaywright('Status: \u2713 Connected', 1), 'unavailable');+ assert.strictEqual(+ probePlaywright('Status: \u2718 Failed to connect\nStatus: \u2713 Connected'),+ 'unavailable'+ );+ }),++ test('evaluator tools follow mode and reuse the approved agent contract', () => {+ assert.deepStrictEqual(evaluatorToolsForMode('playwright'), declaredEvaluatorTools());+ for (const mode of ['screenshot', 'code-only']) {+ assert.deepStrictEqual(+ evaluatorToolsForMode(mode),+ ['Read', 'Write', 'Bash', 'Grep', 'Glob']+ );+ }+ }),++ test('Playwright is checked before setup and again before evaluator launch', () => {+ const preflightCall = harnessSource.indexOf('if ! playwright_mcp_is_connected');+ const setupMutation = harnessSource.indexOf('mkdir -p "$FEEDBACK_DIR"');+ const runtimeCheck = harnessSource.indexOf('[ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected');+ const evaluatorLaunch = harnessSource.indexOf('claude -p --model "$EVALUATOR_MODEL"');++ assert.ok(preflightCall >= 0 && preflightCall < setupMutation);+ assert.ok(runtimeCheck >= 0 && runtimeCheck < evaluatorLaunch);+ assert.match(harnessSource, /--allowedTools "\$EVALUATOR_TOOLS"/);+ assert.match(harnessSource, /Unsupported GAN_EVAL_MODE/);+ }),+test('final score lookup is compatible with the macOS Bash 3.2 runtime', () => {const finalScoreBlock = harnessSource.match(/NUM_ITERATIONS=\$\{#SCORES\[@\]\}\nif \[ "\$NUM_ITERATIONS"[\s\S]*?\nfi/