Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(gan): verify responsive and keyboard evaluator capabilities

affaan-m8f7db0f3cf88
diff --cc agents/gan-evaluator.md
index 363e0972b,363e0972b..0c3c0c9ee
--- a/agents/gan-evaluator.md
+++ b/agents/gan-evaluator.md
@@@ -1,7 -1,7 +1,7 @@@
---
name: gan-evaluator
description: "GAN Harness — Evaluator agent. Tests the live running application via Playwright, scores against rubric, and provides actionable feedback to the Generator."
--tools: Read, Write, Bash, Grep, Glob, mcp__playwright__browser_navigate, mcp__playwright__browser_click, mcp__playwright__browser_take_screenshot, mcp__playwright__browser_snapshot, mcp__playwright__browser_type, mcp__playwright__browser_fill_form
++tools: Read, Write, Bash, Grep, Glob, mcp__playwright__browser_navigate, mcp__playwright__browser_click, mcp__playwright__browser_take_screenshot, mcp__playwright__browser_snapshot, mcp__playwright__browser_type, mcp__playwright__browser_fill_form, mcp__playwright__browser_resize, mcp__playwright__browser_press_key
model: sonnet
color: red
---
diff --cc scripts/gan-harness.sh
index fbab88742,79dd5038f..79e081620
--- a/scripts/gan-harness.sh
+++ b/scripts/gan-harness.sh
@@@ -97,38 -96,6 +97,38 @@@ score_passes()
awk -v s="$score" -v t="$threshold" 'BEGIN { exit !(s >= t) }'
}
+playwright_mcp_is_connected() {
+ local status
+ local status_value
+ local check_mark
+ local heavy_check_mark
+ status=$(NO_COLOR=1 claude mcp get playwright 2>/dev/null) || return 1
+ status_value=$(printf '%s\n' "$status" | awk '
+ /^[[:space:]]*Status:[[:space:]]*/ {
+ sub(/^[[:space:]]*Status:[[:space:]]*/, "")
+ sub(/[[:space:]]*$/, "")
+ print
+ exit
+ }
+ ')
+ check_mark=$(printf '\342\234\223')
+ heavy_check_mark=$(printf '\342\234\224')
+
+ [ "$status_value" = "$check_mark Connected" ] || \
+ [ "$status_value" = "$heavy_check_mark Connected" ]
+}
+
+evaluator_tools_for_mode() {
+ local base_tools="Read,Write,Bash,Grep,Glob"
- local playwright_tools="mcp__playwright__browser_navigate,mcp__playwright__browser_click,mcp__playwright__browser_take_screenshot,mcp__playwright__browser_snapshot,mcp__playwright__browser_type,mcp__playwright__browser_fill_form"
++ local playwright_tools="mcp__playwright__browser_navigate,mcp__playwright__browser_click,mcp__playwright__browser_take_screenshot,mcp__playwright__browser_snapshot,mcp__playwright__browser_type,mcp__playwright__browser_fill_form,mcp__playwright__browser_resize,mcp__playwright__browser_press_key"
+
+ if [ "$1" = "playwright" ]; then
+ printf '%s,%s\n' "$base_tools" "$playwright_tools"
+ else
+ printf '%s\n' "$base_tools"
+ fi
+}
+
elapsed() {
local now=$(date +%s)
local diff=$((now - START_TIME))
diff --cc tests/gan-harness.test.js
index 6a5041f92,36c7255ce..7bc05cba5
--- a/tests/gan-harness.test.js
+++ b/tests/gan-harness.test.js
@@@ -36,20 -34,13 +36,137 @@@ function test(name, fn)
}
}
-function runHarnessScript(script, args = []) {
- const bashExecutable = process.platform === 'win32' ? 'bash' : '/bin/bash';
- const result = spawnSync(bashExecutable, ['-c', script, 'gan-harness-test', ...args], {
- encoding: 'utf8',
++function withShellFixture(fn) {
++ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-gan-shell-'));
++ try {
++ const bin = path.join(root, 'bin');
++ const home = path.join(root, 'home');
++ const project = path.join(root, 'project');
++ for (const directory of [bin, home, project]) fs.mkdirSync(directory);
++ // Only inert system utilities and the explicit fake CLI are reachable.
++ for (const command of ['awk', 'date', 'mkdir', 'cat', 'tee', 'wc']) {
++ const executable = ['/usr/bin', '/bin'].map(dir => path.join(dir, command)).find(fs.existsSync);
++ assert.ok(executable, `missing system utility: ${command}`);
++ fs.symlinkSync(executable, path.join(bin, command));
++ }
++ return fn({ root, bin, project, env: { PATH: bin, HOME: home, TMPDIR: root, LC_ALL: 'C' } });
++ } finally {
++ fs.rmSync(root, { recursive: true, force: true });
++ }
++}
++
+function runHarnessScript(script, args = [], env = {}) {
- const bashExecutable = process.platform === 'win32' ? 'bash' : '/bin/bash';
- const result = spawnSync(bashExecutable, ['-c', script, 'gan-harness-test', ...args], {
- encoding: 'utf8',
- env: { ...process.env, ...env },
++ return withShellFixture(fixture => {
++ const result = spawnSync('/bin/bash', ['--noprofile', '--norc', '-c', script, 'gan-harness-test', ...args], {
++ encoding: 'utf8',
++ cwd: fixture.project,
++ env: { ...fixture.env, ...env },
++ timeout: 5000,
++ });
++ assert.ifError(result.error);
++ assert.strictEqual(result.status, 0, result.stderr || 'GAN harness script failed');
++ return result.stdout.trim();
+ });
- assert.strictEqual(result.status, 0, result.stderr || 'GAN harness script failed');
- return result.stdout.trim();
++}
++
++const fakeClaude = `#!/bin/bash
++set -euo pipefail
++printf '%s\\0' "$@" >> "$GAN_TEST_CALLS"
++printf '\\0' >> "$GAN_TEST_CALLS"
++if [ "$#" -eq 3 ] && [ "$1" = mcp ] && [ "$2" = get ] && [ "$3" = playwright ]; then
++ [ "$NO_COLOR" = 1 ] || exit 65
++ count=0
++ if [ -f "$GAN_TEST_PROBES" ]; then read -r count < "$GAN_TEST_PROBES"; fi
++ printf '%s\\n' "$((count + 1))" > "$GAN_TEST_PROBES"
++ if [ "$count" -eq 0 ]; then
++ printf '%s\\n' "$GAN_TEST_FIRST_STATUS"
++ exit "$GAN_TEST_FIRST_EXIT"
++ fi
++ printf '%s\\n' "$GAN_TEST_SECOND_STATUS"
++ exit "$GAN_TEST_SECOND_EXIT"
++fi
++[ "$1" = -p ] && [ "$2" = --model ] && [ "$3" = fixture-model ] || exit 66
++for prompt in "$@"; do :; done
++case "$prompt" in
++ 'You are the Planner'*)
++ printf 'Inert spec\\n' > gan-harness/spec.md
++ printf 'Inert rubric\\n' > gan-harness/eval-rubric.md
++ ;;
++ 'You are the Generator'*) ;;
++ 'You are the Evaluator'*)
++ printf '| **TOTAL** | | | **9.0** |\\n' > gan-harness/feedback/feedback-001.md
++ ;;
++ *) exit 67 ;;
++esac
++`;
++
++function withHarnessRun(options, check) {
++ return withShellFixture(fixture => {
++ const callsPath = path.join(fixture.root, 'calls');
++ const gitCallsPath = path.join(fixture.root, 'git-calls');
++ fs.writeFileSync(path.join(fixture.bin, 'claude'), fakeClaude, { mode: 0o700 });
++ fs.writeFileSync(path.join(fixture.bin, 'git'), '#!/bin/bash\nprintf unexpected > "$GAN_TEST_GIT_CALLS"\nexit 68\n', { mode: 0o700 });
++ // A directory is sufficient to bypass initialization; no real Git command runs.
++ fs.mkdirSync(path.join(fixture.project, '.git'));
++ const result = spawnSync('/bin/bash', ['--noprofile', '--norc', harnessPath, 'Inert fixture brief'], {
++ encoding: 'utf8',
++ cwd: fixture.project,
++ timeout: 5000,
++ env: {
++ ...fixture.env,
++ GAN_PROJECT_DIR: fixture.project,
++ GAN_MAX_ITERATIONS: '1',
++ GAN_PLANNER_MODEL: 'fixture-model',
++ GAN_GENERATOR_MODEL: 'fixture-model',
++ GAN_EVALUATOR_MODEL: 'fixture-model',
++ GAN_EVAL_MODE: options.mode || 'playwright',
++ GAN_TEST_CALLS: callsPath,
++ GAN_TEST_GIT_CALLS: gitCallsPath,
++ GAN_TEST_PROBES: path.join(fixture.root, 'probes'),
++ GAN_TEST_FIRST_STATUS: options.firstStatus ?? 'Status: \u2713 Connected',
++ GAN_TEST_FIRST_EXIT: String(options.firstExit || 0),
++ GAN_TEST_SECOND_STATUS: options.secondStatus ?? 'Status: \u2713 Connected',
++ GAN_TEST_SECOND_EXIT: String(options.secondExit || 0),
++ },
++ });
++ assert.ifError(result.error);
++ assert.strictEqual(result.signal, null);
++ assert.strictEqual(fs.existsSync(gitCallsPath), false, 'must never invoke real or fake Git');
++ const calls = fs.existsSync(callsPath)
++ ? fs.readFileSync(callsPath, 'utf8').split('\0\0').filter(Boolean).map(call => call.split('\0'))
++ : [];
++ check({ result, calls, project: fixture.project });
+ });
- assert.strictEqual(result.status, 0, result.stderr || 'GAN harness script failed');
- return result.stdout.trim();
++}
++
++function evaluatorCalls(calls) {
++ return calls.filter(args => args[args.length - 1].startsWith('You are the Evaluator'));
++}
++
++const baseTools = ['Read', 'Write', 'Bash', 'Grep', 'Glob'];
++const browserTools = [
++ 'mcp__playwright__browser_navigate',
++ 'mcp__playwright__browser_click',
++ 'mcp__playwright__browser_take_screenshot',
++ 'mcp__playwright__browser_snapshot',
++ 'mcp__playwright__browser_type',
++ 'mcp__playwright__browser_fill_form',
++ 'mcp__playwright__browser_resize',
++ 'mcp__playwright__browser_press_key',
++];
++
++function assertEvaluatorTools(calls, expected) {
++ const launches = evaluatorCalls(calls);
++ assert.strictEqual(launches.length, 1);
++ const args = launches[0];
++ assert.strictEqual(args.filter(arg => arg === '--allowedTools').length, 1);
++ assert.deepStrictEqual(args[args.indexOf('--allowedTools') + 1].split(','), expected);
+}
+
+function extractShellFunction(name) {
+ const functionMatch = harnessSource.match(new RegExp(`${name}\\(\\) \\{[\\s\\S]*?\\n\\}`));
+ assert.ok(functionMatch, `expected scripts/gan-harness.sh to define ${name}`);
+ return functionMatch[0];
}
function extractScore(feedback) {
@@@ -211,6 -127,6 +328,61 @@@ const results = Object.freeze(
assert.match(output, /Score:\s+8\.7\s+\/\s+10\.0/);
}),
++
++ test('declared evaluator tools cover responsive and keyboard tasks', () => {
++ assert.deepStrictEqual(declaredEvaluatorTools(), [...baseTools, ...browserTools]);
++ }),
++
++ test('actual Playwright evaluator launch includes responsive and keyboard tools', () => {
++ withHarnessRun({}, ({ result, calls }) => {
++ assert.strictEqual(result.status, 0, result.stderr);
++ assertEvaluatorTools(calls, [...baseTools, ...browserTools]);
++ assert.strictEqual(calls.filter(args => args[0] === 'mcp').length, 2);
++ });
++ }),
++
++ test('actual preflight errors and disconnected states refuse before setup writes', () => {
++ for (const options of [
++ { firstStatus: 'Status: \u2718 Failed to connect' },
++ { firstStatus: 'Status: ! Connected \u00b7 tools fetch failed' },
++ { firstStatus: '' },
++ { firstExit: 1 },
++ ]) {
++ withHarnessRun(options, ({ result, calls, project }) => {
++ assert.strictEqual(result.status, 1);
++ assert.deepStrictEqual(calls, [['mcp', 'get', 'playwright']]);
++ assert.deepStrictEqual(fs.readdirSync(project), ['.git']);
++ });
++ }
++ }),
++
++ test('unknown mode refuses before CLI calls and setup writes', () => {
++ withHarnessRun({ mode: 'unknown' }, ({ result, calls, project }) => {
++ assert.strictEqual(result.status, 1);
++ assert.deepStrictEqual(calls, []);
++ assert.deepStrictEqual(fs.readdirSync(project), ['.git']);
++ });
++ }),
++
++ test('lost connection and command errors refuse the actual evaluator launch', () => {
++ for (const options of [{ secondStatus: 'Status: \u2718 Failed to connect' }, { secondExit: 1 }]) {
++ withHarnessRun(options, ({ result, calls, project }) => {
++ assert.strictEqual(result.status, 1);
++ assert.strictEqual(calls.filter(args => args[0] === 'mcp').length, 2);
++ assert.strictEqual(calls.filter(args => args[args.length - 1].startsWith('You are the Generator')).length, 1);
++ assert.deepStrictEqual(evaluatorCalls(calls), []);
++ assert.strictEqual(fs.existsSync(path.join(project, 'gan-harness', 'evaluator-1.log')), false);
++ });
++ }
++ }),
++
++ ...['screenshot', 'code-only'].map(mode => test(`actual ${mode} launch keeps base tools without MCP probing`, () => {
++ withHarnessRun({ mode, firstExit: 1, secondExit: 1 }, ({ result, calls }) => {
++ assert.strictEqual(result.status, 0, result.stderr);
++ assert.strictEqual(calls.some(args => args[0] === 'mcp'), false);
++ assertEvaluatorTools(calls, baseTools);
++ });
++ })),
]);
const passed = results.filter(Boolean).length;