The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix: validate evaluator files before pipe reads and strengthen grader controls
haelyra6ef38a299301
diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjsindex d5e8c377d..82ed3f44d 100644--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs@@ -7,6 +7,7 @@ const path = require('path');const CASES = [[{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }],+ [{ name: 'Fay', city: 'Delhi' }, { name: 'Gus', city: 'Seoul' }],[{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }]];const CHECKS = 4;diff --git a/manifests/context-packs/[email protected] b/manifests/context-packs/[email protected]index c967d831f..a563863c7 100644--- a/manifests/context-packs/[email protected]+++ b/manifests/context-packs/[email protected]@@ -10,7 +10,7 @@"id": "hand-seeded","source": "manual-curation-pending-regeneration"},- "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca",+ "registryDigest": "7249c1a4f4fc6673d3bf62d4800b995740a8a322b9d6cb62f1069c58e2ad5f26","schemaVersion": 1,"triggers": {"skill:agent-runtime-gateway-smoke-test": [diff --git a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py b/pi/core/skills/agent-self-evaluation/scripts/evaluate.pyindex e231ac3ce..1cd6fb51e 100755--- a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py+++ b/pi/core/skills/agent-self-evaluation/scripts/evaluate.py@@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]:print("Paste agent output (Ctrl+D to finish):")return task, sys.stdin.read()if args.output:+ output = _read_file_or_text(args.output, required=True) or ""task = _read_file_or_text(args.task)if task is None and not sys.stdin.isatty():task = sys.stdin.read()- return task, _read_file_or_text(args.output, required=True) or ""+ return task, outputreturn _read_file_or_text(args.task), sys.stdin.read()diff --git a/skills/agent-self-evaluation/scripts/evaluate.py b/skills/agent-self-evaluation/scripts/evaluate.pyindex e231ac3ce..1cd6fb51e 100755--- a/skills/agent-self-evaluation/scripts/evaluate.py+++ b/skills/agent-self-evaluation/scripts/evaluate.py@@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]:print("Paste agent output (Ctrl+D to finish):")return task, sys.stdin.read()if args.output:+ output = _read_file_or_text(args.output, required=True) or ""task = _read_file_or_text(args.task)if task is None and not sys.stdin.isatty():task = sys.stdin.read()- return task, _read_file_or_text(args.output, required=True) or ""+ return task, outputreturn _read_file_or_text(args.task), sys.stdin.read()diff --git a/skills/skill-comply/tests/test_runner.py b/skills/skill-comply/tests/test_runner.pyindex 448d990f4..94f6c4bdd 100644--- a/skills/skill-comply/tests/test_runner.py+++ b/skills/skill-comply/tests/test_runner.py@@ -50,7 +50,9 @@ class TestSafeSandboxDirectory:def test_valid_id_keeps_existing_sanitization(self) -> None:assert _safe_sandbox_dir("scenario/one") == SANDBOX_BASE / "scenario_one"- def test_resolved_escape_is_still_rejected(self, tmp_path, monkeypatch) -> None:+ def test_resolved_escape_is_still_rejected(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:base = tmp_path / "sandbox"monkeypatch.setattr("scripts.runner.SANDBOX_BASE", base)with patch.object(Path, "resolve", side_effect=[tmp_path / "outside", base]):diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.jsindex 308340f34..3edfb006d 100644--- a/tests/docker/gateguard-intent.test.js+++ b/tests/docker/gateguard-intent.test.js@@ -219,6 +219,21 @@ test('CSV grading rejects a constant exporter that matches only the first input'} finally { fs.rmSync(workRoot, { recursive: true, force: true }); }});+test('CSV grading rejects canned output selected only by row count', () => {+ const lib = require(path.join(DIR, 'lib'));+ const workRoot = tempDir('gg-csv-rowcount-');+ try {+ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');+ const dest = path.join(workRoot, 'workspace');+ lib.prepareWorkspace(csv, dest);+ fs.writeFileSync(path.join(dest, 'src/export.js'),+ `exports.exportCsv = rows => rows.length === 2+ ? '"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n'+ : '"Cy";"Oslo"\\r\\n"Dee";"Lima"\\r\\n"Eli";"Tokyo"\\r\\n';\n`);+ assert.ok(lib.grade(csv, dest).score < 1, 'row count alone cannot earn a full score');+ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }+});+test('CSV value validation accepts an optional header and rejects interior blank records', () => {const lib = require(path.join(DIR, 'lib'));const workRoot = tempDir('gg-csv-records-');@@ -636,6 +651,20 @@ function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) {return { copied, out, calls, config, scenario, run };}+// These roots are private mkdtemp fixtures. Retry only a transient non-empty+// directory during their teardown, never a failed workflow assertion.+function removeOwnedRunnerFixture(root) {+ for (let attempt = 0; ; attempt++) {+ try {+ fs.rmSync(root, { recursive: true, force: true });+ return;+ } catch (error) {+ if (error.code !== 'ENOTEMPTY' || attempt === 3) throw error;+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);+ }+ }+}+test('actual runner stops before recording a gated trial whose hook never ran', () => {const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-'));try {@@ -647,7 +676,7 @@ test('actual runner stops before recording a gated trial whose hook never ran',const resultsFile = path.join(out, 'results.jsonl');const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : [];assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data');- } finally { fs.rmSync(root, { recursive: true, force: true }); }+ } finally { removeOwnedRunnerFixture(root); }});test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {@@ -672,7 +701,7 @@ test('actual runner freezes arm-patch source and refuses changed-input resume beassert.match(changed.stderr, /different experiment configuration/);assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore, 'changed input rejected before provider dispatch');assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);- } finally { fs.rmSync(root, { recursive: true, force: true }); }+ } finally { removeOwnedRunnerFixture(root); }});test('the config directory is never isolated, because that hides the credentials', () => {diff --git a/tests/test_agent_self_evaluation.py b/tests/test_agent_self_evaluation.pyindex bc31f0d34..9e245ac34 100644--- a/tests/test_agent_self_evaluation.py+++ b/tests/test_agent_self_evaluation.py@@ -72,6 +72,22 @@ def test_missing_output_file_still_fails(evaluator: ModuleType, tmp_path: Path,assert "not found" in capsys.readouterr().err[email protected]("output_kind", ["missing", "directory"])+def test_invalid_output_is_rejected_before_reading_pipe(+ evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output_kind: str+) -> None:+ output = tmp_path / "output.txt"+ if output_kind == "directory":+ output.mkdir()+ monkeypatch.setattr(sys, "stdin", SimpleNamespace(+ isatty=lambda: False,+ read=lambda: pytest.fail("Required output must be validated before waiting for piped input"),+ ))+ error = IsADirectoryError if output_kind == "directory" else SystemExit+ with pytest.raises(error):+ evaluator._read_input(args(output=str(output)))++def test_cli_uses_piped_task_for_conciseness_score(evaluator: ModuleType, tmp_path: Path) -> None:output = tmp_path / "response.txt"output.write_text("Verified with tests. " * 25, encoding="utf-8")