Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix: validate evaluator files before pipe reads and strengthen grader controls

haelyra6ef38a299301
diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
index d5e8c377d..82ed3f44d 100644
--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
@@ -7,6 +7,7 @@ const path = require('path');
const CASES = [
[{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }],
+ [{ name: 'Fay', city: 'Delhi' }, { name: 'Gus', city: 'Seoul' }],
[{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }]
];
const CHECKS = 4;
diff --git a/manifests/context-packs/[email protected] b/manifests/context-packs/[email protected]
index c967d831f..a563863c7 100644
--- a/manifests/context-packs/[email protected]
+++ b/manifests/context-packs/[email protected]
@@ -10,7 +10,7 @@
"id": "hand-seeded",
"source": "manual-curation-pending-regeneration"
},
- "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca",
+ "registryDigest": "7249c1a4f4fc6673d3bf62d4800b995740a8a322b9d6cb62f1069c58e2ad5f26",
"schemaVersion": 1,
"triggers": {
"skill:agent-runtime-gateway-smoke-test": [
diff --git a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py b/pi/core/skills/agent-self-evaluation/scripts/evaluate.py
index e231ac3ce..1cd6fb51e 100755
--- a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py
+++ b/pi/core/skills/agent-self-evaluation/scripts/evaluate.py
@@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]:
print("Paste agent output (Ctrl+D to finish):")
return task, sys.stdin.read()
if args.output:
+ output = _read_file_or_text(args.output, required=True) or ""
task = _read_file_or_text(args.task)
if task is None and not sys.stdin.isatty():
task = sys.stdin.read()
- return task, _read_file_or_text(args.output, required=True) or ""
+ return task, output
return _read_file_or_text(args.task), sys.stdin.read()
diff --git a/skills/agent-self-evaluation/scripts/evaluate.py b/skills/agent-self-evaluation/scripts/evaluate.py
index e231ac3ce..1cd6fb51e 100755
--- a/skills/agent-self-evaluation/scripts/evaluate.py
+++ b/skills/agent-self-evaluation/scripts/evaluate.py
@@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]:
print("Paste agent output (Ctrl+D to finish):")
return task, sys.stdin.read()
if args.output:
+ output = _read_file_or_text(args.output, required=True) or ""
task = _read_file_or_text(args.task)
if task is None and not sys.stdin.isatty():
task = sys.stdin.read()
- return task, _read_file_or_text(args.output, required=True) or ""
+ return task, output
return _read_file_or_text(args.task), sys.stdin.read()
diff --git a/skills/skill-comply/tests/test_runner.py b/skills/skill-comply/tests/test_runner.py
index 448d990f4..94f6c4bdd 100644
--- a/skills/skill-comply/tests/test_runner.py
+++ b/skills/skill-comply/tests/test_runner.py
@@ -50,7 +50,9 @@ class TestSafeSandboxDirectory:
def test_valid_id_keeps_existing_sanitization(self) -> None:
assert _safe_sandbox_dir("scenario/one") == SANDBOX_BASE / "scenario_one"
- def test_resolved_escape_is_still_rejected(self, tmp_path, monkeypatch) -> None:
+ def test_resolved_escape_is_still_rejected(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
base = tmp_path / "sandbox"
monkeypatch.setattr("scripts.runner.SANDBOX_BASE", base)
with patch.object(Path, "resolve", side_effect=[tmp_path / "outside", base]):
diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js
index 308340f34..3edfb006d 100644
--- a/tests/docker/gateguard-intent.test.js
+++ b/tests/docker/gateguard-intent.test.js
@@ -219,6 +219,21 @@ test('CSV grading rejects a constant exporter that matches only the first input'
} finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
});
+test('CSV grading rejects canned output selected only by row count', () => {
+ const lib = require(path.join(DIR, 'lib'));
+ const workRoot = tempDir('gg-csv-rowcount-');
+ try {
+ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+ const dest = path.join(workRoot, 'workspace');
+ lib.prepareWorkspace(csv, dest);
+ fs.writeFileSync(path.join(dest, 'src/export.js'),
+ `exports.exportCsv = rows => rows.length === 2
+ ? '"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n'
+ : '"Cy";"Oslo"\\r\\n"Dee";"Lima"\\r\\n"Eli";"Tokyo"\\r\\n';\n`);
+ assert.ok(lib.grade(csv, dest).score < 1, 'row count alone cannot earn a full score');
+ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+});
+
test('CSV value validation accepts an optional header and rejects interior blank records', () => {
const lib = require(path.join(DIR, 'lib'));
const workRoot = tempDir('gg-csv-records-');
@@ -636,6 +651,20 @@ function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) {
return { copied, out, calls, config, scenario, run };
}
+// These roots are private mkdtemp fixtures. Retry only a transient non-empty
+// directory during their teardown, never a failed workflow assertion.
+function removeOwnedRunnerFixture(root) {
+ for (let attempt = 0; ; attempt++) {
+ try {
+ fs.rmSync(root, { recursive: true, force: true });
+ return;
+ } catch (error) {
+ if (error.code !== 'ENOTEMPTY' || attempt === 3) throw error;
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);
+ }
+ }
+}
+
test('actual runner stops before recording a gated trial whose hook never ran', () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-'));
try {
@@ -647,7 +676,7 @@ test('actual runner stops before recording a gated trial whose hook never ran',
const resultsFile = path.join(out, 'results.jsonl');
const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : [];
assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data');
- } finally { fs.rmSync(root, { recursive: true, force: true }); }
+ } finally { removeOwnedRunnerFixture(root); }
});
test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
@@ -672,7 +701,7 @@ test('actual runner freezes arm-patch source and refuses changed-input resume be
assert.match(changed.stderr, /different experiment configuration/);
assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore, 'changed input rejected before provider dispatch');
assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
- } finally { fs.rmSync(root, { recursive: true, force: true }); }
+ } finally { removeOwnedRunnerFixture(root); }
});
test('the config directory is never isolated, because that hides the credentials', () => {
diff --git a/tests/test_agent_self_evaluation.py b/tests/test_agent_self_evaluation.py
index bc31f0d34..9e245ac34 100644
--- a/tests/test_agent_self_evaluation.py
+++ b/tests/test_agent_self_evaluation.py
@@ -72,6 +72,22 @@ def test_missing_output_file_still_fails(evaluator: ModuleType, tmp_path: Path,
assert "not found" in capsys.readouterr().err
[email protected]("output_kind", ["missing", "directory"])
+def test_invalid_output_is_rejected_before_reading_pipe(
+ evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output_kind: str
+) -> None:
+ output = tmp_path / "output.txt"
+ if output_kind == "directory":
+ output.mkdir()
+ monkeypatch.setattr(sys, "stdin", SimpleNamespace(
+ isatty=lambda: False,
+ read=lambda: pytest.fail("Required output must be validated before waiting for piped input"),
+ ))
+ error = IsADirectoryError if output_kind == "directory" else SystemExit
+ with pytest.raises(error):
+ evaluator._read_input(args(output=str(output)))
+
+
def test_cli_uses_piped_task_for_conciseness_score(evaluator: ModuleType, tmp_path: Path) -> None:
output = tmp_path / "response.txt"
output.write_text("Verified with tests. " * 25, encoding="utf-8")