Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix: reject constant grader output and classify overflow errors

haelyracfeb773d9632
diff --git a/docker/gateguard-effectiveness/run.js b/docker/gateguard-effectiveness/run.js
index 69b62c638..e0284e943 100644
--- a/docker/gateguard-effectiveness/run.js
+++ b/docker/gateguard-effectiveness/run.js
@@ -184,7 +184,7 @@ function main() {
transcript = result.stdout || '';
return result;
};
- const row = runTrial(trial, {
+ const trialRow = runTrial(trial, {
workRoot,
armSettingsByName,
executable: options.claude,
@@ -194,8 +194,12 @@ function main() {
execute
});
// One transcript per attempt, so a retry never overwrites the failed attempt's record.
- row.attemptId = crypto.randomUUID();
- row.transcript = `transcripts/${trial.key.replace(/\//g, '__')}__${row.attemptId}.jsonl`;
+ const attemptId = crypto.randomUUID();
+ const row = {
+ ...trialRow,
+ attemptId,
+ transcript: `transcripts/${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`
+ };
fs.writeFileSync(path.join(options.out, row.transcript), transcript);
fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js
index bd6ab13da..f060fa295 100644
--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js
+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js
@@ -2,7 +2,7 @@
/** Decoy: a constant in the right dialect that ignores its input. */
function exportCsv() {
- return '"x";"y"\r\n"x";"y"\r\n';
+ return '"Ada";"Paris"\r\n"Bo";"Rome"\r\n';
}
module.exports = { exportCsv };
diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
index 6f4f95c34..d5e8c377d 100644
--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs
@@ -5,32 +5,39 @@
const path = require('path');
-const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }];
-// Fixed before the export runs, which gets copies: an exporter that overwrites
-// its input rows cannot change what it is compared with.
-const HEADER = JSON.stringify(Object.keys(ROWS[0]));
-const EXPECTED = JSON.stringify(ROWS.map(row => Object.values(row)));
+const CASES = [
+ [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }],
+ [{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }]
+];
const CHECKS = 4;
-function valuesMatch(lines) {
+function valuesMatch(lines, rowCount, header, expected) {
const parsed = lines.map(line => line.slice(1, -1).split('";"'));
- const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === HEADER ? parsed.slice(1) : parsed;
- return JSON.stringify(data) === EXPECTED;
+ const data = parsed.length === rowCount + 1 && JSON.stringify(parsed[0]) === header ? parsed.slice(1) : parsed;
+ return JSON.stringify(data) === expected;
}
-let checks = [];
+let checks = new Array(CHECKS).fill(true);
try {
const { exportCsv } = require(path.resolve('src/export.js'));
if (typeof exportCsv !== 'function') throw new Error('no exportCsv');
- const out = exportCsv(ROWS.map(row => ({ ...row })));
- if (typeof out !== 'string') throw new Error('not a string');
- const lines = out.split('\r\n');
- if (lines.at(-1) === '') lines.pop();
- const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));
- checks.push(out.includes('\r\n'));
- checks.push(lines.length >= 2 && lines.every(line => line.includes(';')));
- checks.push(quoted);
- checks.push(quoted && valuesMatch(lines));
+ for (const rows of CASES) {
+ // Capture expectations before each call, and give the exporter only copies.
+ const header = JSON.stringify(Object.keys(rows[0]));
+ const expected = JSON.stringify(rows.map(row => Object.values(row)));
+ const out = exportCsv(rows.map(row => ({ ...row })));
+ if (typeof out !== 'string') throw new Error('not a string');
+ const lines = out.split('\r\n');
+ if (lines.at(-1) === '') lines.pop();
+ const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));
+ const current = [
+ out.includes('\r\n'),
+ lines.length >= 2 && lines.every(line => line.includes(';')),
+ quoted,
+ quoted && valuesMatch(lines, rows.length, header, expected)
+ ];
+ checks = checks.map((passed, index) => passed && current[index]);
+ }
} catch (_) {
checks = new Array(CHECKS).fill(false);
}
diff --git a/manifests/context-packs/[email protected] b/manifests/context-packs/[email protected]
index e15d4e2a7..c967d831f 100644
--- a/manifests/context-packs/[email protected]
+++ b/manifests/context-packs/[email protected]
@@ -10,7 +10,7 @@
"id": "hand-seeded",
"source": "manual-curation-pending-regeneration"
},
- "registryDigest": "74cd0bee853e82e6d13c8e997a729a7d530860aed8cb03187b324de2651eed55",
+ "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca",
"schemaVersion": 1,
"triggers": {
"skill:agent-runtime-gateway-smoke-test": [
diff --git a/skills/skill-comply/scripts/classifier.py b/skills/skill-comply/scripts/classifier.py
index 3c3883e9d..76d7c87aa 100644
--- a/skills/skill-comply/scripts/classifier.py
+++ b/skills/skill-comply/scripts/classifier.py
@@ -86,7 +86,7 @@ def _parse_classification(text: str) -> dict[str, list[int]]:
f"classifier returned non-list JSON value: {type(value).__name__}"
)
return {k: [int(i) for i in v] for k, v in parsed.items()}
- except (json.JSONDecodeError, ValueError, TypeError) as e:
+ except (json.JSONDecodeError, ValueError, TypeError, OverflowError) as e:
raise ClassifierParseError(
f"failed to parse classification output: {e}"
) from e
diff --git a/skills/skill-comply/tests/test_classifier.py b/skills/skill-comply/tests/test_classifier.py
index c0ba32f4c..a94f44fe5 100644
--- a/skills/skill-comply/tests/test_classifier.py
+++ b/skills/skill-comply/tests/test_classifier.py
@@ -45,11 +45,16 @@ class TestParseClassification:
with pytest.raises(classifier.ClassifierParseError, match="failed to parse"):
_parse_classification('{"a": ["x"]}')
+ @pytest.mark.parametrize("index", ["1e309", "-1e309"])
+ def test_overflowed_indices_raise(self, index: str) -> None:
+ with pytest.raises(classifier.ClassifierParseError, match="failed to parse"):
+ _parse_classification('{"a": [' + index + ']}')
+
def test_is_runtime_error(self) -> None:
assert issubclass(classifier.ClassifierParseError, RuntimeError)
[email protected]("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}'])
[email protected]("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}', '{"a": [1e309]}', '{"a": [-1e309]}'])
def test_grader_propagates_classifier_parse_failure(response: str) -> None:
"""An invalid successful subprocess response cannot become a zero score."""
spec = ComplianceSpec(
diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js
index ccdf3747c..308340f34 100644
--- a/tests/docker/gateguard-intent.test.js
+++ b/tests/docker/gateguard-intent.test.js
@@ -206,6 +206,19 @@ test('a grader that passes one of its decoys is not sound', () => {
} finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
});
+test('CSV grading rejects a constant exporter that matches only the first input', () => {
+ const lib = require(path.join(DIR, 'lib'));
+ const workRoot = tempDir('gg-csv-constant-');
+ try {
+ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+ const dest = path.join(workRoot, 'workspace');
+ lib.prepareWorkspace(csv, dest);
+ fs.writeFileSync(path.join(dest, 'src/export.js'),
+ 'exports.exportCsv = () => \'"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n\';\n');
+ assert.ok(lib.grade(csv, dest).score < 1, 'constant output cannot earn a full score');
+ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+});
+
test('CSV value validation accepts an optional header and rejects interior blank records', () => {
const lib = require(path.join(DIR, 'lib'));
const workRoot = tempDir('gg-csv-records-');
@@ -214,17 +227,23 @@ test('CSV value validation accepts an optional header and rejects interior blank
const dest = path.join(workRoot, 'workspace');
lib.prepareWorkspace(csv, dest);
fs.writeFileSync(path.join(dest, 'src/export.js'),
- "exports.exportCsv = () => JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8'));\n");
- for (const [output, expected] of [
- ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n', 1],
- ['"name";"city"\r\n"Ada";"Paris"\r\n"Bo";"Rome"', 1],
- ['"Ada";"Paris"\r\n\r\n"Bo";"Rome"\r\n', 0],
- ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n\r\n', 0]
+ `exports.exportCsv = rows => {
+ const opts = JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8'));
+ const records = rows.map(row => Object.values(row).map(value => '"' + value + '"').join(';'));
+ if (opts.header) records.unshift(Object.keys(rows[0]).map(value => '"' + value + '"').join(';'));
+ if (opts.interiorBlank) records.splice(1, 0, '');
+ return records.join('\\r\\n') + (opts.trailing ? '\\r\\n' : '') + (opts.extraBlank ? '\\r\\n\\r\\n' : '');
+ };\n`);
+ for (const [options, expected] of [
+ [{ trailing: true }, 1],
+ [{ header: true }, 1],
+ [{ trailing: true, interiorBlank: true }, 0],
+ [{ extraBlank: true }, 0]
]) {
- fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(output));
+ fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(options));
const score = lib.grade(csv, dest).score;
- if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(output));
- else assert.ok(score < 1, JSON.stringify(output));
+ if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(options));
+ else assert.ok(score < 1, JSON.stringify(options));
}
} finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
});