The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix: reject constant grader output and classify overflow errors
haelyracfeb773d9632
diff --git a/docker/gateguard-effectiveness/run.js b/docker/gateguard-effectiveness/run.jsindex 69b62c638..e0284e943 100644--- a/docker/gateguard-effectiveness/run.js+++ b/docker/gateguard-effectiveness/run.js@@ -184,7 +184,7 @@ function main() {transcript = result.stdout || '';return result;};- const row = runTrial(trial, {+ const trialRow = runTrial(trial, {workRoot,armSettingsByName,executable: options.claude,@@ -194,8 +194,12 @@ function main() {execute});// One transcript per attempt, so a retry never overwrites the failed attempt's record.- row.attemptId = crypto.randomUUID();- row.transcript = `transcripts/${trial.key.replace(/\//g, '__')}__${row.attemptId}.jsonl`;+ const attemptId = crypto.randomUUID();+ const row = {+ ...trialRow,+ attemptId,+ transcript: `transcripts/${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`+ };fs.writeFileSync(path.join(options.out, row.transcript), transcript);fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.jsindex bd6ab13da..f060fa295 100644--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js@@ -2,7 +2,7 @@/** Decoy: a constant in the right dialect that ignores its input. */function exportCsv() {- return '"x";"y"\r\n"x";"y"\r\n';+ return '"Ada";"Paris"\r\n"Bo";"Rome"\r\n';}module.exports = { exportCsv };diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjsindex 6f4f95c34..d5e8c377d 100644--- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs+++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs@@ -5,32 +5,39 @@const path = require('path');-const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }];-// Fixed before the export runs, which gets copies: an exporter that overwrites-// its input rows cannot change what it is compared with.-const HEADER = JSON.stringify(Object.keys(ROWS[0]));-const EXPECTED = JSON.stringify(ROWS.map(row => Object.values(row)));+const CASES = [+ [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }],+ [{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }]+];const CHECKS = 4;-function valuesMatch(lines) {+function valuesMatch(lines, rowCount, header, expected) {const parsed = lines.map(line => line.slice(1, -1).split('";"'));- const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === HEADER ? parsed.slice(1) : parsed;- return JSON.stringify(data) === EXPECTED;+ const data = parsed.length === rowCount + 1 && JSON.stringify(parsed[0]) === header ? parsed.slice(1) : parsed;+ return JSON.stringify(data) === expected;}-let checks = [];+let checks = new Array(CHECKS).fill(true);try {const { exportCsv } = require(path.resolve('src/export.js'));if (typeof exportCsv !== 'function') throw new Error('no exportCsv');- const out = exportCsv(ROWS.map(row => ({ ...row })));- if (typeof out !== 'string') throw new Error('not a string');- const lines = out.split('\r\n');- if (lines.at(-1) === '') lines.pop();- const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));- checks.push(out.includes('\r\n'));- checks.push(lines.length >= 2 && lines.every(line => line.includes(';')));- checks.push(quoted);- checks.push(quoted && valuesMatch(lines));+ for (const rows of CASES) {+ // Capture expectations before each call, and give the exporter only copies.+ const header = JSON.stringify(Object.keys(rows[0]));+ const expected = JSON.stringify(rows.map(row => Object.values(row)));+ const out = exportCsv(rows.map(row => ({ ...row })));+ if (typeof out !== 'string') throw new Error('not a string');+ const lines = out.split('\r\n');+ if (lines.at(-1) === '') lines.pop();+ const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));+ const current = [+ out.includes('\r\n'),+ lines.length >= 2 && lines.every(line => line.includes(';')),+ quoted,+ quoted && valuesMatch(lines, rows.length, header, expected)+ ];+ checks = checks.map((passed, index) => passed && current[index]);+ }} catch (_) {checks = new Array(CHECKS).fill(false);}diff --git a/manifests/context-packs/[email protected] b/manifests/context-packs/[email protected]index e15d4e2a7..c967d831f 100644--- a/manifests/context-packs/[email protected]+++ b/manifests/context-packs/[email protected]@@ -10,7 +10,7 @@"id": "hand-seeded","source": "manual-curation-pending-regeneration"},- "registryDigest": "74cd0bee853e82e6d13c8e997a729a7d530860aed8cb03187b324de2651eed55",+ "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca","schemaVersion": 1,"triggers": {"skill:agent-runtime-gateway-smoke-test": [diff --git a/skills/skill-comply/scripts/classifier.py b/skills/skill-comply/scripts/classifier.pyindex 3c3883e9d..76d7c87aa 100644--- a/skills/skill-comply/scripts/classifier.py+++ b/skills/skill-comply/scripts/classifier.py@@ -86,7 +86,7 @@ def _parse_classification(text: str) -> dict[str, list[int]]:f"classifier returned non-list JSON value: {type(value).__name__}")return {k: [int(i) for i in v] for k, v in parsed.items()}- except (json.JSONDecodeError, ValueError, TypeError) as e:+ except (json.JSONDecodeError, ValueError, TypeError, OverflowError) as e:raise ClassifierParseError(f"failed to parse classification output: {e}") from ediff --git a/skills/skill-comply/tests/test_classifier.py b/skills/skill-comply/tests/test_classifier.pyindex c0ba32f4c..a94f44fe5 100644--- a/skills/skill-comply/tests/test_classifier.py+++ b/skills/skill-comply/tests/test_classifier.py@@ -45,11 +45,16 @@ class TestParseClassification:with pytest.raises(classifier.ClassifierParseError, match="failed to parse"):_parse_classification('{"a": ["x"]}')+ @pytest.mark.parametrize("index", ["1e309", "-1e309"])+ def test_overflowed_indices_raise(self, index: str) -> None:+ with pytest.raises(classifier.ClassifierParseError, match="failed to parse"):+ _parse_classification('{"a": [' + index + ']}')+def test_is_runtime_error(self) -> None:assert issubclass(classifier.ClassifierParseError, RuntimeError)[email protected]("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}'])[email protected]("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}', '{"a": [1e309]}', '{"a": [-1e309]}'])def test_grader_propagates_classifier_parse_failure(response: str) -> None:"""An invalid successful subprocess response cannot become a zero score."""spec = ComplianceSpec(diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.jsindex ccdf3747c..308340f34 100644--- a/tests/docker/gateguard-intent.test.js+++ b/tests/docker/gateguard-intent.test.js@@ -206,6 +206,19 @@ test('a grader that passes one of its decoys is not sound', () => {} finally { fs.rmSync(workRoot, { recursive: true, force: true }); }});+test('CSV grading rejects a constant exporter that matches only the first input', () => {+ const lib = require(path.join(DIR, 'lib'));+ const workRoot = tempDir('gg-csv-constant-');+ try {+ const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');+ const dest = path.join(workRoot, 'workspace');+ lib.prepareWorkspace(csv, dest);+ fs.writeFileSync(path.join(dest, 'src/export.js'),+ 'exports.exportCsv = () => \'"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n\';\n');+ assert.ok(lib.grade(csv, dest).score < 1, 'constant output cannot earn a full score');+ } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }+});+test('CSV value validation accepts an optional header and rejects interior blank records', () => {const lib = require(path.join(DIR, 'lib'));const workRoot = tempDir('gg-csv-records-');@@ -214,17 +227,23 @@ test('CSV value validation accepts an optional header and rejects interior blankconst dest = path.join(workRoot, 'workspace');lib.prepareWorkspace(csv, dest);fs.writeFileSync(path.join(dest, 'src/export.js'),- "exports.exportCsv = () => JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8'));\n");- for (const [output, expected] of [- ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n', 1],- ['"name";"city"\r\n"Ada";"Paris"\r\n"Bo";"Rome"', 1],- ['"Ada";"Paris"\r\n\r\n"Bo";"Rome"\r\n', 0],- ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n\r\n', 0]+ `exports.exportCsv = rows => {+ const opts = JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8'));+ const records = rows.map(row => Object.values(row).map(value => '"' + value + '"').join(';'));+ if (opts.header) records.unshift(Object.keys(rows[0]).map(value => '"' + value + '"').join(';'));+ if (opts.interiorBlank) records.splice(1, 0, '');+ return records.join('\\r\\n') + (opts.trailing ? '\\r\\n' : '') + (opts.extraBlank ? '\\r\\n\\r\\n' : '');+ };\n`);+ for (const [options, expected] of [+ [{ trailing: true }, 1],+ [{ header: true }, 1],+ [{ trailing: true, interiorBlank: true }, 0],+ [{ extraBlank: true }, 0]]) {- fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(output));+ fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(options));const score = lib.grade(csv, dest).score;- if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(output));- else assert.ok(score < 1, JSON.stringify(output));+ if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(options));+ else assert.ok(score < 1, JSON.stringify(options));}} finally { fs.rmSync(workRoot, { recursive: true, force: true }); }});