Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: preserve binary grader fixture bytes

louisacdcd8b21d76
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 2fd5ead3f..75888f302 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -609,3 +609,15 @@ materialized only for grading. These are corpus and grader checks, not model
trials, native catalog persistence, stdio dispatch, authentication or bounded
stalled-response evidence. No real recordings, credentials or provider calls
are used; execution isolation and model improvement remain unproven.
+
+### Binary source fixtures
+
+`bun test evals/coding-agent/binary-fixtures.test.ts` verifies the actual runner
+with a temporary synthetic Git repository. Git-backed fixtures must retain all
+256 byte values, including NUL and invalid UTF-8, and report hashes of those exact
+bytes. The control also preserves local binary fixtures, default and explicit
+source refs, UTF-8 text, empty files, executable flags and binary oracle patches.
+The synthetic broken state fails its intended assertion; the reference passes.
+The command adapter retains Buffers when `encoding: null` is requested and keeps
+normal command output as text. This is harness calibration, not model evaluation
+or proof of process isolation; no real recordings or customer files are used.
diff --git a/evals/coding-agent/binary-fixtures.test.ts b/evals/coding-agent/binary-fixtures.test.ts
new file mode 100644
index 000000000..0b37d3789
--- /dev/null
+++ b/evals/coding-agent/binary-fixtures.test.ts
@@ -0,0 +1,76 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+test('Git fixtures and binary oracle patches preserve exact bytes and provenance', () => {
+ const repo = mkdtempSync(join(tmpdir(), 'eval-binary-fixtures-'));
+ const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, encoding: 'utf8' }).trim();
+ const binary = Buffer.from(Array.from({ length: 256 }, (_, index) => index));
+ const text = Buffer.from('Synthetic text: café 日本語\n');
+ const hash = (bytes: Buffer) => createHash('sha256').update(bytes).digest('hex');
+ try {
+ git('init', '-q'); git('config', 'user.name', 'Synthetic eval control');
+ git('config', 'user.email', '[email protected]'); git('config', 'commit.gpgsign', 'false');
+ writeFileSync(join(repo, 'state.bin'), Buffer.from([0]));
+ writeFileSync(join(repo, 'source.bin'), Buffer.from([1]));
+ writeFileSync(join(repo, 'source.txt'), text);
+ writeFileSync(join(repo, 'empty.bin'), Buffer.alloc(0));
+ writeFileSync(join(repo, 'executable.sh'), '#!/bin/sh\nexit 0\n');
+ git('add', 'state.bin', 'source.bin', 'source.txt', 'empty.bin', 'executable.sh');
+ git('commit', '-qm', 'Synthetic broken state'); const base = git('rev-parse', 'HEAD');
+ writeFileSync(join(repo, 'state.bin'), binary); writeFileSync(join(repo, 'source.bin'), binary);
+ git('commit', '-qam', 'Synthetic fixed state'); const fix = git('rev-parse', 'HEAD');
+ writeFileSync(join(repo, 'local.bin'), binary);
+ writeFileSync(join(repo, 'grade.mjs'), `
+import assert from 'node:assert/strict';
+import { readFileSync, statSync } from 'node:fs';
+const expected = Buffer.from(Array.from({length:256}, (_,i)=>i));
+for (const path of ['git.bin','explicit.bin','local.bin']) assert.deepEqual(readFileSync(path), expected, path);
+assert.equal(readFileSync('text.txt','utf8'), ${JSON.stringify(text.toString())});
+assert.equal(readFileSync('empty.out').length, 0);
+assert.ok(statSync('executable.out').mode & 0o111);
+assert.deepEqual(readFileSync('state.bin'), expected, 'intended synthetic product state');
+`);
+ const fixtures = [
+ { source_path: 'source.bin', destination_path: 'git.bin' },
+ { source_ref: fix, source_path: 'source.bin', destination_path: 'explicit.bin' },
+ { local_path: 'local.bin', destination_path: 'local.bin' },
+ { source_path: 'source.txt', destination_path: 'text.txt' },
+ { source_path: 'empty.bin', destination_path: 'empty.out' },
+ { source_path: 'executable.sh', destination_path: 'executable.out', executable: true },
+ { local_path: 'grade.mjs', destination_path: 'grade.mjs' },
+ ];
+ const manifest = join(repo, 'cases.json');
+ writeFileSync(manifest, JSON.stringify({ schema_version: 1, suite: 'synthetic-binary-fixtures', dataset_version: '1', cases: [{
+ id: 'binary-fixtures', name: 'Synthetic byte-preservation control', base_ref: base, oracle_ref: fix,
+ oracle_paths: ['state.bin'], source: { kind: 'git_regression', fix_commit: fix },
+ prompt: 'Synthetic runner calibration. No model or live service.',
+ grader: { command: 'exec node grade.mjs', timeout_seconds: 5, fixtures },
+ }] }));
+ const output = join(repo, 'results');
+ const result = spawnSync('node', [new URL('./run.mjs', import.meta.url).pathname, '--repo', repo,
+ '--manifest', manifest, '--verify', '--results-dir', output], { encoding: 'utf8', timeout: 20000 });
+ expect(result.error).toBeUndefined();
+ const [verification] = JSON.parse(readFileSync(join(output, 'verification.json'), 'utf8'));
+ expect(result.status).toBe(0);
+ expect(verification.valid).toBe(true);
+ expect(verification.baseline.outcome).toBe('fail');
+ expect(verification.oracle.outcome).toBe('pass');
+ expect(readFileSync(join(output, 'baseline/binary-fixtures/trial-1/grader.stderr.log'), 'utf8')).toContain('intended synthetic product state');
+ for (const trial of [verification.baseline, verification.oracle]) {
+ expect(trial.harness_error).toBeUndefined(); expect(trial.grader_error_kind).toBeNull();
+ for (const destination of ['git.bin', 'explicit.bin', 'local.bin']) {
+ expect(trial.grader_provenance.fixtures.find((f: any) => f.destination_path === destination).sha256).toBe(hash(binary));
+ }
+ for (const f of trial.grader_provenance.fixtures.filter((f: any) => f.source_path)) expect(f.source_commit).toBe(fix);
+ expect(trial.grader_provenance.fixtures.find((f: any) => f.destination_path === 'text.txt').sha256).toBe(hash(text));
+ expect(trial.grader_provenance.fixtures.find((f: any) => f.destination_path === 'empty.out').sha256).toBe(hash(Buffer.alloc(0)));
+ }
+ expect(verification.baseline.grader_provenance.fingerprint).toBe(verification.oracle.grader_provenance.fingerprint);
+ } finally { rmSync(repo, { recursive: true, force: true }); }
+}, 30000);
diff --git a/evals/coding-agent/run.mjs b/evals/coding-agent/run.mjs
index a2c85980c..ec9a27597 100644
--- a/evals/coding-agent/run.mjs
+++ b/evals/coding-agent/run.mjs
@@ -68,7 +68,7 @@ function parseArgs(argv) {
function command(program, args, options = {}) {
const result = spawnSync(program, args, {
cwd: options.cwd ?? REPO,
- encoding: options.encoding ?? "utf8",
+ encoding: options.encoding === undefined ? "utf8" : options.encoding,
input: options.input,
env: options.env ?? process.env,
maxBuffer: MAX_BUFFER,
@@ -77,7 +77,8 @@ function command(program, args, options = {}) {
return {
status: result.status,
signal: result.signal,
- stdout: result.stdout?.toString() ?? "",
+ // An explicit null requests raw Git blob/patch bytes, including invalid UTF-8.
+ stdout: options.encoding === null ? (result.stdout ?? Buffer.alloc(0)) : (result.stdout?.toString() ?? ""),
stderr: result.stderr?.toString() ?? "",
error: result.error?.message ?? null,
};