Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: fingerprint actual grader and shared harness inputs

Louis Beaumont03abee04bfb9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 26944dea2..4568b04d7 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -168,3 +168,29 @@ plan truth, identity binding and cache admission. It does not test live billing,
cryptographic verification, provider settlement or cross-service deployment.
Run `bun test evals/coding-agent/calibrate-billing-capacity.test.js` to check
correct, broken, equivalent and bypass implementations of the grader contract.
+
+## Execution provenance
+
+Each graded trial records SHA-256 hashes of the exact fixture bytes materialized
+into the workspace, their destinations and executable flags, resolved Git source
+commits where applicable, and the grader command hash and timeout. It also records
+the startup bytes of `run.mjs` and `grader-outcome.mjs`, independently of the repo
+being evaluated. These records are retained in verification results too.
+
+`evaluation_fingerprint` identifies the case definition, resolved base, hidden
+grader and harness inputs. The report combines selected case fingerprints with
+the manifest and runtime fingerprints. It remains stable across output-directory
+changes and equivalent runner relocation; changed grader or harness bytes change
+it. `dataset_fingerprint` continues to mean the manifest bytes alone. If setup
+prevents collecting a trial's grader provenance, the report marks
+`provenance_complete: false` and leaves its evaluation fingerprint null.
+
+Run `bun test evals/coding-agent/fingerprints.test.ts` for actual caller controls
+covering changed local/Git grader inputs, moved refs, imported classifier and
+runner changes, repeated runs, relocation, selection and setup failure. These
+use synthetic repositories without model calls.
+
+This is input provenance, not a complete reproducibility or isolation guarantee.
+Dependency directories, compilers, external services, ambient environment and
+sandbox enforcement are not attested by these hashes. Do not use a matching
+fingerprint alone to claim matched agent trials or unchanged product behavior.
diff --git a/evals/coding-agent/fingerprints.test.ts b/evals/coding-agent/fingerprints.test.ts
new file mode 100644
index 000000000..d60eb3b70
--- /dev/null
+++ b/evals/coding-agent/fingerprints.test.ts
@@ -0,0 +1,85 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { createHash } from 'node:crypto';
+const root=mkdtempSync(join(tmpdir(),'eval-fingerprints-'));
+const repo=join(root,'repo');mkdirSync(repo);
+const runnerRoot=join(root,'runner');mkdirSync(runnerRoot);
+for(const p of ['run.mjs','grader-outcome.mjs'])copyFileSync(join(import.meta.dir,p),join(runnerRoot,p));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const git=(...args:string[])=>execFileSync('git',args,{cwd:repo,encoding:'utf8'}).trim();
+const hash=(bytes:any)=>createHash('sha256').update(bytes).digest('hex');
+const correct='import {readFileSync} from "node:fs"; process.exit(readFileSync("state.txt","utf8")==="broken" ? 1 : 0);\n';
+git('init','-q');git('config','user.name','Synthetic eval control');git('config','user.email','[email protected]');git('config','commit.gpgsign','false');
+writeFileSync(join(repo,'state.txt'),'broken');writeFileSync(join(repo,'git-grader.mjs'),correct);git('add','state.txt','git-grader.mjs');git('commit','-qm','synthetic broken state');const base=git('rev-parse','HEAD');
+writeFileSync(join(repo,'state.txt'),'fixed');git('commit','-qam','synthetic reference');const fix=git('rev-parse','HEAD');
+writeFileSync(join(repo,'git-grader.mjs'),'process.exit(0);\n');git('commit','-qam','synthetic alternative grader');const other=git('rev-parse','HEAD');git('checkout','-q','--detach',fix);git('branch','fixture-source',fix);
+writeFileSync(join(repo,'local-grader.mjs'),correct);
+const cases=['local','git'].map(id=>({id,name:id,base_ref:base,oracle_ref:fix,oracle_paths:['state.txt'],source:{kind:'git_regression',fix_commit:fix},prompt:'Synthetic fingerprint control; no model runs.',grader:{timeout_seconds:5,command:'exec node hidden.mjs',fixtures:[id==='local'?{local_path:'local-grader.mjs',destination_path:'hidden.mjs'}:{source_ref:'fixture-source',source_path:'git-grader.mjs',destination_path:'hidden.mjs'}]}}));
+const manifest=join(repo,'cases.json');writeFileSync(manifest,JSON.stringify({schema_version:1,suite:'synthetic-fingerprint-controls',dataset_version:'1',cases}));
+let seq=0;
+function run(runner=join(runnerRoot,'run.mjs'),extra:string[]=[]){
+ const output=join(root,`out-${++seq}`);
+ const result=spawnSync('node',[runner,'--repo',repo,'--manifest',manifest,'--mode','baseline','--results-dir',output,...extra],{cwd:repo,encoding:'utf8',timeout:20000});
+ expect(result.error).toBeUndefined();expect(result.status).toBe(0);
+ return {report:JSON.parse(readFileSync(join(output,'summary.json'),'utf8')),output};
+}
+test('executed grader and shared harness bytes identify reports without changing scoring',()=>{
+ const first=run();const second=run();
+ // First establish the observable collision: same committed repo and manifest,
+ // changed hidden grader, different outcome. The new execution fingerprint must differ.
+ writeFileSync(join(repo,'local-grader.mjs'),'process.exit(0);\n');const changed=run();
+ expect(first.report.trials.find((t:any)=>t.case_id==='local').outcome).toBe('fail');
+ expect(changed.report.trials.find((t:any)=>t.case_id==='local').outcome).toBe('pass');
+ expect(changed.report.dataset_fingerprint).toBe(first.report.dataset_fingerprint);
+ expect(changed.report.runtime.repo_head).toBe(first.report.runtime.repo_head);
+ expect(changed.report.runtime.fingerprint).toBe(first.report.runtime.fingerprint);
+ expect(first.report.evaluation_fingerprint).toMatch(/^[a-f0-9]{64}$/);
+ expect(changed.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);
+ expect(second.report.evaluation_fingerprint).toBe(first.report.evaluation_fingerprint);
+ expect(second.report.runtime.fingerprint).toBe(first.report.runtime.fingerprint);
+ const trial=changed.report.trials.find((t:any)=>t.case_id==='local');
+ expect(trial.grader_provenance.fixtures[0]).toMatchObject({destination_path:'hidden.mjs',sha256:hash('process.exit(0);\n')});
+ expect(trial.evaluation_fingerprint).not.toBe(first.report.trials.find((t:any)=>t.case_id==='local').evaluation_fingerprint);
+ expect(trial.harness_provenance.files['grader-outcome.mjs']).toBe(hash(readFileSync(join(runnerRoot,'grader-outcome.mjs'))));
+ expect(changed.report.provenance_complete).toBe(true);
+ // Restore exact bytes: output paths, elapsed time and run IDs are not identity.
+ writeFileSync(join(repo,'local-grader.mjs'),correct);const restored=run();
+ expect(restored.report.evaluation_fingerprint).toBe(first.report.evaluation_fingerprint);
+ // Git fixture aliases are resolved and content-recorded without moving HEAD.
+ git('update-ref','refs/heads/fixture-source',other);const moved=run();
+ expect(moved.report.runtime.repo_head).toBe(first.report.runtime.repo_head);
+ expect(moved.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);
+ const gitTrial=moved.report.trials.find((t:any)=>t.case_id==='git');
+ expect(gitTrial.grader_provenance.fixtures[0]).toMatchObject({source_commit:other,sha256:hash('process.exit(0);\n')});
+ git('update-ref','refs/heads/fixture-source',fix);
+ // Imported classifier implementation must count even when repo is elsewhere.
+ const originalHelper=readFileSync(join(runnerRoot,'grader-outcome.mjs'),'utf8');
+ writeFileSync(join(runnerRoot,'grader-outcome.mjs'),originalHelper+'\n// synthetic implementation version\n');const helper=run();
+ expect(helper.report.runtime.fingerprint).not.toBe(first.report.runtime.fingerprint);
+ expect(helper.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);
+ writeFileSync(join(runnerRoot,'grader-outcome.mjs'),originalHelper);
+ const originalRunner=readFileSync(join(runnerRoot,'run.mjs'),'utf8');
+ writeFileSync(join(runnerRoot,'run.mjs'),originalRunner+'\n// synthetic runner version\n');const changedRunner=run();
+ expect(changedRunner.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);
+ writeFileSync(join(runnerRoot,'run.mjs'),originalRunner);
+ const copied=join(root,'copied');mkdirSync(copied);for(const p of ['run.mjs','grader-outcome.mjs'])copyFileSync(join(runnerRoot,p),join(copied,p));
+ expect(run(join(copied,'run.mjs')).report.evaluation_fingerprint).toBe(first.report.evaluation_fingerprint);
+ const selected=run(undefined,['--case','local']);expect(selected.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);
+ expect(selected.report.trials).toHaveLength(1);
+ // Verification has no summary.json; its actual per-trial provenance must persist.
+ const output=join(root,'verified');const verified=spawnSync('node',[join(runnerRoot,'run.mjs'),'--repo',repo,'--manifest',manifest,'--verify','--results-dir',output],{cwd:repo,encoding:'utf8',timeout:20000});
+ expect(verified.status).toBe(0);const records=JSON.parse(readFileSync(join(output,'verification.json'),'utf8'));
+ for(const record of records){expect(record.valid).toBe(true);expect(record.baseline.evaluation_fingerprint).toMatch(/^[a-f0-9]{64}$/);expect(record.oracle.evaluation_fingerprint).toBe(record.baseline.evaluation_fingerprint);}
+ // A setup failure must not acquire a complete execution fingerprint.
+ git('config','remote.origin.promisor','true');
+ git('config','remote.origin.url',join(root,'missing-extraction-source'));
+ const unavailable=run();
+ expect(unavailable.report.provenance_complete).toBe(false);
+ expect(unavailable.report.evaluation_fingerprint).toBeNull();
+ for(const trial of unavailable.report.trials){expect(trial.outcome).toBe('error');expect(trial.grader_provenance).toBeUndefined();}
+},60000);
diff --git a/evals/coding-agent/run.mjs b/evals/coding-agent/run.mjs
index d21c5b697..a2c85980c 100644
--- a/evals/coding-agent/run.mjs
+++ b/evals/coding-agent/run.mjs
@@ -21,6 +21,17 @@ import process from "node:process";
import { classifyGraderError } from "./grader-outcome.mjs";
const HERE = dirname(new URL(import.meta.url).pathname);
+const sha256 = (body) => createHash("sha256").update(body).digest("hex");
+// Record these implementation bytes at startup, independently of --repo.
+// This is provenance, not an isolation or dependency/environment attestation.
+const harnessFiles = Object.fromEntries(["run.mjs", "grader-outcome.mjs"].map(
+ (path) => [path, sha256(readFileSync(join(HERE, path)))],
+));
+const HARNESS_PROVENANCE = {
+ schema_version: 1,
+ files: harnessFiles,
+ fingerprint: sha256(JSON.stringify(harnessFiles)),
+};
let REPO = resolve(HERE, "../..");
let MANIFEST = join(HERE, "cases.json");
let EXTRACTION_SOURCE;
@@ -185,6 +196,7 @@ function extractBase(evalCase, workspace) {
});
mustRun("git", ["config", "user.email", "[email protected]"], { cwd: workspace });
mustRun("git", ["config", "user.name", "screenpipe eval harness"], { cwd: workspace });
+ return baseSha;
}
function linkDependencies(evalCase, workspace, field = "dependency_links") {
@@ -220,13 +232,16 @@ function applySavedCandidate(path, workspace) {
}
function materializeGraders(evalCase, workspace) {
+ const fixtures = [];
for (const fixture of evalCase.grader.fixtures ?? []) {
let body;
+ let sourceCommit;
if (fixture.local_path) {
body = readFileSync(join(dirname(MANIFEST), fixture.local_path));
} else {
const sourceRef = fixture.source_ref ?? evalCase.oracle_ref;
- body = mustRun("git", ["show", `${sourceRef}:${fixture.source_path}`], {
+ sourceCommit = mustRun("git", ["rev-parse", `${sourceRef}^{commit}`]).stdout.trim();
+ body = mustRun("git", ["show", `${sourceCommit}:${fixture.source_path}`], {
encoding: null,
}).stdout;
}
@@ -237,7 +252,23 @@ function materializeGraders(evalCase, workspace) {
mkdirSync(dirname(destination), { recursive: true });
writeFileSync(destination, Buffer.from(body));
if (fixture.executable) chmodSync(destination, 0o755);
+ // Hash the same bytes we wrote, not a later read of mutable source files.
+ fixtures.push({
+ local_path: fixture.local_path,
+ source_path: fixture.source_path,
+ source_commit: sourceCommit,
+ destination_path: fixture.destination_path ?? fixture.source_path ?? fixture.local_path,
+ executable: Boolean(fixture.executable),
+ sha256: sha256(Buffer.from(body)),
+ });
}
+ const provenance = {
+ schema_version: 1,
+ command_sha256: sha256(evalCase.grader.command),
+ timeout_seconds: evalCase.grader.timeout_seconds ?? 600,
+ fixtures,
+ };
+ return { ...provenance, fingerprint: sha256(JSON.stringify(provenance)) };
}
function renderAgentCommand(template, paths, evalCase) {
@@ -312,8 +343,10 @@ function runTrial(evalCase, mode, trial, options, runDir) {
let agent = { status: 0, stdout: "", stderr: "", signal: null, error: null };
const started = Date.now();
+ let graderProvenance;
+ let evaluationFingerprint;
try {
- extractBase(evalCase, workspace);
+ const baseCommit = extractBase(evalCase, workspace);
linkDependencies(evalCase, workspace);
if (mode === "oracle") applyOracle(evalCase, workspace);
if (mode === "regrade") {
@@ -362,7 +395,14 @@ function runTrial(evalCase, mode, trial, options, runDir) {
// Grader-only caches are withheld until the agent trajectory is complete,
// so future build artifacts cannot leak information into the trial.
linkDependencies(evalCase, workspace, "grader_dependency_links");
- materializeGraders(evalCase, workspace);
+ graderProvenance = materializeGraders(evalCase, workspace);
+ evaluationFingerprint = sha256(JSON.stringify({
+ schema_version: 1,
+ case_definition_sha256: sha256(JSON.stringify(evalCase)),
+ base_commit: baseCommit,
+ harness: HARNESS_PROVENANCE.fingerprint,
+ grader: graderProvenance.fingerprint,
+ }));
const grader = command("/bin/bash", ["-lc", evalCase.grader.command], {
cwd: workspace,
timeout: (evalCase.grader.timeout_seconds ?? 600) * 1_000,
@@ -389,6 +429,9 @@ function runTrial(evalCase, mode, trial, options, runDir) {
grader_error: grader.error,
grader_error_kind: graderErrorKind,
changed_files: candidate.changedFiles,
+ harness_provenance: HARNESS_PROVENANCE,
+ grader_provenance: graderProvenance,
+ evaluation_fingerprint: evaluationFingerprint,
workspace: options.keep ? workspace : undefined,
};
writeFileSync(join(trialDir, "result.json"), JSON.stringify(result, null, 2));
@@ -402,6 +445,9 @@ function runTrial(evalCase, mode, trial, options, runDir) {
outcome: "error",
duration_ms: Date.now() - started,
harness_error: error instanceof Error ? error.message : String(error),
+ harness_provenance: HARNESS_PROVENANCE,
+ grader_provenance: graderProvenance,
+ evaluation_fingerprint: evaluationFingerprint,
workspace: options.keep ? workspace : undefined,
};
writeFileSync(join(trialDir, "result.json"), JSON.stringify(result, null, 2));
@@ -435,11 +481,24 @@ function summarize(manifest, mode, results, options, runDir) {
release: release(),
arch: arch(),
node: process.version,
+ harness_fingerprint: HARNESS_PROVENANCE.fingerprint,
repo_head: mustRun("git", ["rev-parse", "HEAD"]).stdout.trim(),
agent_command: mode === "agent" ? options.agentCommand : undefined,
candidate_patch: mode === "regrade" ? options.candidatePatch : undefined,
};
+ const runtimeFingerprint = sha256(JSON.stringify(runtime));
+ const provenanceComplete = results.every((result) => Boolean(result.evaluation_fingerprint));
+ const evaluatedCases = [...new Set(results.map((result) =>
+ JSON.stringify([result.case_id, result.evaluation_fingerprint ?? null]),
+ ))].sort();
const report = {
+ provenance_complete: provenanceComplete,
+ evaluation_fingerprint: provenanceComplete ? sha256(JSON.stringify({
+ schema_version: 1,
+ dataset_fingerprint: manifest.dataset_fingerprint,
+ runtime_fingerprint: runtimeFingerprint,
+ evaluated_cases: evaluatedCases,
+ })) : null,
suite: manifest.suite,
dataset_version: manifest.dataset_version,
dataset_fingerprint: manifest.dataset_fingerprint,
@@ -447,7 +506,7 @@ function summarize(manifest, mode, results, options, runDir) {
created_at: new Date().toISOString(),
runtime: {
...runtime,
- fingerprint: createHash("sha256").update(JSON.stringify(runtime)).digest("hex"),
+ fingerprint: runtimeFingerprint,
},
cases,
trials: results,