YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: fingerprint actual grader and shared harness inputs
Louis Beaumont03abee04bfb9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 26944dea2..4568b04d7 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -168,3 +168,29 @@ plan truth, identity binding and cache admission. It does not test live billing,cryptographic verification, provider settlement or cross-service deployment.Run `bun test evals/coding-agent/calibrate-billing-capacity.test.js` to checkcorrect, broken, equivalent and bypass implementations of the grader contract.++## Execution provenance++Each graded trial records SHA-256 hashes of the exact fixture bytes materialized+into the workspace, their destinations and executable flags, resolved Git source+commits where applicable, and the grader command hash and timeout. It also records+the startup bytes of `run.mjs` and `grader-outcome.mjs`, independently of the repo+being evaluated. These records are retained in verification results too.++`evaluation_fingerprint` identifies the case definition, resolved base, hidden+grader and harness inputs. The report combines selected case fingerprints with+the manifest and runtime fingerprints. It remains stable across output-directory+changes and equivalent runner relocation; changed grader or harness bytes change+it. `dataset_fingerprint` continues to mean the manifest bytes alone. If setup+prevents collecting a trial's grader provenance, the report marks+`provenance_complete: false` and leaves its evaluation fingerprint null.++Run `bun test evals/coding-agent/fingerprints.test.ts` for actual caller controls+covering changed local/Git grader inputs, moved refs, imported classifier and+runner changes, repeated runs, relocation, selection and setup failure. These+use synthetic repositories without model calls.++This is input provenance, not a complete reproducibility or isolation guarantee.+Dependency directories, compilers, external services, ambient environment and+sandbox enforcement are not attested by these hashes. Do not use a matching+fingerprint alone to claim matched agent trials or unchanged product behavior.diff --git a/evals/coding-agent/fingerprints.test.ts b/evals/coding-agent/fingerprints.test.tsnew file mode 100644index 000000000..d60eb3b70--- /dev/null+++ b/evals/coding-agent/fingerprints.test.ts@@ -0,0 +1,85 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { tmpdir } from 'node:os';+import { createHash } from 'node:crypto';+const root=mkdtempSync(join(tmpdir(),'eval-fingerprints-'));+const repo=join(root,'repo');mkdirSync(repo);+const runnerRoot=join(root,'runner');mkdirSync(runnerRoot);+for(const p of ['run.mjs','grader-outcome.mjs'])copyFileSync(join(import.meta.dir,p),join(runnerRoot,p));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+const git=(...args:string[])=>execFileSync('git',args,{cwd:repo,encoding:'utf8'}).trim();+const hash=(bytes:any)=>createHash('sha256').update(bytes).digest('hex');+const correct='import {readFileSync} from "node:fs"; process.exit(readFileSync("state.txt","utf8")==="broken" ? 1 : 0);\n';+git('init','-q');git('config','user.name','Synthetic eval control');git('config','user.email','[email protected]');git('config','commit.gpgsign','false');+writeFileSync(join(repo,'state.txt'),'broken');writeFileSync(join(repo,'git-grader.mjs'),correct);git('add','state.txt','git-grader.mjs');git('commit','-qm','synthetic broken state');const base=git('rev-parse','HEAD');+writeFileSync(join(repo,'state.txt'),'fixed');git('commit','-qam','synthetic reference');const fix=git('rev-parse','HEAD');+writeFileSync(join(repo,'git-grader.mjs'),'process.exit(0);\n');git('commit','-qam','synthetic alternative grader');const other=git('rev-parse','HEAD');git('checkout','-q','--detach',fix);git('branch','fixture-source',fix);+writeFileSync(join(repo,'local-grader.mjs'),correct);+const cases=['local','git'].map(id=>({id,name:id,base_ref:base,oracle_ref:fix,oracle_paths:['state.txt'],source:{kind:'git_regression',fix_commit:fix},prompt:'Synthetic fingerprint control; no model runs.',grader:{timeout_seconds:5,command:'exec node hidden.mjs',fixtures:[id==='local'?{local_path:'local-grader.mjs',destination_path:'hidden.mjs'}:{source_ref:'fixture-source',source_path:'git-grader.mjs',destination_path:'hidden.mjs'}]}}));+const manifest=join(repo,'cases.json');writeFileSync(manifest,JSON.stringify({schema_version:1,suite:'synthetic-fingerprint-controls',dataset_version:'1',cases}));+let seq=0;+function run(runner=join(runnerRoot,'run.mjs'),extra:string[]=[]){+ const output=join(root,`out-${++seq}`);+ const result=spawnSync('node',[runner,'--repo',repo,'--manifest',manifest,'--mode','baseline','--results-dir',output,...extra],{cwd:repo,encoding:'utf8',timeout:20000});+ expect(result.error).toBeUndefined();expect(result.status).toBe(0);+ return {report:JSON.parse(readFileSync(join(output,'summary.json'),'utf8')),output};+}+test('executed grader and shared harness bytes identify reports without changing scoring',()=>{+ const first=run();const second=run();+ // First establish the observable collision: same committed repo and manifest,+ // changed hidden grader, different outcome. The new execution fingerprint must differ.+ writeFileSync(join(repo,'local-grader.mjs'),'process.exit(0);\n');const changed=run();+ expect(first.report.trials.find((t:any)=>t.case_id==='local').outcome).toBe('fail');+ expect(changed.report.trials.find((t:any)=>t.case_id==='local').outcome).toBe('pass');+ expect(changed.report.dataset_fingerprint).toBe(first.report.dataset_fingerprint);+ expect(changed.report.runtime.repo_head).toBe(first.report.runtime.repo_head);+ expect(changed.report.runtime.fingerprint).toBe(first.report.runtime.fingerprint);+ expect(first.report.evaluation_fingerprint).toMatch(/^[a-f0-9]{64}$/);+ expect(changed.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);+ expect(second.report.evaluation_fingerprint).toBe(first.report.evaluation_fingerprint);+ expect(second.report.runtime.fingerprint).toBe(first.report.runtime.fingerprint);+ const trial=changed.report.trials.find((t:any)=>t.case_id==='local');+ expect(trial.grader_provenance.fixtures[0]).toMatchObject({destination_path:'hidden.mjs',sha256:hash('process.exit(0);\n')});+ expect(trial.evaluation_fingerprint).not.toBe(first.report.trials.find((t:any)=>t.case_id==='local').evaluation_fingerprint);+ expect(trial.harness_provenance.files['grader-outcome.mjs']).toBe(hash(readFileSync(join(runnerRoot,'grader-outcome.mjs'))));+ expect(changed.report.provenance_complete).toBe(true);+ // Restore exact bytes: output paths, elapsed time and run IDs are not identity.+ writeFileSync(join(repo,'local-grader.mjs'),correct);const restored=run();+ expect(restored.report.evaluation_fingerprint).toBe(first.report.evaluation_fingerprint);+ // Git fixture aliases are resolved and content-recorded without moving HEAD.+ git('update-ref','refs/heads/fixture-source',other);const moved=run();+ expect(moved.report.runtime.repo_head).toBe(first.report.runtime.repo_head);+ expect(moved.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);+ const gitTrial=moved.report.trials.find((t:any)=>t.case_id==='git');+ expect(gitTrial.grader_provenance.fixtures[0]).toMatchObject({source_commit:other,sha256:hash('process.exit(0);\n')});+ git('update-ref','refs/heads/fixture-source',fix);+ // Imported classifier implementation must count even when repo is elsewhere.+ const originalHelper=readFileSync(join(runnerRoot,'grader-outcome.mjs'),'utf8');+ writeFileSync(join(runnerRoot,'grader-outcome.mjs'),originalHelper+'\n// synthetic implementation version\n');const helper=run();+ expect(helper.report.runtime.fingerprint).not.toBe(first.report.runtime.fingerprint);+ expect(helper.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);+ writeFileSync(join(runnerRoot,'grader-outcome.mjs'),originalHelper);+ const originalRunner=readFileSync(join(runnerRoot,'run.mjs'),'utf8');+ writeFileSync(join(runnerRoot,'run.mjs'),originalRunner+'\n// synthetic runner version\n');const changedRunner=run();+ expect(changedRunner.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);+ writeFileSync(join(runnerRoot,'run.mjs'),originalRunner);+ const copied=join(root,'copied');mkdirSync(copied);for(const p of ['run.mjs','grader-outcome.mjs'])copyFileSync(join(runnerRoot,p),join(copied,p));+ expect(run(join(copied,'run.mjs')).report.evaluation_fingerprint).toBe(first.report.evaluation_fingerprint);+ const selected=run(undefined,['--case','local']);expect(selected.report.evaluation_fingerprint).not.toBe(first.report.evaluation_fingerprint);+ expect(selected.report.trials).toHaveLength(1);+ // Verification has no summary.json; its actual per-trial provenance must persist.+ const output=join(root,'verified');const verified=spawnSync('node',[join(runnerRoot,'run.mjs'),'--repo',repo,'--manifest',manifest,'--verify','--results-dir',output],{cwd:repo,encoding:'utf8',timeout:20000});+ expect(verified.status).toBe(0);const records=JSON.parse(readFileSync(join(output,'verification.json'),'utf8'));+ for(const record of records){expect(record.valid).toBe(true);expect(record.baseline.evaluation_fingerprint).toMatch(/^[a-f0-9]{64}$/);expect(record.oracle.evaluation_fingerprint).toBe(record.baseline.evaluation_fingerprint);}+ // A setup failure must not acquire a complete execution fingerprint.+ git('config','remote.origin.promisor','true');+ git('config','remote.origin.url',join(root,'missing-extraction-source'));+ const unavailable=run();+ expect(unavailable.report.provenance_complete).toBe(false);+ expect(unavailable.report.evaluation_fingerprint).toBeNull();+ for(const trial of unavailable.report.trials){expect(trial.outcome).toBe('error');expect(trial.grader_provenance).toBeUndefined();}+},60000);diff --git a/evals/coding-agent/run.mjs b/evals/coding-agent/run.mjsindex d21c5b697..a2c85980c 100644--- a/evals/coding-agent/run.mjs+++ b/evals/coding-agent/run.mjs@@ -21,6 +21,17 @@ import process from "node:process";import { classifyGraderError } from "./grader-outcome.mjs";const HERE = dirname(new URL(import.meta.url).pathname);+const sha256 = (body) => createHash("sha256").update(body).digest("hex");+// Record these implementation bytes at startup, independently of --repo.+// This is provenance, not an isolation or dependency/environment attestation.+const harnessFiles = Object.fromEntries(["run.mjs", "grader-outcome.mjs"].map(+ (path) => [path, sha256(readFileSync(join(HERE, path)))],+));+const HARNESS_PROVENANCE = {+ schema_version: 1,+ files: harnessFiles,+ fingerprint: sha256(JSON.stringify(harnessFiles)),+};let REPO = resolve(HERE, "../..");let MANIFEST = join(HERE, "cases.json");let EXTRACTION_SOURCE;@@ -185,6 +196,7 @@ function extractBase(evalCase, workspace) {});mustRun("git", ["config", "user.email", "[email protected]"], { cwd: workspace });mustRun("git", ["config", "user.name", "screenpipe eval harness"], { cwd: workspace });+ return baseSha;}function linkDependencies(evalCase, workspace, field = "dependency_links") {@@ -220,13 +232,16 @@ function applySavedCandidate(path, workspace) {}function materializeGraders(evalCase, workspace) {+ const fixtures = [];for (const fixture of evalCase.grader.fixtures ?? []) {let body;+ let sourceCommit;if (fixture.local_path) {body = readFileSync(join(dirname(MANIFEST), fixture.local_path));} else {const sourceRef = fixture.source_ref ?? evalCase.oracle_ref;- body = mustRun("git", ["show", `${sourceRef}:${fixture.source_path}`], {+ sourceCommit = mustRun("git", ["rev-parse", `${sourceRef}^{commit}`]).stdout.trim();+ body = mustRun("git", ["show", `${sourceCommit}:${fixture.source_path}`], {encoding: null,}).stdout;}@@ -237,7 +252,23 @@ function materializeGraders(evalCase, workspace) {mkdirSync(dirname(destination), { recursive: true });writeFileSync(destination, Buffer.from(body));if (fixture.executable) chmodSync(destination, 0o755);+ // Hash the same bytes we wrote, not a later read of mutable source files.+ fixtures.push({+ local_path: fixture.local_path,+ source_path: fixture.source_path,+ source_commit: sourceCommit,+ destination_path: fixture.destination_path ?? fixture.source_path ?? fixture.local_path,+ executable: Boolean(fixture.executable),+ sha256: sha256(Buffer.from(body)),+ });}+ const provenance = {+ schema_version: 1,+ command_sha256: sha256(evalCase.grader.command),+ timeout_seconds: evalCase.grader.timeout_seconds ?? 600,+ fixtures,+ };+ return { ...provenance, fingerprint: sha256(JSON.stringify(provenance)) };}function renderAgentCommand(template, paths, evalCase) {@@ -312,8 +343,10 @@ function runTrial(evalCase, mode, trial, options, runDir) {let agent = { status: 0, stdout: "", stderr: "", signal: null, error: null };const started = Date.now();+ let graderProvenance;+ let evaluationFingerprint;try {- extractBase(evalCase, workspace);+ const baseCommit = extractBase(evalCase, workspace);linkDependencies(evalCase, workspace);if (mode === "oracle") applyOracle(evalCase, workspace);if (mode === "regrade") {@@ -362,7 +395,14 @@ function runTrial(evalCase, mode, trial, options, runDir) {// Grader-only caches are withheld until the agent trajectory is complete,// so future build artifacts cannot leak information into the trial.linkDependencies(evalCase, workspace, "grader_dependency_links");- materializeGraders(evalCase, workspace);+ graderProvenance = materializeGraders(evalCase, workspace);+ evaluationFingerprint = sha256(JSON.stringify({+ schema_version: 1,+ case_definition_sha256: sha256(JSON.stringify(evalCase)),+ base_commit: baseCommit,+ harness: HARNESS_PROVENANCE.fingerprint,+ grader: graderProvenance.fingerprint,+ }));const grader = command("/bin/bash", ["-lc", evalCase.grader.command], {cwd: workspace,timeout: (evalCase.grader.timeout_seconds ?? 600) * 1_000,@@ -389,6 +429,9 @@ function runTrial(evalCase, mode, trial, options, runDir) {grader_error: grader.error,grader_error_kind: graderErrorKind,changed_files: candidate.changedFiles,+ harness_provenance: HARNESS_PROVENANCE,+ grader_provenance: graderProvenance,+ evaluation_fingerprint: evaluationFingerprint,workspace: options.keep ? workspace : undefined,};writeFileSync(join(trialDir, "result.json"), JSON.stringify(result, null, 2));@@ -402,6 +445,9 @@ function runTrial(evalCase, mode, trial, options, runDir) {outcome: "error",duration_ms: Date.now() - started,harness_error: error instanceof Error ? error.message : String(error),+ harness_provenance: HARNESS_PROVENANCE,+ grader_provenance: graderProvenance,+ evaluation_fingerprint: evaluationFingerprint,workspace: options.keep ? workspace : undefined,};writeFileSync(join(trialDir, "result.json"), JSON.stringify(result, null, 2));@@ -435,11 +481,24 @@ function summarize(manifest, mode, results, options, runDir) {release: release(),arch: arch(),node: process.version,+ harness_fingerprint: HARNESS_PROVENANCE.fingerprint,repo_head: mustRun("git", ["rev-parse", "HEAD"]).stdout.trim(),agent_command: mode === "agent" ? options.agentCommand : undefined,candidate_patch: mode === "regrade" ? options.candidatePatch : undefined,};+ const runtimeFingerprint = sha256(JSON.stringify(runtime));+ const provenanceComplete = results.every((result) => Boolean(result.evaluation_fingerprint));+ const evaluatedCases = [...new Set(results.map((result) =>+ JSON.stringify([result.case_id, result.evaluation_fingerprint ?? null]),+ ))].sort();const report = {+ provenance_complete: provenanceComplete,+ evaluation_fingerprint: provenanceComplete ? sha256(JSON.stringify({+ schema_version: 1,+ dataset_fingerprint: manifest.dataset_fingerprint,+ runtime_fingerprint: runtimeFingerprint,+ evaluated_cases: evaluatedCases,+ })) : null,suite: manifest.suite,dataset_version: manifest.dataset_version,dataset_fingerprint: manifest.dataset_fingerprint,@@ -447,7 +506,7 @@ function summarize(manifest, mode, results, options, runDir) {created_at: new Date().toISOString(),runtime: {...runtime,- fingerprint: createHash("sha256").update(JSON.stringify(runtime)).digest("hex"),+ fingerprint: runtimeFingerprint,},cases,trials: results,