YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(evals): always execute hidden retention capture assertions
Louis Beaumont0c4fcd23cb7d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b096754d0..a61ba5ac2 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -2103,3 +2103,13 @@ metadata and input-mutation controls, and a missing-source infrastructure error.The grader executes the production edit projection with synthetic documents.UI navigation, backend persistence, authorization, native execution, enforcedagent isolation and model improvement are outside this case's evidence.++### Retention capture contract dispatch++The retention capture grader always installs its hidden assertions under a+reserved test name. A candidate's same-named product test cannot replace the+cleanup-start deadline, capture deadline, deletion, watermark or reopen checks.+Run `bun test evals/coding-agent/retention-grader-adapter.test.js` for ten adapter+controls. These use a Cargo dispatch shim and do not establish native behavior;+the existing historical `--verify` contrast and current-source Cargo execution+remain separate evidence. Fixtures still appear only at grading.diff --git a/evals/coding-agent/graders/retention-capture-durable.mjs b/evals/coding-agent/graders/retention-capture-durable.mjsindex cb43b2ce5..36a58f843 100644--- a/evals/coding-agent/graders/retention-capture-durable.mjs+++ b/evals/coding-agent/graders/retention-capture-durable.mjs@@ -6,18 +6,26 @@ import { readFileSync, writeFileSync } from "node:fs";import { spawnSync } from "node:child_process";const sourcePath = "crates/screenpipe-engine/src/retention.rs";-const testName = "all_retention_yields_archived_history_to_durable_capture";-let source = readFileSync(sourcePath, "utf8");-if (!source.includes(`async fn ${testName}()`)) {- const fixture = readFileSync(".eval-retention/retention-capture-durable.rs", "utf8");- const moduleStart = source.indexOf("#[cfg(test)]\nmod tests {");- const moduleEnd = source.lastIndexOf("\n}");- if (moduleStart < 0 || moduleEnd <= moduleStart) {- throw new Error("retention unit-test module boundary not found");- }- source = `${source.slice(0, moduleEnd)}\n${fixture}\n${source.slice(moduleEnd)}`;- writeFileSync(sourcePath, source);+const testName = "eval_hidden_retention_capture_contract";+const source = readFileSync(sourcePath, "utf8");+const originalName = "all_retention_yields_archived_history_to_durable_capture";+const fixture = readFileSync(".eval-retention/retention-capture-durable.rs", "utf8");+const declaration = `async fn ${originalName}()`;+if (fixture.split(declaration).length !== 2) {+ throw new Error("retention fixture must declare exactly one contract test");}+if (source.includes(testName)) {+ throw new Error("retention source collides with reserved hidden test name");+}+const moduleStart = source.indexOf("#[cfg(test)]\nmod tests {");+const moduleEnd = source.lastIndexOf("\n}");+if (moduleStart < 0 || moduleEnd <= moduleStart) {+ throw new Error("retention unit-test module boundary not found");+}+// Always run the hidden contract, even when the candidate retains a same-named+// product test. Candidate assertions cannot replace the disclosed outcomes.+const hidden = fixture.replace(declaration, `async fn ${testName}()`);+writeFileSync(sourcePath, `${source.slice(0, moduleEnd)}\n${hidden}\n${source.slice(moduleEnd)}`);const result = spawnSync("cargo", ["test", "--locked", "-p", "screenpipe-engine",testName, "--", "--nocapture",diff --git a/evals/coding-agent/retention-grader-adapter.test.js b/evals/coding-agent/retention-grader-adapter.test.jsnew file mode 100644index 000000000..231d3ac79--- /dev/null+++ b/evals/coding-agent/retention-grader-adapter.test.js@@ -0,0 +1,55 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, chmodSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { spawnSync } from 'node:child_process';++// Adapter calibration only. The Cargo shim records dispatch and injected bytes;+// actual broken/reference/current behavior is verified separately with Cargo.+const adapter = readFileSync(join(import.meta.dir, 'graders/retention-capture-durable.mjs'), 'utf8');+const fixture = readFileSync(join(import.meta.dir, 'graders/retention-capture-durable.rs.txt'), 'utf8');+const original = 'all_retention_yields_archived_history_to_durable_capture';+const hidden = 'eval_hidden_retention_capture_contract';+const source = body => `#[cfg(test)]\nmod tests {\n${body}\n}\n`;+function run(text, fixtureText = fixture, exit = 0, script = adapter) {+ const root = mkdtempSync(join(tmpdir(), 'retention-adapter-'));+ try {+ for (const path of ['crates/screenpipe-engine/src', '.eval-retention', 'bin']) mkdirSync(join(root, path), { recursive: true });+ writeFileSync(join(root, 'crates/screenpipe-engine/src/retention.rs'), text);+ if (fixtureText !== null) writeFileSync(join(root, '.eval-retention/retention-capture-durable.rs'), fixtureText);+ writeFileSync(join(root, 'adapter.mjs'), script);+ writeFileSync(join(root, 'bin/cargo'), `#!${process.execPath}\nconst fs=require('node:fs'); fs.writeFileSync('dispatch.json',JSON.stringify({args:process.argv.slice(2),source:fs.readFileSync('crates/screenpipe-engine/src/retention.rs','utf8')})); process.exit(${exit});\n`);+ chmodSync(join(root, 'bin/cargo'), 0o755);+ const result = spawnSync(process.execPath, ['adapter.mjs'], { cwd: root, env: { ...process.env, PATH: `${join(root, 'bin')}:${process.env.PATH}` }, encoding: 'utf8', timeout: 10000 });+ let dispatch = null;+ try { dispatch = JSON.parse(readFileSync(join(root, 'dispatch.json'), 'utf8')); } catch {}+ return { ...result, dispatch };+ } finally { rmSync(root, { recursive: true, force: true }); }+}+for (const [name, body] of [+ ['absent product test', ''],+ ['empty same-name test', `async fn ${original}() {}`],+ ['weakened same-name test', `async fn ${original}() { let _ = 15; }`],+ ['unrelated preserved test', '#[test] fn ordinary() { assert!(true); }'],+]) test(name, () => {+ const result = run(source(body));+ expect(result.status).toBe(0);+ expect(result.dispatch.args).toEqual(['test', '--locked', '-p', 'screenpipe-engine', hidden, '--', '--nocapture']);+ expect(result.dispatch.source).toContain(body);+ expect(result.dispatch.source).toContain(fixture.replace(`async fn ${original}()`, `async fn ${hidden}()`));+ expect(result.dispatch.source).toContain('retention cleanup must begin processing within 15 seconds');+});+test('does not turn a Cargo failure into success', () => expect(run(source(''), fixture, 101).status).toBe(101));+for (const [name, text, input] of [+ ['reserved collision', source(`async fn ${hidden}() {}`), fixture],+ ['missing test module', 'fn main() {}', fixture],+ ['missing fixture', source(''), null],+ ['malformed fixture', source(''), '// no contract'],+ ['duplicate fixture declarations', source(''), fixture + fixture],+]) test(name, () => {+ const result = run(text, input);+ expect(result.status).not.toBe(0);+ expect(result.dispatch).toBeNull();+});