Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(evals): always execute hidden retention capture assertions

Louis Beaumont0c4fcd23cb7d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b096754d0..a61ba5ac2 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -2103,3 +2103,13 @@ metadata and input-mutation controls, and a missing-source infrastructure error.
The grader executes the production edit projection with synthetic documents.
UI navigation, backend persistence, authorization, native execution, enforced
agent isolation and model improvement are outside this case's evidence.
+
+### Retention capture contract dispatch
+
+The retention capture grader always installs its hidden assertions under a
+reserved test name. A candidate's same-named product test cannot replace the
+cleanup-start deadline, capture deadline, deletion, watermark or reopen checks.
+Run `bun test evals/coding-agent/retention-grader-adapter.test.js` for ten adapter
+controls. These use a Cargo dispatch shim and do not establish native behavior;
+the existing historical `--verify` contrast and current-source Cargo execution
+remain separate evidence. Fixtures still appear only at grading.
diff --git a/evals/coding-agent/graders/retention-capture-durable.mjs b/evals/coding-agent/graders/retention-capture-durable.mjs
index cb43b2ce5..36a58f843 100644
--- a/evals/coding-agent/graders/retention-capture-durable.mjs
+++ b/evals/coding-agent/graders/retention-capture-durable.mjs
@@ -6,18 +6,26 @@ import { readFileSync, writeFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
const sourcePath = "crates/screenpipe-engine/src/retention.rs";
-const testName = "all_retention_yields_archived_history_to_durable_capture";
-let source = readFileSync(sourcePath, "utf8");
-if (!source.includes(`async fn ${testName}()`)) {
- const fixture = readFileSync(".eval-retention/retention-capture-durable.rs", "utf8");
- const moduleStart = source.indexOf("#[cfg(test)]\nmod tests {");
- const moduleEnd = source.lastIndexOf("\n}");
- if (moduleStart < 0 || moduleEnd <= moduleStart) {
- throw new Error("retention unit-test module boundary not found");
- }
- source = `${source.slice(0, moduleEnd)}\n${fixture}\n${source.slice(moduleEnd)}`;
- writeFileSync(sourcePath, source);
+const testName = "eval_hidden_retention_capture_contract";
+const source = readFileSync(sourcePath, "utf8");
+const originalName = "all_retention_yields_archived_history_to_durable_capture";
+const fixture = readFileSync(".eval-retention/retention-capture-durable.rs", "utf8");
+const declaration = `async fn ${originalName}()`;
+if (fixture.split(declaration).length !== 2) {
+ throw new Error("retention fixture must declare exactly one contract test");
}
+if (source.includes(testName)) {
+ throw new Error("retention source collides with reserved hidden test name");
+}
+const moduleStart = source.indexOf("#[cfg(test)]\nmod tests {");
+const moduleEnd = source.lastIndexOf("\n}");
+if (moduleStart < 0 || moduleEnd <= moduleStart) {
+ throw new Error("retention unit-test module boundary not found");
+}
+// Always run the hidden contract, even when the candidate retains a same-named
+// product test. Candidate assertions cannot replace the disclosed outcomes.
+const hidden = fixture.replace(declaration, `async fn ${testName}()`);
+writeFileSync(sourcePath, `${source.slice(0, moduleEnd)}\n${hidden}\n${source.slice(moduleEnd)}`);
const result = spawnSync("cargo", [
"test", "--locked", "-p", "screenpipe-engine",
testName, "--", "--nocapture",
diff --git a/evals/coding-agent/retention-grader-adapter.test.js b/evals/coding-agent/retention-grader-adapter.test.js
new file mode 100644
index 000000000..231d3ac79
--- /dev/null
+++ b/evals/coding-agent/retention-grader-adapter.test.js
@@ -0,0 +1,55 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, chmodSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+// Adapter calibration only. The Cargo shim records dispatch and injected bytes;
+// actual broken/reference/current behavior is verified separately with Cargo.
+const adapter = readFileSync(join(import.meta.dir, 'graders/retention-capture-durable.mjs'), 'utf8');
+const fixture = readFileSync(join(import.meta.dir, 'graders/retention-capture-durable.rs.txt'), 'utf8');
+const original = 'all_retention_yields_archived_history_to_durable_capture';
+const hidden = 'eval_hidden_retention_capture_contract';
+const source = body => `#[cfg(test)]\nmod tests {\n${body}\n}\n`;
+function run(text, fixtureText = fixture, exit = 0, script = adapter) {
+ const root = mkdtempSync(join(tmpdir(), 'retention-adapter-'));
+ try {
+ for (const path of ['crates/screenpipe-engine/src', '.eval-retention', 'bin']) mkdirSync(join(root, path), { recursive: true });
+ writeFileSync(join(root, 'crates/screenpipe-engine/src/retention.rs'), text);
+ if (fixtureText !== null) writeFileSync(join(root, '.eval-retention/retention-capture-durable.rs'), fixtureText);
+ writeFileSync(join(root, 'adapter.mjs'), script);
+ writeFileSync(join(root, 'bin/cargo'), `#!${process.execPath}\nconst fs=require('node:fs'); fs.writeFileSync('dispatch.json',JSON.stringify({args:process.argv.slice(2),source:fs.readFileSync('crates/screenpipe-engine/src/retention.rs','utf8')})); process.exit(${exit});\n`);
+ chmodSync(join(root, 'bin/cargo'), 0o755);
+ const result = spawnSync(process.execPath, ['adapter.mjs'], { cwd: root, env: { ...process.env, PATH: `${join(root, 'bin')}:${process.env.PATH}` }, encoding: 'utf8', timeout: 10000 });
+ let dispatch = null;
+ try { dispatch = JSON.parse(readFileSync(join(root, 'dispatch.json'), 'utf8')); } catch {}
+ return { ...result, dispatch };
+ } finally { rmSync(root, { recursive: true, force: true }); }
+}
+for (const [name, body] of [
+ ['absent product test', ''],
+ ['empty same-name test', `async fn ${original}() {}`],
+ ['weakened same-name test', `async fn ${original}() { let _ = 15; }`],
+ ['unrelated preserved test', '#[test] fn ordinary() { assert!(true); }'],
+]) test(name, () => {
+ const result = run(source(body));
+ expect(result.status).toBe(0);
+ expect(result.dispatch.args).toEqual(['test', '--locked', '-p', 'screenpipe-engine', hidden, '--', '--nocapture']);
+ expect(result.dispatch.source).toContain(body);
+ expect(result.dispatch.source).toContain(fixture.replace(`async fn ${original}()`, `async fn ${hidden}()`));
+ expect(result.dispatch.source).toContain('retention cleanup must begin processing within 15 seconds');
+});
+test('does not turn a Cargo failure into success', () => expect(run(source(''), fixture, 101).status).toBe(101));
+for (const [name, text, input] of [
+ ['reserved collision', source(`async fn ${hidden}() {}`), fixture],
+ ['missing test module', 'fn main() {}', fixture],
+ ['missing fixture', source(''), null],
+ ['malformed fixture', source(''), '// no contract'],
+ ['duplicate fixture declarations', source(''), fixture + fixture],
+]) test(name, () => {
+ const result = run(text, input);
+ expect(result.status).not.toBe(0);
+ expect(result.dispatch).toBeNull();
+});