YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: page all reachable history during regression discovery
Louis Beaumontf7b147ea545c
diff --git a/evals/coding-agent/DESIGN.md b/evals/coding-agent/DESIGN.mdindex 2897fb7e1..fcec1d58f 100644--- a/evals/coding-agent/DESIGN.md+++ b/evals/coding-agent/DESIGN.md@@ -43,11 +43,31 @@ may begin with a deliberately lower success rate.node evals/coding-agent/mine-history.mjs \--repo /path/to/repository \--ref origin/main \- --since '18 months ago' > /tmp/eval-candidates.json+ --limit 100 --skip 0 > /tmp/eval-candidates.json```+The miner defaults to all history reachable from the selected ref, including+merged branch commits and merges, with no date cutoff. Each invocation is bounded+by `--limit` (default 500); `--skip` selects a page. Continue with the returned+`resolved_ref` SHA and `next_skip` so a moving branch cannot shift the page+boundary. `next_skip: null` means this traversal is exhausted, not that all+behavior has been reviewed. An explicit `--since` remains available for a+purposefully narrower scan.+The miner finds fix-shaped commits that changed potential deterministic graders.Promotion is intentionally reviewed: confirm the original failure from thecommit/PR, write a sanitized outcome-focused prompt, select graders that coverboth the fix and preserved behavior, then add the exact parent/fix pair to themanifest and run `--verify`.++Merge candidates include changed paths against every parent and+`requires_merge_review: true`. Their first `broken_parent` is only a discovery+hint: inspect the resolution, establish the appropriate broken state, and+deduplicate constituent or cherry-picked fixes before promotion. Root commits+have no broken parent and are not promoted. `scanned_commit_ids` tracks metadata+visited, not source review or behavior verification. Non-fix subjects and fixes+without recognizable test paths still require separate path-based review.++Run `bun test evals/coding-agent/mine-history.test.ts` for synthetic old-history,+merged-branch, merge-resolution, bounded-page and explicit-filter controls. They+exercise discovery without models, network access or changes to product code.diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 797f92d06..33746a96d 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -109,3 +109,17 @@ The five controls require the historical contrast, accept equivalent helperrenaming, reject an identifier-trusting fast path, and reject blanket anonymousfallback. These are grader controls, not model trials. The July 10 historicalcase does not require machine-service-token support introduced afterward.++## Bounded history discovery++`mine-history.mjs` scans all reachable history by default, including merged+branches and merge commits. Page with `--limit` and `--skip`; continue using the+returned `resolved_ref` SHA and `next_skip` to keep page boundaries stable.+An explicit `--since` narrows the scan. Merge candidates expose each parent diff+and require resolution review before choosing a broken baseline. The miner is+still a subject/test-path heuristic, so its output is discovery evidence, not+complete source review or verified regressions. See [DESIGN.md](./DESIGN.md).++`bun test evals/coding-agent/mine-history.test.ts` runs four synthetic controls+covering old and merged fixes, per-parent merge paths, complete bounded paging,+root/non-fix exclusions, explicit date filtering and invalid inputs.diff --git a/evals/coding-agent/mine-history.mjs b/evals/coding-agent/mine-history.mjsindex 93eedd988..0de100654 100644--- a/evals/coding-agent/mine-history.mjs+++ b/evals/coding-agent/mine-history.mjs@@ -13,7 +13,8 @@ function parseArgs(argv) {const options = {repo: resolve(new URL("../..", import.meta.url).pathname),ref: "origin/main",- since: "18 months ago",+ since: null,+ skip: 0,limit: 500,};for (let index = 0; index < argv.length; index += 1) {@@ -22,11 +23,15 @@ function parseArgs(argv) {else if (value === "--ref") options.ref = argv[++index];else if (value === "--since") options.since = argv[++index];else if (value === "--limit") options.limit = Number(argv[++index]);+ else if (value === "--skip") options.skip = Number(argv[++index]);else throw new Error(`unknown argument: ${value}`);}- if (!Number.isInteger(options.limit) || options.limit < 1) {+ if (!Number.isSafeInteger(options.limit) || options.limit < 1 || options.limit >= Number.MAX_SAFE_INTEGER) {throw new Error("--limit must be a positive integer");}+ if (!Number.isSafeInteger(options.skip) || options.skip < 0) {+ throw new Error("--skip must be a nonnegative safe integer");+ }return options;}@@ -44,33 +49,41 @@ function git(repo, args) {function main() {const options = parseArgs(process.argv.slice(2));- const rows = git(options.repo, [+ const resolvedRef = git(options.repo, ["rev-parse", "--verify", `${options.ref}^{commit}`]);+ const allRows = git(options.repo, ["log",- options.ref,- "--first-parent",- "--no-merges",- `--since=${options.since}`,- `--max-count=${options.limit}`,+ resolvedRef,+ "--topo-order",+ ...(options.since === null ? [] : [`--since-as-filter=${options.since}`]),+ `--skip=${options.skip}`,+ `--max-count=${options.limit + 1}`,"--format=%H%x09%P%x09%s",]).split("\n").filter(Boolean);+ const rows = allRows.slice(0, options.limit);const candidates = [];for (const row of rows) {const [commit, parents, ...subjectParts] = row.split("\t");const subject = subjectParts.join("\t");if (!parents || !FIX_PATTERN.test(subject)) continue;- const changedPaths = git(options.repo, [- "diff-tree",- "--no-commit-id",- "--name-only",- "-r",- commit,- ]).split("\n").filter(Boolean);+ const parentIds = parents.split(" ");+ // A merge can introduce behavior relative to either parent. These diffs are+ // discovery evidence, not proof of a merge-only defect or a valid baseline.+ const parentComparisons = parentIds.map((parent) => ({+ parent,+ changed_paths: git(options.repo, [+ "diff-tree", "--no-commit-id", "--name-only", "-r", parent, commit,+ ]).split("\n").filter(Boolean),+ }));+ const changedPaths = [...new Set(parentComparisons.flatMap((p) => p.changed_paths))].sort();const graderCandidates = changedPaths.filter((path) => GRADER_PATTERN.test(path));if (!graderCandidates.length) continue;candidates.push({fix_commit: commit,- broken_parent: parents.split(" ")[0],+ broken_parent: parentIds[0],+ parents: parentIds,+ requires_merge_review: parentIds.length > 1,+ parent_comparisons: parentComparisons,subject,grader_candidates: graderCandidates,changed_paths: changedPaths,@@ -80,6 +93,12 @@ function main() {process.stdout.write(`${JSON.stringify({repository: git(options.repo, ["remote", "get-url", "origin"]),ref: options.ref,+ resolved_ref: resolvedRef,+ skip: options.skip,+ limit: options.limit,+ next_skip: allRows.length > options.limit ? options.skip + rows.length : null,+ history_scope: "all reachable commits; candidates remain subject/test-path heuristics",+ scanned_commit_ids: rows.map((row) => row.split("\t")[0]),since: options.since,scanned_commits: rows.length,candidates,diff --git a/evals/coding-agent/mine-history.test.ts b/evals/coding-agent/mine-history.test.tsnew file mode 100644index 000000000..e9f165525--- /dev/null+++ b/evals/coding-agent/mine-history.test.ts@@ -0,0 +1,87 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { expect, test } from "bun:test";+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { spawnSync } from "node:child_process";++const miner = new URL("./mine-history.mjs", import.meta.url).pathname;+function fixture(run: (repo: string, ids: Record<string, string>) => void) {+ const repo = mkdtempSync(join(tmpdir(), "eval-history-control-"));+ const git = (...args: string[]) => {+ const r = spawnSync("git", args, { cwd: repo, encoding: "utf8", timeout: 10_000 });+ if (r.status !== 0) throw new Error(r.stderr || String(r.error));+ return r.stdout.trim();+ };+ const commit = (subject: string, date: string, path: string) => {+ writeFileSync(join(repo, path), subject + "\n"); git("add", path);+ const r = spawnSync("git", ["commit", "-qm", subject], { cwd: repo, encoding: "utf8", env: { ...process.env, GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date }, timeout: 10_000 });+ if (r.status !== 0) throw new Error(r.stderr);+ return git("rev-parse", "HEAD");+ };+ try {+ git("init", "-q", "-b", "main"); git("config", "user.name", "Synthetic eval control");+ git("config", "user.email", "[email protected]"); git("config", "commit.gpgsign", "false");+ git("remote", "add", "origin", "https://example.invalid/synthetic.git");+ const root = commit("fix initial fixture", "2020-01-01T00:00:00Z", "root.test.js");+ const old = commit("fix old escaped failure", "2021-01-01T00:00:00Z", "old.test.js");+ git("checkout", "-qb", "side");+ const side = commit("fix merged branch failure", "2026-09-01T00:00:00Z", "side.test.js");+ git("checkout", "-q", "main");+ const unrelated = commit("maintain examples", "2026-09-02T00:00:00Z", "example.test.js");+ git("merge", "--no-ff", "--no-commit", "side");+ const merge = commit("fix merge resolution behavior", "2026-09-03T00:00:00Z", "resolution.test.js");+ run(repo, { root, old, side, unrelated, merge });+ } finally { rmSync(repo, { recursive: true, force: true }); }+}+function invoke(repo: string, ...args: string[]) {+ return spawnSync(process.execPath, [miner, "--repo", repo, "--ref", "main", ...args], { encoding: "utf8", timeout: 20_000 });+}+function scan(repo: string, ...args: string[]) {+ const result = invoke(repo, ...args); expect(result.status).toBe(0);+ return JSON.parse(result.stdout);+}++test("default discovery reaches old fixes and merged branches without promoting roots or non-fixes", () => fixture((repo, ids) => {+ const page = scan(repo);+ expect(page.candidates.map((c: any) => c.fix_commit).sort()).toEqual([ids.old, ids.side, ids.merge].sort());+ expect(page.scanned_commits).toBe(5);+ expect(page.resolved_ref).toBe(ids.merge);+ expect(page.since).toBeNull();+}));++test("merge candidates expose each parent comparison and require resolution review", () => fixture((repo, ids) => {+ const merge = scan(repo).candidates.find((c: any) => c.fix_commit === ids.merge);+ expect(merge).toBeDefined();+ expect(merge.requires_merge_review).toBe(true);+ expect(merge.parents).toEqual([ids.unrelated, ids.side]);+ expect(merge.parent_comparisons).toHaveLength(2);+ expect(merge.parent_comparisons[0].changed_paths).toContain("side.test.js");+ expect(merge.parent_comparisons[1].changed_paths).toContain("example.test.js");+ expect(merge.parent_comparisons.every((p: any) => p.changed_paths.includes("resolution.test.js"))).toBe(true);+}));++test("bounded pages on a pinned head neither omit nor duplicate reachable commits", () => fixture((repo, ids) => {+ const seen: string[] = []; const candidates: string[] = []; let skip = 0;+ for (let n = 0; n < 3; n++) {+ const page = scan(repo, "--ref", ids.merge, "--limit", "2", "--skip", String(skip));+ expect(page.scanned_commits).toBeLessThanOrEqual(2);+ seen.push(...page.scanned_commit_ids); candidates.push(...page.candidates.map((c: any) => c.fix_commit));+ if (n < 2) { expect(page.next_skip).toBe(skip + 2); skip = page.next_skip; }+ else expect(page.next_skip).toBeNull();+ }+ expect(seen.sort()).toEqual(Object.values(ids).sort());+ expect(new Set(candidates).size).toBe(3);+ const empty = scan(repo, "--skip", "5");+ expect(empty.scanned_commits).toBe(0); expect(empty.next_skip).toBeNull();+}));++test("explicit date filters remain supported and invalid paging fails closed", () => fixture((repo, ids) => {+ const page = scan(repo, "--since", "2026-01-01");+ expect(page.candidates.map((c: any) => c.fix_commit).sort()).toEqual([ids.side, ids.merge].sort());+ for (const args of [["--skip", "-1"], ["--skip", "1.5"], ["--skip"], ["--limit", "0"], ["--ref", "does-not-exist"]]) {+ expect(invoke(repo, ...args).status).not.toBe(0);+ }+}));