Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: page all reachable history during regression discovery

Louis Beaumontf7b147ea545c
diff --git a/evals/coding-agent/DESIGN.md b/evals/coding-agent/DESIGN.md
index 2897fb7e1..fcec1d58f 100644
--- a/evals/coding-agent/DESIGN.md
+++ b/evals/coding-agent/DESIGN.md
@@ -43,11 +43,31 @@ may begin with a deliberately lower success rate.
node evals/coding-agent/mine-history.mjs \
--repo /path/to/repository \
--ref origin/main \
- --since '18 months ago' > /tmp/eval-candidates.json
+ --limit 100 --skip 0 > /tmp/eval-candidates.json
```
+The miner defaults to all history reachable from the selected ref, including
+merged branch commits and merges, with no date cutoff. Each invocation is bounded
+by `--limit` (default 500); `--skip` selects a page. Continue with the returned
+`resolved_ref` SHA and `next_skip` so a moving branch cannot shift the page
+boundary. `next_skip: null` means this traversal is exhausted, not that all
+behavior has been reviewed. An explicit `--since` remains available for a
+purposefully narrower scan.
+
The miner finds fix-shaped commits that changed potential deterministic graders.
Promotion is intentionally reviewed: confirm the original failure from the
commit/PR, write a sanitized outcome-focused prompt, select graders that cover
both the fix and preserved behavior, then add the exact parent/fix pair to the
manifest and run `--verify`.
+
+Merge candidates include changed paths against every parent and
+`requires_merge_review: true`. Their first `broken_parent` is only a discovery
+hint: inspect the resolution, establish the appropriate broken state, and
+deduplicate constituent or cherry-picked fixes before promotion. Root commits
+have no broken parent and are not promoted. `scanned_commit_ids` tracks metadata
+visited, not source review or behavior verification. Non-fix subjects and fixes
+without recognizable test paths still require separate path-based review.
+
+Run `bun test evals/coding-agent/mine-history.test.ts` for synthetic old-history,
+merged-branch, merge-resolution, bounded-page and explicit-filter controls. They
+exercise discovery without models, network access or changes to product code.
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 797f92d06..33746a96d 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -109,3 +109,17 @@ The five controls require the historical contrast, accept equivalent helper
renaming, reject an identifier-trusting fast path, and reject blanket anonymous
fallback. These are grader controls, not model trials. The July 10 historical
case does not require machine-service-token support introduced afterward.
+
+## Bounded history discovery
+
+`mine-history.mjs` scans all reachable history by default, including merged
+branches and merge commits. Page with `--limit` and `--skip`; continue using the
+returned `resolved_ref` SHA and `next_skip` to keep page boundaries stable.
+An explicit `--since` narrows the scan. Merge candidates expose each parent diff
+and require resolution review before choosing a broken baseline. The miner is
+still a subject/test-path heuristic, so its output is discovery evidence, not
+complete source review or verified regressions. See [DESIGN.md](./DESIGN.md).
+
+`bun test evals/coding-agent/mine-history.test.ts` runs four synthetic controls
+covering old and merged fixes, per-parent merge paths, complete bounded paging,
+root/non-fix exclusions, explicit date filtering and invalid inputs.
diff --git a/evals/coding-agent/mine-history.mjs b/evals/coding-agent/mine-history.mjs
index 93eedd988..0de100654 100644
--- a/evals/coding-agent/mine-history.mjs
+++ b/evals/coding-agent/mine-history.mjs
@@ -13,7 +13,8 @@ function parseArgs(argv) {
const options = {
repo: resolve(new URL("../..", import.meta.url).pathname),
ref: "origin/main",
- since: "18 months ago",
+ since: null,
+ skip: 0,
limit: 500,
};
for (let index = 0; index < argv.length; index += 1) {
@@ -22,11 +23,15 @@ function parseArgs(argv) {
else if (value === "--ref") options.ref = argv[++index];
else if (value === "--since") options.since = argv[++index];
else if (value === "--limit") options.limit = Number(argv[++index]);
+ else if (value === "--skip") options.skip = Number(argv[++index]);
else throw new Error(`unknown argument: ${value}`);
}
- if (!Number.isInteger(options.limit) || options.limit < 1) {
+ if (!Number.isSafeInteger(options.limit) || options.limit < 1 || options.limit >= Number.MAX_SAFE_INTEGER) {
throw new Error("--limit must be a positive integer");
}
+ if (!Number.isSafeInteger(options.skip) || options.skip < 0) {
+ throw new Error("--skip must be a nonnegative safe integer");
+ }
return options;
}
@@ -44,33 +49,41 @@ function git(repo, args) {
function main() {
const options = parseArgs(process.argv.slice(2));
- const rows = git(options.repo, [
+ const resolvedRef = git(options.repo, ["rev-parse", "--verify", `${options.ref}^{commit}`]);
+ const allRows = git(options.repo, [
"log",
- options.ref,
- "--first-parent",
- "--no-merges",
- `--since=${options.since}`,
- `--max-count=${options.limit}`,
+ resolvedRef,
+ "--topo-order",
+ ...(options.since === null ? [] : [`--since-as-filter=${options.since}`]),
+ `--skip=${options.skip}`,
+ `--max-count=${options.limit + 1}`,
"--format=%H%x09%P%x09%s",
]).split("\n").filter(Boolean);
+ const rows = allRows.slice(0, options.limit);
const candidates = [];
for (const row of rows) {
const [commit, parents, ...subjectParts] = row.split("\t");
const subject = subjectParts.join("\t");
if (!parents || !FIX_PATTERN.test(subject)) continue;
- const changedPaths = git(options.repo, [
- "diff-tree",
- "--no-commit-id",
- "--name-only",
- "-r",
- commit,
- ]).split("\n").filter(Boolean);
+ const parentIds = parents.split(" ");
+ // A merge can introduce behavior relative to either parent. These diffs are
+ // discovery evidence, not proof of a merge-only defect or a valid baseline.
+ const parentComparisons = parentIds.map((parent) => ({
+ parent,
+ changed_paths: git(options.repo, [
+ "diff-tree", "--no-commit-id", "--name-only", "-r", parent, commit,
+ ]).split("\n").filter(Boolean),
+ }));
+ const changedPaths = [...new Set(parentComparisons.flatMap((p) => p.changed_paths))].sort();
const graderCandidates = changedPaths.filter((path) => GRADER_PATTERN.test(path));
if (!graderCandidates.length) continue;
candidates.push({
fix_commit: commit,
- broken_parent: parents.split(" ")[0],
+ broken_parent: parentIds[0],
+ parents: parentIds,
+ requires_merge_review: parentIds.length > 1,
+ parent_comparisons: parentComparisons,
subject,
grader_candidates: graderCandidates,
changed_paths: changedPaths,
@@ -80,6 +93,12 @@ function main() {
process.stdout.write(`${JSON.stringify({
repository: git(options.repo, ["remote", "get-url", "origin"]),
ref: options.ref,
+ resolved_ref: resolvedRef,
+ skip: options.skip,
+ limit: options.limit,
+ next_skip: allRows.length > options.limit ? options.skip + rows.length : null,
+ history_scope: "all reachable commits; candidates remain subject/test-path heuristics",
+ scanned_commit_ids: rows.map((row) => row.split("\t")[0]),
since: options.since,
scanned_commits: rows.length,
candidates,
diff --git a/evals/coding-agent/mine-history.test.ts b/evals/coding-agent/mine-history.test.ts
new file mode 100644
index 000000000..e9f165525
--- /dev/null
+++ b/evals/coding-agent/mine-history.test.ts
@@ -0,0 +1,87 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { expect, test } from "bun:test";
+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const miner = new URL("./mine-history.mjs", import.meta.url).pathname;
+function fixture(run: (repo: string, ids: Record<string, string>) => void) {
+ const repo = mkdtempSync(join(tmpdir(), "eval-history-control-"));
+ const git = (...args: string[]) => {
+ const r = spawnSync("git", args, { cwd: repo, encoding: "utf8", timeout: 10_000 });
+ if (r.status !== 0) throw new Error(r.stderr || String(r.error));
+ return r.stdout.trim();
+ };
+ const commit = (subject: string, date: string, path: string) => {
+ writeFileSync(join(repo, path), subject + "\n"); git("add", path);
+ const r = spawnSync("git", ["commit", "-qm", subject], { cwd: repo, encoding: "utf8", env: { ...process.env, GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date }, timeout: 10_000 });
+ if (r.status !== 0) throw new Error(r.stderr);
+ return git("rev-parse", "HEAD");
+ };
+ try {
+ git("init", "-q", "-b", "main"); git("config", "user.name", "Synthetic eval control");
+ git("config", "user.email", "[email protected]"); git("config", "commit.gpgsign", "false");
+ git("remote", "add", "origin", "https://example.invalid/synthetic.git");
+ const root = commit("fix initial fixture", "2020-01-01T00:00:00Z", "root.test.js");
+ const old = commit("fix old escaped failure", "2021-01-01T00:00:00Z", "old.test.js");
+ git("checkout", "-qb", "side");
+ const side = commit("fix merged branch failure", "2026-09-01T00:00:00Z", "side.test.js");
+ git("checkout", "-q", "main");
+ const unrelated = commit("maintain examples", "2026-09-02T00:00:00Z", "example.test.js");
+ git("merge", "--no-ff", "--no-commit", "side");
+ const merge = commit("fix merge resolution behavior", "2026-09-03T00:00:00Z", "resolution.test.js");
+ run(repo, { root, old, side, unrelated, merge });
+ } finally { rmSync(repo, { recursive: true, force: true }); }
+}
+function invoke(repo: string, ...args: string[]) {
+ return spawnSync(process.execPath, [miner, "--repo", repo, "--ref", "main", ...args], { encoding: "utf8", timeout: 20_000 });
+}
+function scan(repo: string, ...args: string[]) {
+ const result = invoke(repo, ...args); expect(result.status).toBe(0);
+ return JSON.parse(result.stdout);
+}
+
+test("default discovery reaches old fixes and merged branches without promoting roots or non-fixes", () => fixture((repo, ids) => {
+ const page = scan(repo);
+ expect(page.candidates.map((c: any) => c.fix_commit).sort()).toEqual([ids.old, ids.side, ids.merge].sort());
+ expect(page.scanned_commits).toBe(5);
+ expect(page.resolved_ref).toBe(ids.merge);
+ expect(page.since).toBeNull();
+}));
+
+test("merge candidates expose each parent comparison and require resolution review", () => fixture((repo, ids) => {
+ const merge = scan(repo).candidates.find((c: any) => c.fix_commit === ids.merge);
+ expect(merge).toBeDefined();
+ expect(merge.requires_merge_review).toBe(true);
+ expect(merge.parents).toEqual([ids.unrelated, ids.side]);
+ expect(merge.parent_comparisons).toHaveLength(2);
+ expect(merge.parent_comparisons[0].changed_paths).toContain("side.test.js");
+ expect(merge.parent_comparisons[1].changed_paths).toContain("example.test.js");
+ expect(merge.parent_comparisons.every((p: any) => p.changed_paths.includes("resolution.test.js"))).toBe(true);
+}));
+
+test("bounded pages on a pinned head neither omit nor duplicate reachable commits", () => fixture((repo, ids) => {
+ const seen: string[] = []; const candidates: string[] = []; let skip = 0;
+ for (let n = 0; n < 3; n++) {
+ const page = scan(repo, "--ref", ids.merge, "--limit", "2", "--skip", String(skip));
+ expect(page.scanned_commits).toBeLessThanOrEqual(2);
+ seen.push(...page.scanned_commit_ids); candidates.push(...page.candidates.map((c: any) => c.fix_commit));
+ if (n < 2) { expect(page.next_skip).toBe(skip + 2); skip = page.next_skip; }
+ else expect(page.next_skip).toBeNull();
+ }
+ expect(seen.sort()).toEqual(Object.values(ids).sort());
+ expect(new Set(candidates).size).toBe(3);
+ const empty = scan(repo, "--skip", "5");
+ expect(empty.scanned_commits).toBe(0); expect(empty.next_skip).toBeNull();
+}));
+
+test("explicit date filters remain supported and invalid paging fails closed", () => fixture((repo, ids) => {
+ const page = scan(repo, "--since", "2026-01-01");
+ expect(page.candidates.map((c: any) => c.fix_commit).sort()).toEqual([ids.side, ids.merge].sort());
+ for (const args of [["--skip", "-1"], ["--skip", "1.5"], ["--skip"], ["--limit", "0"], ["--ref", "does-not-exist"]]) {
+ expect(invoke(repo, ...args).status).not.toBe(0);
+ }
+}));