Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

feat(pair): measure tokens on the last merged PR

Shrish Dwivediae79acbcd8ba
diff --git a/src/cloud/onboarding.ts b/src/cloud/onboarding.ts
index f9b4cb92..c28b7578 100644
--- a/src/cloud/onboarding.ts
+++ b/src/cloud/onboarding.ts
@@ -20,7 +20,9 @@
* says nothing about it.
*/
import { execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
import { hostname } from "node:os";
+import { join } from "node:path";
import { formatAsk, ask } from "../ask/ask.js";
import { contextDirFor } from "../context/node-file.js";
import { loadGraphCached } from "../graph/load.js";
@@ -210,6 +212,22 @@ export interface TokensSummary {
pct: number;
/** How many files the baseline reads whole. */
files: number;
+ /**
+ * When the question was the last merged PR's title: that PR, at most
+ * {@link PR_FILES_SENT} of the code files it changed, how many it changed
+ * in all, and how many of them the answer pointed to.
+ */
+ pr?: LastPr & { changed: number; found: number };
+}
+
+/** A merged pull request, as git's history on the default branch tells it. */
+export interface LastPr {
+ number: number;
+ title: string;
+ /** The PR on GitHub, when the repo is there. */
+ url?: string;
+ /** The product code files it changed that are still in the repo. */
+ files: string[];
}
export interface LearningSummary {
@@ -361,6 +379,8 @@ export function meaningfulSymbol(name: string): boolean {
* files). Null without a graph.
*/
export function tokensOnOneLookup(repo: string, opts: { contextDir?: string; tries?: number } = {}): TokensSummary | null {
+ const pr = tokensOnLastPr(repo, opts);
+ if (pr) return pr;
try {
const graph = loadGraphCached(contextDirFor(repo, opts.contextDir));
if (!graph) return null;
@@ -386,6 +406,72 @@ export function tokensOnOneLookup(repo: string, opts: { contextDir?: string; tri
}
}
+/** The most of a PR's changed files sent to Trail; `changed` says how many there were. */
+export const PR_FILES_SENT = 20;
+
+/** `Merge pull request #12 from acme/retry-backoff` (a merge), with the title on the body's first line. */
+const MERGED_PR = /^Merge pull request #(\d+) from (\S+)/;
+/** `Retry with backoff (#12)` (a squash or rebase merge). */
+const SQUASHED_PR = /^(.+?) \(#(\d+)\)$/;
+/** Bots' PRs and release chores, which say nothing about the work. */
+const NOT_WORK_PR = /^(bump |chore\(release\)|release[: ]|\[bot\]|merge (branch|remote-tracking))/i;
+
+/**
+ * The merged PRs on the default branch, newest first, read from git alone:
+ * GitHub's merge commits and squash merges both name the PR. Only PRs that
+ * changed product code still in the repo; at most `limit` of them.
+ */
+export function lastMergedPrs(repo: string, limit = 5): LastPr[] {
+ const base = git(repo, ["rev-parse", "--abbrev-ref", "origin/HEAD"])?.trim() || "HEAD";
+ const log = git(repo, ["log", "-n", "300", "--first-parent", "--format=%H%x1f%s%x1f%b%x1e", base]);
+ if (!log) return [];
+ const key = repoPlace(repo).key;
+ const out: LastPr[] = [];
+ for (const rec of log.split("\x1e")) {
+ const [sha, subject = "", body = ""] = rec.trim().split("\x1f");
+ if (!sha) continue;
+ const merged = subject.match(MERGED_PR);
+ const squashed = merged ? null : subject.match(SQUASHED_PR);
+ if (!merged && !squashed) continue;
+ const number = Number(merged ? merged[1] : squashed![2]);
+ const title = cleanSubject(merged ? body.split("\n").find((l) => l.trim()) || merged[2]!.split("/").pop()!.replace(/[-_]/g, " ") : squashed![1]!);
+ if (!title || NOT_WORK_PR.test(title)) continue;
+ const changed = git(repo, ["diff", "--name-only", "--diff-filter=AMR", `${sha}^1`, sha]) ?? "";
+ const files = changed
+ .split("\n")
+ .map((f) => f.trim())
+ .filter((f) => f && !notProductCode(f) && !docOrConfig(f) && existsSync(join(repo, f)));
+ if (!files.length) continue;
+ const url = key.startsWith("github.com/") ? `https://${key}/pull/${number}` : undefined;
+ out.push({ number, title, ...(url ? { url } : {}), files });
+ if (out.length >= limit) break;
+ }
+ return out;
+}
+
+/**
+ * The lookup an agent starting the last merged PR would make: its title as
+ * the question, answered with the code inlined, against reading the files
+ * the answer covers whole (ask's own estimate). The next PR back is tried
+ * when one saves nothing. Null with no PR in git's history or no graph.
+ */
+export function tokensOnLastPr(repo: string, opts: { contextDir?: string; tries?: number } = {}): TokensSummary | null {
+ try {
+ if (!loadGraphCached(contextDirFor(repo, opts.contextDir))) return null;
+ for (const pr of lastMergedPrs(repo, Math.max(1, opts.tries ?? 3))) {
+ const r = ask(repo, pr.title, { source: true, limit: 5, contextDir: opts.contextDir });
+ const got = parseAskSavings(formatAsk(r));
+ if (!got || got.saved <= 0) continue;
+ const pointed = new Set(r.hits.map((h) => h.pointer.replace(/:L\d+(-L?\d+)?$/, "")));
+ const found = pr.files.filter((f) => pointed.has(f)).length;
+ return { question: pr.title, symbol: "", ...got, pr: { ...pr, files: pr.files.slice(0, PR_FILES_SENT), changed: pr.files.length, found } };
+ }
+ return null;
+ } catch {
+ return null;
+ }
+}
+
/* learnings ---------------------------------------------------------------- */
/** How many sessions a learning read from past sessions was said in, from the line init writes under it. */
diff --git a/src/cloud/pair.ts b/src/cloud/pair.ts
index 341c66a7..77fef08d 100644
--- a/src/cloud/pair.ts
+++ b/src/cloud/pair.ts
@@ -157,7 +157,9 @@ export class PairRun {
"tokens",
"done",
tokens ? `~${tokens.pct}% fewer tokens` : "skipped",
- tokens ? `One lookup used ~${tokens.pct}% fewer tokens than reading ${tokens.files} file${tokens.files === 1 ? "" : "s"} (estimate)` : undefined,
+ tokens
+ ? `${tokens.pr ? `Your last PR, #${tokens.pr.number}: one` : "One"} lookup used ~${tokens.pct}% fewer tokens than reading ${tokens.files} file${tokens.files === 1 ? "" : "s"} (estimate)`
+ : undefined,
);
}
diff --git a/test/trail-connect.test.ts b/test/trail-connect.test.ts
index 5791c240..fdded852 100644
--- a/test/trail-connect.test.ts
+++ b/test/trail-connect.test.ts
@@ -33,6 +33,7 @@ import {
meaningfulSymbol,
openBranch,
tokensOnOneLookup,
+ lastMergedPrs,
} from "../src/cloud/onboarding.js";
import { readCloud, writeCloud, type Cloud } from "../src/cloud/workspace.js";
import { inWorkspace, recordActivity } from "../src/cloud/workspace-sync.js";
@@ -211,6 +212,48 @@ test("one real lookup on the fresh map: the most central symbol, and what readin
assert.equal(tokensOnOneLookup(mkdtempSync(join(tmpdir(), "connect-nograph-"))), null);
});
+test("the last merged PR is the question: its title, measured, with how many of its files the answer found", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "connect-last-pr-"));
+ git(dir, "init", "-q", "-b", "main");
+ git(dir, "config", "user.email", "[email protected]");
+ git(dir, "config", "user.name", "Sam");
+ git(dir, "remote", "add", "origin", "https://github.com/acme/api.git");
+ const filler = Array.from({ length: 400 }, (_, i) => `export function helper${i}(x: number): number {\n return x * ${i} + ${i};\n}\n`).join("\n");
+ writeFileSync(join(dir, "client.ts"), `export function retryDo(attempts: number): number {\n return attempts;\n}\n\n${filler}`);
+ writeFileSync(join(dir, "a.ts"), 'import { retryDo } from "./client";\nexport function useA() {\n return retryDo(4);\n}\n');
+ git(dir, "add", ".");
+ git(dir, "commit", "-qm", "first");
+ // The PR that changed the code, squash-merged.
+ writeFileSync(join(dir, "client.ts"), `export function retryDo(attempts: number): number {\n return attempts * 2;\n}\n\n${filler}`);
+ git(dir, "commit", "-qam", "Double the attempts in retryDo (#12)");
+ // Newer, but docs only, and a release chore: neither is the work.
+ writeFileSync(join(dir, "README.md"), "# api\n");
+ git(dir, "add", ".");
+ git(dir, "commit", "-qm", "Say what the api is (#13)");
+ writeFileSync(join(dir, "a.ts"), 'import { retryDo } from "./client";\nexport function useA() {\n return retryDo(5);\n}\n');
+ git(dir, "commit", "-qam", "chore(release): 1.2.0 (#14)");
+ const prs = lastMergedPrs(dir);
+ assert.deepEqual(prs.map((p) => p.number), [12]);
+ assert.deepEqual(prs[0], { number: 12, title: "Double the attempts in retryDo", url: "https://github.com/acme/api/pull/12", files: ["client.ts"] });
+ await buildGraph(dir);
+ const t = tokensOnOneLookup(dir);
+ assert.ok(t?.pr, "measured on the last PR");
+ assert.equal(t.question, "Double the attempts in retryDo");
+ assert.equal(t.pr.number, 12);
+ assert.equal(t.pr.found, 1, "the answer pointed at the file the PR changed");
+ assert.equal(t.pr.changed, 1);
+ assert.ok(t.saved > 0 && t.baseline > t.pack);
+ // A merge commit names the PR too, with its title on the body's first line.
+ writeFileSync(join(dir, "a.ts"), 'import { retryDo } from "./client";\nexport function useA() {\n return retryDo(6);\n}\n');
+ git(dir, "checkout", "-qb", "acme/use-six");
+ git(dir, "commit", "-qam", "use six");
+ git(dir, "checkout", "-q", "main");
+ git(dir, "merge", "-q", "--no-ff", "acme/use-six", "-m", "Merge pull request #15 from acme/use-six", "-m", "Use six attempts in useA");
+ assert.deepEqual(lastMergedPrs(dir)[0], { number: 15, title: "Use six attempts in useA", url: "https://github.com/acme/api/pull/15", files: ["a.ts"] });
+ // No PR in the history: the central-symbol lookup, as before.
+ assert.deepEqual(lastMergedPrs(mkdtempSync(join(tmpdir(), "connect-no-git-"))), []);
+});
+
test("tests, fixtures, vendored and generated code are not product code", () => {
for (const p of [
"client_test.go", "pkg/retry/client_test.go", "src/a.test.ts", "src/a.spec.js", "lib/a_spec.rb", "tests/test_client.py", "test_client.py",