Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

feat(trail): count a learning's cost from where its work began (#620)

anirudhkumar-nanonets8b231846640e
diff --git a/src/cli.ts b/src/cli.ts
index 6db77e6a..eeb47230 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -56,7 +56,7 @@ import { normalizePathPrefix } from "./util/paths.js";
import { latestSession, formatSessionStats, sessionInputRate } from "./claude/session-metrics.js";
import { listSessionIds, readSession, sessionDir } from "./claude/state.js";
import { costLabel, listNotes, noteName, notePlace, noteTargets, readingTokens, savedByNote, commitLines, PERSONAL_SAVED_LINE, PR_SPEEDUP, SECTIONS, tokensWorthSaying, sectionLead, shortDate, tokensLabel, writeNote, type Note, mentionedFiles } from "./notes/notes.js";
-import { currentSessionCost } from "./notes/session-cost.js";
+import { aboutTerms, currentSessionCost } from "./notes/session-cost.js";
import { beginPastRead, finishPastRead, type Finding } from "./notes/cold-start.js";
import { beginReviewRead, finishReviewRead, type ReviewFinding } from "./notes/pr-reviews.js";
import { currentBranch, noteAuthor, teammatesOn } from "./notes/git-facts.js";
@@ -1338,9 +1338,11 @@ async function runNoteCommand(repo: string, opts: NoteOptions): Promise<void> {
};
const minutes = wholeNumber(opts.minutes, "--minutes");
const tokens = wholeNumber(opts.tokens, "--tokens");
- // What the session spent, from the agent's own transcript when there is one;
- // the flags win, since only the agent knows where its task began.
- const spent = currentSessionCost(repo);
+ // What the session spent, from the agent's own transcript when there is one,
+ // counted from where the work the note names first came up; the flags win,
+ // since only the agent knows for sure where its task began.
+ const named = [...(opts.touches?.split(",").map((t) => t.trim()).filter(Boolean) ?? []), ...mentionedFiles(repo, body)];
+ const spent = currentSessionCost(repo, undefined, Date.now(), aboutTerms(body, named));
const cost =
minutes !== undefined || tokens !== undefined || spent
? { minutes: minutes ?? spent?.minutes, tokens: tokens ?? spent?.tokens }
diff --git a/src/notes/session-cost.ts b/src/notes/session-cost.ts
index 04820982..7845ccc3 100644
--- a/src/notes/session-cost.ts
+++ b/src/notes/session-cost.ts
@@ -9,7 +9,12 @@
* the agent passes `--minutes` / `--tokens`.
*
* Counted from the session's last finished `trail note` (or its start), so two
- * notes from one long session don't both claim the same work. Tokens are the
+ * notes from one long session don't both claim the same work. Within that,
+ * from the turn where the work the note names first came up: the first prompt
+ * or tool call that mentions one of its files, symbols or commands. A long
+ * session that did other things first (a Slack search, a merge) doesn't charge
+ * them to this learning. When nothing matches, only the last 30 active
+ * minutes count, never the whole stretch. Tokens are the
* ones the model actually processed fresh: input, cache writes and output.
* Cache reads are left out, since they re-read the same context every turn and
* would make an hour's work look like millions. Minutes are active minutes: a
@@ -23,13 +28,17 @@ import { dirname, join, resolve } from "node:path";
export interface SessionCost {
minutes: number;
tokens: number;
- /** When the counted stretch began, epoch ms. */
+ /** When the stretch since the last note began, epoch ms. */
startedAt: number;
+ /** When the counted work began, epoch ms: the first mention of what the note is about. */
+ countedFrom: number;
/** Files the agent edited or wrote in that stretch, absolute, in order. */
edited: string[];
}
const IDLE_MS = 10 * 60 * 1000;
+/** With nothing in the transcript naming what the note is about, at most this much active time counts. */
+export const FALLBACK_ACTIVE_MS = 30 * 60 * 1000;
/** A transcript untouched this long belongs to some earlier session. */
const ACTIVE_MS = 30 * 60 * 1000;
@@ -92,12 +101,84 @@ const EDIT_TOOLS = new Set(["Edit", "MultiEdit", "Write", "NotebookEdit"]);
export const NOTE_CALL = /(^|[|&;]\s*|\s)(npx\s+(-y\s+)?(@trailhq\/|@nanonets\/)?)?(trail|graft)\s+note\b/;
+/** Words too common to say what a note is about: they'd match the first turn of any session. */
+const TOO_COMMON = new Set(["trail", "graft", "claude", "true", "false", "null", "main", "master", "node", "git", "npm", "test", "tests"]);
+
+/**
+ * What a note is about, as strings a transcript would show when that work
+ * began: the files it names (path and file name) and anything it puts in
+ * backticks, such as a symbol, a command or a setting.
+ */
+export function aboutTerms(body: string, files: string[] = []): string[] {
+ const terms = new Set<string>();
+ for (const f of files) {
+ const path = f.split("#")[0]!.trim();
+ if (!path) continue;
+ terms.add(path);
+ const name = path.split("/").pop()!;
+ if (name.length >= 4) terms.add(name);
+ }
+ for (const m of body.matchAll(/`([^`\n]{3,80})`/g)) {
+ const t = m[1]!.trim().replace(/\(\)$/, "");
+ if (t.length >= 4 && !TOO_COMMON.has(t.toLowerCase()) && specific(t)) terms.add(t);
+ }
+ return [...terms];
+}
+
+/**
+ * Whether a backticked term names something specific enough to date the work:
+ * a path or file (`a/b`, `x.ts`), or an identifier (`dropHtmlComments`,
+ * `repo_id`, `check-ignore`). A phrase or a plain word (`trail init`,
+ * `publish`) comes up in any session about the product, so it would date
+ * the work to the session's first turn.
+ */
+function specific(t: string): boolean {
+ if (/\s/.test(t)) return false;
+ return /[\/._]|[a-z][A-Z]|\w-\w/.test(t);
+}
+
+/** A prompt the person typed, as opposed to a tool's result coming back. */
+function promptText(e: Entry): string | null {
+ if (e.type !== "user") return null;
+ const c = e.message?.content;
+ if (typeof c === "string") return c;
+ if (!Array.isArray(c) || c.some((b) => (b as { type?: string }).type === "tool_result")) return null;
+ const text = c.filter((b) => (b as { type?: string }).type === "text").map((b) => String((b as { text?: unknown }).text ?? "")).join("\n");
+ return text || null;
+}
+
+/**
+ * Index into `entries` where the work on a note began: the prompt of the turn
+ * whose prompt or tool calls first mention one of `terms`, never earlier than
+ * `from`. A `trail note` call doesn't count: the one being saved names them all.
+ */
+function firstMention(entries: Entry[], from: number, terms: string[]): number | null {
+ if (!terms.length) return null;
+ let turn = from;
+ for (let i = from; i < entries.length; i++) {
+ const e = entries[i]!;
+ const prompt = promptText(e);
+ if (prompt !== null) turn = i;
+ const said = [prompt ?? ""];
+ for (const b of blocks(e)) {
+ if (b.type !== "tool_use") continue;
+ const input = JSON.stringify(b.input ?? {});
+ if (NOTE_CALL.test(input)) continue;
+ said.push(input);
+ }
+ if (said.some((s) => s && terms.some((t) => s.includes(t)))) return turn;
+ }
+ return null;
+}
+
/**
* The cost of one transcript since its last finished note. Pure, for tests:
* `text` is the JSONL, `now` closes the last stretch when the transcript ends
- * mid-turn (it always does: the note call itself is in flight).
+ * mid-turn (it always does: the note call itself is in flight). `terms` are
+ * what the note is about (see `aboutTerms`); without them the whole stretch
+ * since the last note counts, as before.
*/
-export function costFromTranscript(text: string, now = Date.now()): SessionCost | null {
+export function costFromTranscript(text: string, now = Date.now(), terms?: string[]): SessionCost | null {
const entries: Entry[] = [];
for (const line of text.split("\n")) {
if (!line.trim()) continue;
@@ -123,10 +204,19 @@ export function costFromTranscript(text: string, now = Date.now()): SessionCost
});
const after = entries.slice(boundary + 1);
- const counted = after.filter((e) => e.timestamp);
- if (counted.length === 0) return null;
- const times = counted.map((e) => Date.parse(e.timestamp!)).filter((t) => Number.isFinite(t));
- if (times.length === 0) return null;
+ const since = after.filter((e) => e.timestamp && Number.isFinite(Date.parse(e.timestamp)));
+ if (since.length === 0) return null;
+ const startedAt = Date.parse(since[0]!.timestamp!);
+
+ // Where this note's work began within the stretch.
+ let counted = since;
+ if (terms) {
+ const at = firstMention(entries, boundary + 1, terms);
+ const stretch = new Set(since);
+ counted = at !== null ? entries.slice(at).filter((e) => stretch.has(e)) : lastActive(since, now, FALLBACK_ACTIVE_MS);
+ if (counted.length === 0) counted = since.slice(-1);
+ }
+ const times = counted.map((e) => Date.parse(e.timestamp!));
times.push(now);
let activeMs = 0;
for (let i = 1; i < times.length; i++) {
@@ -157,15 +247,29 @@ export function costFromTranscript(text: string, now = Date.now()): SessionCost
if (typeof file === "string" && file && !edited.includes(file)) edited.push(file);
}
}
- return { minutes: Math.max(1, Math.round(activeMs / 60_000)), tokens, startedAt: times[0]!, edited };
+ return { minutes: Math.max(1, Math.round(activeMs / 60_000)), tokens, startedAt, countedFrom: times[0]!, edited };
+}
+
+/** The entries in the last `budgetMs` of active time before `now`, oldest first. */
+function lastActive(entries: Entry[], now: number, budgetMs: number): Entry[] {
+ let used = 0;
+ let prev = now;
+ for (let i = entries.length - 1; i >= 0; i--) {
+ const t = Date.parse(entries[i]!.timestamp!);
+ const gap = prev - t;
+ if (gap > 0 && gap <= IDLE_MS) used += gap;
+ if (used > budgetMs) return entries.slice(i + 1);
+ prev = t;
+ }
+ return entries;
}
/** The running session's cost in `repo`, or null when there is no transcript to read. */
-export function currentSessionCost(repo: string, home?: string, now = Date.now()): SessionCost | null {
+export function currentSessionCost(repo: string, home?: string, now = Date.now(), terms?: string[]): SessionCost | null {
const path = currentTranscript(repo, home, now);
if (!path) return null;
try {
- return costFromTranscript(readFileSync(path, "utf8"), now);
+ return costFromTranscript(readFileSync(path, "utf8"), now, terms);
} catch {
return null;
}
diff --git a/test/trail-notes.test.ts b/test/trail-notes.test.ts
index 878e64f2..65a44992 100644
--- a/test/trail-notes.test.ts
+++ b/test/trail-notes.test.ts
@@ -23,7 +23,7 @@ import {
writeNote,
type Note,
} from "../src/notes/notes.js";
-import { costFromTranscript } from "../src/notes/session-cost.js";
+import { aboutTerms, costFromTranscript, FALLBACK_ACTIVE_MS } from "../src/notes/session-cost.js";
import { ensureRepoHome, repoPlace } from "../src/notes/home.js";
import { homeEnv, tmpRepo } from "./helpers.js";
@@ -198,6 +198,51 @@ test("the cost carries the files the session edited since its last note, for the
]);
});
+test("a note names its work by its files and specific backticked terms, not by phrases", () => {
+ const body = "## Decided\n- `dropHtmlComments` in `src/notes/pr-reviews.ts` repeats until stable.\n- Run `trail init` and `publish`, keep `repo_id`, see `git check-ignore`.\n- `main` is fine.";
+ assert.deepEqual(aboutTerms(body, ["internal/ocr/bbox.go#NormalizeBox"]), [
+ "internal/ocr/bbox.go",
+ "bbox.go",
+ "dropHtmlComments",
+ "src/notes/pr-reviews.ts",
+ "repo_id",
+ ]);
+});
+
+test("a note's cost starts at the turn where its work first came up, not at the session's start", () => {
+ const t = [
+ // An hour on something else first.
+ line({ type: "user", timestamp: at(0), message: { content: "search slack for the release" } }),
+ line({ type: "assistant", timestamp: at(5), message: { id: "s", usage: usage(0, 50_000, 1000), content: [{ type: "tool_use", id: "x", name: "Bash", input: { command: "slack search release" } }] } }),
+ line({ type: "assistant", timestamp: at(9), message: { id: "s2", usage: usage(0, 40_000, 1000), content: [] } }),
+ // The turn the learning is about: its prompt counts, then the tool call that names the file.
+ line({ type: "user", timestamp: at(10), message: { content: "why is the box rotated?" } }),
+ line({ type: "assistant", timestamp: at(11), message: { id: "w", usage: usage(0, 3000, 500), content: [{ type: "tool_use", id: "r", name: "Read", input: { file_path: "/repo/internal/ocr/bbox.go" } }] } }),
+ line({ type: "assistant", timestamp: at(14), message: { id: "w2", usage: usage(0, 2000, 500), content: [] } }),
+ // The note being saved names everything: it never counts as where the work began.
+ line({ type: "assistant", timestamp: at(15), message: { id: "n", usage: usage(0, 0, 100), content: [{ type: "tool_use", id: "n1", name: "Bash", input: { command: "trail note --title x <<'EOF'\nslack bbox.go\nEOF" } }] } }),
+ ].join("\n");
+ const now = Date.parse(at(15));
+ const whole = costFromTranscript(t, now)!;
+ const scoped = costFromTranscript(t, now, aboutTerms("see `internal/ocr/bbox.go`"))!;
+ assert.equal(whole.tokens, 51_000 + 41_000 + 3500 + 2500 + 100);
+ assert.equal(scoped.tokens, 3500 + 2500 + 100);
+ assert.equal(scoped.minutes, 5);
+ assert.equal(scoped.countedFrom, Date.parse(at(10)));
+ assert.equal(scoped.startedAt, Date.parse(at(0)), "the stretch since the last note still starts at the top");
+});
+
+test("when nothing in the transcript names the work, only the last 30 active minutes count", () => {
+ const t = Array.from({ length: 13 }, (_, i) =>
+ line({ type: "assistant", timestamp: at(i * 5), message: { id: `m${i}`, usage: usage(0, 1000, 0), content: [] } }),
+ ).join("\n");
+ const now = Date.parse(at(60));
+ assert.equal(costFromTranscript(t, now)!.minutes, 60);
+ const capped = costFromTranscript(t, now, ["nowhere/in/it.go"])!;
+ assert.equal(capped.minutes, FALLBACK_ACTIVE_MS / 60_000);
+ assert.equal(capped.tokens, 7000, "the entries at 30, 35, …, 60 minutes");
+});
+
test("no transcript entries, no cost", () => {
assert.equal(costFromTranscript("", Date.now()), null);
assert.equal(costFromTranscript("not json\n", Date.now()), null);