Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
/**
* Typed preferences, found without a model (design 4.4, P3): the standing
* or corrective things the person told the agent on more than one day.
* This replaces phase 0's find call, which sent their typed text to Claude.
*
* Pure and in memory: the lines come from the replay (pointers plus text),
* the text never leaves the worker, and what is found is clusters of
* pointers with the shortest line to show. A line counts only when it is a
* preference's shape (4.4 step 1); lines cluster by their content words
* (lowercased, stop words out, a small stemmer so "published" meets
* "publish"); a cluster counts on 2+ local days that are not one 8 h
* sitting (phase 0's rule, D46).
*/
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { readdirSync } from 'node:fs';
import { join } from 'node:path';
import { secretReason } from '../notes/secrets.js';
import { askText, isStatusOrHistory, matchJob, MAX_ASK_WORDS, wordCount } from '../trace/jobs.js';
import { PASTE_MARK } from '../trace/claude.js';
import { isCorrection } from '../trace/typed.js';
import { readPlain } from './plain-file.js';
import type { TypedRef } from './types.js';
/** Two lines whose times all fall closer than this are one sitting that crossed midnight (phase 0, D46). */
export const SITTING_MS = 8 * 60 * 60 * 1000;
/** Content-word Jaccard at or over this joins two lines (4.4 step 3). */
export const SAME_WORDS = 0.5;
const TITLE_CHARS = 120;
/** package.json or a Makefile read for the repo's names, at most this big. */
const REPO_FILE_MAX = 1024 * 1024;
/** Preference-shaped lines clustered at most, the newest: single-link clustering is quadratic in them (b5 review). */
export const MAX_PREF_LINES = 2000;
const MAX_REFS = 3;
/** Verbs a preference opens with once a correction's lead word ("no,", "actually,") is set aside. */
const IMPERATIVE = new Set([
'use', 'run', 'keep', 'write', 'say', 'call', 'prefer', 'put', 'add', 'make', 'check', 'ask', 'push', 'commit', 'test', 'read', 'show',
'explain', 'open', 'stop', 'start', 'leave', 'avoid', 'try', 'do', "don't", 'dont', 'never', 'always', 'give', 'tell', 'let', 'wait',
'merge', 'rebase', 'deploy', 'publish', 'build', 'install', 'remove', 'delete', 'move', 'rename', 'name', 'format', 'lint', 'quote',
'include', 'skip', 'follow', 'stay', 'be', 'go', 'set', 'pin', 'reply', 'answer', 'ship', 'release', 'branch', 'squash', 'sign', 'mention',
]);
const STANDING = /^(?:always|never|don'?t|do not|stop|please (?:always|never|don'?t))\b/i;
const NOT_THAT = /\b(?:instead of|rather than|not)\b .{1,40}$/i;
const LEADS = /^(?:use|keep|write|say|call it|prefer)\b/i;
/** A correction's lead, set aside to read the rest: "no,", "nope", "actually,", "wrong,". */
const CORRECTION_LEAD = /^(?:no|nope|actually|wrong|not like that|that'?s (?:wrong|not right))[,.!:;\s-]+/i;
const STOP = new Set([
'a', 'an', 'the', 'to', 'of', 'in', 'on', 'for', 'and', 'or', 'it', 'its', 'this', 'that', 'these', 'those', 'is', 'are', 'be', 'with',
'at', 'by', 'from', 'as', 'please', 'i', 'you', 'we', 'me', 'my', 'your', 'our', 'just', 'so', 'then', 'here', 'there', 'when', 'if',
'can', 'will', 'should', 'would', 'pl', 'all', 'any', 'some', 'up', 'out', 'too', 'also', 'ok', 'okay',
]);
/**
* D102: how the person wants to be talked to ("explain it simply", "keep it short", "use plain words", "no jargon",
* "be brief" and close variants). A preference shape, and about the person, so it applies in every repo.
*/
const STYLE = [
/^(?:please,?\s+)?(?:explain|say|describe|write|answer|reply|talk|put)\b.{0,60}\b(?:simply|simpler|plainly|briefly|concisely|in plain (?:words|english|language)|in simple (?:words|terms|language))\b/i,
/^(?:please,?\s+)?(?:keep|make)\s+(?:it|this|that|them|things|(?:your\s+)?(?:answers?|replies|reply|responses?|explanations?|messages?|summaries|summary))\s+(?:short|shorter|brief|briefer|simple|simpler|concise|plain|tight)\b/i,
/^(?:please,?\s+)?(?:use|in|speak|write in)\s+(?:plain|simple|simpler|everyday|short)\s+(?:words|english|language|terms|sentences)\b/i,
/^(?:plain|simple)\s+(?:english|words|language)\b/i,
/^(?:please,?\s+)?(?:no|less|fewer|avoid|skip|without|drop the)\s+(?:the\s+)?jargon\b/i,
/^(?:please,?\s+)?be\s+(?:brief|briefer|concise|terse|short|succinct|direct)\b/i,
/^(?:shorter|simpler|briefer|plainer)\s+(?:answers?|replies|responses?|explanations?|words|sentences|please)\b/i,
/^(?:please,?\s+)?(?:one|an?) example first\b/i,
];
/** The person's own words in a typed line: no quoted (`>`) lines, no code block, no paste marker, one line. */
export function prefText(text: string): string {
return askText(String(text ?? '')).split(PASTE_MARK).join(' ').replace(/\s+/g, ' ').trim();
}
/** D102: a style preference ("explain it simply", "keep it short", "no jargon"), which applies in every repo. */
export function isStylePreference(text: string): boolean {
const t = prefText(text).replace(/[.!]+$/, '');
return STYLE.some((re) => re.test(t));
}
const firstWord = (t: string) => (/^[\s"'(]*([a-z][a-z']*)/i.exec(t)?.[1] ?? '').toLowerCase();
/** 4.4 step 1: a standing or corrective line of 25 words or fewer that is not a job ask, a question or a secret. */
export function isPreferenceShape(text: string): boolean {
const t = prefText(text);
if (!t || wordCount(t) > MAX_ASK_WORDS || /\?\s*$/.test(t)) return false;
if (matchJob(t) || isStatusOrHistory(t) || secretReason(t) !== null) return false;
if (STANDING.test(t) || LEADS.test(t) || isStylePreference(t)) return true;
if (NOT_THAT.test(t) && IMPERATIVE.has(firstWord(t))) return true;
if (isCorrection(t)) {
const rest = t.replace(CORRECTION_LEAD, '');
return rest !== t && IMPERATIVE.has(firstWord(rest));
}
return false;
}
/** A small stemmer: possessive, -ing, -ed, -es after a hiss, plural -s, then a final -e, so forms of one verb meet. */
function stem(w: string): string {
let s = w.replace(/'s$/, '');
if (s.length > 5 && s.endsWith('ing')) s = s.slice(0, -3);
else if (s.length > 4 && s.endsWith('ed')) s = s.slice(0, -2);
else if (s.length > 4 && /(?:sh|ch|x|ss|z)es$/.test(s)) s = s.slice(0, -2);
else if (s.length > 3 && s.endsWith('s') && !s.endsWith('ss')) s = s.slice(0, -1);
if (s.length > 2 && s.endsWith('e')) s = s.slice(0, -1);
return s;
}
/** Content words, stemmed, sorted, once each (4.4 step 2). */
export function contentWords(text: string): string[] {
const words = String(text ?? '').toLowerCase().split(/[^a-z0-9'+-]+/).map((w) => w.replace(/^['-]+|['-]+$/g, '')).filter((w) => w && !STOP.has(w));
return [...new Set(words.map(stem))].filter(Boolean).sort();
}
/** |a ∩ b| / |a ∪ b|; 0 when both are empty. */
export function jaccard(a: readonly string[], b: readonly string[]): number {
if (!a.length && !b.length) return 0;
const sa = new Set(a);
const inter = b.filter((w) => sa.has(w)).length;
return inter / (sa.size + new Set(b).size - inter);
}
type Line = { ref: TypedRef; day: string; text: string; at?: string };
/** D135 (2): a near-repeat is short: at most this many words, at least NEAR_MIN_WORDS content words. */
export const NEAR_MAX_WORDS = 12;
const NEAR_MIN_WORDS = 3;
/** Almost word for word: content-word Jaccard at or over this. */
export const NEAR_WORDS = 0.8;
/** On this many local days or more. */
export const NEAR_DAYS = 3;
/** A request for something now, not a way of working: aimed at the person ("show me"), or tied to a time ("today"). */
const NEAR_REQUEST = /^\S+\s+(?:me|us)\b|\b(?:today|tonight|yesterday|tomorrow|right now|this (?:morning|afternoon|evening|week))\b/i;
/**
* D135 (2): a short imperative that 4.4's shape doesn't read ("add a changelog entry for each fix"): an imperative
* verb first, once a correction's lead is set aside; not a question, a job ask, a status question, a request for
* something now, or a secret.
*/
export function isNearRepeatShape(text: string): boolean {
const t = prefText(text);
if (!t || wordCount(t) > NEAR_MAX_WORDS || /\?\s*$/.test(t) || contentWords(t).length < NEAR_MIN_WORDS) return false;
if (matchJob(t) || isStatusOrHistory(t) || NEAR_REQUEST.test(t) || secretReason(t) !== null) return false;
return IMPERATIVE.has(firstWord(t.replace(CORRECTION_LEAD, '')));
}
/**
* Typed preference clusters (4.4): standing or corrective shapes, on 2+ local days, not one 8 h sitting. Text stays
* in memory. `refs`: the shortest line's pointer first (the title the screen reads again), then the newest, at most
* 3. `sha`: 16 hex of the cluster's oldest line's content words, so it holds as the person says it again. `lines`: every
* line of the cluster (in memory only: phase 0's closed quote hashes are checked against them). `near`: a D135 (2)
* near-repeat (a short imperative typed almost word for word on 3+ days), which the screen shows unticked.
*/
export type PrefCluster = { sha: string; refs: TypedRef[]; days: string[]; shortest: string; lines: string[]; near?: true };
export function minePreferences(typed: Line[], _now: Date = new Date()): PrefCluster[] {
const shaped = clusterLines(typed, isPreferenceShape, SAME_WORDS, 2);
// D135 (2): near-repeats, unless a shaped cluster already says it.
const said = shaped.map((c) => contentWords(c.shortest));
const near = clusterLines(typed, (t) => !isPreferenceShape(t) && isNearRepeatShape(t), NEAR_WORDS, NEAR_DAYS)
.filter((c) => !said.some((w) => jaccard(w, contentWords(c.shortest)) >= SAME_WORDS))
.map((c) => ({ ...c, near: true as const }));
return [...shaped, ...near];
}
function clusterLines(typed: Line[], shape: (text: string) => boolean, same: number, minDays: number): PrefCluster[] {
// Only the person's own words: a quoted block, a code block or a paste is never a preference's (D99, b5 review).
const lines = typed.filter((l) => l && typeof l.text === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(l.day) && shape(l.text))
.map((l, i) => ({ ...l, raw: l.text, text: prefText(l.text), i, words: contentWords(prefText(l.text)), time: l.at ? Date.parse(l.at) : NaN }))
.filter((l) => l.words.length > 0);
// Newest first: by time where known, else by order of reading.
lines.sort((a, b) => (Number.isNaN(a.time) || Number.isNaN(b.time) ? b.i - a.i : b.time - a.time || b.i - a.i));
const clusters: Array<typeof lines> = [];
for (const l of lines.slice(0, MAX_PREF_LINES)) {
const home = clusters.find((c) => c.some((m) => jaccard(m.words, l.words) >= same));
if (home) home.push(l);
else clusters.push([l]);
}
const out: PrefCluster[] = [];
for (const c of clusters) {
const days = [...new Set(c.map((l) => l.day))].sort();
if (days.length < minDays) continue;
const times = c.map((l) => l.time).filter((t) => !Number.isNaN(t));
if (times.length === c.length && Math.max(...times) - Math.min(...times) < SITTING_MS) continue;
const shortest = [...c].sort((a, b) => a.text.length - b.text.length || b.i - a.i)[0]!;
const oldest = c[c.length - 1]!;
const refs = [shortest.ref, ...c.filter((l) => l !== shortest).map((l) => l.ref)].slice(0, MAX_REFS);
out.push({
sha: createHash('sha256').update(oldest.words.join(' ')).digest('hex').slice(0, 16),
refs, days, shortest: shortest.text.replace(/\s+/g, ' ').trim().slice(0, TITLE_CHARS), lines: [...new Set(c.flatMap((l) => [l.text, l.raw]))],
});
}
return out;
}
/** Words and path pieces of a line, lowercased, as `preferenceApplies` compares them. */
function tokens(text: string): string[] {
const out: string[] = [];
for (const raw of String(text ?? '').toLowerCase().split(/[\s,;:!?()"`]+/)) {
const t = raw.replace(/'s$/, '').replace(/^['.]+|['.]+$/g, '');
if (!t) continue;
out.push(t);
for (const piece of t.split(/[/\\]/)) if (piece && piece !== t) out.push(piece.replace(/^\.+|\.+$/g, ''));
}
return out.filter(Boolean);
}
/**
* D102: programs every machine runs. Naming one ("use pnpm", "always git rebase") says nothing of this repo, so it
* never makes a line "this repo", whichever source (a folder, a target, a session's command) put the word in.
*/
export const COMMON_PROGRAMS: ReadonlySet<string> = new Set([
'git', 'gh', 'hub', 'npm', 'npx', 'pnpm', 'pnpx', 'yarn', 'bun', 'bunx', 'deno', 'node', 'nvm', 'volta', 'corepack', 'tsx', 'ts-node', 'tsc',
'python', 'python3', 'pip', 'pip3', 'pipx', 'uv', 'uvx', 'poetry', 'pyenv', 'conda', 'pytest', 'ruff', 'black', 'mypy',
'go', 'gofmt', 'cargo', 'rustc', 'rustup', 'make', 'cmake', 'ninja', 'bazel', 'docker', 'docker-compose', 'compose', 'podman', 'colima',
'kubectl', 'helm', 'terraform', 'java', 'javac', 'mvn', 'gradle', 'ruby', 'gem', 'bundle', 'bundler', 'rake', 'rails', 'php', 'composer',
'dotnet', 'swift', 'xcodebuild', 'brew', 'apt', 'apt-get', 'curl', 'wget', 'ssh', 'scp', 'rsync', 'jq', 'sed', 'awk', 'grep', 'rg', 'find',
'ls', 'cat', 'echo', 'cd', 'rm', 'cp', 'mv', 'mkdir', 'chmod', 'sh', 'bash', 'zsh', 'fish', 'sudo', 'env', 'timeout', 'eslint', 'prettier',
'jest', 'vitest', 'mocha', 'playwright', 'cypress', 'webpack', 'vite', 'esbuild', 'turbo', 'nx', 'lerna', 'aws', 'gcloud', 'az', 'vercel',
'netlify', 'fly', 'flyctl', 'heroku', 'supabase', 'psql', 'mysql', 'sqlite3', 'redis-cli', 'code', 'vim', 'nvim', 'emacs', 'tmux', 'claude', 'trail',
]);
/** The line says it is about this repo in so many words ("here", "in this repo", "this project"). */
const HERE = /\b(?:here|in this (?:repo|repository|project|codebase|checkout)|this (?:repo|repository|project|codebase)(?:'s)?)\b/i;
/**
* 'this repo' when the line names something specific to the repo (a script, path, branch, package or file name, or
* the repo itself: "here"), else 'every repo' (spec 2, R50). A common program never counts, and a style preference
* ("explain it simply") is always every repo (D102).
*/
export function preferenceApplies(text: string, repoWords: Set<string>): 'this repo' | 'every repo' {
if (isStylePreference(text)) return 'every repo';
const own = prefText(text);
if (HERE.test(own)) return 'this repo';
const named = (t: string) => !COMMON_PROGRAMS.has(t) && repoWords.has(t);
return tokens(own).some((t) => named(t) || named(t.replace(/\.[a-z0-9]+$/, '')) || (t.includes('/') && t.length > 1 && !/^https?:/.test(t)))
? 'this repo' : 'every repo';
}
/**
* The repo's own names (4.4 step 7): top-level files and folders, `package.json` name and scripts, Makefile
* targets, branch names, and the script files its sessions ran; never a common program (D102). Lowercased. Reads
* only; never throws.
*/
export function repoWordsOf(checkout: string, programs: Iterable<string> = []): Set<string> {
const out = new Set<string>();
const add = (w: unknown) => {
if (typeof w !== 'string') return;
const t = w.trim().toLowerCase();
if (t.length >= 2 && t.length <= 80 && !COMMON_PROGRAMS.has(t)) out.add(t);
};
if (!checkout) return out;
try {
for (const name of readdirSync(checkout)) {
if (name.startsWith('.')) continue;
add(name);
add(name.replace(/\.[a-z0-9]+$/i, ''));
}
} catch { /* no folder */ }
try {
// Never a link or a FIFO (a planted one would hang init), never a huge file (b5 review).
const pkg = JSON.parse(readPlain(join(checkout, 'package.json'), REPO_FILE_MAX) ?? 'null') as { name?: unknown; scripts?: unknown };
add(pkg.name);
if (typeof pkg.name === 'string' && pkg.name.includes('/')) add(pkg.name.split('/').pop());
if (pkg.scripts && typeof pkg.scripts === 'object') for (const k of Object.keys(pkg.scripts)) add(k);
} catch { /* none */ }
for (const file of ['Makefile', 'makefile', 'GNUmakefile']) {
try {
for (const m of (readPlain(join(checkout, file), REPO_FILE_MAX) ?? '').matchAll(/^([A-Za-z0-9][\w.-]*)\s*:(?!=)/gm)) add(m[1]);
} catch { /* none */ }
}
const r = spawnSync('git', ['-C', checkout, 'for-each-ref', '--format=%(refname:short)', '--count=200', 'refs/heads'], { encoding: 'utf8', timeout: 5000 });
if (r.status === 0) for (const b of r.stdout.split('\n')) add(b);
// A program the sessions ran names the repo only when it is a script file of its own (`seed.sh`), never a program
// every machine has (D102).
for (const p of programs) if (typeof p === 'string' && /^[\w.+-]+\.[a-z0-9]{1,5}$/i.test(p)) add(p);
return out;
}