Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
feat(graph): scope discovery — project markers become ranking scopes in graph meta
Shrish Dwivedi992bb2c0435d
diff --git a/src/graph/build.ts b/src/graph/build.tsindex ee43c4af..38c69a35 100644--- a/src/graph/build.ts+++ b/src/graph/build.ts@@ -17,9 +17,35 @@ import { enrichGraph, type EnrichStats } from "./enrich.js";import { readGraph, writeGraph, wiringPath } from "./write.js";import { writeCards, writeIndex, writeCovers } from "./cards.js";import { writeAskIndex } from "../ask/index-file.js";-import type { GraphV1, Kind, NodeV1, Relation } from "./types.js";+import { discoverScopes, scopeOf } from "./scopes.js";+import type { GraphV1, Kind, NodeV1, Relation, ScopeV1 } from "./types.js";import type { CruxSummarizer } from "../ai/crux.js";+/** Minimum non-file node count for a discovered sub-scope to stand on its own+ * (over-split guard 3). A scope with fewer nodes than this is folded into the+ * root scope — node counts aren't known until after the graph is assembled,+ * so this runs here rather than in `discoverScopes`. */+const MIN_SCOPE_NODES = 5;++/** Guard 5: merge scopes with too few non-file nodes into the root scope, then+ * re-apply the canonical single-scope collapse (rule 6) if only root is left. */+function applyMinSubstanceGuard(scopes: ScopeV1[], nodes: NodeV1[]): ScopeV1[] {+ if (scopes.length <= 1) return scopes;+ const counts = new Map<string, number>();+ for (const scope of scopes) counts.set(scope.prefix, 0);+ for (const node of nodes) {+ if (node.kind === "file") continue;+ const scope = scopeOf(node.path, scopes);+ counts.set(scope.prefix, (counts.get(scope.prefix) ?? 0) + 1);+ }+ const kept = scopes.filter((s) => s.prefix === "" || (counts.get(s.prefix) ?? 0) >= MIN_SCOPE_NODES);+ if (kept.length === 0) return [{ prefix: "", label: "", markers: [] }];+ if (kept.length === 1 && kept[0].prefix === "") {+ return [{ prefix: "", label: "", markers: kept[0].markers }];+ }+ return kept;+}+export interface GraphBuildOptions {/** Override the output dir (default: `<root>/.context`). */contextDir?: string;@@ -82,6 +108,7 @@ export async function buildGraph(const root = resolve(dir);const outDir = contextDirFor(root, opts.contextDir);const files = listSourceFiles(root, outDir);+ const discoveredScopes = discoverScopes(root);const nodes: NodeV1[] = [];const rawEdges: RawEdge[] = [];@@ -125,12 +152,17 @@ export async function buildGraph(});errors.push(...meaning.errors);+ // Guard 5 (minimum-substance): node counts aren't known until nodes are+ // assembled, so the merge-tiny-scopes-into-root guard runs here.+ const scopes = applyMinSubstanceGuard(discoveredScopes, nodes);+const graph: GraphV1 = {meta: {version: 1,nodeCount: nodes.length,edgeCount: edges.length,languages: [...langs].sort(),+ scopes,},nodes,edges,diff --git a/src/graph/scopes.ts b/src/graph/scopes.tsnew file mode 100644index 00000000..32b2e980--- /dev/null+++ b/src/graph/scopes.ts@@ -0,0 +1,278 @@+/**+ * Scope discovery — finds sub-project boundaries inside a repo so ranking can+ * stay per-scope instead of pooling a monorepo's biggest sub-project against+ * everything else. A "scope" is rooted at a directory that carries a+ * project-marker file (`package.json`, `go.mod`, `pyproject.toml`, `setup.py`,+ * `Cargo.toml`).+ *+ * Discovery rules (see task brief for the numbered spec):+ * 1. Any directory with >=1 marker file is a scope candidate.+ * 2. Workspace-config-as-intent: `pnpm-workspace.yaml` or root `package.json`+ * `workspaces` glob(s) are resolved and become the ONLY JS-family+ * sub-scopes considered — other `package.json` dirs are ignored.+ * 3. Depth guard: candidates deeper than 2 path segments below root are+ * ignored unless matched by a workspace glob.+ * 4. Nesting collapse: a candidate nested inside another is dropped (keep+ * the shallower one) — except a workspace-glob match wins over its parent.+ * 5. Minimum-substance guard (< 5 non-file nodes merged into root) runs in+ * `build.ts`, since node counts aren't known at walk time.+ * 6. If exactly the root scope survives (or nothing does), the canonical+ * single-scope form `[{ prefix: "", label: "", markers: [...] }]` is emitted.+ * 7. `label` = `prefix` (both "" for root); ordering is prefix-length desc,+ * then lexicographic.+ */+import { existsSync, readdirSync, readFileSync, type Dirent } from "node:fs";+import { join, resolve } from "node:path";+import { SKIP_DIRS } from "../ingest/fs.js";+import type { GraphV1, ScopeV1 } from "./types.js";++/** Project-marker files, checked in this order (also the order `markers` is built in). */+const MARKERS = ["package.json", "go.mod", "pyproject.toml", "setup.py", "Cargo.toml"];++const CANONICAL_ROOT: ScopeV1[] = [{ prefix: "", label: "", markers: [] }];++/** Recursively collect every directory under `root` (posix rel path, "" = root itself),+ * reusing `walkDir`'s skip rules (dot-dirs, `SKIP_DIRS`) so build-output and dependency+ * trees are never scanned for markers. */+function collectDirs(root: string): string[] {+ const out: string[] = [""];+ const walk = (absDir: string, relDir: string): void => {+ let entries: Dirent[];+ try {+ entries = readdirSync(absDir, { withFileTypes: true });+ } catch {+ return;+ }+ for (const entry of entries) {+ if (!entry.isDirectory()) continue;+ if (entry.name.startsWith(".") || SKIP_DIRS.has(entry.name)) continue;+ const rel = relDir === "" ? entry.name : `${relDir}/${entry.name}`;+ out.push(rel);+ walk(join(absDir, entry.name), rel);+ }+ };+ walk(root, "");+ return out;+}++/** Minimal `packages:` list parser for `pnpm-workspace.yaml` — no YAML dependency,+ * handles the `packages:\n - 'glob'\n - "glob"` form pnpm actually generates. */+function parsePnpmPackagesList(text: string): string[] | null {+ const lines = text.split(/\r?\n/);+ const idx = lines.findIndex((l) => /^packages\s*:/.test(l.trim()));+ if (idx === -1) return null;+ const globs: string[] = [];+ for (let i = idx + 1; i < lines.length; i++) {+ const line = lines[i];+ if (line.trim() === "") continue;+ if (!/^\s*-\s*/.test(line)) break; // dedented — list ended+ const val = line.replace(/^\s*-\s*/, "").trim().replace(/^['"]|['"]$/g, "");+ if (val) globs.push(val);+ }+ return globs.length ? globs : null;+}++/** Rule 2: resolve workspace-config-as-intent globs. Returns null when the repo has+ * no workspace config (pnpm-workspace.yaml wins over `package.json#workspaces` when+ * both exist). */+function readWorkspaceGlobs(root: string): string[] | null {+ const pnpmPath = join(root, "pnpm-workspace.yaml");+ if (existsSync(pnpmPath)) {+ try {+ const globs = parsePnpmPackagesList(readFileSync(pnpmPath, "utf8"));+ if (globs) return globs;+ } catch {+ /* malformed pnpm-workspace.yaml — fall through to package.json */+ }+ }+ const pkgPath = join(root, "package.json");+ if (existsSync(pkgPath)) {+ try {+ const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));+ const ws = pkg.workspaces;+ if (Array.isArray(ws)) return ws as string[];+ if (ws && Array.isArray(ws.packages)) return ws.packages as string[];+ } catch {+ /* malformed package.json — no workspace intent */+ }+ }+ return null;+}++/** Resolve one workspace glob to matched dir paths (posix, rel to `root`). Supports+ * only `dir/*` (immediate subdirs) and `dir/**` (any nested depth) forms, implemented+ * with plain `readdir` — no glob dependency, per spec. Unsupported forms resolve empty. */+function resolveGlob(root: string, pattern: string): string[] {+ const norm = pattern.replace(/\\/g, "/").replace(/\/$/, "");+ let base: string;+ let recursive: boolean;+ if (norm === "**") {+ base = "";+ recursive = true;+ } else if (norm === "*") {+ base = "";+ recursive = false;+ } else if (norm.endsWith("/**")) {+ base = norm.slice(0, -3);+ recursive = true;+ } else if (norm.endsWith("/*")) {+ base = norm.slice(0, -2);+ recursive = false;+ } else {+ return []; // unsupported glob form+ }++ const results: string[] = [];+ const listDirs = (absDir: string, relDir: string): string[] => {+ let entries: Dirent[];+ try {+ entries = readdirSync(absDir, { withFileTypes: true });+ } catch {+ return [];+ }+ return entries+ .filter((e) => e.isDirectory() && !e.name.startsWith(".") && !SKIP_DIRS.has(e.name))+ .map((e) => (relDir === "" ? e.name : `${relDir}/${e.name}`));+ };++ const baseAbs = base === "" ? root : join(root, base);+ if (!recursive) {+ results.push(...listDirs(baseAbs, base));+ } else {+ const stack = [base];+ while (stack.length) {+ const relDir = stack.pop()!;+ const absDir = relDir === "" ? root : join(root, relDir);+ const children = listDirs(absDir, relDir);+ results.push(...children);+ stack.push(...children);+ }+ }+ return results;+}++interface Candidate {+ markers: string[];+ isWorkspace: boolean;+}++/** Walk the tree (reusing `walkDir`'s skip rules) and find project-marker dirs. */+export function discoverScopes(root: string): ScopeV1[] {+ const absRoot = resolve(root);+ const dirs = collectDirs(absRoot);++ const markerMap = new Map<string, string[]>();+ for (const dir of dirs) {+ const found = MARKERS.filter((m) => existsSync(join(absRoot, dir, m)));+ if (found.length) markerMap.set(dir, found);+ }++ const workspaceGlobs = readWorkspaceGlobs(absRoot);+ const workspaceMatches = new Set<string>();+ if (workspaceGlobs) {+ for (const glob of workspaceGlobs) {+ for (const dir of resolveGlob(absRoot, glob)) workspaceMatches.add(dir);+ }+ }++ const candidates = new Map<string, Candidate>();++ for (const [dir, rawMarkers] of markerMap) {+ const isWorkspace = workspaceMatches.has(dir);+ let markers = rawMarkers;++ // Rule 2: with workspace config present, a non-matched dir's package.json+ // marker doesn't count — the globs are the only JS-family intent honored.+ if (workspaceGlobs && !isWorkspace && markers.includes("package.json")) {+ markers = markers.filter((m) => m !== "package.json");+ if (markers.length === 0) continue;+ }++ // Rule 3: depth guard (workspace matches are exempt).+ const depth = dir === "" ? 0 : dir.split("/").length;+ if (depth > 2 && !isWorkspace) continue;++ candidates.set(dir, { markers, isWorkspace });+ }++ // Workspace-matched dirs are candidates even without a marker file of their own.+ for (const dir of workspaceMatches) {+ const existing = candidates.get(dir);+ if (existing) existing.isWorkspace = true;+ else candidates.set(dir, { markers: markerMap.get(dir) ?? [], isWorkspace: true });+ }++ // Rule 4: nesting collapse.+ for (const prefix of [...candidates.keys()]) {+ if (prefix === "" || !candidates.has(prefix)) continue;+ const entry = candidates.get(prefix)!;+ const segs = prefix.split("/");+ for (let i = segs.length - 1; i >= 1; i--) {+ const ancestor = segs.slice(0, i).join("/");+ const ancestorEntry = candidates.get(ancestor);+ if (!ancestorEntry) continue;+ if (entry.isWorkspace && !ancestorEntry.isWorkspace) {+ candidates.delete(ancestor); // workspace-glob match wins over its parent+ } else {+ candidates.delete(prefix); // keep the shallower ancestor+ }+ break; // only the nearest ancestor candidate matters+ }+ }++ const survivors = [...candidates.entries()].map(([prefix, c]) => ({ prefix, ...c }));++ // Rule 6: canonical single-scope form when only root survives, or nothing does.+ if (survivors.length === 0) return CANONICAL_ROOT;+ if (survivors.length === 1 && survivors[0].prefix === "") {+ return [{ prefix: "", label: "", markers: survivors[0].markers }];+ }++ const scopes: ScopeV1[] = survivors.map((s) => ({+ prefix: s.prefix,+ label: s.prefix,+ markers: s.markers,+ }));+ // Rule 7: deterministic ordering — prefix-length desc, then lexicographic.+ scopes.sort((a, b) => b.prefix.length - a.prefix.length || a.prefix.localeCompare(b.prefix));+ return scopes;+}++/** Nearest-prefix owner. `scopes` MUST be sorted prefix-length desc; "" matches all.+ * Falls back to a synthetic root scope when `scopes` has no explicit "" entry and+ * nothing else matches — every path resolves to some scope. */+export function scopeOf(path: string, scopes: ScopeV1[]): ScopeV1 {+ for (const s of scopes) {+ if (s.prefix === "") continue; // root is the fallback, checked last+ if (path === s.prefix || path.startsWith(`${s.prefix}/`)) return s;+ }+ return scopes.find((s) => s.prefix === "") ?? { prefix: "", label: "", markers: [] };+}++/** Immediate subdirs of `root` that are themselves git repos (have `.git`).+ * Used by workspace federation (Task 5). */+export function discoverWorkspaceChildren(root: string): string[] {+ const absRoot = resolve(root);+ let entries: Dirent[];+ try {+ entries = readdirSync(absRoot, { withFileTypes: true });+ } catch {+ return [];+ }+ return entries+ .filter(+ (e) =>+ e.isDirectory() &&+ !e.name.startsWith(".") &&+ !SKIP_DIRS.has(e.name) &&+ existsSync(join(absRoot, e.name, ".git")),+ )+ .map((e) => e.name);+}++/** Every consumer's entry point to a graph's scopes: absent `meta.scopes` (old+ * graphs, or graphs where discovery found nothing but the root) defaults to the+ * canonical single-scope form. Never read `graph.meta.scopes` directly. */+export function scopesOfGraph(graph: GraphV1): ScopeV1[] {+ return graph.meta.scopes ?? CANONICAL_ROOT;+}diff --git a/src/graph/types.ts b/src/graph/types.tsindex 5584a691..e42e84e5 100644--- a/src/graph/types.ts+++ b/src/graph/types.ts@@ -77,12 +77,25 @@ export interface EdgeV1 {confidence: Confidence;}+/** A ranking scope: a sub-project discovered by project-marker files (`package.json`,+ * `go.mod`, ...). `prefix` is a posix path relative to the graph root ("" = root scope);+ * `label` is the same value without a trailing slash (also "" for root); `markers` lists+ * which marker file(s) were found in that directory. See `src/graph/scopes.ts`. */+export interface ScopeV1 {+ prefix: string;+ label: string;+ markers: string[];+}+export interface GraphV1 {meta: {version: 1;nodeCount: number;edgeCount: number;languages: string[];+ /** Ranking scopes: posix path prefixes relative to the graph root, "" = root scope.+ * Absent (old graphs) ≡ [{ prefix: "", label: "" }]. Sorted by prefix length desc. */+ scopes?: ScopeV1[];};nodes: NodeV1[];edges: EdgeV1[];diff --git a/test/graph-scopes.test.ts b/test/graph-scopes.test.tsnew file mode 100644index 00000000..0558d048--- /dev/null+++ b/test/graph-scopes.test.ts@@ -0,0 +1,125 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import {+ discoverScopes,+ scopeOf,+ discoverWorkspaceChildren,+ scopesOfGraph,+} from "../src/graph/scopes.js";+import { buildGraph } from "../src/graph/build.js";+import { readGraph, wiringPath, writeGraph } from "../src/graph/write.js";+import { loadGraphCached } from "../src/graph/load.js";+import type { GraphV1 } from "../src/graph/types.js";++function fx(layout: Record<string, string>): string {+ const dir = mkdtempSync(join(tmpdir(), "scopes-"));+ for (const [p, content] of Object.entries(layout)) {+ mkdirSync(join(dir, p, ".."), { recursive: true });+ writeFileSync(join(dir, p), content);+ }+ return dir;+}++test("frontend/backend markers under one git root -> two scopes", () => {+ const d = fx({+ "frontend/package.json": "{}", "frontend/src/app.ts": "export const a = 1;",+ "backend/go.mod": "module m", "backend/main.go": "package main",+ });+ const scopes = discoverScopes(d);+ assert.deepEqual(scopes.map((s) => s.prefix).sort(), ["backend", "frontend"]);+ assert.equal(scopeOf("frontend/src/app.ts", scopes).label, "frontend");+ assert.equal(scopeOf("README.md", scopes).prefix, ""); // root scope always exists as fallback+ rmSync(d, { recursive: true, force: true });+});++test("workspace globs are intent: packages/* honored, deeper ignored", () => {+ const d = fx({+ "package.json": JSON.stringify({ workspaces: ["packages/*"] }),+ "packages/core/package.json": "{}", "packages/core/i.ts": "1",+ "packages/cli/package.json": "{}", "packages/cli/i.ts": "1",+ "packages/cli/nested/package.json": "{}", // deeper than glob -> ignored+ });+ const prefixes = discoverScopes(d).map((s) => s.prefix).sort();+ assert.deepEqual(prefixes, ["packages/cli", "packages/core"]);+ rmSync(d, { recursive: true, force: true });+});++test("root-only marker -> canonical single scope", () => {+ const d = fx({ "package.json": "{}", "src/a.ts": "1" });+ const scopes = discoverScopes(d);+ assert.deepEqual(scopes, [{ prefix: "", label: "", markers: ["package.json"] }]);+ rmSync(d, { recursive: true, force: true });+});++test("depth guard: markers 3 levels down ignored without workspace glob", () => {+ const d = fx({ "a/b/c/package.json": "{}", "src/x.ts": "1" });+ assert.equal(discoverScopes(d).length, 1); // root only+ rmSync(d, { recursive: true, force: true });+});++test("nesting collapse keeps shallower candidate", () => {+ const d = fx({ "svc/package.json": "{}", "svc/sub/package.json": "{}", "svc/i.ts": "1" });+ assert.deepEqual(discoverScopes(d).filter((s) => s.prefix).map((s) => s.prefix), ["svc"]);+ rmSync(d, { recursive: true, force: true });+});++test("discoverWorkspaceChildren finds immediate git children only", () => {+ const d = fx({ "repoA/x.ts": "1", "repoB/y.py": "1", "plain/z.go": "1" });+ mkdirSync(join(d, "repoA/.git"), { recursive: true });+ mkdirSync(join(d, "repoB/.git"), { recursive: true });+ mkdirSync(join(d, "repoB/vendored/.git"), { recursive: true }); // nested: not a child of d+ assert.deepEqual(discoverWorkspaceChildren(d).sort(), ["repoA", "repoB"]);+ rmSync(d, { recursive: true, force: true });+});++function tsFns(n: number): string {+ return Array.from({ length: n }, (_, i) => `export function fn${i}() { return ${i}; }`).join("\n");+}++function pyFns(n: number): string {+ return Array.from({ length: n }, (_, i) => `def fn${i}():\n return ${i}\n`).join("\n");+}++test("buildGraph wires meta.scopes: substantial scopes survive, tiny scopes merge into root", async () => {+ const d = fx({+ "frontend/package.json": "{}",+ "frontend/app.ts": tsFns(6),+ "backend/pyproject.toml": '[project]\nname = "backend"\n',+ "backend/app.py": pyFns(6),+ "tiny/package.json": "{}",+ "tiny/app.ts": tsFns(1),+ });+ try {+ await buildGraph(d);+ const graph = readGraph(wiringPath(join(d, "graft")));+ assert.ok(graph, "wiring graph should be written");+ const prefixes = (graph!.meta.scopes ?? []).map((s) => s.prefix).sort();+ assert.deepEqual(prefixes, ["backend", "frontend"]); // tiny (1 symbol) merged into root+ } finally {+ rmSync(d, { recursive: true, force: true });+ }+});++test("old graphs without meta.scopes fall back to the canonical root scope via scopesOfGraph", () => {+ const d = mkdtempSync(join(tmpdir(), "scopes-oldgraph-"));+ try {+ const graph: GraphV1 = {+ meta: { version: 1, nodeCount: 0, edgeCount: 0, languages: [] }, // no `scopes` field+ nodes: [],+ edges: [],+ };+ const outDir = join(d, "graft");+ writeGraph(graph, outDir);++ const loaded = loadGraphCached(outDir);+ assert.ok(loaded);+ const scopes = scopesOfGraph(loaded!);+ assert.deepEqual(scopes, [{ prefix: "", label: "", markers: [] }]);+ assert.equal(scopeOf("anything/at/all.ts", scopes).prefix, "");+ } finally {+ rmSync(d, { recursive: true, force: true });+ }+});