Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
// Copyright 2026 Ix Infrastructure Inc.
import { createHash } from "node:crypto";
import * as fs from "node:fs";
import * as path from "node:path";
import { ingestSymbolsJournalPath, ingestSymbolsPath } from "./config.js";
/**
* The symbol table an incremental `ix map` resolves against.
*
* Resolving a changed file needs to know what every OTHER file defines,
* exports and imports. Without a record of that, every run re-read and
* re-parsed the whole repository's JS/TS, PHP, R and SAS just to build the
* index -- and knew nothing at all about Python, Java and the rest, so an edit
* to one of those resolved its calls against itself alone and lost its
* cross-file edges. This keeps one `FileSummary` (core-ingestion's
* `summarizeParseResult`) per file, keyed by workspace-relative path, beside
* the ingest baseline.
*
* An entry is usable when its file is unchanged since the last run (mtime-clean
* in the baseline) or its content hash matches, and only for the extractor that
* wrote it.
*/
/**
* Bump when the stored shape or meaning changes; an older table is ignored,
* not migrated. 2: the table may be extended by a journal (below), so a CLI
* that reads the table alone would see a stale "before" for the run's
* dependents step.
*/
const TABLE_VERSION = 2;
/** What core-ingestion's `FileSummary` looks like from here: opaque except for the path. */
export interface StoredSummary {
filePath: string;
[key: string]: unknown;
}
export interface SymbolEntry {
/** sha256 of the file the summary describes. */
hash: string;
/**
* The file's mtime when it was read for this summary, stat'd before the read.
* An entry is trusted without reading its file only while the file still has
* this mtime: the baseline's mtimes say nothing about this table, which is
* saved separately and can be older, and a scoped run never stats the files
* outside its scope.
*/
mtime?: number;
summary: StoredSummary;
/**
* What the file's patch resolved its edges to, recorded when the patch is
* built from these bytes: a hash of its edge ids, their targets and their
* tier/confidence (`resolutionHash`). A re-resolution's re-send records it only once it lands. An unchanged file is
* re-sent when names it uses change elsewhere, and only if this differs.
*/
res?: string;
/**
* Picked for re-resolution (a name it uses changed) and not yet sent. Set
* before the re-send and cleared when it lands or turns out unneeded, so a
* failed re-send is retried by the next run rather than forgotten: by then
* nothing it depends on looks changed any more.
*/
pending?: true;
}
/**
* A hash of a patch's edges, where they point and how they were resolved.
* Same bytes resolved against the same names gives the same value, so it
* changes exactly when something the file refers to moved, appeared or went
* away -- or when an edge keeps its id and target but its `tier`/`confidence`
* changes (a receiver's type learned or lost through another file).
*/
export function resolutionHash(ops: ReadonlyArray<{ type?: unknown; id?: unknown; dst?: unknown; attrs?: unknown }>): string {
const pairs = ops
.filter(op => op.type === "UpsertEdge")
.map(op => {
const { tier, confidence } = (op.attrs ?? {}) as { tier?: unknown; confidence?: unknown };
const how = tier === undefined && confidence === undefined ? "" : ` ${String(tier)}/${String(confidence)}`;
return `${String(op.id)}>${String(op.dst)}${how}`;
})
.sort();
return createHash("sha256").update(pairs.join("\n")).digest("hex");
}
interface SerializedSymbolTable {
version: number;
root: string;
extractor: string;
/**
* Which write of the table this is. The journal names the generation it
* extends, so it is applied only on top of the table it was written against.
* Optional: a table written before the journal existed has none, and then
* no journal applies to it.
*/
gen?: string;
files: Record<string, SymbolEntry>;
}
/**
* The table is written in full only now and then. In between, a run that
* changed a few entries appends them to a journal beside it, one JSON record
* per line, and the next load applies the journal over the table. A table
* read without its journal is not merely older: the dependents step diffs the
* names a file exported before this run against after it, and a "before"
* several journaled runs behind the backend misses renames that already
* landed. So the table is version 2, which a CLI that does not know about the
* journal ignores (it re-parses) rather than reading alone.
*
* Line 1 is a header naming the table it extends; each line after it is
* `{"k": path, "e": entry}` (set) or `{"k": path, "d": 1}` (removed).
*/
interface JournalHeader {
journal: number;
root: string;
extractor: string;
gen: string;
}
type JournalRecord = { k: string; e: SymbolEntry } | { k: string; d: 1 };
const JOURNAL_VERSION = 1;
/** The journal is rewritten into the table once it would pass this share of the table's size. */
const JOURNAL_MAX_SHARE = 0.5;
/** What a load saw on disk, so a save can tell whether appending to the journal is safe. */
interface LoadedState {
gen?: string;
tableBytes: number;
journalBytes: number;
/** Every journal line parsed and the file ended in a newline: appending to it is safe. */
journalIntact: boolean;
}
const loadedStates = new WeakMap<Map<string, SymbolEntry>, LoadedState>();
function validEntry(entry: unknown): entry is SymbolEntry {
const e = entry as SymbolEntry | undefined;
return typeof e?.hash === "string" && !!e.summary && typeof e.summary.filePath === "string";
}
/** Apply the journal for `gen` to `table`. Returns its size and whether it can be appended to. */
function applyJournal(
projectRoot: string, extractor: string, gen: string, table: Map<string, SymbolEntry>,
): { bytes: number; intact: boolean } {
let text: string;
try {
text = fs.readFileSync(ingestSymbolsJournalPath(projectRoot), "utf-8");
} catch {
return { bytes: 0, intact: true };
}
const bytes = Buffer.byteLength(text);
const lines = text.split("\n");
// A journal that does not end in a newline was cut off mid-append.
let intact = text.length === 0 || text.endsWith("\n");
if (lines[lines.length - 1] === "") lines.pop();
let header: JournalHeader | undefined;
try {
header = lines.length > 0 ? JSON.parse(lines[0]) as JournalHeader : undefined;
} catch {
header = undefined;
}
if (
header?.journal !== JOURNAL_VERSION || header.root !== projectRoot ||
header.extractor !== extractor || header.gen !== gen
) {
// Another table's journal, or none worth reading: it is replaced, never appended to.
return { bytes, intact: false };
}
for (let i = 1; i < lines.length; i++) {
let record: JournalRecord;
try {
record = JSON.parse(lines[i]) as JournalRecord;
} catch {
intact = false;
continue;
}
if (typeof record?.k !== "string") { intact = false; continue; }
if ("d" in record) table.delete(record.k);
else if (validEntry(record.e)) table.set(record.k, record.e);
}
return { bytes, intact };
}
/** The table for `projectRoot` and `extractor`, or an empty one. */
export function loadIngestSymbols(projectRoot: string, extractor: string): Map<string, SymbolEntry> {
const table = new Map<string, SymbolEntry>();
let state: LoadedState = { tableBytes: 0, journalBytes: 0, journalIntact: false };
try {
const text = fs.readFileSync(ingestSymbolsPath(projectRoot), "utf-8");
const data = JSON.parse(text) as SerializedSymbolTable;
if (
data.version === TABLE_VERSION && data.root === projectRoot && data.extractor === extractor &&
data.files && typeof data.files === "object"
) {
for (const [rel, entry] of Object.entries(data.files)) {
if (validEntry(entry)) table.set(rel, entry);
}
state = { tableBytes: Buffer.byteLength(text), journalBytes: 0, journalIntact: false };
if (typeof data.gen === "string") {
const journal = applyJournal(projectRoot, extractor, data.gen, table);
state = { gen: data.gen, tableBytes: state.tableBytes, journalBytes: journal.bytes, journalIntact: journal.intact };
}
}
} catch {
table.clear();
state = { tableBytes: 0, journalBytes: 0, journalIntact: false };
}
loadedStates.set(table, state);
return table;
}
/** Write `contents` to `target` via a temp file and a rename. */
function writeAtomically(target: string, contents: string): void {
const tmp = `${target}.${process.pid}.tmp`;
fs.writeFileSync(tmp, contents);
fs.renameSync(tmp, target);
}
/**
* Save the table.
*
* With `changed` -- the paths whose entries this run set, altered or removed --
* nothing is written when it is empty, and a few changes are appended to the
* journal when the table on disk is the one this table was loaded from. The
* table itself is rewritten (via a temp file and a rename, so a process killed
* mid-write leaves the previous one) when there is no such table, the journal
* is not intact, or it would grow past half the table's size. Without
* `changed`, the table is rewritten.
*/
export function saveIngestSymbols(
projectRoot: string,
extractor: string,
table: Map<string, SymbolEntry>,
changed?: ReadonlySet<string>,
): void {
if (changed !== undefined && changed.size === 0) return;
try {
const target = ingestSymbolsPath(projectRoot);
const journalPath = ingestSymbolsJournalPath(projectRoot);
fs.mkdirSync(path.dirname(target), { recursive: true });
const state = loadedStates.get(table);
if (changed !== undefined && state?.gen !== undefined && state.journalIntact && onDiskGen(target) === state.gen) {
const records = [...changed].sort().map(k => {
const e = table.get(k);
return JSON.stringify(e !== undefined ? { k, e } : { k, d: 1 }) + "\n";
}).join("");
const addBytes = Buffer.byteLength(records);
if (state.journalBytes + addBytes <= state.tableBytes * JOURNAL_MAX_SHARE) {
if (state.journalBytes === 0) {
const header: JournalHeader = { journal: JOURNAL_VERSION, root: projectRoot, extractor, gen: state.gen };
const headerLine = JSON.stringify(header) + "\n";
writeAtomically(journalPath, headerLine + records);
state.journalBytes = Buffer.byteLength(headerLine) + addBytes;
} else {
fs.appendFileSync(journalPath, records);
state.journalBytes += addBytes;
}
return;
}
}
const gen = createHash("sha256").update(`${process.pid}:${Date.now()}:${Math.random()}`).digest("hex").slice(0, 16);
const data: SerializedSymbolTable = {
version: TABLE_VERSION,
root: projectRoot,
extractor,
gen,
files: Object.fromEntries(table),
};
const text = JSON.stringify(data);
writeAtomically(target, text);
// After the table: a journal left behind names the old generation and is ignored.
try { fs.rmSync(journalPath, { force: true }); } catch { /* ignored by gen */ }
loadedStates.set(table, { gen, tableBytes: Buffer.byteLength(text), journalBytes: 0, journalIntact: true });
} catch {
// An optimization: losing it costs the next run a parse of what it lacks.
}
}
/**
* The generation of the table on disk, read from the start of the file
* without parsing the rest: `gen` is written before `files`.
*/
function onDiskGen(target: string): string | undefined {
let fd: number | undefined;
try {
fd = fs.openSync(target, "r");
const buf = Buffer.alloc(1024);
const n = fs.readSync(fd, buf, 0, buf.length, 0);
const head = buf.subarray(0, n).toString("utf-8");
const filesAt = head.indexOf('"files":');
const m = /"gen":"([0-9a-f]+)"/.exec(filesAt === -1 ? head : head.slice(0, filesAt));
return m?.[1];
} catch {
return undefined;
} finally {
if (fd !== undefined) try { fs.closeSync(fd); } catch { /* nothing to do */ }
}
}
// ── Dependents (IN-11) ──────────────────────────────────────────────────
// A file that did not change can still need re-sending: a call it makes may
// now resolve somewhere else, or nowhere, because a name it uses was added,
// renamed or removed in another file. The table knows what every file defines
// and refers to, so the run can find those files without reading the rest.
type SummaryFields = {
language?: string;
qkeys?: Array<[string, string]>;
exportPublicNames?: Array<[string, string]>;
phpTypes?: Array<[string, string]>;
receiverFactories?: Array<[string, string]>;
receiverHeritage?: Array<[string, string]>;
receiverPackage?: string;
importAliases?: Record<string, string>;
refs?: string[];
imports?: Array<{ dstName?: string; importRaw?: string }>;
sig?: string;
};
/** The names other files can resolve to in a summary. */
export function definedNames(summary: StoredSummary | undefined): Set<string> {
const names = new Set<string>();
if (!summary) return names;
const s = summary as SummaryFields;
for (const [name, qkey] of s.qkeys ?? []) { names.add(name); names.add(qkey); }
for (const [publicName] of s.exportPublicNames ?? []) names.add(publicName);
for (const [fqcn, typeName] of s.phpTypes ?? []) { names.add(fqcn); names.add(typeName); }
return names;
}
/**
* What other files can bind to in a summary, as `key -> name`: one key per
* entity name and qualified key, per `[public, local]` export pair and per PHP
* type. The key carries what the name resolves to, so `export { a as foo }`
* becoming `export { b as foo }` changes a key although no name came or went.
*/
function bindings(summary: StoredSummary | undefined): Map<string, string[]> {
const out = new Map<string, string[]>();
if (!summary) return out;
const s = summary as SummaryFields;
for (const [name, qkey] of s.qkeys ?? []) out.set(`q\0${s.receiverPackage ?? ''}\0${name}\0${qkey}`, [name, qkey]);
for (const [publicName, local] of s.exportPublicNames ?? []) out.set(`e\0${publicName}\0${local}`, [publicName]);
for (const [fqcn, typeName] of s.phpTypes ?? []) out.set(`p\0${fqcn}\0${typeName}`, [fqcn, typeName]);
// A superclass and a factory's return type resolve through this file's
// imports (and Go import aliases), so they are part of the key.
const heritageImports = [...new Set((s.imports ?? []).map(i => i.dstName ?? ""))].sort().join("\0");
const aliases = Object.entries(s.importAliases ?? {}).map(([a, m]) => `${a}=${m}`).sort().join("\0");
// A Java method result (`URL.getUrl`) types a chained call, which names the
// method only (`x.getUrl().m()`): the method name counts too.
for (const [name, type] of s.receiverFactories ?? []) out.set(`f\0${name}\0${type}\0${heritageImports}\0\0${aliases}`, javaMember(s, name));
for (const [name, base] of s.receiverHeritage ?? []) out.set(`h\0${name}\0${base}\0${heritageImports}`, [name]);
return out;
}
/** `[Class.method, method]` for a Java method result, else `[name]`. */
function javaMember(s: SummaryFields, name: string): string[] {
const dot = name.lastIndexOf(".");
return s.language === "java" && dot > 0 ? [name, name.slice(dot + 1)] : [name];
}
/**
* Names whose binding differs before and after: defined in one but not the
* other, or now pointing at something else. None when the signature is the same.
*/
export function changedNames(before: StoredSummary | undefined, after: StoredSummary | undefined): Set<string> {
const b = before as SummaryFields | undefined;
const a = after as SummaryFields | undefined;
if (b?.sig !== undefined && b.sig === a?.sig) return new Set();
const was = bindings(before);
const now = bindings(after);
const out = new Set<string>();
for (const [key, names] of was) if (!now.has(key)) for (const n of names) out.add(n);
for (const [key, names] of now) if (!was.has(key)) for (const n of names) out.add(n);
return out;
}
/**
* Whether a file's imports differ before and after. A file that re-exports
* (`export { foo } from './x'`, a Python `__init__`) passes names through its
* imports, which its signature does not cover: when they change, files that
* import it may resolve differently, so it counts as a path that changed.
*/
export function importsChanged(before: StoredSummary | undefined, after: StoredSummary | undefined): boolean {
if (!before || !after) return false;
const key = (s: StoredSummary) =>
JSON.stringify(((s as SummaryFields).imports ?? []).map(i => `${i.dstName ?? ""}\0${i.importRaw ?? ""}`).sort());
return key(before) !== key(after);
}
/** The stem an import of `relPath` would name: `math` for `web/math.ts`, `web` for `web/index.ts`. */
function importStem(relPath: string): string {
const posix = relPath.replace(/\\/g, "/");
const stem = path.posix.basename(posix).replace(/\.[^.]+$/, "");
if (stem === "index" || stem === "__init__" || stem === "mod") return path.posix.basename(path.posix.dirname(posix));
return stem;
}
const REF_SEPARATOR = /::|->|[.#\\]/;
/** The segments of an import specifier: `./math.js` -> `math`, `js`; `pkg.util` -> `pkg`, `util`. */
function importSegments(spec: string): string[] {
return spec.split(/[/\\.:]+/).filter(Boolean);
}
/**
* Files in `table` that may resolve differently now: a name they refer to is in
* `names` (also by its first or last segment, for `obj.method` and `Mod::f`), or
* an import of theirs names the stem of a path in `paths` (a file that appeared,
* went away, or changed what it imports). Files in `exclude` -- the ones the
* run already sent -- are left out. Over-matching costs a parse; the resolution hash then decides whether
* anything is sent.
*/
export function findDependents(
table: ReadonlyMap<string, SymbolEntry>,
names: ReadonlySet<string>,
paths: Iterable<string>,
exclude: ReadonlySet<string>,
): string[] {
const stems = new Set<string>();
for (const p of paths) {
const stem = importStem(p);
if (stem.length > 0 && stem !== ".") stems.add(stem);
}
if (names.size === 0 && stems.size === 0) return [];
// A changed superclass changes the members visible through every subclass,
// even when those subclass files were not edited (or were already sent).
// Likewise a changed type changes what every factory returning it gives a
// caller's receiver (a Scala `apply`, a Go `NewX`, a Java method result), so
// the factory counts too.
const inheritors = new Map<string, Set<string>>();
const dependsOn = (type: string, name: string) => {
for (const key of new Set([type, type.split(".").pop()!])) {
const children = inheritors.get(key) ?? new Set<string>();
children.add(name);
inheritors.set(key, children);
}
};
for (const entry of table.values()) {
const s = entry.summary as SummaryFields;
for (const [child, parent] of s.receiverHeritage ?? []) {
dependsOn(parent, child);
if (s.receiverPackage) dependsOn(parent, `${s.receiverPackage}.${child}`);
}
for (const [factory, type] of s.receiverFactories ?? []) if (type) for (const name of javaMember(s, factory)) dependsOn(type, name);
}
const affectedNames = new Set(names);
const pending = [...affectedNames];
for (let i = 0; i < pending.length; i++) {
for (const child of inheritors.get(pending[i]) ?? []) {
if (affectedNames.has(child)) continue;
affectedNames.add(child);
pending.push(child);
}
}
const out: string[] = [];
for (const [rel, entry] of table) {
if (exclude.has(rel)) continue;
const s = entry.summary as SummaryFields;
const byName = affectedNames.size > 0 && (s.refs ?? []).some(ref => {
if (affectedNames.has(ref)) return true;
// Most refs are plain names; splitting each costs more than the rest of the scan.
if (!REF_SEPARATOR.test(ref)) return false;
const parts = ref.split(/::|->|\.|#|\\/).filter(Boolean);
return parts.length > 1 && (affectedNames.has(parts[0]) || affectedNames.has(parts[parts.length - 1]));
});
// By path segment, not substring: an added `a.ts` must not match every
// import with an "a" in it.
const byImport = !byName && stems.size > 0 && (s.imports ?? []).some(imp => {
const spec = imp.importRaw ?? imp.dstName ?? "";
for (const stem of stems) {
if (spec.includes(stem) && importSegments(spec).includes(stem)) return true;
}
return false;
});
if (byName || byImport) out.push(rel);
}
return out.sort();
}