Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf(ingest): the symbol table writes only what changed

KageBinaryb35375ad40f9
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index 37248988..b89a814e 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -8,11 +8,12 @@ import { execFileSync } from "node:child_process";
import { ingestFiles, ingestPathSingleFlight } from "../commands/ingest.js";
import { acquireMapLock, takeMapRerun } from "../single-flight.js";
-import { ingestMtimeCachePath, ingestRebuildPath, ingestSymbolsPath, loadConfig } from "../config.js";
+import { ingestMtimeCachePath, ingestRebuildPath, ingestSymbolsJournalPath, ingestSymbolsPath, loadConfig } from "../config.js";
import { workspaceIdForPath } from "../system.js";
import { FakeBackend } from "./helpers/fake-backend.js";
import { runWithSignal } from "../../client/run-signal.js";
import { loadIngestBaseline } from "../ingest-baseline.js";
+import { loadIngestSymbols } from "../ingest-symbols.js";
import { detectStaleFiles } from "../stale.js";
import { renderStatusLlm } from "../commands/status.js";
@@ -1559,6 +1560,44 @@ describe("ingestFiles against a fake backend", () => {
expect(tableFiles()).toEqual(["src/a.ts", "src/b.py"]);
});
+ it("writes the symbol table only when an entry changed, and then only the change", async () => {
+ // It was rewritten in full on every run that parsed anything.
+ fixture(20);
+ const table = () => readFileSync(ingestSymbolsPath(repo), "utf8");
+ const journal = ingestSymbolsJournalPath(repo);
+
+ await ingestFiles(repo, quiet);
+ const written = table();
+ const writtenAt = statSync(ingestSymbolsPath(repo)).mtimeMs;
+ expect(Object.keys((JSON.parse(written) as { files: object }).files)).toHaveLength(20);
+ expect(existsSync(journal)).toBe(false);
+
+ // Nothing changed: an incremental run and a forced re-parse of every file
+ // leave both files as they were.
+ await ingestFiles(repo, quiet);
+ await ingestFiles(repo, { ...quiet, force: true });
+ expect(table()).toBe(written);
+ expect(statSync(ingestSymbolsPath(repo)).mtimeMs).toBe(writtenAt);
+ expect(existsSync(journal)).toBe(false);
+
+ // One file changed: the table stays, and the journal carries that entry.
+ writeFileSync(join(repo, "src", "m007.ts"), "export function f7(): number { return 70; }\nexport const g7 = 1;\n", "utf8");
+ await ingestFiles(repo, quiet);
+ expect(table()).toBe(written);
+ const lines = readFileSync(journal, "utf8").trim().split("\n").map(l => JSON.parse(l) as { k?: string });
+ expect(lines.slice(1).map(l => l.k)).toEqual(["src/m007.ts"]);
+
+ // And what the next run loads is the change, with every other entry.
+ const loaded = loadIngestSymbols(repo, (JSON.parse(written) as { extractor: string }).extractor);
+ expect(loaded.size).toBe(20);
+ expect(JSON.stringify(loaded.get("src/m007.ts")?.summary)).toContain("g7");
+
+ // Unchanged again: nothing more is appended.
+ const journalText = readFileSync(journal, "utf8");
+ await ingestFiles(repo, quiet);
+ expect(readFileSync(journal, "utf8")).toBe(journalText);
+ });
+
it("writes no baseline from a subdirectory run when the workspace has none", async () => {
// One written from `src/` alone would report the whole graph complete.
fixture(2);
diff --git a/ix-cli/src/cli/__tests__/ingest-symbols.test.ts b/ix-cli/src/cli/__tests__/ingest-symbols.test.ts
index 28ce15eb..d2905357 100644
--- a/ix-cli/src/cli/__tests__/ingest-symbols.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-symbols.test.ts
@@ -1,11 +1,11 @@
// Copyright 2026 Ix Infrastructure Inc.
import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { appendFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { ingestSymbolsPath } from "../config.js";
+import { clearIngestMtimeCache, ingestSymbolsJournalPath, ingestSymbolsPath } from "../config.js";
import {
changedNames, definedNames, findDependents, importsChanged, loadIngestSymbols, resolutionHash, saveIngestSymbols, type SymbolEntry,
} from "../ingest-symbols.js";
@@ -52,6 +52,127 @@ describe("ingest symbol table", () => {
saveIngestSymbols(root, "x", new Map([["a.py", { ...entry("a.py"), res: "r1" }]]));
expect(loadIngestSymbols(root, "x").get("a.py")?.res).toBe("r1");
});
+
+ describe("writing only what changed", () => {
+ const tablePath = () => ingestSymbolsPath(root);
+ const journalPath = () => ingestSymbolsJournalPath(root);
+ /** A table of `n` entries on disk, written in full. */
+ const seed = (n: number) => {
+ const t = new Map<string, SymbolEntry>();
+ for (let i = 0; i < n; i++) t.set(`f${i}.py`, entry(`f${i}.py`));
+ saveIngestSymbols(root, "x", t);
+ };
+
+ it("writes nothing when no entry changed", () => {
+ seed(10);
+ const before = readFileSync(tablePath(), "utf8");
+ saveIngestSymbols(root, "x", loadIngestSymbols(root, "x"), new Set());
+ expect(readFileSync(tablePath(), "utf8")).toBe(before);
+ expect(existsSync(journalPath())).toBe(false);
+ // And nothing at all when there was no table.
+ rmSync(tablePath());
+ saveIngestSymbols(root, "x", new Map([["a.py", entry("a.py")]]), new Set());
+ expect(existsSync(tablePath())).toBe(false);
+ });
+
+ it("appends a few changes to the journal, leaves the table, and loads them back", () => {
+ seed(10);
+ const before = readFileSync(tablePath(), "utf8");
+ const t = loadIngestSymbols(root, "x");
+ t.set("f3.py", { ...entry("f3.py"), hash: "h2", res: "r" });
+ t.delete("f4.py");
+ saveIngestSymbols(root, "x", t, new Set(["f3.py", "f4.py"]));
+ expect(readFileSync(tablePath(), "utf8")).toBe(before);
+ expect(readFileSync(journalPath(), "utf8").trim().split("\n")).toHaveLength(3);
+
+ const again = loadIngestSymbols(root, "x");
+ expect(again.size).toBe(9);
+ expect(again.get("f3.py")).toEqual({ ...entry("f3.py"), hash: "h2", res: "r" });
+ expect(again.has("f4.py")).toBe(false);
+
+ // A second run appends to the same journal.
+ again.set("f5.py", { ...entry("f5.py"), mtime: 7 });
+ saveIngestSymbols(root, "x", again, new Set(["f5.py"]));
+ expect(readFileSync(journalPath(), "utf8").trim().split("\n")).toHaveLength(4);
+ expect(loadIngestSymbols(root, "x").get("f5.py")?.mtime).toBe(7);
+ expect(readFileSync(tablePath(), "utf8")).toBe(before);
+ });
+
+ it("keeps the table readable without the journal: version 1, every field it had", () => {
+ seed(3);
+ const data = JSON.parse(readFileSync(tablePath(), "utf8")) as Record<string, unknown>;
+ expect(data).toMatchObject({ version: 1, root, extractor: "x" });
+ expect(Object.keys(data.files as object)).toEqual(["f0.py", "f1.py", "f2.py"]);
+ });
+
+ it("rewrites the table in full once the journal would pass half its size", () => {
+ seed(4);
+ const t = loadIngestSymbols(root, "x");
+ for (const k of t.keys()) t.set(k, { ...entry(k), hash: "h9" });
+ saveIngestSymbols(root, "x", t, new Set(t.keys()));
+ expect(existsSync(journalPath())).toBe(false);
+ const data = JSON.parse(readFileSync(tablePath(), "utf8")) as { files: Record<string, SymbolEntry> };
+ expect(Object.values(data.files).map(e => e.hash)).toEqual(["h9", "h9", "h9", "h9"]);
+ });
+
+ it("applies the intact lines of a torn journal, and rewrites the table on the next save", () => {
+ seed(10);
+ const t = loadIngestSymbols(root, "x");
+ t.set("f1.py", { ...entry("f1.py"), hash: "h2" });
+ saveIngestSymbols(root, "x", t, new Set(["f1.py"]));
+ appendFileSync(journalPath(), '{"k":"f2.py","e":{"hash":"cut'); // killed mid-append
+
+ const loaded = loadIngestSymbols(root, "x");
+ expect(loaded.get("f1.py")?.hash).toBe("h2");
+ expect(loaded.get("f2.py")?.hash).toBe("h1");
+ loaded.set("f6.py", { ...entry("f6.py"), hash: "h3" });
+ saveIngestSymbols(root, "x", loaded, new Set(["f6.py"]));
+ expect(existsSync(journalPath())).toBe(false);
+ const after = loadIngestSymbols(root, "x");
+ expect(after.get("f1.py")?.hash).toBe("h2");
+ expect(after.get("f6.py")?.hash).toBe("h3");
+ });
+
+ it("ignores a journal written against another table", () => {
+ seed(10);
+ const t = loadIngestSymbols(root, "x");
+ t.set("f1.py", { ...entry("f1.py"), hash: "h2" });
+ saveIngestSymbols(root, "x", t, new Set(["f1.py"]));
+ // A CLI that does not know the journal rewrites the table without a generation.
+ writeFileSync(tablePath(), JSON.stringify({ version: 1, root, extractor: "x", files: { "f1.py": entry("f1.py") } }));
+ const loaded = loadIngestSymbols(root, "x");
+ expect(loaded.get("f1.py")?.hash).toBe("h1");
+ // And the save after that replaces it rather than appending to it.
+ loaded.set("f1.py", { ...entry("f1.py"), hash: "h4" });
+ saveIngestSymbols(root, "x", loaded, new Set(["f1.py"]));
+ expect(existsSync(journalPath())).toBe(false);
+ expect(loadIngestSymbols(root, "x").get("f1.py")?.hash).toBe("h4");
+ });
+
+ it("does not append when another process rewrote the table since this one loaded it", () => {
+ seed(10);
+ const mine = loadIngestSymbols(root, "x");
+ const theirs = loadIngestSymbols(root, "x");
+ for (const k of theirs.keys()) theirs.set(k, { ...entry(k), hash: "t" });
+ saveIngestSymbols(root, "x", theirs, new Set(theirs.keys())); // rewritten in full
+ mine.set("f1.py", { ...entry("f1.py"), hash: "m" });
+ saveIngestSymbols(root, "x", mine, new Set(["f1.py"]));
+ // Mine was written in full over theirs, never as a journal against their table.
+ expect(existsSync(journalPath())).toBe(false);
+ expect(loadIngestSymbols(root, "x").get("f1.py")?.hash).toBe("m");
+ });
+
+ it("is cleared with the mtime cache, journal and all", () => {
+ seed(10);
+ const t = loadIngestSymbols(root, "x");
+ t.set("f1.py", { ...entry("f1.py"), hash: "h2" });
+ saveIngestSymbols(root, "x", t, new Set(["f1.py"]));
+ expect(existsSync(journalPath())).toBe(true);
+ clearIngestMtimeCache(root);
+ expect(existsSync(tablePath())).toBe(false);
+ expect(existsSync(journalPath())).toBe(false);
+ });
+ });
});
describe("dependents (IN-11)", () => {
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 976171a5..f2db579f 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -2486,7 +2486,8 @@ export async function ingestFiles(
let loadedSymbolTable: Map<string, SymbolEntry> | undefined;
const symbolTable = (): Map<string, SymbolEntry> =>
(loadedSymbolTable ??= loadIngestSymbols(projectRoot, currentExtractor));
- let symbolTableChanged = false;
+ /** Paths whose entries this run set, altered or removed: what the save writes. */
+ const symbolTableChanges = new Set<string>();
/** sha256 of each file this run has read, by absolute path. */
const currentHashes = new Map<string, string>();
const mtimeChangedSet = new Set(mtimeChangedPaths);
@@ -2500,10 +2501,14 @@ export async function ingestFiles(
// The resolution hash describes the patch built from these bytes; it is
// kept until that patch is built again (noteResolution).
const res = before?.hash === hash ? before.res : undefined;
- table.set(relFilePath, {
+ const next: SymbolEntry = {
hash, summary: summarize(parsed), ...(mtime !== undefined ? { mtime } : {}), ...(res !== undefined ? { res } : {}),
- });
- symbolTableChanged = true;
+ };
+ // The same bytes parse to the same summary: a forced or touched-only
+ // re-parse leaves the entry as it was, and so leaves nothing to write.
+ if (before !== undefined && JSON.stringify(before) === JSON.stringify(next)) return;
+ table.set(relFilePath, next);
+ symbolTableChanges.add(relFilePath);
};
/**
* What each file this run parsed defined before the run, by relative path
@@ -2518,7 +2523,7 @@ export async function ingestFiles(
const res = resolutionHash(patch.ops);
if (entry.res !== res) {
entry.res = res;
- symbolTableChanged = true;
+ symbolTableChanges.add(relFilePath);
}
};
/** The file's mtime now: from this run's stat loop, or a stat for a file outside it. */
@@ -2544,7 +2549,7 @@ export async function ingestFiles(
const mtime = currentMtimes.get(absFilePath);
if (mtime !== undefined && entry.mtime !== mtime) {
entry.mtime = mtime;
- symbolTableChanged = true;
+ symbolTableChanges.add(toWorkspaceRelative(absFilePath));
}
return true;
}
@@ -3976,7 +3981,7 @@ export async function ingestFiles(
const entry = table.get(rel);
if (entry && !entry.pending) {
entry.pending = true;
- symbolTableChanged = true;
+ symbolTableChanges.add(rel);
}
}
if (debug && dependents.length > 0) {
@@ -4031,7 +4036,7 @@ export async function ingestFiles(
if (entry && res !== undefined) {
entry.res = res;
delete entry.pending;
- symbolTableChanged = true;
+ symbolTableChanges.add(item.filePath);
}
},
});
@@ -4134,11 +4139,18 @@ export async function ingestFiles(
// not every commit landed -- pruned to the files that still exist. A
// `--lang` run's resolution paths hold its languages only; the files it
// left out still exist and keep their entries, as they keep their baseline.
- if (symbolTableChanged) {
+ // Only what changed is written: nothing on a run that changed no entry,
+ // and on most runs a few journal lines rather than the whole table.
+ if (symbolTableChanges.size > 0) {
const existing = new Set([...resolutionPaths, ...langExcluded].map(toWorkspaceRelative));
const table = symbolTable();
- for (const rel of [...table.keys()]) if (!existing.has(rel)) table.delete(rel);
- saveIngestSymbols(projectRoot, currentExtractor, table);
+ for (const rel of [...table.keys()]) {
+ if (!existing.has(rel)) {
+ table.delete(rel);
+ symbolTableChanges.add(rel);
+ }
+ }
+ saveIngestSymbols(projectRoot, currentExtractor, table, symbolTableChanges);
}
if (rebuildProgress !== null) {
// Finished: the baseline now records the new extractor. Otherwise keep
diff --git a/ix-cli/src/cli/config.ts b/ix-cli/src/cli/config.ts
index cf788040..f22eee4e 100644
--- a/ix-cli/src/cli/config.ts
+++ b/ix-cli/src/cli/config.ts
@@ -45,13 +45,18 @@ export function ingestRebuildPath(projectRoot: string): string {
/**
* Path to the per-file symbol summaries an incremental map resolves against
- * (see `ingest-symbols.ts`). Cleared with the mtime cache.
+ * (see `ingest-symbols.ts`). Cleared, with its journal, with the mtime cache.
*/
export function ingestSymbolsPath(projectRoot: string): string {
const key = rootKey(projectRoot);
return join(ixHome(), `ingest_symbols_${key}.json`);
}
+/** The symbol table's journal: entries changed since the table was last written in full. */
+export function ingestSymbolsJournalPath(projectRoot: string): string {
+ return `${ingestSymbolsPath(projectRoot)}.journal`;
+}
+
/** Path to the architecture-map completion marker for one project root. */
export function mapBaselinePath(projectRoot: string): string {
const key = rootKey(projectRoot);
@@ -145,6 +150,7 @@ export function clearIngestMtimeCache(projectRoot: string): void {
try { rmSync(ingestMtimeCachePath(projectRoot), { force: true }); } catch { /* non-critical */ }
try { rmSync(ingestRebuildPath(projectRoot), { force: true }); } catch { /* non-critical */ }
try { rmSync(ingestSymbolsPath(projectRoot), { force: true }); } catch { /* non-critical */ }
+ try { rmSync(ingestSymbolsJournalPath(projectRoot), { force: true }); } catch { /* non-critical */ }
clearMapBaseline(projectRoot);
}
diff --git a/ix-cli/src/cli/ingest-symbols.ts b/ix-cli/src/cli/ingest-symbols.ts
index 9ac155d5..ab399751 100644
--- a/ix-cli/src/cli/ingest-symbols.ts
+++ b/ix-cli/src/cli/ingest-symbols.ts
@@ -3,7 +3,7 @@
import { createHash } from "node:crypto";
import * as fs from "node:fs";
import * as path from "node:path";
-import { ingestSymbolsPath } from "./config.js";
+import { ingestSymbolsJournalPath, ingestSymbolsPath } from "./config.js";
/**
* The symbol table an incremental `ix map` resolves against.
@@ -76,49 +76,217 @@ interface SerializedSymbolTable {
version: number;
root: string;
extractor: string;
+ /**
+ * Which write of the table this is. The journal names the generation it
+ * extends, so it is applied only on top of the table it was written against.
+ * Optional: a table written before the journal existed has none, and then
+ * no journal applies to it.
+ */
+ gen?: string;
files: Record<string, SymbolEntry>;
}
+/**
+ * The table is written in full only now and then. In between, a run that
+ * changed a few entries appends them to a journal beside it, one JSON record
+ * per line, and the next load applies the journal over the table. The table
+ * keeps its version-1 shape, so a CLI that does not know about the journal
+ * still reads it; the entries it lacks are then merely older, and an older
+ * entry is checked against its file's hash and mtime before it is used.
+ *
+ * Line 1 is a header naming the table it extends; each line after it is
+ * `{"k": path, "e": entry}` (set) or `{"k": path, "d": 1}` (removed).
+ */
+interface JournalHeader {
+ journal: number;
+ root: string;
+ extractor: string;
+ gen: string;
+}
+type JournalRecord = { k: string; e: SymbolEntry } | { k: string; d: 1 };
+
+const JOURNAL_VERSION = 1;
+
+/** The journal is rewritten into the table once it would pass this share of the table's size. */
+const JOURNAL_MAX_SHARE = 0.5;
+
+/** What a load saw on disk, so a save can tell whether appending to the journal is safe. */
+interface LoadedState {
+ gen?: string;
+ tableBytes: number;
+ journalBytes: number;
+ /** Every journal line parsed and the file ended in a newline: appending to it is safe. */
+ journalIntact: boolean;
+}
+
+const loadedStates = new WeakMap<Map<string, SymbolEntry>, LoadedState>();
+
+function validEntry(entry: unknown): entry is SymbolEntry {
+ const e = entry as SymbolEntry | undefined;
+ return typeof e?.hash === "string" && !!e.summary && typeof e.summary.filePath === "string";
+}
+
+/** Apply the journal for `gen` to `table`. Returns its size and whether it can be appended to. */
+function applyJournal(
+ projectRoot: string, extractor: string, gen: string, table: Map<string, SymbolEntry>,
+): { bytes: number; intact: boolean } {
+ let text: string;
+ try {
+ text = fs.readFileSync(ingestSymbolsJournalPath(projectRoot), "utf-8");
+ } catch {
+ return { bytes: 0, intact: true };
+ }
+ const bytes = Buffer.byteLength(text);
+ const lines = text.split("\n");
+ // A journal that does not end in a newline was cut off mid-append.
+ let intact = text.length === 0 || text.endsWith("\n");
+ if (lines[lines.length - 1] === "") lines.pop();
+ let header: JournalHeader | undefined;
+ try {
+ header = lines.length > 0 ? JSON.parse(lines[0]) as JournalHeader : undefined;
+ } catch {
+ header = undefined;
+ }
+ if (
+ header?.journal !== JOURNAL_VERSION || header.root !== projectRoot ||
+ header.extractor !== extractor || header.gen !== gen
+ ) {
+ // Another table's journal, or none worth reading: it is replaced, never appended to.
+ return { bytes, intact: false };
+ }
+ for (let i = 1; i < lines.length; i++) {
+ let record: JournalRecord;
+ try {
+ record = JSON.parse(lines[i]) as JournalRecord;
+ } catch {
+ intact = false;
+ continue;
+ }
+ if (typeof record?.k !== "string") { intact = false; continue; }
+ if ("d" in record) table.delete(record.k);
+ else if (validEntry(record.e)) table.set(record.k, record.e);
+ }
+ return { bytes, intact };
+}
+
/** The table for `projectRoot` and `extractor`, or an empty one. */
export function loadIngestSymbols(projectRoot: string, extractor: string): Map<string, SymbolEntry> {
+ const table = new Map<string, SymbolEntry>();
+ let state: LoadedState = { tableBytes: 0, journalBytes: 0, journalIntact: false };
try {
- const data = JSON.parse(fs.readFileSync(ingestSymbolsPath(projectRoot), "utf-8")) as SerializedSymbolTable;
- if (data.version !== TABLE_VERSION || data.root !== projectRoot || data.extractor !== extractor) return new Map();
- if (!data.files || typeof data.files !== "object") return new Map();
- const table = new Map<string, SymbolEntry>();
- for (const [rel, entry] of Object.entries(data.files)) {
- if (typeof entry?.hash === "string" && entry.summary && typeof entry.summary.filePath === "string") {
- table.set(rel, entry);
+ const text = fs.readFileSync(ingestSymbolsPath(projectRoot), "utf-8");
+ const data = JSON.parse(text) as SerializedSymbolTable;
+ if (
+ data.version === TABLE_VERSION && data.root === projectRoot && data.extractor === extractor &&
+ data.files && typeof data.files === "object"
+ ) {
+ for (const [rel, entry] of Object.entries(data.files)) {
+ if (validEntry(entry)) table.set(rel, entry);
+ }
+ state = { tableBytes: Buffer.byteLength(text), journalBytes: 0, journalIntact: false };
+ if (typeof data.gen === "string") {
+ const journal = applyJournal(projectRoot, extractor, data.gen, table);
+ state = { gen: data.gen, tableBytes: state.tableBytes, journalBytes: journal.bytes, journalIntact: journal.intact };
}
}
- return table;
} catch {
- return new Map();
+ table.clear();
+ state = { tableBytes: 0, journalBytes: 0, journalIntact: false };
}
+ loadedStates.set(table, state);
+ return table;
+}
+
+/** Write `contents` to `target` via a temp file and a rename. */
+function writeAtomically(target: string, contents: string): void {
+ const tmp = `${target}.${process.pid}.tmp`;
+ fs.writeFileSync(tmp, contents);
+ fs.renameSync(tmp, target);
}
/**
- * Write the table via a temp file and a rename, so a process killed mid-write
- * leaves the previous table rather than a truncated one.
+ * Save the table.
+ *
+ * With `changed` -- the paths whose entries this run set, altered or removed --
+ * nothing is written when it is empty, and a few changes are appended to the
+ * journal when the table on disk is the one this table was loaded from. The
+ * table itself is rewritten (via a temp file and a rename, so a process killed
+ * mid-write leaves the previous one) when there is no such table, the journal
+ * is not intact, or it would grow past half the table's size. Without
+ * `changed`, the table is rewritten.
*/
-export function saveIngestSymbols(projectRoot: string, extractor: string, table: Map<string, SymbolEntry>): void {
+export function saveIngestSymbols(
+ projectRoot: string,
+ extractor: string,
+ table: Map<string, SymbolEntry>,
+ changed?: ReadonlySet<string>,
+): void {
+ if (changed !== undefined && changed.size === 0) return;
try {
const target = ingestSymbolsPath(projectRoot);
+ const journalPath = ingestSymbolsJournalPath(projectRoot);
fs.mkdirSync(path.dirname(target), { recursive: true });
+ const state = loadedStates.get(table);
+
+ if (changed !== undefined && state?.gen !== undefined && state.journalIntact && onDiskGen(target) === state.gen) {
+ const records = [...changed].sort().map(k => {
+ const e = table.get(k);
+ return JSON.stringify(e !== undefined ? { k, e } : { k, d: 1 }) + "\n";
+ }).join("");
+ const addBytes = Buffer.byteLength(records);
+ if (state.journalBytes + addBytes <= state.tableBytes * JOURNAL_MAX_SHARE) {
+ if (state.journalBytes === 0) {
+ const header: JournalHeader = { journal: JOURNAL_VERSION, root: projectRoot, extractor, gen: state.gen };
+ const headerLine = JSON.stringify(header) + "\n";
+ writeAtomically(journalPath, headerLine + records);
+ state.journalBytes = Buffer.byteLength(headerLine) + addBytes;
+ } else {
+ fs.appendFileSync(journalPath, records);
+ state.journalBytes += addBytes;
+ }
+ return;
+ }
+ }
+
+ const gen = createHash("sha256").update(`${process.pid}:${Date.now()}:${Math.random()}`).digest("hex").slice(0, 16);
const data: SerializedSymbolTable = {
version: TABLE_VERSION,
root: projectRoot,
extractor,
+ gen,
files: Object.fromEntries(table),
};
- const tmp = `${target}.${process.pid}.tmp`;
- fs.writeFileSync(tmp, JSON.stringify(data));
- fs.renameSync(tmp, target);
+ const text = JSON.stringify(data);
+ writeAtomically(target, text);
+ // After the table: a journal left behind names the old generation and is ignored.
+ try { fs.rmSync(journalPath, { force: true }); } catch { /* ignored by gen */ }
+ loadedStates.set(table, { gen, tableBytes: Buffer.byteLength(text), journalBytes: 0, journalIntact: true });
} catch {
// An optimization: losing it costs the next run a parse of what it lacks.
}
}
+/**
+ * The generation of the table on disk, read from the start of the file
+ * without parsing the rest: `gen` is written before `files`.
+ */
+function onDiskGen(target: string): string | undefined {
+ let fd: number | undefined;
+ try {
+ fd = fs.openSync(target, "r");
+ const buf = Buffer.alloc(1024);
+ const n = fs.readSync(fd, buf, 0, buf.length, 0);
+ const head = buf.subarray(0, n).toString("utf-8");
+ const filesAt = head.indexOf('"files":');
+ const m = /"gen":"([0-9a-f]+)"/.exec(filesAt === -1 ? head : head.slice(0, filesAt));
+ return m?.[1];
+ } catch {
+ return undefined;
+ } finally {
+ if (fd !== undefined) try { fs.closeSync(fd); } catch { /* nothing to do */ }
+ }
+}
+
// ── Dependents (IN-11) ──────────────────────────────────────────────────
// A file that did not change can still need re-sending: a call it makes may
// now resolve somewhere else, or nowhere, because a name it uses was added,