Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(map): stop blaming the language when the ingest was deduplicated (#570)
KageBinary084f472bcde7
diff --git a/ix-cli/src/cli/__tests__/map-auto-skip.test.ts b/ix-cli/src/cli/__tests__/map-auto-skip.test.tsindex 56909b60..173f41aa 100644--- a/ix-cli/src/cli/__tests__/map-auto-skip.test.ts+++ b/ix-cli/src/cli/__tests__/map-auto-skip.test.ts@@ -110,6 +110,8 @@ describe('describeEmptyCompletedMap', () => {const message = describeEmptyCompletedMap(emptyResult, {filesDiscovered: 260,patchesApplied: 260,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,});@@ -120,10 +122,84 @@ describe('describeEmptyCompletedMap', () => {expect(message).toContain("the next 'ix map' can reuse unchanged files");});+ it('blames deduplication, not the language, when every commit was a replay (#527)', () => {+ const message = describeEmptyCompletedMap(emptyResult, {+ filesDiscovered: 1,+ patchesApplied: 1,+ idempotentPatches: 1,+ filesSkippedAsUnchanged: 0,+ parseErrors: 0,+ commitErrors: 0,+ });++ expect(message).toContain("answered every one of the 1 committed patch with 'already applied'");+ expect(message).toContain('this run wrote nothing');+ expect(message).toContain('expire within 24 hours');+ // The old diagnosis is actively wrong here: the source parsed fine, and+ // nothing about it reached the backend's graph to be mapped.+ expect(message).not.toContain('may not map this source language');+ expect(message).not.toContain('The source graph was ingested');+ expect(message).toContain("the next 'ix map' can reuse unchanged files");+ });++ it('does not blame deduplication when the run skipped files as unchanged', () => {+ // Joey's case on #570: a workspace on a language the backend cannot build a+ // hierarchy for. 99 files are mtime-clean and never submitted — which is+ // itself proof the backend still holds their source hashes, and so their+ // nodes. The user reverts one file to content ingested earlier, its patch id+ // matches a record that is still there, and the backend answers Idempotent.+ // Every patch this run submitted was a replay, but the graph is intact and+ // waiting 24 hours changes nothing.+ const message = describeEmptyCompletedMap(emptyResult, {+ filesDiscovered: 100,+ patchesApplied: 1,+ idempotentPatches: 1,+ filesSkippedAsUnchanged: 99,+ parseErrors: 0,+ commitErrors: 0,+ });++ expect(message).not.toContain('deduplicated');+ expect(message).not.toContain('expire within 24 hours');+ expect(message).toContain('may not map this source language');+ });++ it('keeps the language diagnosis when only some commits were replays', () => {+ const message = describeEmptyCompletedMap(emptyResult, {+ filesDiscovered: 10,+ patchesApplied: 10,+ idempotentPatches: 9,+ filesSkippedAsUnchanged: 0,+ parseErrors: 0,+ commitErrors: 0,+ });++ expect(message).toContain('may not map this source language');+ expect(message).not.toContain("already applied");+ });++ it('does not claim deduplication when nothing was committed at all', () => {+ // patchesApplied 0 with files discovered is the hash-skip path, not a+ // replay; `idempotentPatches >= patchesApplied` is trivially true there.+ const message = describeEmptyCompletedMap(emptyResult, {+ filesDiscovered: 30,+ patchesApplied: 0,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,+ parseErrors: 0,+ commitErrors: 0,+ });++ expect(message).toContain('may not map this source language');+ expect(message).not.toContain("already applied");+ });+it('does not reject an actually empty workspace', () => {expect(describeEmptyCompletedMap(emptyResult, {filesDiscovered: 0,patchesApplied: 0,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,})).toBeUndefined();@@ -133,12 +209,16 @@ describe('describeEmptyCompletedMap', () => {expect(describeEmptyCompletedMap(emptyResult, {filesDiscovered: 12,patchesApplied: 12,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 1,commitErrors: 0,})).toBeUndefined();expect(describeEmptyCompletedMap(emptyResult, {filesDiscovered: 12,patchesApplied: 12,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 1,})).toBeUndefined();@@ -150,6 +230,8 @@ describe('describeEmptyCompletedMap', () => {expect(describeEmptyCompletedMap({ ...emptyResult, outcome: 'ok' }, {filesDiscovered: 260,patchesApplied: 260,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,})).toBeUndefined();@@ -160,6 +242,8 @@ describe('describeEmptyCompletedMap', () => {expect(describeEmptyCompletedMap({ ...emptyResult, outcome }, {filesDiscovered: 260,patchesApplied: 260,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,})).toBeUndefined();@@ -170,18 +254,24 @@ describe('describeEmptyCompletedMap', () => {expect(describeEmptyCompletedMap({ ...emptyResult, outcome: 'local_map_not_recommended' }, {filesDiscovered: 12,patchesApplied: 12,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,})).toBeUndefined();expect(describeEmptyCompletedMap({ ...emptyResult, file_count: 12 }, {filesDiscovered: 12,patchesApplied: 12,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,})).toBeUndefined();expect(describeEmptyCompletedMap({ ...emptyResult, region_count: 1 }, {filesDiscovered: 12,patchesApplied: 12,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,})).toBeUndefined();@@ -192,6 +282,8 @@ describe('describeEmptyCompletedMap', () => {const message = invalidateBaselineForIncompleteCompletedMap(emptyResult, {filesDiscovered: 260,patchesApplied: 0,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,}, '/workspace/account', invalidate);@@ -206,6 +298,8 @@ describe('describeEmptyCompletedMap', () => {expect(describeEmptyCompletedMap({ ...emptyResult, outcome: 'coupling_unchanged' }, {filesDiscovered: 260,patchesApplied: 0,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,})).toBeDefined();@@ -216,6 +310,8 @@ describe('describeRegionlessCompletedMap', () => {const cleanIngest = {filesDiscovered: 200,patchesApplied: 200,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,};@@ -272,6 +368,8 @@ describe('describeRegionlessCompletedMap', () => {}, {filesDiscovered: 2,patchesApplied: 2,+ idempotentPatches: 0,+ filesSkippedAsUnchanged: 0,parseErrors: 0,commitErrors: 0,}, '/workspace/mixed', invalidate);diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex cc7c3e7b..7c77f889 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -912,10 +912,63 @@ export type CommitOutcome =| { kind: "warn"; message: string }| { kind: "fatal"; message: string };+/**+ * The wire value the backend sends for a commit it short-circuited because it+ * had already recorded the patch id — `CommitStatus.Idempotent.toString` in+ * `PatchCommitRoutes` / `BulkPatchCommitRoutes`.+ */+const COMMIT_STATUS_IDEMPOTENT = 'Idempotent';++/**+ * The wire value for a commit the backend refused because the optimistic+ * base-rev check lost a race — `CommitStatus.BaseRevMismatch`. It writes+ * nothing, exactly like `Idempotent`, but unlike `Idempotent` it means the+ * work still needs doing.+ */+const COMMIT_STATUS_BASE_REV_MISMATCH = 'BaseRevMismatch';+/** Minimal local-ingest facts needed by commands that continue after ingestion. */export interface IngestFilesSummary {filesDiscovered: number;patchesApplied: number;+ /**+ * How many of `patchesApplied` the backend answered `Idempotent` — it had+ * already recorded that patch id, so the commit wrote nothing.+ *+ * Normally uninteresting: `ix ingest --force` re-submits every file (and a+ * lost or blank local baseline has the same effect against a backend that+ * still holds the patch records), so a repo whose graph is already correct+ * legitimately deduplicates everything. Note `ix map --full` is NOT such a+ * case — it forwards `full` to the backend map and never sets `force` on the+ * ingest below it, so it re-submits nothing.+ *+ * It matters only when the graph turns out to be EMPTY afterwards, because+ * then the deduplication is the reason nothing was written rather than a+ * sign that nothing needed to be (#527).+ */+ idempotentPatches: number;+ /**+ * Files this run skipped because they were already unchanged — mtime-clean+ * against the local baseline, or hash-clean against the backend.+ *+ * Load-bearing for the #527 diagnosis, not informational. `idempotentPatches`+ * only describes the patches a run actually submitted; it says nothing about+ * the files the run never submitted, and those are what separate a graph+ * deleted under its own patch records from a graph that is simply fine. A run+ * in the #527 state skips nothing here: the DB-reset guard clears the mtime+ * cache and the hash lookup comes back empty, so every file is re-submitted+ * and every one is answered `Idempotent`. A run that skipped even one file as+ * unchanged has proof the backend still holds that file's source hash, and+ * therefore its nodes.+ *+ * Counts only the unchanged skips — empty, oversized and minified files are+ * skipped for reasons that carry no such proof. It is the same counter the+ * stitch-completeness gate reads, which also means it inherits that counter's+ * Path-B reset: a run with no baseline re-reads and re-hashes every file, so+ * the stat loop's mtime skips did not actually happen and are subtracted+ * back out. That is the shape a #527 run has, and it is exactly right here.+ */+ filesSkippedAsUnchanged: number;parseErrors: number;commitErrors: number;stitchErrors: number;@@ -1568,6 +1621,7 @@ export async function ingestFiles(let filesDiscovered = 0;let filesChanged = 0;let patchesApplied = 0;+ let idempotentPatches = 0;let filesSkipped = 0;/*** Files skipped because we ASSUMED THEY WERE UNCHANGED. (Ix#568)@@ -2105,9 +2159,16 @@ export async function ingestFiles(recordMs(chunkMs);timings.fallbackCommitMs += chunkMs;latestRev = advanceRev(latestRev, result.rev);+ // See onBulkCommitted: a lost base-rev race wrote nothing and+ // must not be reported as applied.+ if (result.status === COMMIT_STATUS_BASE_REV_MISMATCH) {+ commitErrors++;+ continue;+ }patchesApplied++;patchesTheBackendTook++;commitBreaker.recordSuccess();+ if (result.status === COMMIT_STATUS_IDEMPOTENT) idempotentPatches++;opts?.onCommitted?.(item, result.rev);} catch (commitErr) {// Counted apart from parseErrors: a patch that parsed fine and@@ -2255,6 +2316,13 @@ export async function ingestFiles(patchesApplied += chunk.length;patchesTheBackendTook += chunk.length;commitBreaker.recordSuccess();+ // The cutoff drain is a third commit site, added after this counter+ // was written (Ix#571). It is a plain bulk commit carrying a status,+ // so it counts like the other one -- a run whose held patches were+ // all placed here and all deduplicated would otherwise report+ // `idempotentPatches: 0` and fall back to the language hypothesis+ // this change exists to retire.+ if (result.status === COMMIT_STATUS_IDEMPOTENT) idempotentPatches += chunk.length;for (const item of chunk) opts?.onCommitted?.(item, result.rev);if (debug) process.stderr.write(` [cutoff] bulk placed ${chunk.length} of ${items.length} held`);@@ -2414,9 +2482,27 @@ export async function ingestFiles(},onBulkCommitted: (items, result) => {latestRev = advanceRev(latestRev, result.rev);+ // `BaseRevMismatch` wrote nothing: the backend read the latest rev+ // outside the transaction and it moved before the commit ran. It+ // is a lost race, not a replay, so it must not count as applied —+ // and it belongs in commitErrors, because that is what stops+ // `persistIngestBaselineIfClean` caching these files as unchanged+ // and leaving them missing from the graph until the next edit.+ if (result.status === COMMIT_STATUS_BASE_REV_MISMATCH) {+ commitErrors += items.length;+ if (debug) {+ process.stderr.write(+ `\n [commit lost the base-rev race] ${items.length} patches wrote nothing; they will be re-sent next run\n`+ );+ }+ return;+ }patchesApplied += items.length;patchesTheBackendTook += items.length;commitBreaker.recordSuccess();+ // One status covers the whole bulk: the backend commits the chunk+ // as a unit, so `Idempotent` means every patch in it was a replay.+ if (result.status === COMMIT_STATUS_IDEMPOTENT) idempotentPatches += items.length;for (const item of items) opts?.onCommitted?.(item, result.rev);},commitIndividually,@@ -3328,6 +3414,8 @@ export async function ingestFiles(const summary: IngestFilesSummary = {filesDiscovered,patchesApplied,+ idempotentPatches,+ filesSkippedAsUnchanged,// `+ crashedParses()`, as the baseline and delete guards already do. Files// lost to a dead parse pool raise `filesSkippedUnparsed`, never// `parseErrors`, so without this everything downstream read the run as@@ -3409,6 +3497,7 @@ export async function ingestFiles(filesChanged,patchesApplied,filesSkipped,+ idempotentPatches,entitiesParsed,latestRev,// `unchanged` is the files we ASSUMED unchanged, not every skip. It useddiff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.tsindex 735d972c..fbe83c12 100644--- a/ix-cli/src/cli/commands/map.ts+++ b/ix-cli/src/cli/commands/map.ts@@ -151,7 +151,12 @@ const COMPLETED_MAP_OUTCOMES = new Set([*/export function describeEmptyCompletedMap(result: Pick<MapResult, "file_count" | "region_count" | "regions" | "outcome">,- ingest: Pick<IngestFilesSummary, "filesDiscovered" | "patchesApplied" | "parseErrors" | "commitErrors"> | undefined,+ ingest:+ | Pick<+ IngestFilesSummary,+ "filesDiscovered" | "patchesApplied" | "idempotentPatches" | "filesSkippedAsUnchanged" | "parseErrors" | "commitErrors"+ >+ | undefined,): string | undefined {if (!ingest || ingest.filesDiscovered <= 0 || ingest.parseErrors > 0 || ingest.commitErrors > 0) {return undefined;@@ -161,7 +166,33 @@ export function describeEmptyCompletedMap(const sourceFiles = ingest.filesDiscovered;const patches = ingest.patchesApplied;- return `Backend reported ${result.outcome}, but mapped 0 files after local ingest found ${sourceFiles} supported source ${sourceFiles === 1 ? "file" : "files"} (${patches} ${patches === 1 ? "patch" : "patches"} committed). The source graph was ingested, but no architecture hierarchy was created. The active backend may not map this source language yet. The source ingest baseline was preserved, so the next 'ix map' can reuse unchanged files.`;+ const files = `${sourceFiles} supported source ${sourceFiles === 1 ? "file" : "files"}`;+ const patchWord = patches === 1 ? "patch" : "patches";+ const tail = "The source ingest baseline was preserved, so the next 'ix map' can reuse unchanged files.";++ // The backend deduplicated every commit, so this run wrote nothing at all —+ // and the two sentences below are then both false: nothing was ingested, and+ // the language had no chance to be the reason (#527). The observed cause is a+ // graph deleted out from under the backend's own patch records: a scoped+ // reset used to remove a workspace's nodes and patches while leaving its+ // idempotency keys, so the identical re-ingest was accepted as a replay and+ // committed nothing (Ix-memory#174). Those keys expire after 24h, which is+ // why waiting is a real workaround and worth saying out loud.+ // `filesSkippedAsUnchanged === 0` is load-bearing, not a belt-and-braces extra.+ // `idempotentPatches >= patches` only says every patch this run *submitted*+ // was a replay; it says nothing about the files the run never submitted, and+ // those are what disprove the diagnosis. A workspace on a language the+ // backend cannot build a hierarchy for skips its 99 clean files, and one+ // reverted file whose content was ingested before commits as `Idempotent` —+ // giving patches === 1, idempotentPatches === 1, file_count === 0 with a+ // graph that is perfectly intact and a 24-hour wait that changes nothing.+ // A real #527 run skips nothing: the DB-reset guard clears the mtime cache+ // and the hash lookup comes back empty, so every file is re-submitted.+ if (patches > 0 && ingest.idempotentPatches >= patches && ingest.filesSkippedAsUnchanged === 0) {+ return `Backend reported ${result.outcome}, but mapped 0 files after local ingest found ${files}. The backend answered every one of the ${patches} committed ${patchWord} with 'already applied', so this run wrote nothing — the graph is empty because the commits were deduplicated against patch records the backend still holds, not because the source could not be parsed. That is what a workspace whose graph was deleted without its patch records looks like; the records expire within 24 hours, or an upgraded backend clears them with the reset. ${tail}`;+ }++ return `Backend reported ${result.outcome}, but mapped 0 files after local ingest found ${files} (${patches} ${patchWord} committed). The source graph was ingested, but no architecture hierarchy was created. The active backend may not map this source language yet. ${tail}`;}/**@@ -249,7 +280,12 @@ function emitDroppedFileWarning(*/export function invalidateBaselineForIncompleteCompletedMap(result: Pick<MapResult, "file_count" | "region_count" | "regions" | "outcome">,- ingest: Pick<IngestFilesSummary, "filesDiscovered" | "patchesApplied" | "parseErrors" | "commitErrors"> | undefined,+ ingest:+ | Pick<+ IngestFilesSummary,+ "filesDiscovered" | "patchesApplied" | "idempotentPatches" | "filesSkippedAsUnchanged" | "parseErrors" | "commitErrors"+ >+ | undefined,projectRoot: string,invalidate: (root: string) => void = clearMapBaseline,): string | undefined {