Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
// Copyright 2026 Ix Infrastructure Inc.
import * as nodePath from 'node:path';
import * as fs from 'node:fs';
import * as crypto from 'node:crypto';
import * as os from 'node:os';
import { fileURLToPath } from 'node:url';
import type { Command } from 'commander';
import { ParsePool } from './parse-pool.js';
// import { ResolveWorker } from './resolve-pool.js';
import chalk from 'chalk';
import { IxClient } from '../../client/api.js';
import { combineSignals, currentRunSignal } from '../../client/run-signal.js';
import type { GraphNode, GraphPatchPayload } from '../../client/types.js';
import { canonicalWorkspacePath, isPathInside, resolveWorkspaceRoot, clearMapResultCache, clearStitchScopeCache } from '../config.js';
import { resolveIngestRootWithSource } from '../map-root.js';
import { acquireMapLock, requestMapRerun, takeMapRerun } from '../single-flight.js';
import { applyRequestedMapCoalesceExitCode } from './map.js';
import {
canonicalizeDiscoveredFilePath, discoverSourceFiles, emptyDiscoveryCounts, isSupportedSourceFile,
} from '../file-discovery.js';
import {
clearRebuildProgress, extractorChanged, isRev, loadIngestBaseline, loadRebuildProgress,
saveIngestBaseline, saveRebuildProgress, type BaselineFileNotes,
loadOrphanSweepPending, saveOrphanSweepPending, type OrphanSweepPending,
} from '../ingest-baseline.js';
import { resolveGitHubToken } from '../github/auth.js';
import { parseGitHubRepo, fetchGitHubData } from '../github/fetch.js';
import { loadIngestionModules } from './ingestion-loader.js';
import { declaredPackageDirs } from '../package-dirs.js';
import { ensureWorkspaceIdState, workspaceStateFor } from '../bootstrap.js';
import {
changedNames, findDependents, importsChanged, loadIngestSymbols, resolutionHash, saveIngestSymbols,
type StoredSummary, type SymbolEntry,
} from '../ingest-symbols.js';
import { detectSystem, fileIdentity, lookupPackage, readPackageNames, readPackageDeps } from '../system.js';
import { CLIENT_EXPECTED_SCHEMA_VERSION } from '../backend-status.js';
import { admitStitchWaiting, connectionNeverEstablished, type StitchRefusal } from '../stitch-guard.js';
import {
createCommitBreaker,
createDrainGate,
describeCommitCutoff,
drainFailureBudget,
drainInPasses,
perFileAction,
} from '../commit-breaker.js';
import { retryCommit, TRANSIENT_RETRY_BUDGET_MS } from '../commit-retry.js';
import { createDatabaseProbe, probeDatabase } from '../database-probe.js';
// Moved to commit-retry.ts with the retry loop that uses them; re-exported so
// existing importers keep working.
export { isAbortError, isRetryableCommitConflict } from '../commit-retry.js';
import { readBackendHealth } from './upgrade.js';
import {
createIgnoreMatcher, parseIgnoreFile, parseIgnorePattern, type IgnorePattern,
} from '../ignore-globs.js';
import { canRenderProgress } from '../stderr.js';
import { createTypeScriptModuleResolver } from '../ts-module-resolution.js';
import {
deterministicId,
transformIssue,
transformIssueComment,
transformPR,
transformPRComment,
transformCommit,
} from '../github/transform.js';
import { printJson } from '../format.js';
import { describeReplayedChanges } from '../graph-health.js';
import { createClient } from "../../client/factory.js";
import { mapLimit } from "../../client/request-memo.js";
import { createLimiter, type Limiter } from "../tree-walk.js";
// ---------------------------------------------------------------------------
// File discovery
// ---------------------------------------------------------------------------
export { isSupportedSourceFile, tryGitLsFiles, walkFiles } from '../file-discovery.js';
// ---------------------------------------------------------------------------
// Language filter helpers
// ---------------------------------------------------------------------------
/** Normalise user-supplied language names to their canonical enum values. */
const LANG_ALIASES: Record<string, string> = {
'c++': 'cpp',
'cplusplus': 'cpp',
'c#': 'csharp',
'cs': 'csharp',
'js': 'javascript',
'ts': 'typescript',
'py': 'python',
'rb': 'ruby',
'rs': 'rust',
'kt': 'kotlin',
};
function parseLangs(raw: string): Set<string> {
return new Set(
raw.split(',')
.map(s => s.trim().toLowerCase())
.map(s => LANG_ALIASES[s] ?? s)
.filter(Boolean),
);
}
const MAX_FILE_BYTES = 1024 * 1024; // 1 MB
/** Parse workers at most, unless IX_PARSE_WORKERS says otherwise. */
const DEFAULT_PARSE_WORKERS = 8;
/** Files a parse worker should have before another one is worth starting. */
const FILES_PER_PARSE_WORKER = 50;
/**
* How many parse workers `expectedTasks` files deserve: one per 50 files,
* never more than the cores less one or IX_PARSE_WORKERS (default 8), never
* fewer than one. Each worker loads every grammar, about 70 MB, so a worker
* per core for a one-file edit cost more than the parse.
*/
export function parsePoolSize(
expectedTasks: number,
cores: number = os.availableParallelism(),
configured: string | undefined = process.env.IX_PARSE_WORKERS,
): number {
const fromEnv = Number(configured);
const cap = configured !== undefined && Number.isInteger(fromEnv) && fromEnv > 0 ? fromEnv : DEFAULT_PARSE_WORKERS;
return Math.max(1, Math.min(cores - 1, cap, Math.ceil(expectedTasks / FILES_PER_PARSE_WORKER)));
}
const MINIFIED_BYTES_THRESHOLD = 200 * 1024;
const MINIFIED_MAX_LINE_THRESHOLD = 20_000;
const MINIFIED_AVG_LINE_THRESHOLD = 2_000;
const SLOW_WORK_LOG_MS = 5_000;
const SLOW_WORK_REPEAT_MS = 10_000;
export function parsePositiveIntEnv(name: string, fallback: number): number {
const raw = process.env[name]?.trim();
if (!raw) return fallback;
// Matched, not parsed, as in single-flight's lockMaxMs: `parseInt` reads any
// numeric prefix, so `1e3` meant 1 and `2k` meant 2.
if (!/^\d+$/.test(raw)) return fallback;
const parsed = Number.parseInt(raw, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
/**
* `parsePositiveIntEnv`, but 0 is a value rather than "unset". For a count of
* RETRIES, where 0 means "do not retry" and reading it as the default made the
* setting impossible to turn off (IX_COMMIT_CONFLICT_RETRIES=0 retried 6 times).
*/
export function parseNonNegativeIntEnv(name: string, fallback: number): number {
const raw = process.env[name]?.trim();
if (!raw) return fallback;
if (!/^\d+$/.test(raw)) return fallback;
const parsed = Number.parseInt(raw, 10);
return Number.isSafeInteger(parsed) ? parsed : fallback;
}
/**
* The patterns in force for this ingest: `--exclude` flags first, then the
* `.ixignore` at the root.
*
* `.ixignore` is read from the ingest root only. A per-directory cascade is
* what `.gitignore` does and it is a lot of stat calls for a file almost
* nobody will write twice in one repository.
*/
export function collectExcludePatterns(root: string, cliPatterns: string[] = []): IgnorePattern[] {
const patterns = cliPatterns
.map((p) => parseIgnorePattern(p))
.filter((p): p is IgnorePattern => p !== null);
try {
const text = fs.readFileSync(nodePath.join(root, '.ixignore'), 'utf-8');
patterns.push(...parseIgnoreFile(text));
} catch {
// No .ixignore is the ordinary case.
}
return patterns;
}
/**
* True when `candidate` lies inside `root`.
*
* Both arguments must ALREADY be canonical (realpath'd). This is pure path
* arithmetic and so cannot see through a symlink itself — resolving first is
* what gives it meaning, and resolving both sides is what keeps it from
* rejecting everything when the root is itself reached through a link (macOS
* `/var` -> `/private/var`, a home on a network mount, a `~/code` symlink).
*/
export function isWithinDiscoveryRoot(root: string, candidate: string): boolean {
const relative = nodePath.relative(root, candidate);
// Compare path *segments*: a bare `startsWith('..')` also rejects a
// legitimate sibling whose name merely begins with dots, e.g. `..shared/`.
// `isAbsolute` covers Windows, where `relative` across drives is absolute.
return (
relative !== '' &&
relative !== '..' &&
!relative.startsWith(`..${nodePath.sep}`) &&
!nodePath.isAbsolute(relative)
);
}
export function dedupeDiscoveredFilePaths(
filePaths: string[],
canonicalize: (filePath: string) => string = canonicalizeDiscoveredFilePath,
): string[] {
const deduped: string[] = [];
const seenCanonical = new Set<string>();
for (const filePath of filePaths) {
const canonicalPath = canonicalize(filePath);
if (seenCanonical.has(canonicalPath)) continue;
seenCanonical.add(canonicalPath);
deduped.push(canonicalPath);
}
return deduped;
}
/**
* Canonicalize, de-duplicate, and confine discovered paths to `root`.
*
* Discovery resolves symlinks, so a repository can name files outside itself:
* `git ls-files` reports a committed symlink as an ordinary entry and the
* `statSync` behind it describes the *target*, so `src/notes.ts` ->
* `~/.ssh/id_rsa` is discovered, read, parsed into the graph and sent to the
* configured endpoint. `ix ingest --github owner/repo` runs this over
* repositories nobody has read, which makes repository contents attacker input.
*
* (`walkFiles` is not the hole — readdir Dirents describe the link itself, so
* `isFile()` is false for a symlink and it is skipped there. Git's list is
* the gap.)
*
* A symlink *within* the tree still resolves and is kept, which is what
* monorepos that link packages around depend on. `root` is `undefined` when a
* single file was named explicitly: that IS the request, and there is nothing
* for it to escape from.
*
* Kept as one function rather than a filter at the call site so the confinement
* cannot be dropped without a test noticing.
*/
export function discoverIngestFilePaths(
discovered: string[],
root: string | undefined,
canonicalize: (filePath: string) => string = canonicalizeDiscoveredFilePath,
): { files: string[]; outsideRoot: number } {
const canonical = dedupeDiscoveredFilePaths(discovered, canonicalize);
if (root === undefined) return { files: canonical, outsideRoot: 0 };
// Canonicalize the root too. Comparing a resolved file against an unresolved
// root drops everything whenever the root is itself reached through a link —
// macOS `/var` -> `/private/var`, a home on a network mount, a `~/code`
// symlink — which would look like "this repo has no source files".
const canonicalRoot = canonicalizeDiscoveredFilePath(root);
const files = canonical.filter((candidate) => isWithinDiscoveryRoot(canonicalRoot, candidate));
return { files, outsideRoot: canonical.length - files.length };
}
function sha256(content: Buffer): string {
return crypto.createHash('sha256').update(content).digest('hex');
}
function isLikelyMinifiedSource(source: string): boolean {
if (source.length < MINIFIED_BYTES_THRESHOLD) return false;
let lineCount = 1;
let currentLineLength = 0;
let maxLineLength = 0;
for (let i = 0; i < source.length; i++) {
if (source.charCodeAt(i) === 10) {
lineCount++;
if (currentLineLength > maxLineLength) maxLineLength = currentLineLength;
currentLineLength = 0;
} else {
currentLineLength++;
}
}
if (currentLineLength > maxLineLength) maxLineLength = currentLineLength;
const averageLineLength = source.length / Math.max(lineCount, 1);
return maxLineLength >= MINIFIED_MAX_LINE_THRESHOLD || (lineCount <= 50 && averageLineLength >= MINIFIED_AVG_LINE_THRESHOLD);
}
/**
* Strip chunks and claims from patches — mapMode only needs nodes + edges.
*
* Every edge that touches a dropped chunk goes too, not only the chunk
* predicates: a named chunk also DEFINES its symbol, and keeping that edge
* once its chunk is gone left one dangling DEFINES per function in every
* `ix map` graph.
*/
export function stripMapModeOps(patch: GraphPatchPayload): GraphPatchPayload {
const droppedNodes = new Set<unknown>(
patch.ops.filter(op => op.type === 'UpsertNode' && op['kind'] === 'chunk').map(op => op['id']),
);
return {
...patch,
ops: patch.ops.filter(op => {
if (op.type === 'AssertClaim' || op.type === 'RetractClaim') return false;
if (op.type === 'UpsertNode' && op['kind'] === 'chunk') return false;
if (op.type === 'UpsertEdge' && (droppedNodes.has(op['src']) || droppedNodes.has(op['dst']))) return false;
return true;
}),
};
}
interface StoredPatchEntities {
nodeIds: string[];
/** Node kinds the stored patch wrote, when it kept its ops; empty otherwise. */
kinds: Map<string, string>;
/** External package nodes (`attrs.external`) the stored patch wrote. */
external: Set<string>;
/** Ids only a later window manifest named; an absent one gets no DeleteNode. */
extraOnly?: Set<string>;
/** Why a later window manifest was left out: a backend error reading it. */
extraSkipped?: string;
}
/**
* Whether a node is an external package node: `external://<pkg>` keyed by the
* workspace, not the file, so every file that calls into the package writes
* the same one.
*/
function isExternalNode(entityNode: unknown): boolean {
return (entityNode as { attrs?: { external?: unknown } } | undefined)?.attrs?.external === true;
}
function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {
const data = (raw as { data?: unknown } | null)?.data as {
entityIds?: unknown;
nodeOpCount?: unknown;
ops?: unknown;
} | undefined;
if (!data) return null;
if (Array.isArray(data.ops)) {
const ops = data.ops as Array<{ type?: unknown; id?: unknown; kind?: unknown; attrs?: unknown }>;
const kinds = new Map<string, string>();
const external = new Set<string>();
for (const op of ops) {
if (op.type !== 'UpsertNode' || typeof op.id !== 'string') continue;
if (typeof op.kind === 'string') kinds.set(op.id, op.kind);
if (isExternalNode(op)) external.add(op.id);
}
return {
nodeIds: ops
.filter(op => op.type === 'UpsertNode' || op.type === 'DeleteNode')
.map(op => op.id)
.filter((id): id is string => typeof id === 'string'),
kinds,
external,
};
}
if (!Array.isArray(data.entityIds)) return null;
if (!Number.isInteger(data.nodeOpCount)) return null;
const nodeCount = data.nodeOpCount as number;
const entityIds = data.entityIds.filter((id): id is string => typeof id === 'string');
if (nodeCount < 0 || entityIds.length < nodeCount) return null;
return {
nodeIds: entityIds.slice(0, nodeCount),
kinds: new Map(),
external: new Set(),
};
}
/**
* How many leading candidates `loadStoredPatchEntities` reads and merges: the
* current extractor and the one before it. An extractor rebuild leaves its
* orphans in the previous extractor's patch, and a graph a pre-fix rebuild
* polluted holds them there too. There is no batch patch GET, so merging all
* ~30 candidates would cost ~30 reads per reconciled file.
*/
const RECONCILE_MANIFEST_WINDOW = 2;
async function loadStoredPatchEntities(
client: Pick<IxClient, 'getPatch'>,
patchIds: string[],
window: number = RECONCILE_MANIFEST_WINDOW,
): Promise<StoredPatchEntities> {
const windowIds = patchIds.slice(0, window);
const windowResults = await Promise.all(windowIds.map(async patchId => {
try {
return { patchId, raw: await client.getPatch(patchId) as unknown, failed: false, error: undefined as unknown };
} catch (err) {
if (isNotFoundError(err)) return { patchId, raw: null, failed: false, error: undefined as unknown };
return { patchId, raw: null, failed: true, error: err };
}
}));
// Results are walked in candidate order. A backend error before any patch
// was found fails the reconcile, as the first-hit walk always did. One on a
// later window patch only loses its extra ids: the patch found first is the
// one a first-hit walk would have used, so the file reconciles at least as
// fully as it did before the merge, and the skip is reported, not fatal.
let primary: StoredPatchEntities | null = null;
for (const result of windowResults) {
if (result.failed) {
if (primary === null) throw result.error;
primary.extraSkipped = `patch ${result.patchId} unread: ${result.error instanceof Error ? result.error.message : String(result.error)}`;
continue;
}
if (result.raw === null) continue;
const entities = readStoredPatchEntities(result.raw);
if (primary === null) {
if (!entities) throw new Error(`Patch ${result.patchId} has no entity manifest`);
primary = {
nodeIds: [...entities.nodeIds],
kinds: new Map(entities.kinds),
external: new Set(entities.external),
extraOnly: new Set(),
};
continue;
}
if (!entities) continue;
const seen = new Set(primary.nodeIds);
for (const id of entities.nodeIds) {
if (seen.has(id)) continue;
seen.add(id);
primary.nodeIds.push(id);
primary.extraOnly!.add(id);
}
for (const [id, kind] of entities.kinds) if (!primary.kinds.has(id)) primary.kinds.set(id, kind);
for (const id of entities.external) primary.external.add(id);
}
if (primary) return primary;
for (const patchId of patchIds.slice(window)) {
try {
const entities = readStoredPatchEntities(await client.getPatch(patchId));
if (entities) return entities;
throw new Error(`Patch ${patchId} has no entity manifest`);
} catch (err) {
if (!isNotFoundError(err)) throw err;
}
}
throw new Error('Previous source patch was not found');
}
interface ReconcileOptions {
/** Told when a later stored manifest could not be read and was left out. */
onWindowSkipped?: (reason: string) => void;
/**
* How many leading candidates to read and merge (default
* RECONCILE_MANIFEST_WINDOW). 1 is the first-hit walk: one read when the
* current candidate exists, which is what an ordinary edit pays.
*/
window?: number;
}
export async function reconcileRemovedEntities(
client: Pick<IxClient, 'getPatch' | 'entity'>,
patch: GraphPatchPayload,
previousPatchIds: string[],
dependentSourceUris?: Set<string>,
// `ix map` does not write chunks: stripMapModeOps drops every chunk UpsertNode
// from the patch. It does NOT drop DeleteNode, and chunk ids hash the chunk's
// start line, so any edit that shifts lines makes every downstream chunk look
// removed. Reconciling without this flag therefore had `ix map` after an
// `ix ingest` delete the file's chunks and never recreate them — the cheaper
// command silently destroying embedding data the expensive one built. Chunks
// are not map mode's to reconcile, so in map mode they are left alone.
mapMode = false,
// Bounds the entity lookups. One limiter shared by every file a run
// reconciles keeps the whole run to its slots, however many files run at once.
lookupLimit: Limiter = task => task(),
options: ReconcileOptions = {},
): Promise<GraphPatchPayload> {
const previous = await loadStoredPatchEntities(client, previousPatchIds, options.window);
if (previous.extraSkipped) options.onWindowSkipped?.(previous.extraSkipped);
const currentNodeIds = new Set(
patch.ops
.filter(op => op.type === 'UpsertNode')
.map(op => op['id'])
.filter((id): id is string => typeof id === 'string'),
);
const currentEdgeIds = new Set(
patch.ops
.filter(op => op.type === 'UpsertEdge')
.map(op => op['id'])
.filter((id): id is string => typeof id === 'string'),
);
// A chunk the stored patch names as one is skipped in map mode without a
// lookup; patches stored without their ops still need the lookup for it.
const candidateNodeIds = previous.nodeIds.filter(id =>
!currentNodeIds.has(id) && !(mapMode && previous.kinds.get(id) === 'chunk'));
const removedNodeIds: string[] = [];
const removedEdgeIds = new Set<string>();
const CHUNK_PREDICATES = new Set(['CONTAINS_CHUNK', 'NEXT']);
// The lookups run concurrently; their results are walked in candidate order,
// so the patch comes out the same however they complete.
const lookups = await Promise.all(candidateNodeIds.map(nodeId => lookupLimit(async () => {
try {
return { nodeId, entity: await client.entity(nodeId) };
} catch (err) {
if (!isNotFoundError(err)) throw err;
return { nodeId, entity: null };
}
})));
for (const { nodeId, entity } of lookups) {
if (entity === null) {
// An id only the second window manifest named, already absent: nothing
// to delete, and the patch stays free of extra deletes.
if (previous.extraOnly?.has(nodeId)) continue;
// Already absent. Emitting the delete anyway keeps the patch a complete
// statement of intent and costs nothing.
removedNodeIds.push(nodeId);
continue;
}
// The lookup is already being made for the incident edges, so the kind
// costs nothing extra.
if (mapMode && (entity.node as { kind?: unknown } | undefined)?.kind === 'chunk') {
continue;
}
const edges = (entity.edges ?? []).map(edge => {
const edgeRecord = edge as {
id?: unknown;
predicate?: unknown;
provenance?: { sourceUri?: unknown; source_uri?: unknown };
};
return {
id: edgeRecord.id,
predicate: edgeRecord.predicate,
sourceUri: edgeRecord.provenance?.sourceUri ?? edgeRecord.provenance?.source_uri,
};
});
// An external package node is shared by every file in the workspace that
// calls into the package. This file no longer writing it does not remove
// it: while another file's edges still reach it, the node and those edges
// stay, and only this file's own edges to it go.
const shared = (previous.external.has(nodeId) || isExternalNode(entity.node))
&& edges.some(edge => edge.sourceUri !== patch.source.uri);
if (shared) {
for (const edge of edges) {
if (typeof edge.id === 'string' && edge.sourceUri === patch.source.uri && !currentEdgeIds.has(edge.id)) {
removedEdgeIds.add(edge.id);
}
}
continue;
}
removedNodeIds.push(nodeId);
for (const { id: edgeId, predicate, sourceUri } of edges) {
const isChunkEdge = typeof predicate === 'string' && CHUNK_PREDICATES.has(predicate);
if (typeof edgeId === 'string' && !currentEdgeIds.has(edgeId) && !(mapMode && isChunkEdge)) {
removedEdgeIds.add(edgeId);
}
if (
dependentSourceUris &&
typeof sourceUri === 'string' &&
sourceUri !== patch.source.uri
) {
dependentSourceUris.add(sourceUri);
}
}
}
const nodeOps = patch.ops.filter(op => op.type === 'UpsertNode' || op.type === 'DeleteNode');
const edgeOps = patch.ops.filter(op => op.type === 'UpsertEdge' || op.type === 'DeleteEdge');
const otherOps = patch.ops.filter(op =>
op.type !== 'UpsertNode' && op.type !== 'DeleteNode' &&
op.type !== 'UpsertEdge' && op.type !== 'DeleteEdge'
);
return {
...patch,
ops: [
...removedNodeIds.map(id => ({ type: 'DeleteNode', id })),
...nodeOps,
...[...removedEdgeIds].map(id => ({ type: 'DeleteEdge', id })),
...edgeOps,
...otherOps,
],
};
}
/**
* The same-hash orphan sweep (IN-20). An extractor change rebuilds every file
* with unchanged bytes, and the nodes the old extractor wrote and the new one
* does not stay live. The sweep finds them from the old extractor's stored
* patch for the file -- one read per file -- and checks every candidate of a
* batch with one batched node read, then adds a DeleteNode for each that is
* still live, was last written by this file, and is neither an external
* package node (shared by every file calling into the package) nor, under
* `ix map`, a chunk. It costs nothing per node, and a run's budget bounds it.
*/
export interface OrphanSweepBudget {
maxMs: number;
maxRequests: number;
/** Wall time the sweep has spent so far in this run. */
usedMs: number;
/** Stored-patch and batched node reads made so far in this run. */
requests: number;
/** Files the sweep skipped because the budget ran out. */
skipped: number;
}
/** Default time budget of a run's orphan sweep (`IX_ORPHAN_SWEEP_MAX_MS`). */
export const ORPHAN_SWEEP_MAX_MS = 60_000;
/** Default request budget of a run's orphan sweep (`IX_ORPHAN_SWEEP_MAX_REQUESTS`). */
export const ORPHAN_SWEEP_MAX_REQUESTS = 25_000;
export function orphanSweepBudget(env: NodeJS.ProcessEnv = process.env): OrphanSweepBudget {
const read = (raw: string | undefined, fallback: number): number => {
if (raw === undefined || raw.trim() === '') return fallback;
const n = Number(raw);
return Number.isFinite(n) && n >= 0 ? n : fallback;
};
return {
maxMs: read(env.IX_ORPHAN_SWEEP_MAX_MS, ORPHAN_SWEEP_MAX_MS),
maxRequests: read(env.IX_ORPHAN_SWEEP_MAX_REQUESTS, ORPHAN_SWEEP_MAX_REQUESTS),
usedMs: 0,
requests: 0,
skipped: 0,
};
}
export interface OrphanSweepFile {
/** The file's patch as built, before map mode strips it. */
patch: GraphPatchPayload;
/**
* The stored patch ids, one per earlier extractor, that may name the file's
* orphans for its (unchanged) hash: usually just the previous extractor's.
*/
previousPatchIds: string[];
}
/**
* The backend does not serve `POST /v1/nodes`, or answers it without the
* provenance the ownership check needs: the sweep is off for the run.
*/
export class BatchNodeReadsUnsupported extends Error {}
/** `POST /v1/nodes` takes at most this many ids. */
const NODES_PER_READ = 1000;
/** Node ids are UUIDs; the batched read refuses the whole request over one that is not. */
const NODE_ID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
/**
* The node ids a stored patch upserted, with the kinds and external flags it
* recorded. A patch stored with its ops (`/v1/patch`) has all three; a bulk
* patch keeps only ids (`nodeIds`, or the older `entityIds` manifest), which
* is why the batched read checks kind and ownership.
*/
function storedUpserts(raw: unknown): { ids: string[]; kinds: Map<string, string>; external: Set<string> } | null {
const data = (raw as { data?: unknown } | null)?.data as { ops?: unknown; nodeIds?: unknown } | undefined;
if (!data) return null;
if (Array.isArray(data.ops)) {
const ids: string[] = [];
const kinds = new Map<string, string>();
const external = new Set<string>();
for (const op of data.ops as Array<{ type?: unknown; id?: unknown; kind?: unknown; attrs?: unknown }>) {
if (op.type !== 'UpsertNode' || typeof op.id !== 'string') continue;
ids.push(op.id);
if (typeof op.kind === 'string') kinds.set(op.id, op.kind);
if (isExternalNode(op)) external.add(op.id);
}
return { ids, kinds, external };
}
if (Array.isArray(data.nodeIds)) {
return { ids: data.nodeIds.filter((id): id is string => typeof id === 'string'), kinds: new Map(), external: new Set() };
}
const manifest = readStoredPatchEntities(raw);
return manifest ? { ids: manifest.nodeIds, kinds: manifest.kinds, external: manifest.external } : null;
}
/** The file that last wrote a node, from the provenance the backend keeps. */
function nodeSourceUri(node: GraphNode): unknown {
const provenance = node.provenance as { sourceUri?: unknown; source_uri?: unknown } | undefined;
return provenance?.sourceUri ?? provenance?.source_uri;
}
/**
* One node the file's patch upserts, read beside the orphan candidates to tell
* whether a patch for these very bytes and this extractor already landed: the
* file node if there is one, else the first other non-chunk node.
*/
function currentPatchProbe(patch: GraphPatchPayload): string | undefined {
let fallback: string | undefined;
for (const op of patch.ops) {
if (op.type !== 'UpsertNode' || typeof op['id'] !== 'string' || !NODE_ID_SHAPE.test(op['id'])) continue;
if (op['kind'] === 'chunk' || isExternalNode(op)) continue;
if (op['kind'] === 'file') return op['id'];
fallback ??= op['id'];
}
return fallback;
}
/**
* Whether the backend reports `node` last written by a patch for `patch`'s
* file, hash and extractor. The bulk route rewrites every node a patch
* upserts with that patch's provenance, so this holds once such a patch has
* landed -- under its deterministic id or a salted one.
*/
function writtenBy(node: GraphNode | undefined, patch: GraphPatchPayload): boolean {
if (!node || (node.deletedRev !== undefined && node.deletedRev !== null)) return false;
const provenance = node.provenance as { extractor?: unknown; sourceHash?: unknown; source_hash?: unknown } | undefined;
return nodeSourceUri(node) === patch.source.uri
&& provenance?.extractor === patch.source.extractor
&& (provenance?.sourceHash ?? provenance?.source_hash) === patch.source.sourceHash;
}
function isUnsupportedRouteError(error: unknown): boolean {
return /^(?:Error:\s*)?(?:404|405|501):/.test(String(error));
}
/**
* The orphan node ids to delete for each file, in `files` order. Never one a
* patch of this batch upserts (`batchUpserts`), and never one the backend does
* not report live, last written by this very file, non-external and (in map
* mode) not a chunk. A file whose reads fail, or that the budget no longer
* covers, gets none: the sweep is best effort and the file commits as built.
* Throws `BatchNodeReadsUnsupported` when the backend has no batched node read.
*/
export async function sweepOrphanNodes(
client: Pick<IxClient, 'getPatch' | 'nodes'>,
files: OrphanSweepFile[],
opts: {
mapMode: boolean;
budget: OrphanSweepBudget;
batchUpserts?: Set<string>;
concurrency?: number;
onSkip?: (file: OrphanSweepFile, reason: string) => void;
/**
* Told of each file the sweep did not finish: past the budget, or a read
* failed. Every other file was swept (or had nothing to sweep).
*/
onUnswept?: (file: OrphanSweepFile) => void;
/**
* Told of each file given orphans whose current patch the backend may
* already hold: one node of it (`currentPatchProbe`, read in the same
* batched node read) was last written by this file at this hash and
* extractor, or could not be read. A patch id the backend already holds
* is answered `Idempotent` and its deletes dropped, so such a file's
* deletes need a salted id.
*/
onWrittenBefore?: (file: OrphanSweepFile) => void;
now?: () => number;
},
): Promise<string[][]> {
const now = opts.now ?? (() => performance.now());
const { budget } = opts;
const started = now();
const spent = (): number => budget.usedMs + (now() - started);
const result: string[][] = files.map(() => []);
try {
const candidates = await mapLimit(files, opts.concurrency ?? RECONCILE_CONCURRENCY, async (file): Promise<string[] | null> => {
if (file.previousPatchIds.length === 0) return null;
if (budget.requests >= budget.maxRequests || spent() >= budget.maxMs) {
budget.skipped++;
opts.onUnswept?.(file);
return null;
}
budget.requests += file.previousPatchIds.length;
const reads = await Promise.all(file.previousPatchIds.map(async patchId => {
try {
return { raw: await client.getPatch(patchId) as unknown, error: null as string | null };
} catch (err) {
return { raw: null, error: isNotFoundError(err) ? null : (err instanceof Error ? err.message : String(err)) };
}
}));
const failed = reads.find(r => r.error !== null);
if (failed) {
opts.onSkip?.(file, failed.error!);
opts.onUnswept?.(file);
return null;
}
const current = new Set(
file.patch.ops.filter(op => op.type === 'UpsertNode').map(op => op['id']).filter((id): id is string => typeof id === 'string'),
);
const ids = new Set<string>();
for (const { raw } of reads) {
const stored = raw === null ? null : storedUpserts(raw);
if (!stored) continue;
for (const id of stored.ids) {
if (!current.has(id)
&& !opts.batchUpserts?.has(id)
&& NODE_ID_SHAPE.test(id)
&& !stored.external.has(id)
&& !(opts.mapMode && stored.kinds.get(id) === 'chunk')) ids.add(id);
}
}
return [...ids];
});
const unique = [...new Set(candidates.flatMap(c => c ?? []))];
if (unique.length === 0) return result;
const probes = files.map((file, i) =>
opts.onWrittenBefore && candidates[i] && candidates[i]!.length > 0 ? currentPatchProbe(file.patch) : undefined);
const seen = new Set(unique);
for (const probe of probes) {
if (probe !== undefined && !seen.has(probe)) {
seen.add(probe);
unique.push(probe);
}
}
const live = new Map<string, GraphNode>();
const unread = new Set<string>();
let unreadReason = '';
for (let i = 0; i < unique.length; i += NODES_PER_READ) {
const ids = unique.slice(i, i + NODES_PER_READ);
budget.requests++;
try {
const answer = await client.nodes(ids, 'full');
for (const node of answer?.nodes ?? []) {
if (node && typeof node.id === 'string') live.set(node.id, node);
}
} catch (err) {
if (isUnsupportedRouteError(err)) {
throw new BatchNodeReadsUnsupported('the backend does not serve the batched node read (POST /v1/nodes)');
}
for (const id of ids) unread.add(id);
unreadReason = err instanceof Error ? err.message : String(err);
}
}
// Ownership is read from each node's provenance. A backend whose batched
// read leaves it out would make every candidate look foreign, and the
// sweep a silent no-op; say so and turn it off instead.
for (const node of live.values()) {
if (typeof nodeSourceUri(node) !== 'string') {
throw new BatchNodeReadsUnsupported('the batched node read (POST /v1/nodes) returned nodes without provenance.sourceUri');
}
}
files.forEach((file, i) => {
const ids = candidates[i];
if (!ids || ids.length === 0) return;
if (ids.some(id => unread.has(id))) {
opts.onSkip?.(file, `node read failed: ${unreadReason}`);
opts.onUnswept?.(file);
return;
}
result[i] = ids.filter(id => {
const node = live.get(id);
if (!node || (node.deletedRev !== undefined && node.deletedRev !== null)) return false;
if (isExternalNode(node)) return false;
if (opts.mapMode && node.kind === 'chunk') return false;
return nodeSourceUri(node) === file.patch.source.uri;
});
const probe = probes[i];
if (result[i]!.length > 0 && probe !== undefined && (unread.has(probe) || writtenBy(live.get(probe), file.patch))) {
opts.onWrittenBefore?.(file);
}
});
return result;
} finally {
budget.usedMs += now() - started;
}
}
/**
* Files reconciled at once, and entity lookups in flight across all of them:
* the graph walkers' cap (`EXPAND_CONCURRENCY`), for the same reason.
*/
const RECONCILE_CONCURRENCY = 8;
/**
* Whether `patch` must go to `/v1/patch` on its own. Only a patch that deletes,
* and only against a backend whose bulk route is not known to apply deletes.
* `/v1/patch` does not sweep the file's old edges, so a removed call between
* two surviving symbols stayed live there; the bulk route sweeps them.
*/
export function patchRequiresPerFileCommit(patch: GraphPatchPayload, bulkDeletes = false): boolean {
if (bulkDeletes) return false;
return patch.ops.some(op => op.type === 'DeleteNode' || op.type === 'DeleteEdge');
}
/**
* The first release whose bulk route applies DeleteNode/DeleteEdge in the
* patch's own workspace, by indexed sweep and primary key (Ix-memory #211,
* #237): checked live on that release before delete-bearing patches moved there.
*/
export const BULK_DELETES_SINCE = '1.0.32';
/**
* Whether the backend's bulk route can take delete-bearing patches. Decided by
* the release the backend reports; one that reports none (older than
* Ix-memory#157, or not built by the release pipeline) keeps the per-file
* route. `IX_BULK_DELETES=1` or `=0` overrides for such a build.
*/
export function backendAppliesBulkDeletes(
health: { release_version?: unknown } | null | undefined,
env: NodeJS.ProcessEnv = process.env,
): boolean {
const forced = env.IX_BULK_DELETES;
if (forced === '1') return true;
if (forced === '0') return false;
return releaseAtLeast(health, BULK_DELETES_SINCE) === true;
}
/**
* Whether the release the backend reports is at least `min`: null when it
* reports none, or one that does not parse.
*/
function releaseAtLeast(health: { release_version?: unknown } | null | undefined, min: string): boolean | null {
const release = health?.release_version;
if (typeof release !== 'string') return null;
const parse = (v: string): number[] | null => {
const m = /^v?(\d+)\.(\d+)\.(\d+)/.exec(v.trim());
return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : null;
};
const have = parse(release);
const need = parse(min)!;
if (have === null) return null;
for (let i = 0; i < 3; i++) if (have[i] !== need[i]) return have[i] > need[i];
return true;
}
/** The first release that serves `POST /v1/nodes` (Ix-memory batch reads). */
export const BATCH_NODE_READS_SINCE = '1.0.33';
/**
* Why the same-hash orphan sweep (IN-20) cannot run against this backend, or
* null when it can. The sweep adds DeleteNode ops to rebuild patches, which
* only the bulk route of a backend that applies deletes takes without falling
* back to one `/v1/patch` per file, and it checks its candidates with one
* batched `POST /v1/nodes`. A backend that reports no release but is forced
* onto bulk deletes (`IX_BULK_DELETES=1`) is tried; if it does not serve the
* batch read, the sweep turns itself off for the run.
*/
export function orphanSweepUnsupportedReason(
health: { release_version?: unknown } | null | undefined,
env: NodeJS.ProcessEnv = process.env,
): string | null {
const release = health?.release_version;
const described = typeof release === 'string' ? `release ${release}` : 'unversioned';
if (!backendAppliesBulkDeletes(health, env)) {
return `the backend (${described}) is older than ${BATCH_NODE_READS_SINCE} or does not apply deletes in bulk`;
}
if (releaseAtLeast(health, BATCH_NODE_READS_SINCE) === false) {
return `the backend (${described}) is older than ${BATCH_NODE_READS_SINCE}`;
}
return null;
}
export function planDeletedFileRecovery(
projectRoot: string,
filePaths: string[],
deletedFiles: Map<string, string[]>,
): {
previousDeletedFiles: Map<string, string[]>;
nextDeletedFiles: Map<string, string[]>;
recreatedPaths: string[];
forceReingestPaths: Set<string>;
} {
const previousDeletedFiles = new Map<string, string[]>();
for (const [deletedPath, dependents] of deletedFiles) {
const absolutePath = nodePath.isAbsolute(deletedPath)
? nodePath.resolve(deletedPath)
: nodePath.resolve(projectRoot, deletedPath);
previousDeletedFiles.set(absolutePath, dependents);
}
const nextDeletedFiles = new Map(previousDeletedFiles);
const filePathSet = new Set(filePaths.map(filePath => nodePath.resolve(filePath)));
const recreatedPaths = [...previousDeletedFiles.keys()].filter(filePath =>
filePathSet.has(filePath)
);
const forceReingestPaths = new Set<string>();
for (const recreatedPath of recreatedPaths) {
for (const dependent of previousDeletedFiles.get(recreatedPath) ?? []) {
const absoluteDependent = nodePath.isAbsolute(dependent)
? nodePath.resolve(dependent)
: nodePath.resolve(projectRoot, dependent);
if (filePathSet.has(absoluteDependent)) forceReingestPaths.add(absoluteDependent);
}
nextDeletedFiles.delete(recreatedPath);
}
return { previousDeletedFiles, nextDeletedFiles, recreatedPaths, forceReingestPaths };
}
/**
* A bulk commit the server accepted only in part, or whose patches all landed
* under a different grouping.
*
* Re-sending the same patches cannot fix this: the group id is derived from the
* patch ids, so an identical retry lands on the identical rejected group. The
* way through is to commit only what is missing, which the backend tells us by
* naming the ids that landed (`committed_patch_ids`).
*
* Matched on the message because the client throws `${status}: ${body}`, so the
* JSON body is the tail of the error string.
*/
export function isBulkPartiallyCommittedError(err: unknown): boolean {
return String(err).toLowerCase().includes('partially committed') || isBulkPlanMismatchError(err);
}
/**
* Ix-memory's 409 for a bulk whose every patch id was already committed by a
* different request plan (BulkPlanMismatchException, Ix-memory#273): the body
* is `{error: "conflict", message, committed_patch_ids, committed_count}`,
* without the "partially committed" text, so it is recognised by its status
* and the ids it names. Those ids landed, exactly as in a partial commit, and
* are handled the same way: counted as committed, never re-bulked.
*/
export function isBulkPlanMismatchError(err: unknown): boolean {
return /(^|\s)409: \{/.test(String(err)) && parseBulkCommittedPatchIds(err) !== undefined;
}
/**
* The patch ids the server says already landed, or undefined when it did not
* say — a backend older than this field, or a body we cannot parse. Undefined
* means "fall back", never "nothing landed": treating an unparseable body as an
* empty set would re-send patches that are already committed.
*/
export function parseBulkCommittedPatchIds(err: unknown): Set<string> | undefined {
const text = String(err);
const start = text.indexOf('{');
const end = text.lastIndexOf('}');
if (start === -1 || end <= start) return undefined;
try {
const body = JSON.parse(text.slice(start, end + 1)) as { committed_patch_ids?: unknown };
const ids = body.committed_patch_ids;
if (!Array.isArray(ids)) return undefined;
const parsed = ids.filter((id): id is string => typeof id === 'string');
return parsed.length === ids.length ? new Set(parsed) : undefined;
} catch {
return undefined;
}
}
/**
* Does this commit failure say anything about the BACKEND? (Ix#560)
*
* Only failures that indict the backend may feed the run-wide cutoff, because
* the cutoff abandons every remaining patch in the run.
*
* Excluded, and each for its own reason:
*
* - the run's own deadline. That is our wall clock, not the backend's
* answer, and `describeCommitOutcome` has a whole branch to keep them
* apart. Note this is the DEADLINE, not `isAbortError`: a per-request
* `AbortSignal.timeout` also raises TimeoutError, and a backend that
* HANGS produces nothing else. Excluding those made the cutoff inert in
* exactly the saturation shape it exists for -- a stalled ArangoDB is not
* obliged to answer 500, and when it does not, the timeout IS the signal.
* - payload-too-large, which is about this patch and not the server; five
* oversized generated files in a row must not stop the whole repo.
*
* A lock conflict is NOT excluded, though an earlier revision excluded it.
* `timeout waiting to lock key` and `error: 1200` are what a RocksDB-backed
* ArangoDB emits under exactly the sustained write contention this cutoff is
* for, so excluding them made it inert in a real saturation mode -- and, worse,
* those same patterns are in the RETRY list, so every doomed patch was being
* multiplied by `retryOnConflict` instead of stopped.
*
* The worry that motivated the exclusion -- two overlapping `ix map` runs
* contending -- is already answered upstream of here: by the time an error
* reaches this function, `retryOnConflict` has spent six backed-off attempts
* on it. A conflict that survives all six is not transient contention.
*/
export function commitFailureIndictsBackend(err: unknown, runDeadlineExpired: boolean): boolean {
if (runDeadlineExpired) return false;
if (isPayloadTooLargeError(err)) return false;
return true;
}
/**
* The per-file commit's retry: lock conflicts and transport drops only.
* Transient statuses (502/503/504/429) are retried at the BULK sites, which
* call `retryCommit` themselves; retrying them per patch would multiply a
* per-bulk wait budget by the number of patches.
*/
async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Promise<T> {
return retryCommit(fn, { maxRetries, transient: false });
}
/**
* Re-send a commit that lost the backend's base-rev race.
*
* `BaseRevMismatch` means the backend read the latest rev outside the
* transaction and another commit moved it before this one ran, so nothing was
* written. Patch ids are deterministic, so sending the same patches again is
* safe, and every round of a race has a winner: two `ix map` runs against one
* backend (two repos, two editors, a benchmark at concurrency 2) drain each
* other's contention within a few backed-off attempts. Without this, the loser
* counted the whole chunk as failed and left it for the next run -- 477 to
* 1,730 files of a 1,200-2,500-file repo missing from each of two concurrent
* maps, which then exited 1.
*
* Returns the last result; a caller that still sees `BaseRevMismatch` has
* exhausted `maxRetries` and falls back to counting the chunk as failed.
*/
export async function retryOnBaseRevRace<T extends { status?: string }>(
fn: () => Promise<T>,
maxRetries: number,
onRetry?: (attempt: number) => void,
): Promise<T> {
for (let attempt = 0; ; attempt++) {
const result = await fn();
if (result.status !== COMMIT_STATUS_BASE_REV_MISMATCH || attempt >= maxRetries) return result;
onRetry?.(attempt + 1);
// Jittered so two racing runs stop re-colliding in lockstep. The window is
// the other side's transaction, which for a bulk runs to seconds.
const delay = 50 * (1 << Math.min(attempt, 5)) + Math.random() * 100;
await new Promise<void>(r => setTimeout(r, delay));
}
}
// Every entry here means the same thing operationally: this batch was refused for
// being too big to apply as one unit, so committing fewer patches per request can
// still succeed. That is what makes bisecting the right recovery.
const PAYLOAD_TOO_LARGE_PATTERNS = [
// A proxy / ingress refused the request body before the backend saw it.
'payload too large',
'request entity too large',
'content too large',
'request body too large',
// The backend accepted the body, then ArangoDB refused the write: a bulk commit
// runs as ONE exclusive transaction, and Arango caps a transaction at 512MB by
// default (error 32, surfaced as a 500 — not a 413):
// "AQL: Maximal transaction size limit of 536870912 bytes is reached"
// Bisecting splits the group across two transactions, so each half fits.
// Without this, a big-repo save fell through to the per-file path and a single
// 1,000-file chunk became 1,000 serialized commits — Ix#516.
'maximal transaction size',
];
/** How many oversized files a summary names; the count covers the rest. */
export const FILES_TOO_LARGE_LISTED = 20;
/**
* The oversized files in the stat loop's record, workspace-relative, sorted,
* and cut to `FILES_TOO_LARGE_LISTED`. Exported for tests.
*/
export function listTooLargeFiles(
record: ReadonlyMap<string, 'empty' | 'tooLarge'> | undefined,
toRelative: (absPath: string) => string,
): string[] {
if (record === undefined) return [];
const paths: string[] = [];
for (const [path, why] of record) if (why === 'tooLarge') paths.push(toRelative(path));
return paths.sort().slice(0, FILES_TOO_LARGE_LISTED);
}
export function isPayloadTooLargeError(err: unknown): boolean {
const responseError = err as { status?: unknown; statusCode?: unknown } | null;
if (responseError?.status === 413 || responseError?.statusCode === 413) return true;
const message = String(err).toLowerCase();
if (/^(?:error:\s*)?413(?:\b|:)/.test(message)) return true;
return PAYLOAD_TOO_LARGE_PATTERNS.some(pattern => message.includes(pattern));
}
/**
* What a 413 from the bulk writer's own estimate says one request should hold
* (Ix-memory#232: `suggestedMaxPatches` in the body, and "send at most N
* patches per request" in its message). Absent from a 413 that ArangoDB raised
* mid-write, from the backend's body cap, or from a proxy's.
*
* This is the only 413 that says the request held too many OPS. The others
* are about bytes, or say nothing, and a run must not learn an op count from
* them -- see `learnBulkLimits` in ingestFiles.
*/
export function parseSuggestedMaxPatches(err: unknown): number | undefined {
const valid = (n: unknown): number | undefined =>
typeof n === 'number' && Number.isInteger(n) && n >= 1 ? n : undefined;
const text = String(err);
const start = text.indexOf('{');
const end = text.lastIndexOf('}');
if (start !== -1 && end > start) {
try {
const n = valid((JSON.parse(text.slice(start, end + 1)) as { suggestedMaxPatches?: unknown }).suggestedMaxPatches);
if (n !== undefined) return n;
} catch {
/* not JSON: fall through to the message */
}
}
const stated = /send at most (\d+) patches per request/i.exec(text);
return stated ? valid(Number(stated[1])) : undefined;
}
/**
* Ops per bulk request unless `IX_COMMIT_MAX_OPS` says otherwise. The backend's
* transaction grows with them, about 4.3 KB a document against ArangoDB's
* 512 MB cap, so a 500-file batch of a large repository could not fit and
* failed after 14-20 s.
*
* The bound that matters is the bulk writer's own pre-flight budget (Ix-memory
* `BulkTransactionBudget`, from 1.0.32): 80% of 512 MB at 4,300 bytes per
* upserted document, doubled once the graph holds anything, which is about
* 49,900 documents on every map but a backend's first. Over it, a request is
* refused with a 413 before any write; this default stays under it.
*/
export const DEFAULT_COMMIT_MAX_OPS = 45_000;
/** Per-request bounds on a bulk commit. A run lowers them when a commit is too large. */
export interface BulkLimits {
maxFiles: number;
/** Ops across the request's patches: the backend's transaction grows with them. */
maxOps: number;
/**
* Serialized bytes across the request's patches, once a 413 that did not
* blame the op count has been seen: a body cap (Ix-memory's
* `IX_BULK_MAX_BODY_MB`), a proxy's limit, or one that gave no reason.
* Absent until then.
*/
maxBytes?: number;
}
/**
* Cut `items` into bulk requests in order, each within `limits`. An item over
* `maxOps` (or `maxBytes`) on its own still goes, alone: one patch cannot be
* split. `bytesOf` is consulted only while `limits.maxBytes` is set.
*/
export function cutBulkChunks<T>(
items: T[],
opsOf: (item: T) => number,
limits: BulkLimits,
bytesOf?: (item: T) => number,
): T[][] {
const byteBound = limits.maxBytes !== undefined && bytesOf !== undefined ? limits.maxBytes : undefined;
const chunks: T[][] = [];
let current: T[] = [];
let ops = 0;
let bytes = 0;
for (const item of items) {
const n = opsOf(item);
const b = byteBound !== undefined && bytesOf ? bytesOf(item) : 0;
if (
current.length > 0 &&
(current.length >= limits.maxFiles || ops + n > limits.maxOps || (byteBound !== undefined && bytes + b > byteBound))
) {
chunks.push(current);
current = [];
ops = 0;
bytes = 0;
}
current.push(item);
ops += n;
bytes += b;
}
if (current.length > 0) chunks.push(current);
return chunks;
}
export async function commitBulkWithPayloadSplit<T, R>(
items: T[],
handlers: {
commitBulk: (batch: T[]) => Promise<R>;
onBulkCommitted: (batch: T[], result: R) => void;
commitIndividually: (batch: T[], error: unknown, opts?: { replay?: boolean }) => Promise<void>;
onSplit?: (batch: T[], error: unknown) => void;
/**
* How to cut a group the backend found too large, when the caller knows
* better than halving: what the 413 suggested, or a size the run has
* learned. Pieces go out in order, one at a time. Fewer than two pieces,
* or absent, and the group is bisected.
*/
splitTooLarge?: (batch: T[], error: unknown) => T[][] | undefined;
/** Cut what is left of a split group to the bounds as they stand now. */
recut?: (batch: T[]) => T[][];
/** Patch id of an item, so a partly-committed group can be resumed. */
patchIdOf?: (item: T) => string | undefined;
onPartialBulk?: (landed: T[], missing: T[], error: unknown) => void;
/**
* Has the caller given up on the backend? (Ix#560)
*
* Consulted before the per-file fallback, and ONLY there. The bulk is
* always sent: one request per group is not the amplification, and it is
* how a backend that recovered mid-run gets to prove it.
* The fallback exists for a bulk that failed for a
* reason specific to the GROUP (too large, partly applied), where sending
* the patches separately is a different request that can succeed. When the
* backend itself is refusing writes, none of that holds: the fan-out is one
* doomed request per patch, serialized, against the very backend that is
* the reason.
*
* Absent, nothing is ever abandoned: the behaviour that shipped.
*/
shouldStop?: () => boolean;
/** `batch` was abandoned unsent. `error` is absent when none was seen. */
onAbandoned?: (batch: T[], error?: unknown) => void;
},
): Promise<void> {
if (items.length === 0) return;
// No cutoff check before a bulk that is actually a GROUP. One request per
// group is not the amplification -- the per-file fallback is, at one request
// per patch -- and sending it is the only way a backend that has recovered
// mid-run can prove it and let the rest of the ingest through. Skipping it
// was strictly worse than useless: the group went to the deferred set and
// came back as N SERIALIZED per-file commits instead of one bulk.
//
// A SINGLE-patch chunk is different, and the difference is the whole issue.
// At `IX_COMMIT_HTTP_MAX_FILES=1` -- which #560's reporters tried, and which
// the header's own table measures -- every chunk is one patch, so "one
// request per group" IS one request per patch and the rule above halves
// main's cost instead of bounding it: a 10,000-file map against a dead
// backend still sent ~10,000 doomed bulks. The cutoff applies to those, and
// recovery is still provable -- the drain's gate lets one probe through every
// few batches for exactly this.
if (items.length === 1 && handlers.shouldStop?.()) {
handlers.onAbandoned?.(items);
return;
}
let result: R;
try {
result = await handlers.commitBulk(items);
} catch (err) {
if (isPayloadTooLargeError(err) && items.length > 1) {
handlers.onSplit?.(items, err);
let pieces = handlers.splitTooLarge?.(items, err);
if (pieces && pieces.length > 1) {
// The rest is re-cut after each piece: a piece that is refused again
// lowers the bounds, and the pieces after it should not each be
// refused at the size it already failed at.
while (pieces.length > 0) {
const [head, ...tail] = pieces;
await commitBulkWithPayloadSplit(head, handlers);
const rest = tail.flat();
pieces = rest.length === 0 ? [] : handlers.recut?.(rest) ?? tail;
}
return;
}
const midpoint = Math.ceil(items.length / 2);
await commitBulkWithPayloadSplit(items.slice(0, midpoint), handlers);
await commitBulkWithPayloadSplit(items.slice(midpoint), handlers);
return;
}
// Part of this group landed. Split it by what the server says it already
// has: replay the landed ones individually (idempotent no-ops that return
// their original revision, so the counters and the mtime baseline stay
// truthful) and re-bulk the rest, which is a different id set and so a
// different group. Without this the whole batch degrades to the per-file
// path, which is serialized — hundreds of sequential commits, the several
// minutes reported in Ix#495.
const landedIds = isBulkPartiallyCommittedError(err) ? parseBulkCommittedPatchIds(err) : undefined;
if (landedIds !== undefined && handlers.patchIdOf) {
const idOf = handlers.patchIdOf;
const landed = items.filter(item => { const id = idOf(item); return id !== undefined && landedIds.has(id); });
const missing = items.filter(item => { const id = idOf(item); return id === undefined || !landedIds.has(id); });
// The server says it already holds every one of them. Re-bulking would
// repeat this exact call, so this still falls through to the per-file
// path -- but flagged as a replay, because that is what it is. Without the
// flag, a tripped breaker counts patches the server has CONFIRMED as
// commit errors, the same over-reporting `replay` exists to prevent on
// the sibling branch (Ix#560).
if (landed.length === items.length && items.length > 0) {
await handlers.commitIndividually(items, err, { replay: true });
return;
}
// Only worth it when the split is real. If nothing is claimed as landed,
// re-bulking `missing` would repeat this exact call.
if (landed.length > 0 && missing.length > 0) {
handlers.onPartialBulk?.(landed, missing, err);
// `replay: true` -- the server has confirmed it already holds these.
// Re-sending them is bookkeeping, not a write the breaker should
// refuse: skipping them would count patches that DID land as commit
// errors and drop them from patchesApplied (Ix#495).
await handlers.commitIndividually(landed, undefined, { replay: true });
await commitBulkWithPayloadSplit(missing, handlers);
return;
}
}
if (handlers.shouldStop?.()) {
handlers.onAbandoned?.(items, err);
return;
}
await handlers.commitIndividually(items, err);
return;
}
handlers.onBulkCommitted(items, result);
}
/**
* Advance the running max revision, ignoring anything that is not a revision.
*
* `result.rev` comes off the wire, and a bare `>` is not safe on it. A string
* "9" clears `> 0` by coercion and becomes the max; the next comparison is then
* string-to-string, so `"10" > "9"` is false and the max walks backwards. That
* value also reaches the user verbatim as the `latestRev` field of
* `ix ingest --format json`, where every consumer reads it as a number.
* Screening it here fixes the max, the JSON contract and the persisted baseline
* in one place, rather than papering over the last of the three.
*/
export function advanceRev(current: number, incoming: unknown): number {
return isRev(incoming) && incoming > current ? incoming : current;
}
/** How often, at most, an extractor re-ingest records its progress to disk. */
const REBUILD_PROGRESS_SAVE_MS = 2_000;
// ---------------------------------------------------------------------------
// Mtime cache — skip readFileSync+sha256 for unchanged files
// ---------------------------------------------------------------------------
/**
* The mtimes to record, plus `carried` -- files this run did not process -- at
* their previous mtimes: deleted files whose cleanup had to be skipped (kept so
* the next run finds them deleted and retries) and files a `--lang` run left
* out (kept so it does not reset their state).
*/
export function carryForwardMtimes(
currentMtimes: Map<string, number>,
previousMtimes: Map<string, number>,
carried: readonly string[],
): Map<string, number> {
if (carried.length === 0) return currentMtimes;
const out = new Map(currentMtimes);
for (const filePath of carried) {
const mtime = previousMtimes.get(filePath);
if (mtime !== undefined && !out.has(filePath)) out.set(filePath, mtime);
}
return out;
}
/**
* The mtimes to record, with each of `retried` back at its previous mtime --
* or left out when it had none -- so the next run sees it as changed and sends
* it again. For files whose patch this run could not get applied (F-01): a
* deleted file comes back too, which is what makes the next run find it
* deleted again.
*/
export function retryOnNextRun(
mtimes: Map<string, number>,
previousMtimes: Map<string, number>,
retried: readonly string[],
): Map<string, number> {
if (retried.length === 0) return mtimes;
const out = new Map(mtimes);
for (const filePath of retried) {
const previous = previousMtimes.get(filePath);
if (previous === undefined) out.delete(filePath);
else out.set(filePath, previous);
}
return out;
}
export function persistIngestBaselineIfClean(
projectRoot: string,
mtimes: Map<string, number>,
currentRev: number,
parseErrors: number,
commitErrors: number,
now?: Date,
deletedFiles: Map<string, string[]> = new Map(),
extractor?: string | null,
notes: BaselineFileNotes = {},
): boolean {
if (!ingestCompletedCleanly(parseErrors, commitErrors)) return false;
return persistIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, notes);
}
/**
* Write the baseline -- whatever errors the run had. The caller has already
* left the files it did not settle at their previous mtimes.
*/
export function persistIngestBaseline(
projectRoot: string,
mtimes: Map<string, number>,
currentRev: number,
now?: Date,
deletedFiles: Map<string, string[]> = new Map(),
extractor?: string | null,
notes: BaselineFileNotes = {},
): boolean {
// An empty mtime map is normally a discovery failure, not an empty repo —
// an over-broad ignore rule, the wrong cwd, a glob that matched nothing —
// and persisting it silently discards the baseline, forcing a full re-ingest
// next run. The original guard rejected every empty map for that reason.
//
// But deleting the last mapped file legitimately produces one, and refusing
// to record that leaves the baseline claiming files that are gone. So the
// guard now asks *why* the map is empty: with deletions recorded this run,
// empty is a real state and is persisted; without them, it is still treated
// as discovery having gone wrong.
if (mtimes.size === 0 && deletedFiles.size === 0) return false;
saveIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, notes);
return true;
}
// ---------------------------------------------------------------------------
// Progress bar
// ---------------------------------------------------------------------------
const PROG_BAR_WIDTH = 25;
const PROG_LINE_WIDTH = 72;
function renderProgressLine(phase: string, current: number, total: number): string {
if (total === 0) {
return ` ${phase}...`.padEnd(PROG_LINE_WIDTH);
}
const pct = Math.min(current / total, 1);
const filled = Math.round(pct * PROG_BAR_WIDTH);
const bar = chalk.cyan('█'.repeat(filled)) + chalk.dim('░'.repeat(PROG_BAR_WIDTH - filled));
const pctStr = `${Math.round(pct * 100)}%`.padStart(4);
const w = total.toString().length;
const count = `${current.toString().padStart(w)} / ${total}`;
return ` ${phase.padEnd(8)} ${bar} ${pctStr} ${count}`.padEnd(PROG_LINE_WIDTH);
}
// ---------------------------------------------------------------------------
// Command registration
// ---------------------------------------------------------------------------
type PathIngestOpts = Parameters<typeof ingestFiles>[1];
/**
* The workspace root `target` ingests into, or undefined when it cannot be
* resolved. `looseFile`: the target is a file in no workspace and no
* repository, which is never registered (see ingestFiles).
*/
function ingestLockRoot(target: string, explicitRoot?: string): { root: string; looseFile: boolean } | undefined {
try {
const resolved = nodePath.resolve(target);
const isDirectory = fs.statSync(resolved).isDirectory();
const { root, source } = resolveIngestRootWithSource(resolved, isDirectory, explicitRoot);
return { root, looseFile: source === 'loose' && !isDirectory };
} catch {
// `ingestFiles` reports a missing or unreadable path itself.
return undefined;
}
}
/**
* `ix ingest <path>`, single-flight per workspace under the lock `ix map`
* holds. Two ingests (or an ingest and a map) over one workspace race on the
* mtime baseline: the second to finish overwrites what the first recorded.
*
* A run that finds the lock held coalesces the way `ix map` does: it asks the
* holder for one more pass (`requestMapRerun`) and exits 0, unless
* IX_MAP_COALESCE_EXIT_CODE asks for another code. It used to exit 1, which
* editor hooks and MCP clients that ingest after every edit reported as a
* failed ingest. The holder takes the request before it lets go: `ix map`
* re-ingests the workspace, and so does a holding `ix ingest` when the
* workspace has a baseline -- an incremental pass, so it costs a stat per
* file. Without a baseline that pass would be a first ingest of the whole
* workspace, so it re-ingests only its own path then.
*/
export async function ingestPathSingleFlight(target: string, opts: PathIngestOpts): Promise<void> {
const lockRoot = ingestLockRoot(target, opts.root);
if (lockRoot === undefined) {
await ingestFiles(target, opts);
return;
}
const { root, looseFile } = lockRoot;
const lock = acquireMapLock(root, `ix ingest ${root}`);
if (lock === null) {
requestMapRerun(root);
if (opts.format === 'json') {
// A JSON consumer gets an object, not an empty stdout it cannot parse.
printJson({ coalesced: true, workspace: root });
} else {
process.stderr.write(chalk.dim(' Another ix map or ix ingest is running for this workspace; it will pick up this change before it exits.\n'));
}
applyRequestedMapCoalesceExitCode();
return;
}
// Holding the lock: a request left before now is covered by this run, which
// has not read the tree yet.
takeMapRerun(root);
try {
await ingestFiles(target, opts);
if (takeMapRerun(root)) {
// One pass at most, as in `ix map`. The waiting run's own flags are not
// known here, so this is a plain incremental pass.
const rerunTarget = loadIngestBaseline(root) !== null ? root : target;
try {
// The rerun may widen a loose file to its whole directory; that must
// not register the directory the file's own ingest would not have.
await ingestFiles(rerunTarget, {
recursive: true, format: 'json', printSummary: false, suppressOutput: true, root: opts.root,
neverRegister: looseFile,
});
} catch (err: any) {
if (opts.format !== 'json') {
process.stderr.write(chalk.dim(` Re-run for an ingest that waited on this one failed: ${err?.message ?? err}\n`));
}
}
}
} finally {
lock.release();
}
}
export function registerIngestCommand(program: Command): void {
program
.command('ingest [path]')
.description('Ingest source files or GitHub data into the knowledge graph')
.option('--path <dir>', 'Path to ingest (alternative to positional argument)')
.option('--no-recursive', 'Do not recurse into subdirectories (recursive is on by default)')
.option('--github <owner/repo>', 'Ingest issues, PRs, and commits from a GitHub repository')
.option('--token <pat>', 'GitHub personal access token')
.option('--since <date>', 'Only fetch items updated after this date (ISO 8601)')
.option('--limit <n>', 'Max items per category (default 50)', '50')
.option('--force', 'Force re-ingest even if files are unchanged (useful after parser upgrades)')
.option('--format <fmt>', 'Output format (text|json|llm)', 'text')
.option('--root <dir>', 'Workspace root directory')
.option('--debug', 'Show phase timing breakdown', false)
.option('--lang <langs>', 'Comma-separated languages to include (e.g. cpp,c or typescript). Aliases: c++=cpp, c#=csharp, py=python, ts=typescript, js=javascript')
.option(
'--exclude <glob>',
'Exclude paths matching this glob (repeatable; same syntax as .ixignore)',
(value: string, previous: string[] = []) => [...previous, value],
[] as string[],
)
.addHelpText('after', '\nExamples:\n ix ingest ./src\n ix ingest --path ./src --force\n ix ingest --path ./rocksdb --lang cpp,c\n ix ingest --github owner/repo\n ix ingest --github owner/repo --since 2026-01-01 --limit 20 --format json\n ix ingest --github owner/repo --token ghp_xxxx\n\nA path inside a mapped workspace (or a git repo) refreshes just that part of it, under that\nworkspace; a directory outside every workspace and repo becomes a workspace of its own (never the\ndefault). A loose file outside them is ingested but registers nothing, so ix read cannot open its neighbours.')
.action(async (positionalPath: string | undefined, opts: {
path?: string; recursive?: boolean; force?: boolean; github?: string; token?: string;
since?: string; limit: string; format: string; root?: string; debug?: boolean; lang?: string;
exclude?: string[];
}) => {
const effectivePath = positionalPath ?? opts.path;
if (opts.github) {
await ingestGitHub(opts);
} else if (effectivePath) {
await ingestPathSingleFlight(effectivePath, opts);
} else {
console.error('Error: provide a <path> or use --github <owner/repo>');
process.exit(1);
}
});
}
export type CommitOutcome =
| { kind: "ok" }
| { kind: "warn"; message: string }
| { kind: "fatal"; message: string };
/**
* The wire value the backend sends for a commit it short-circuited because it
* had already recorded the patch id — `CommitStatus.Idempotent.toString` in
* `PatchCommitRoutes` / `BulkPatchCommitRoutes`.
*/
const COMMIT_STATUS_IDEMPOTENT = 'Idempotent';
/**
* The wire value for a commit the backend refused because the optimistic
* base-rev check lost a race — `CommitStatus.BaseRevMismatch`. It writes
* nothing, exactly like `Idempotent`, but unlike `Idempotent` it means the
* work still needs doing.
*/
const COMMIT_STATUS_BASE_REV_MISMATCH = 'BaseRevMismatch';
/** The stderr note for files skipped because their parse ran past the budget. */
export function describeParseTimeouts(files: readonly string[], sample = 5): string {
const shown = files.slice(0, sample).join(", ");
const more = files.length > sample ? ` and ${files.length - sample} more` : "";
return `Skipped ${files.length} file(s) whose parse ran past the per-file budget: ${shown}${more}. ` +
"They are not in the graph. Raise the budget with IX_PARSE_BUDGET_MS (milliseconds, 0 = none).";
}
/** Minimal local-ingest facts needed by commands that continue after ingestion. */
export interface IngestFilesSummary {
filesDiscovered: number;
patchesApplied: number;
/**
* How many of `patchesApplied` the backend answered `Idempotent` — it had
* already recorded that patch id, so the commit wrote nothing.
*
* Normally uninteresting: `ix ingest --force` re-submits every file (and a
* lost or blank local baseline has the same effect against a backend that
* still holds the patch records), so a repo whose graph is already correct
* legitimately deduplicates everything. Note `ix map --full` is NOT such a
* case — it forwards `full` to the backend map and never sets `force` on the
* ingest below it, so it re-submits nothing.
*
* It matters only when the graph turns out to be EMPTY afterwards, because
* then the deduplication is the reason nothing was written rather than a
* sign that nothing needed to be (#527).
*/
idempotentPatches: number;
/**
* Changed files whose patch the backend answered `Idempotent` (F-01), by
* workspace-relative path. Unlike the rest of `idempotentPatches`, these
* are not harmless: the file differs from what the backend last recorded
* for it, yet its patch id was committed before -- a file reverted to
* earlier bytes, or deleted and restored -- so the backend wrote nothing and
* the graph still shows the content in between. Not counted as errors and
* not in the exit code; the run warns, and the baseline keeps these files
* dirty so the next run sends them again.
*/
replayedChanges: string[];
/**
* Files this run skipped because they were already unchanged — mtime-clean
* against the local baseline, or hash-clean against the backend.
*
* Load-bearing for the #527 diagnosis, not informational. `idempotentPatches`
* only describes the patches a run actually submitted; it says nothing about
* the files the run never submitted, and those are what separate a graph
* deleted under its own patch records from a graph that is simply fine. A run
* in the #527 state skips nothing here: the DB-reset guard clears the mtime
* cache and the hash lookup comes back empty, so every file is re-submitted
* and every one is answered `Idempotent`. A run that skipped even one file as
* unchanged has proof the backend still holds that file's source hash, and
* therefore its nodes.
*
* Counts only the unchanged skips — empty, oversized and minified files are
* skipped for reasons that carry no such proof. It is the same counter the
* stitch-completeness gate reads, which also means it inherits that counter's
* Path-B reset: a run with no baseline re-reads and re-hashes every file, so
* the stat loop's mtime skips did not actually happen and are subtracted
* back out. That is the shape a #527 run has, and it is exactly right here.
*/
filesSkippedAsUnchanged: number;
/** Files over the 1 MB parse limit, left out of the graph. Absent where no run counted them. */
filesTooLarge?: number;
/**
* Which files those were, workspace-relative and sorted, at most
* `FILES_TOO_LARGE_LISTED` of them: `filesTooLarge` is the whole count.
* Absent where no run counted them.
*/
filesTooLargePaths?: string[];
parseErrors: number;
commitErrors: number;
stitchErrors: number;
/**
* Why the cross-workspace stitch was not attempted, if it was not (Ix#568).
*
* A skip is not an error and deliberately does not move the exit code -- but
* CLAUDE.md RULE 5 has hooks running `ix map --silent` and reading the exit
* code, so without a field here an automated consumer sees a wholly
* successful map for up to 15 minutes while cross-repo edges are stale, and
* the only signal is an English sentence on stderr.
*/
stitchSkipped?: string;
/**
* Which rule kept the stitch from happening, for a consumer that must branch.
*
* The prose in `stitchSkipped` is for people. `incomplete` -- an incremental
* map with nothing complete to send -- fires on almost every run, so without
* this a hook watching for the #568 cooldown could only tell the two apart by
* substring-matching English, which `StitchRefusal`'s own doc forbids.
*/
stitchSkippedRule?: StitchRefusal;
/**
* True only when this run cannot have changed the graph: nothing was
* forced (by `--force`, a schema mismatch or an extractor change), the
* workspace was not migrated, the DB-reset guard did not fire, no patch or
* deletion was sent, no stitch was sent, and nothing failed.
*
* `ix map` reads it to decide whether it may reuse its last `/v1/map`
* response. It errs towards false: a run that only might have written
* something must not claim the graph is where it was.
*/
graphUnchanged: boolean;
/**
* True only when this run was an ordinary incremental update of a graph
* whose baseline it trusted: a local baseline existed (Path A), nothing was
* forced or migrated, the DB-reset guard and an extractor rebuild did not
* fire, at least one file changed or was deleted, and nothing failed to
* parse or commit.
*
* `ix map` reads it to decide whether it may defer `/v1/map` to the next
* command that reads regions (CL-13). It errs towards false: a run that
* might have left the graph somewhere unexpected computes the map, so the
* completed-map diagnostics still see every run that had errors.
*/
incremental: boolean;
/**
* Files tree-sitter parsed with ERROR or MISSING nodes. They are in the
* graph, but whatever sat in the broken region may not be. Not counted in
* `parseErrors`, which is files that produced nothing.
*/
filesWithParseErrors: number;
/**
* Files skipped because their parse ran past the per-file budget
* (IX_PARSE_BUDGET_MS, default 10 s), workspace-relative. A subset of the
* unparsed skips, named because a file that times out does so on every run.
*/
parseTimeouts: string[];
}
/**
* The HTTP status `IxClient` put at the front of its `${status}: ${body}`
* error, when there is one. Anchored, so a three-digit run anywhere else in the
* message — a port, a byte count, a timestamp — cannot be read as a status.
*/
function stitchFailureStatus(error: unknown): number | null {
const match = /^(?:Error:\s*)?(\d{3}):/.exec(String(error));
return match ? Number(match[1]) : null;
}
/**
* A backend that does not implement `/v1/stitch` is not a failed stitch.
*
* The stitch call has always been best-effort precisely so an older backend
* keeps working, and it is issued on every full local map. Reporting 404/501
* as a degraded graph would make `ix map` warn and exit 1 on every run against
* such a backend, for a step it was never going to perform.
*/
export function isStitchUnsupported(error: unknown): boolean {
const status = stitchFailureStatus(error);
return status === 404 || status === 501;
}
/**
* Is this "the backend does not have it", as opposed to any other failure?
*
* Anchored, the way `stitchFailureStatus` is. A bare
* `String(err).includes('404:')` was safe only while an unparseable body
* surfaced as V8's `SyntaxError`, which quotes about ten characters of input.
* `IxClient` now reports it as `"${status}: response body is not JSON: ..."`
* with a 200-character preview -- so a gateway answering 200 with an HTML error
* page whose title is `404: Not Found` matched, the error was swallowed as
* "absent", and `reconcileRemovedEntities` emitted a DeleteNode for a node that
* exists. That is the #528 proxy shape this file already cites twice.
*/
function isNotFoundError(error: unknown): boolean {
return /^(?:Error:\s*)?404:/.test(String(error));
}
/**
* Ix#528: describe the failure without echoing the response body. The observed
* case is a proxy 504 whose body is a full HTML error page, and pasting that
* into a map summary buries the one line that matters.
*/
export function describeStitchFailure(error: unknown): string {
const status = stitchFailureStatus(error);
// A 2xx here means the request succeeded and something downstream made the
// body unreadable -- the #528 proxy shape -- so "HTTP 200" as the failure
// status is accurate and useless. Say what actually happened.
const detail =
status === null
? "backend request failed"
: status >= 200 && status < 300
? `HTTP ${status} with an unreadable body`
: `HTTP ${status}`;
return `Warning: Cross-workspace stitch failed (${detail}). Source patches were committed, but cross-repository relationships may be incomplete.`;
}
/**
* Ix#568: the stitch was never sent, and why.
*
* Deliberately not phrased as a failure. Nothing broke and the previous
* registration still stands -- the same position a stitch that FAILED already
* left the graph in. Saying only that cross-repo relationships may be
* incomplete, with no reason, is what would make this look like a regression.
*
* It does not promise that the next map re-registers, because that is not
* reliably true and predates this guard: the stitch block is gated on
* `filesSkippedAsUnchanged === 0`, so an incremental map that skips any
* mtime- or hash-unchanged file never reaches it -- and by then it has no
* registration data to send
* either, since it only parsed what changed. Its own comment already names
* the ways back in: a fresh map, `--force`, or a post-reset re-map. What the
* cooldown changes is only how often this path is taken, not where it leads.
*/
/**
* Does a skipped stitch get a human Note, or only the machine surfaces?
*
* Extracted so the rule is pinned by a test rather than by reading a
* conditional buried in `ingestFiles` (Ix#620). Every rule is always on
* `stitchSkipped` / `stitch_skipped`; this decides stderr only.
*/
export function shouldPrintStitchSkipped(
rule: StitchRefusal | undefined,
suppressOutput: boolean,
): boolean {
// Never on `--silent`. `ix map` always passes `suppressOutput`, and that
// surface is deliberately one terse line per run; the machine token still
// carries the rule, so nothing is lost.
if (suppressOutput) return false;
// `incomplete` is the one silent rule, and stays that way: it fires on nearly
// every incremental map, so what it reports is the ordinary steady state
// rather than an event worth a ~250-character Note on every save.
//
// `run-errors` PRINTS (Ix#620). It was suppressed on the grounds that it
// "restates lines the run has already printed", and that is false in both
// halves. What the run actually prints is
//
// Warning: 1 of 2067 file patches failed to commit; the graph is missing
// those files. Re-run 'ix map' to retry, or --debug to see why.
//
// which is about THOSE FILES. It never says the whole workspace's
// cross-workspace registration was withheld -- one file versus every
// cross-repo edge the workspace has. And the remedy it offers is the wrong
// one here: a plain `ix map` re-run is incremental, so
// `filesSkippedAsUnchanged > 0` lands it on `incomplete`, which is silent --
// the user re-runs, sees a clean exit, and the registration stays frozen.
// Only `--force` gets back in, which is what the `run-errors` text says.
//
// Noise is not the objection either. Unlike `incomplete`, this can only fire
// on a run that already had errors. Measured on a 2,066-file ingest: one
// commit error out of 2,067 patches silently cost the largest workspace in
// the graph its entire cross-repo registration, on a run that exited 0.
return rule !== "incomplete";
}
export function describeStitchSkipped(
reason: string,
rule?: StitchRefusal,
/** Whether the caller is showing per-file detail (`--verbose` / `--debug`). */
verbose = false,
): string {
// Short by default. A cooldown lasts 15 minutes, and under CLAUDE.md RULE 5
// an auto-map hook fires after every edit -- so the long form printed three
// sentences of unchanging advice on every map in that window. The reason is
// the part that changes and the part worth reading; the rest is on --verbose,
// and `stitchSkipped` / `stitch_skipped` carry it to machine consumers either
// way.
// The short form has to carry a real remedy, not just the reason. Waiting the
// cooldown out and re-running does NOT recover: `persistIngestBaselineIfClean`
// has already written the mtime baseline, so the next map has
// `filesSkipped > 0` and never enters the stitch block at all -- it prints
// nothing and exits 0, and the user reads that as fixed while cross-repo edges
// stay frozen. Only a run that re-ingests every file gets back in.
// Every branch names `--force`, because nothing else re-registers this
// workspace. A skipped stitch does NOT stop the run persisting its mtime
// baseline -- the baseline records every file the run settled, stitch or
// not -- so an ordinary re-run is incremental, skips unchanged files, and
// never enters the stitch block. "Re-run once that finishes" and "raise
// IX_MAP_DEADLINE_MS" were both promises this code cannot keep: the re-run
// prints nothing, exits 0, and the cross-repo edges stay exactly as stale.
//
// The hedge that used to justify withholding `--force` from the in-flight
// branch is still right, and now lives in the wording rather than in the
// omission: the other run may well be registering this same workspace, in
// which case the force is unnecessary -- but it is the only thing that
// recovers the case where it was not.
const recover =
rule === "deadline"
? " Raise IX_MAP_DEADLINE_MS or map a smaller path, then re-register with `ix ingest <root> --force`."
: rule === "in-flight"
? " Re-run once that finishes; if the cross-repo edges still look stale, `ix ingest <root> --force`."
: rule === "lost-parses"
// Nothing "clears" here, and this rule fires ON a --force run, so the
// cooling remedy told the user to wait for a condition that does not
// exist and then re-run the command they had just run.
? " Re-run `ix ingest <root> --force`; if it keeps happening, the crash is reproducible and IX_PARSE_DEBUG=1 will show it."
: rule === "run-errors"
? " Fix the errors above and re-run `ix ingest <root> --force`."
: " Recover with `ix ingest <root> --force` once it clears.";
const head = `Note: Cross-workspace stitch not started — ${reason}.${recover}`;
if (!verbose) return head;
// The remedy follows the RULE, never the prose. Note the other run may be
// registering this SAME workspace: only `ix map` takes the per-workspace
// lock, so two `ix ingest` runs on one repo, or an `ix ingest` racing an
// `ix map`, both reach here. Hence "may be" rather than a confident
// instruction to force a full re-ingest nobody needs.
const remedy =
rule === "in-flight"
? "That run may be registering a different workspace, in which case this one was not " +
"registered; re-run once it has finished, or force a full re-ingest " +
"(`ix ingest <root> --force`) if the cross-repo edges still look stale."
: rule === "lost-parses"
? "A parse worker crashed, so this run never saw some files and its registration would be " +
"missing them. Nothing is running on the backend and nothing needs to clear: re-run " +
"`ix ingest <root> --force`, and if it recurs the crash is reproducible -- IX_PARSE_DEBUG=1 " +
"will show what the parser choked on."
: rule === "run-errors"
? "The parse or commit errors above mean this run has an incomplete picture of the repo, so " +
"registering from it would overwrite a good registration with a worse one. Fix those and " +
"re-run `ix ingest <root> --force`."
: rule === "deadline"
? "Nothing was sent, so nothing is running -- but nothing was registered either, and a longer " +
"budget does not fix that on its own: the next map is incremental and never reaches the stitch " +
"block. Raise IX_MAP_DEADLINE_MS or map a smaller path, then `ix ingest <root> --force`."
: "Once the cooldown expires and the backend is healthy, re-register with a run that re-ingests " +
"every file (`ix ingest <root> --force`) — an incremental map that skips unchanged files does not " +
"re-attempt the stitch.";
return `${head} Source patches were committed; cross-repository edges are unchanged since the last successful stitch. ${remedy}`;
}
/**
* Decide how loudly a run should complain about patches that failed to commit.
*
* Split out of `ingestFiles` so the rule is testable without a backend: the
* failure it guards against is one nobody can see, so a rule nobody can test
* is not much of an improvement.
*
* Partial failure warns rather than throws — some patches landed, the graph
* moved forward, and the mtime cache is not written on a run with commit
* errors, so the next run retries the rest. Committing nothing is the case
* where carrying on would state something false about the graph, and `map.ts`
* already turns a throw into `emitError` plus exit 1.
*/
export function describeCommitOutcome(
commitErrors: number,
patchesApplied: number,
/**
* The flag that turns on per-file commit detail for the command being run.
* `ix map` has no `--debug` and never passes one — telling its users to use
* it sent them to an option that does not exist, on the one path where the
* failure is otherwise invisible.
*/
debugFlag: string = "--debug",
/** True when the run was cut short by the map deadline rather than rejected. */
deadlineHit: boolean = false,
/** Patches abandoned unsent by the commit cutoff (Ix#560). */
cutoffSkipped: number = 0,
/**
* Did the cutoff fire at any point, whether or not it ended up losing
* patches? When it did, the banner above has already given the reason, and
* this line must not send the reader back to a flag that adds nothing.
*/
cutoffFired: boolean = cutoffSkipped > 0,
/**
* The backend held none of this run's files before it started (a first map,
* or one after `ix reset`). With nothing landed, "the graph is unchanged"
* would read as "the old graph is still there", and there is none.
*/
graphStartedEmpty: boolean = false
): CommitOutcome {
if (commitErrors <= 0) return { kind: "ok" };
const attempted = commitErrors + patchesApplied;
if (deadlineHit) {
// Distinguishable because the two need opposite responses: a rejected
// patch is a backend problem, a deadline is our own wall clock and the fix
// is a longer budget or a smaller repo. Folding both into "failed to
// commit" is the same information loss this change exists to undo.
//
// Unless nothing landed at all. With a 5-minute per-request timeout and a
// 15-minute budget a stalled backend only fits ~3 attempts in, so the
// cutoff's streak cannot reach its limit and the run ends on the deadline
// -- and telling someone to RAISE the budget then is advice to wait longer
// on a backend that answered nothing. A run that committed zero patches
// did not run out of time doing work; it ran out of time waiting.
if (patchesApplied === 0) {
// Deliberately NOT "raising the budget will not help": the deadline
// covers discovery, hashing and parsing too, so a large repo or a short
// IX_MAP_DEADLINE_MS can spend it before the first commit is even
// attempted -- and there raising it is exactly the fix. Both readings
// are live from here, so name both rather than assert the wrong one.
// The cutoff's share is named here too. This is the LIKELIEST shape for a
// dead backend, not a corner: nothing lands, so telling the user all N
// were abandoned by the clock -- and to lengthen the budget -- is the
// wrong remedy for the ones deliberately never sent, and contradicts the
// banner directly above, which has just named the split.
const withheld =
cutoffSkipped > 0
? `${commitErrors - cutoffSkipped} file patches were abandoned when the map deadline fired and ` +
`${cutoffSkipped} more were withheld by the commit cutoff, and none landed first`
: `all ${commitErrors} file patches were abandoned when the map deadline fired, and none landed first`;
return {
kind: "fatal",
message:
`Ingest ran out of time without committing anything: ${withheld}. Either the budget ran out ` +
`before the commits did (raise IX_MAP_DEADLINE_MS or map a smaller path), or the backend ` +
`is not accepting writes — an ArangoDB that cannot begin a transaction is still reachable ` +
`and still consistent, so \`ix doctor\` passes. \`docker stats\` and ` +
`\`/_api/query/current\` tell the two apart.`,
};
}
// The cutoff's share is named rather than folded in. A run that withheld
// 200 patches from a refusing backend and then lost 3 more to the clock
// reported all 203 as "abandoned when the map deadline fired ... raise
// IX_MAP_DEADLINE_MS", which is the wrong remedy for 200 of them. The
// banner above already names the split; this line was the one contradicting
// it.
const clockLost = commitErrors - cutoffSkipped;
const share =
cutoffSkipped > 0
? `${clockLost} of ${attempted} file patches were abandoned when the map deadline fired, and ` +
`${cutoffSkipped} more were withheld earlier by the commit cutoff`
: `${commitErrors} of ${attempted} file patches were abandoned when the map deadline fired`;
const remedy =
cutoffSkipped > 0
? `The graph is missing all of them. Raise IX_MAP_DEADLINE_MS or map a smaller path for the first group; ` +
`see the cutoff above for the second.`
: `The graph is missing those files. Raise IX_MAP_DEADLINE_MS or map a smaller path.`;
return {
kind: "warn",
message: `Ingest ran out of time: ${share}. ${remedy}`,
};
}
// After the deadline branch, deliberately: a run that tripped the cutoff on
// a handful of patches and THEN blew its budget on thousands lost far more
// to the clock, and attributing all of it to the cutoff hides the number
// that would actually change the outcome.
if (cutoffFired) {
// Two shapes, and they want opposite advice about the debug flag.
//
// When patches were held back unsent, they produce no per-file line at all,
// so pointing at the flag sends the reader to output that does not exist --
// the cutoff banner above is the only account of them there is.
//
// When the drain placed everything it held, `cutoffSkipped` is 0 and every
// counted error came from a patch that WAS sent and DID emit a line. The
// banner names only the last error and no filenames, so telling the reader
// the flag adds nothing steers them away from the only per-file detail
// available. That is the shape the 12-file reproduction produces at the
// default limit: five sent failures, seven drained successfully.
if (cutoffSkipped === 0) {
return {
kind: patchesApplied === 0 ? "fatal" : "warn",
message:
`Ingest stopped early: ${commitErrors} of ${attempted} file patches are missing from the graph. ` +
`See the cutoff above for why it stopped; ${debugFlag} lists the individual failures.`,
};
}
return {
kind: patchesApplied === 0 ? "fatal" : "warn",
message:
`Ingest stopped early: ${commitErrors} of ${attempted} file patches are missing from the graph, ` +
`${cutoffSkipped} of them never sent. See the cutoff above for why; ${debugFlag} will not add to it.`,
};
}
if (patchesApplied === 0) {
const patches = commitErrors === 1 ? "patch" : "patches";
return {
kind: "fatal",
message:
`Ingest committed nothing: all ${commitErrors} ${patches} failed to commit. ` +
(graphStartedEmpty
? `The backend held none of these files before this run, so this workspace's graph is still empty. `
: `The graph is unchanged and may now be stale. `) +
`Re-run with ${debugFlag} to see why.`,
};
}
return {
kind: "warn",
message:
`Warning: ${commitErrors} of ${attempted} file patches failed to commit; ` +
`the graph is missing those files. Re-run 'ix map' to retry, or ${debugFlag} to see why.`,
};
}
// Builtin instance-method names that are never a real cross-repo call target.
const STITCH_COMMODITY = new Set(['map', 'forEach', 'reduce', 'reduceRight', 'flatMap', 'flat', 'push', 'pop', 'shift', 'unshift', 'splice', 'slice', 'concat', 'get', 'set', 'has', 'keys', 'values', 'entries', 'then', 'catch', 'finally', 'resolve', 'race', 'allSettled', 'bind', 'call', 'apply', 'stringify', 'toString', 'valueOf', 'hasOwnProperty']);
/**
* The cross-repo stitch entry a parsed relationship contributes, if any. A
* receiver-only typed call (ParsedRelationship.receiverOnly) names a target
* inferred from a local receiver's type; it is written only when it resolves
* in this workspace, so it is never a cross-repo call candidate.
*/
export function stitchCallEntry(
rel: { predicate?: unknown; dstName?: unknown; srcName?: unknown; receiverOnly?: unknown },
filePath: string,
): { symbol: string; caller: string; callerFilePath: string } | undefined {
if (rel.predicate !== 'CALLS' || rel.receiverOnly || typeof rel.dstName !== 'string' || typeof rel.srcName !== 'string') return undefined;
if (STITCH_COMMODITY.has(rel.dstName)) return undefined;
return { symbol: rel.dstName, caller: rel.srcName, callerFilePath: filePath };
}
/**
* Did the run finish with the graph fully matching what it set out to ingest?
*
* Three things are conditional on this: persisting the mtime cache, the
* post-migration cleanup of the old workspace id, and writing the stitch
* registry. All three were previously written as `parseErrors === 0`, which
* worked only because commit failures were folded into `parseErrors`.
*
* The mtime cache is the one that bites. Persisting it after a failed commit
* records the file as ingested at its current mtime, so the next run skips it
* as unchanged and it stays missing from the graph until a `--force` — a worse
* bug than the silent success this change removes, and one introduced *by*
* splitting the counters. Named and exported so the rule itself is pinned by a
* test instead of living as a conjunct three call sites have to remember.
*
* That pins the rule, not its application: nothing yet fails if a call site
* stops calling this. `ingestFiles` has no test at all — it needs a backend —
* so the three call sites below, the `process.exitCode` on a partial failure,
* and the debug-flag wiring are all still verified by reading rather than by
* execution.
*/
export function ingestCompletedCleanly(parseErrors: number, commitErrors: number): boolean {
return parseErrors === 0 && commitErrors === 0;
}
// ---------------------------------------------------------------------------
// File ingestion — parse locally, send via /v1/patch
// ---------------------------------------------------------------------------
export async function ingestFiles(
path: string,
opts: { recursive?: boolean; force?: boolean; format: string; root?: string; debug?: boolean; printSummary?: boolean; suppressOutput?: boolean; lang?: string; mapMode?: boolean; exclude?: string[]; deadlineSignal?: AbortSignal; neverRegister?: boolean }
): Promise<IngestFilesSummary> {
// Under `ix mcp` the tool call's deadline is this run's deadline too. The
// client would see it anyway (createClient combines it), but the commit loop,
// the stitch wait and the outcome report all read `opts.deadlineSignal`: left
// out of it, a timed-out ingest kept fanning out per-file commits that failed
// at once on the aborted signal and charged them to the backend.
opts = { ...opts, deadlineSignal: combineSignals(opts.deadlineSignal, currentRunSignal()) };
const debug = opts.debug || process.env.IX_DEBUG === '1';
const mapMode = opts.mapMode === true;
const trueStart = performance.now();
const [
{ parseFile, resolveEdges },
{
buildPatchWithResolution,
buildDeletionPatch,
sourcePatchIdCandidates,
fileNodeId,
symbolNodeId,
extractorName,
PREVIOUS_EXTRACTORS,
},
{ languageFromPath },
] = await loadIngestionModules();
const moduleLoadMs = Math.round(performance.now() - trueStart);
// Canonical (realpath), because discovery canonicalises every file it finds:
// a root spelled through a symlink, macOS's /tmp or a Windows 8.3 name made
// every source_uri `../<real dir>/...` instead of workspace-relative. `ix
// map` hands in a canonical root already; `ix ingest <path>` did not. A path
// that does not exist stays as given, for the not-found error below.
const resolvedPath = canonicalWorkspacePath(nodePath.isAbsolute(path)
? path
: nodePath.resolve(resolveWorkspaceRoot(opts.root), path));
// Here, not in the discovery phase below: the statSync that picks the
// workspace root runs first, and threw a bare ENOENT before the friendly
// message there could be reached.
if (!fs.existsSync(resolvedPath)) {
throw new Error(`Path not found: ${resolvedPath}`);
}
// Workspace identity for client-agnostic backend.
//
// The backend used to dereference provenance.source_uri against the host
// filesystem (via a broad HOME bind mount). We now emit workspace-relative
// paths as the canonical source_uri, plus a stable workspace_id (persisted in
// ~/.ix/config.yaml, derived from the workspace's absolute path) that
// core-ingestion folds into every node id so two workspaces with the same
// relative layout do not collide. The id is path-based (not random) so a repo
// mapped standalone gets the SAME workspace_id it gets as a member of a system
// (see workspaceIdForPath / repoWorkspaceIdFor), keeping its node identity
// byte-identical across both. The backend treats both as opaque strings; it
// never reads host files.
//
// The workspace is the one the path BELONGS to, not the path itself: see
// `resolveIngestRoot`. A path strictly inside that root is a scoped run --
// discovery, deletion and the stitch are limited to it, and the rest of the
// workspace's baseline is carried forward untouched.
const targetIsDirectory = fs.statSync(resolvedPath).isDirectory();
const { root: workspaceRoot, source: rootSource } = resolveIngestRootWithSource(resolvedPath, targetIsDirectory, opts.root);
// A file in no workspace and no repository is ingested under its directory's
// id but never registered: that would make the whole directory readable.
// `neverRegister`: a coalesced rerun on behalf of such a file ingest.
const registerAfterCommit = opts.neverRegister !== true && !(rootSource === 'loose' && !targetIsDirectory);
// A new workspace is only registered once a commit lands (below, before the
// summary): see workspaceStateFor.
const {
workspaceId,
migrated: workspaceMigrated,
previousWorkspaceId,
name: workspaceName,
pending: registrationPending,
} = workspaceStateFor(workspaceRoot);
// What this run discovers. A migrated workspace is widened to its root: the
// re-key below re-ingests everything under the new id and then deletes the
// old id's graph, which after a scoped run would leave the rest of the
// workspace with no nodes and a baseline vouching for them.
const scopePath = workspaceMigrated ? workspaceRoot : resolvedPath;
const scopedRun = scopePath !== workspaceRoot;
const inScope = (absPath: string): boolean => !scopedRun || isPathInside(scopePath, absPath);
if (workspaceMigrated) {
// Legacy random workspace_id was just re-keyed to the path-based id (Ix#225
// gap 2). Node identity folds the workspace_id, so the previously-ingested
// nodes live under the old id; force a full re-ingest under the new id (below)
// and delete the old id's orphaned nodes after that succeeds.
process.stderr.write(
`Migrated workspace to a stable path-based id; re-ingesting this workspace once.\n`,
);
clearMapResultCache(workspaceRoot);
}
const toWorkspaceRelative = (absPath: string): string => {
// Force workspace-local POSIX separators so IDs are stable across OS.
const rel = nodePath.relative(workspaceRoot, absPath);
return rel.split(nodePath.sep).join('/');
};
// ── Multi-repo co-ingest (Ix#225 Path 1) ──────────────────────────────
// Auto-detected (no flag): if the mapped dir has >= 2 child repo roots, treat
// each as a member repo. Each repo gets its OWN stable workspace_id (so its
// node ids match whether ingested alone or in a system), all share the
// system_id, and each file carries its repoId. A cross-repo edge's dst id is
// resolved in the dst repo's workspace namespace by core-ingestion.
const detectedSystem = detectSystem(workspaceRoot);
// Per-file workspace id + multi-repo context fed to buildPatchWithResolution.
// Shared with `ix verify` (system.ts fileIdentity), which must agree exactly.
const {
systemId,
repoOf,
fileWorkspaceId,
fileMultiRepo,
} = fileIdentity(workspaceRoot, workspaceId, detectedSystem?.systemId);
// The workspace_id a file's nodes are folded under (same value the patch builder
// stamps on source.workspaceId) — used to scope the baseline hash lookup per file.
const sourceWorkspaceIdOf = (absPath: string): string => fileWorkspaceId(toWorkspaceRelative(absPath));
// Cross-repo dependency gate: map an import module name to the member repo that
// publishes that package (exact, or prefix at a package boundary for scoped /
// sub-module / sub-path imports like "@babel/types/lib/x", "tokio::sync",
// "sqlalchemy.orm", "github.com/org/repo/sub"). The resolver uses this to keep a
// cross-repo edge only when the source repo actually imports the target repo.
// packageOf maps an import specifier to the member repo that publishes it
// (exact, lowercased stem, or a package-boundary prefix for scoped / sub-module
// / sub-path imports). declaredRepoDeps is the ground-truth member->member
// dependency graph from manifests; together they decide whether a cross-repo
// edge is dependency-backed. See lookupPackage for the matching rules.
const packageRegistry = detectedSystem?.packageRegistry ?? {};
const packageOf = (mod: string): string | undefined => lookupPackage(packageRegistry, mod);
const declaredRepoDeps = detectedSystem?.repoDeps;
const crossRepoResolveOpts = systemId ? { repoOf, packageOf, declaredRepoDeps } : undefined;
// ── Path-2 separate-ingest stitcher (Ix#225 Half A) ──────────────────
// Co-ingest already forms cross-repo edges in-batch, so the stitcher only runs
// for a SOLO ingest: it registers this repo's published packages (provides) and
// its production-dep external imports (consumes), and the backend joins them
// bidirectionally against other separately-ingested repos to recreate the
// cross-repo IMPORTS edges. Production-dep gated (mirrors #244): an import only
// counts as a consume if the package is a declared production dependency.
const stitchEnabled = systemId === undefined;
const stitchProdDeps = stitchEnabled ? new Set<string>(readPackageDeps(workspaceRoot)) : new Set<string>();
const stitchConsumes: Array<{ name: string; consumerNodeId: string }> = [];
const stitchConsumeSeen = new Set<string>();
const stitchFiles: string[] = [];
// Symbol-level stitch (Ix#225): a provider's exported top-level symbols + a
// consumer's calls (collected raw; resolved post-ingest once the full repo symbol
// set is known, so "external" = a call to a symbol not defined anywhere in this repo).
const stitchDefs = new Map<string, { qk: string; filePath: string }>(); // name -> def (caller-key + local set)
const stitchExportNames = new Set<string>();
const stitchExports: Array<{ name: string; nodeId: string }> = [];
const stitchCalls: Array<{ symbol: string; caller: string; callerFilePath: string }> = [];
// Per-file JS/TS import bindings (localName -> {pkg, public symbol}) so a call to
// a renamed/default-imported local name resolves to the provider's public symbol.
const stitchBindings = new Map<string, Map<string, { pkg: string; imported: string }>>();
// Provider-side public export aliases (default exports, `export { x as y }`):
// {filePath, local entity name, public name} -> add an export entry by public name.
const stitchExportAliases: Array<{ filePath: string; local: string; pub: string }> = [];
const stitchExportAliasSeen = new Set<string>();
const specToPkg = (spec: string | undefined): string | null => {
if (!spec || spec.startsWith('.') || spec.startsWith('/')) return null;
if (spec.startsWith('@')) { const parts = spec.split('/'); return parts.length >= 2 ? `${parts[0]}/${parts[1]}` : null; }
return spec.split('/')[0] || null;
};
const collectStitch = (parsedFiles: Array<{ filePath: string; entities?: Array<{ name: string; kind: string; container?: string }>; relationships?: Array<{ predicate: string; srcName?: string; dstName?: string; importRaw?: string }>; importBindings?: Array<{ pkg: string; local: string; imported: string }>; exportPublicNames?: Array<{ local: string; public: string }> }>): void => {
for (const p of parsedFiles) {
stitchFiles.push(p.filePath);
for (const e of (p.entities ?? [])) {
const qk = e.container ? `${e.container}.${e.name}` : e.name;
stitchDefs.set(e.name, { qk, filePath: p.filePath });
// A call FROM inside a class/namespace method has a QUALIFIED srcName
// (e.g. "Ora.constructor"); also key the def by its qualified key so the
// caller node resolves (else cross-repo calls from methods are dropped).
if (qk !== e.name) stitchDefs.set(qk, { qk, filePath: p.filePath });
if (!e.container && (e.kind === 'function' || e.kind === 'class') && !stitchExportNames.has(e.name)) {
stitchExportNames.add(e.name);
stitchExports.push({ name: e.name, nodeId: symbolNodeId(workspaceId, p.filePath, qk) });
}
}
// JS/TS alias metadata (additive; absent for other languages).
for (const b of (p.importBindings ?? [])) {
let m = stitchBindings.get(p.filePath);
if (!m) { m = new Map(); stitchBindings.set(p.filePath, m); }
if (!m.has(b.local)) m.set(b.local, { pkg: b.pkg, imported: b.imported });
}
for (const x of (p.exportPublicNames ?? [])) {
stitchExportAliases.push({ filePath: p.filePath, local: x.local, pub: x.public });
}
for (const rel of (p.relationships ?? [])) {
const call = stitchCallEntry(rel, p.filePath);
if (call) stitchCalls.push(call);
if (rel.predicate !== 'IMPORTS') continue;
const pkg = specToPkg(typeof rel.importRaw === 'string' ? rel.importRaw : rel.dstName);
if (!pkg || !stitchProdDeps.has(pkg)) continue;
const consumerNodeId = fileNodeId(workspaceId, p.filePath);
const k = `${pkg}\u0000${consumerNodeId}`;
if (stitchConsumeSeen.has(k)) continue;
stitchConsumeSeen.add(k);
stitchConsumes.push({ name: pkg, consumerNodeId });
}
}
};
// Deterministic entry file (index/main basename -> shallowest -> lexical): the
// node a cross-repo import couples to. Mirrors resolveEdges' entryFileOf so a
// stitched edge targets the SAME node a co-ingest would have.
const pickEntryFile = (files: string[]): string | undefined => {
const ENTRY = new Set(['index', 'main', 'mod', 'lib', '__init__', 'app', 'init']);
let best: string | undefined;
let bestScore: [number, number, string] | undefined;
for (const f of files) {
const norm = f.replace(/\\/g, '/');
const base = (norm.split('/').pop() ?? '').replace(/\.[^.]+$/, '').toLowerCase();
const score: [number, number, string] = [ENTRY.has(base) ? 0 : 1, norm.split('/').length, norm];
if (!bestScore || score[0] < bestScore[0] ||
(score[0] === bestScore[0] && (score[1] < bestScore[1] ||
(score[1] === bestScore[1] && score[2] < bestScore[2])))) { best = f; bestScore = score; }
}
return best;
};
if (systemId && debug) {
const depCount = declaredRepoDeps ? Object.values(declaredRepoDeps).reduce((a, d) => a + d.length, 0) : 0;
process.stderr.write(`[multi-repo] system "${detectedSystem!.name}" (${systemId}) members=${detectedSystem!.members.join(', ')} packages=${Object.keys(packageRegistry).length} declaredDeps=${depCount}\n`);
}
// Long-running: `ix ingest` passes no deadline, and the reads it makes per
// changed file (GET /v1/patches/:id) come after a parse that can take
// minutes, past any read deadline counted from here.
const client = createClient({ deadlineSignal: opts.deadlineSignal, longRunning: true });
// Schema-version check forces a clean re-ingest when the backend's graph
// format has changed in a way that invalidates existing node IDs (e.g. the
// absolute→relative source_uri migration, or folding workspace_id into ids).
// CLIENT_EXPECTED_SCHEMA_VERSION is shared with doctor/upgrade (backend-status).
// Read from the same health answer. Unknown (the check failed) keeps the
// per-file route for delete-bearing patches.
let bulkDeletes = false;
// Why the same-hash orphan sweep cannot run here; null when it can.
let orphanSweepBlocker: string | null = 'the backend release could not be read';
const reconcileLookups = createLimiter(RECONCILE_CONCURRENCY);
try {
const health = await readBackendHealth(client);
bulkDeletes = backendAppliesBulkDeletes(health);
orphanSweepBlocker = orphanSweepUnsupportedReason(health);
if (debug) process.stderr.write(`\n [commit] delete-bearing patches via ${bulkDeletes ? 'bulk' : '/v1/patch'}\n`);
const serverVersion = (health as any)?.schema_version;
if (typeof serverVersion === 'number' && serverVersion !== CLIENT_EXPECTED_SCHEMA_VERSION) {
process.stderr.write(
`\n [schema mismatch] backend schema_version=${serverVersion}, ` +
`client expects ${CLIENT_EXPECTED_SCHEMA_VERSION}. Forcing full re-ingest.\n`,
);
opts.force = true;
}
} catch (err) {
if (debug) process.stderr.write(`\n [schema check skipped] ${err}\n`);
}
const start = performance.now();
// Worker pool for parallel file parsing across CPU cores
const workerPath = nodePath.resolve(
nodePath.dirname(fileURLToPath(import.meta.url)),
'../../../../core-ingestion/dist/parse-worker.js',
);
let pool: ParsePool | null = null;
/**
* In-flight parses lost to a crashed worker, read where the pool is still in
* scope. A function rather than a read at the use site because control-flow
* narrowing makes `pool` `never` down there.
*/
const crashedParses = (): number => (pool === null ? 0 : pool.crashedTasks());
/** Files whose parse ran past the budget; see `ParsePool.timedOutFiles`. */
const timedOutParses = (): string[] => (pool === null ? [] : pool.timedOutFiles());
/**
* The pool, sized to the work the caller is about to give it: one worker per
* 50 files, capped by the cores and by IX_PARSE_WORKERS (default 8). A
* one-file edit used to start a worker per core -- 23 threads and about
* 1.6 GB on a 24-core machine -- to parse one file. Workers start lazily, so
* the size is a ceiling, and a later, larger phase can raise it.
*/
const ensureParsePool = (expectedTasks: number): ParsePool => {
const size = parsePoolSize(expectedTasks);
if (pool) {
pool.growTo(size);
return pool;
}
pool = new ParsePool(workerPath, size);
return pool;
};
let progressPhase = 'Scanning';
let progressCurrent = 0;
let progressTotal = 0;
let progressStart = performance.now();
// canRenderProgress, not just the format: `ix map` redirected to a file or
// captured by an agent is still format 'text', and every 80 ms frame is
// retained rather than overwritten.
//
// Kept as two conditions rather than folded into one flag because they gate
// different things below. `interval` gates the redrawing writes; the text
// format alone still gates the newline-terminated summary, which redirected
// output should keep — that is the "final summary" the bar was drowning.
const wantsTextProgress = opts.format === 'text';
const interval = (wantsTextProgress && canRenderProgress()) ? setInterval(() => {
if (debug && currentWorkLabel) {
const workElapsed = performance.now() - currentWorkStart;
if (workElapsed >= SLOW_WORK_LOG_MS && (lastSlowWorkNotice === 0 || workElapsed - lastSlowWorkNotice >= SLOW_WORK_REPEAT_MS)) {
lastSlowWorkNotice = workElapsed;
process.stderr.write(`\n [slow ${progressPhase.toLowerCase()}] ${currentWorkLabel} (${Math.round(workElapsed / 1000)}s)\n`);
}
}
process.stderr.write('\r' + renderProgressLine(progressPhase, progressCurrent, progressTotal));
}, 80) : null;
let filesDiscovered = 0;
/**
* Paths `--exclude` or `.ixignore` kept out of discovery.
*
* Reported, because "that file is not in the graph" and "you excluded that
* file" are different problems with different fixes, and an exclusion nobody
* is told about is indistinguishable from an ingest that missed something.
*/
let filesExcluded = 0;
/** What discovery left out: SKIPPED_DIRS hits and files it could not stat. */
const discoveryCounts = emptyDiscoveryCounts();
let filesChanged = 0;
let patchesApplied = 0;
let idempotentPatches = 0;
/** Workspace-relative paths of changed files the backend answered `Idempotent` (F-01). */
const replayedChanges = new Set<string>();
// Per-file outcomes, so one failing file does not keep the whole baseline
// from being written. A changed file is settled once its patch is committed
// (`committedRel`), its content turns out to be what the backend has, or it
// is skipped for a reason that will not change (minified); a file that
// parsed to nothing is settled only if no worker crashed this run.
const committedRel = new Set<string>();
const settledAbs = new Set<string>();
const unparsedRel = new Set<string>();
/** Files that parsed with ERROR or MISSING nodes: indexed, but possibly incompletely. */
let filesWithParseErrors = 0;
// What `graphUnchanged` is computed from, beside the counters above. Each is
// a way this run can have written to the graph, or have found it not where
// the last run left it, without a patch being counted.
let dbResetDetected = false;
/**
* The backend held none of this run's files when it started: the DB-reset
* guard fired, or a lookup over the whole file set found no hashes (a first
* map, or one after `ix reset` / `reset --ingest`). Only the failure messages
* read it: "the graph is unchanged" is misleading when there was no graph.
* Unset on any path that ends before the lookup.
*/
let graphStartedEmpty: boolean | undefined;
let extractorRebuild = false;
let deletionsFound = false;
// Path A with a local baseline: the run updated a graph it already knew.
let baselineRun = false;
let stitchSent = false;
let filesSkipped = 0;
/**
* Files skipped because we ASSUMED THEY WERE UNCHANGED. (Ix#568)
*
* A subset of `filesSkipped`, and the one the cross-workspace stitch cares
* about. The stitch registers this repo's provides/consumes, collected over
* the files actually parsed this run, so it must only run when that set
* covers the whole repo -- otherwise it overwrites a full registration with
* a partial one.
*
* `filesSkipped` is the wrong test for that, because it also counts files
* that contribute nothing under ANY run: a zero-byte file and one that
* looks minified. Both are decided by the file's own content under a rule
* that does not vary between runs, so the collected set is complete and
* correct without them -- and their presence in the count was silently
* disqualifying repos from ever stitching. A single empty `__init__.py` was
* enough, `--force` or not, which made `ix ingest <root> --force` -- the
* remedy this command prints when it refuses a stitch -- a promise it could
* not keep: the run would print nothing, exit 0, and leave the cross-repo
* edges stale.
*
* An mtime- or hash-clean skip is different in kind: the file has content
* we would have indexed and we did not look at it this run, so the set
* really is missing it. So is a file the parser returned nothing for --
* see `filesSkippedUnparsed`, which the gate reads alongside this.
*/
let filesSkippedAsUnchanged = 0;
/** Zero-byte files. Reported as `skipReasons.emptyFile`, which was hardcoded 0. */
let filesSkippedAsEmpty = 0;
/**
* Files the parse pool returned nothing for.
*
* Reported, but NOT part of the stitch gate -- `ParsePool.crashedTasks()` is.
* `parse()` resolves `null` for two different things and only one of them
* makes the run incomplete:
*
* - `parseFile` returned null, which is deterministic and means the file
* has nothing to give. The dominant cause is an unavailable optional
* grammar -- fourteen are optionalDependencies loaded through a try/catch,
* and `tree-sitter-sas` has no win32 prebuild at all -- and those
* extensions are still in SUPPORTED_EXTENSIONS, so the files are
* discovered, read, dispatched and come back null on EVERY run. Gating on
* this count blocked stitching permanently for any repo containing one,
* `--force` included, since forcing does not change a parse result. That
* is the empty-`__init__.py` bug again with a different file class.
* - a worker CRASHED with the task in flight, which is transient and did
* lose a file we would have indexed. That one belongs in the gate.
*
* A parse that THREW is in the first category, not the second, even though it
* sounds like the second: `parseFile` catches every exception itself and
* returns null (logging only under IX_PARSE_DEBUG=1), so the worker cannot
* tell it from a missing grammar -- and neither can this. It is also
* deterministic for the same bytes in every case we can observe, so gating on
* it would block stitching permanently for that repo, which is exactly the
* bug this counter was taken OUT of the gate to fix. The residue is the
* transient sub-case, an OOM caught by that same handler; it is narrower than
* the bug the alternative creates, and it is a deliberate choice rather than
* an oversight.
*
* Reported as `skipReasons.unparsed`, its own bucket: `parseError` counts
* every parse-stage failure including stat, read and patch-build errors, most
* of which are not skips, so adding to it made that number mean neither thing.
*/
let filesSkippedUnparsed = 0;
let parseErrors = 0;
let commitErrors = 0;
let stitchErrors = 0;
let stitchError: unknown;
// Set when the stitch was refused before it was ever sent (Ix#568).
// Distinct from stitchError: nothing failed, so it must not count as a
// stitch error or move the exit code -- but the user is about to be told
// cross-repo edges may be incomplete and is owed the reason.
let stitchSkipped: string | undefined;
let stitchSkippedRule: StitchRefusal | undefined;
// Ix#560: a backend refusing every commit gets one failed request per
// patch out of the per-file fallback. Stop after a streak instead.
const commitBreaker = createCommitBreaker();
/**
* Whether a batch's held-back patches get a drain. Run-scoped: the decision
* spans batches, and it has to be revocable -- see `createDrainGate`.
*/
const drainGate = createDrainGate();
/**
* Patches the BACKEND accepted, counted only where a request came back OK.
*
* Not `patchesApplied`, which the gate used to read. That counter also rises
* on the `count-applied` path, where up to a thousand patches are counted
* from a 409 body without a single request being sent -- so a full-landed
* replay could reopen a gate that had closed against a dead backend and
* restore the per-batch re-probing the gate exists to prevent.
*/
let patchesTheBackendTook = 0;
/**
* Commit failures that said something about the BACKEND rather than about one
* patch. Counted so the drain gate can tell "refused everything because the
* backend is down" from "refused everything because every patch was too big".
*/
let backendIndictingFailures = 0;
/**
* The RUN deadline, captured here because `commitPreparedPatches` shadows
* `opts` with its own. Not `isAbortError`: a per-request AbortSignal.timeout
* raises TimeoutError too, and that is a saturation signal, not our clock.
*/
const runDeadlineExpired = (): boolean => opts.deadlineSignal?.aborted === true;
/** The same deadline, for the commit retries' waits to abort on. */
const runDeadlineSignal = opts.deadlineSignal;
/**
* The backend's own word on its database, asked only once commits are
* failing (IN-19C). The cutoff judges by a streak, which with ArangoDB down
* costs a streak of doomed requests here and a failure budget per drain
* pass; the backend can simply say so. One probe answers every caller for a
* few seconds, under a 2 s timeout, and only `unreachable` changes anything
* -- see `database-probe.ts`.
*/
const newDatabaseProbe = () => createDatabaseProbe(() => probeDatabase({
// Through the one place health is fetched and its release recorded.
readHealth: timeoutMs => readBackendHealth(client, timeoutMs),
currentRevision: timeoutMs => client.currentRevision(timeoutMs),
}), 3_000);
let askDatabase = newDatabaseProbe();
/** `patchesTheBackendTook` when `askDatabase`'s cache was started; a landed patch is newer evidence than any cached verdict. */
let askedAtPatches = 0;
/** `patchesTheBackendTook` when the backend last reported its database down; undefined since it reported it up. */
let databaseDownAsOf: number | undefined;
/** The last answer was "unreachable", and nothing has landed since to say otherwise. */
const databaseKnownDown = (): boolean =>
databaseDownAsOf !== undefined && databaseDownAsOf === patchesTheBackendTook;
/** Ask (or reuse a recent answer) and record it. True when the database is down. */
const checkDatabase = async (): Promise<boolean> => {
// IX_COMMIT_FAILURE_LIMIT=0 opts out of giving up; the probe never stops a run then.
if (commitBreaker.limit <= 0) return false;
if (runDeadlineExpired()) return databaseKnownDown();
if (askedAtPatches !== patchesTheBackendTook) {
askDatabase = newDatabaseProbe();
askedAtPatches = patchesTheBackendTook;
}
const verdict = await askDatabase();
if (verdict === 'unreachable') databaseDownAsOf = patchesTheBackendTook;
else if (verdict === 'reachable') databaseDownAsOf = undefined;
return verdict === 'unreachable';
};
/**
* After a commit failure that indicts the backend: if its database is down,
* give up now rather than after a streak. A later commit landing clears the
* trip, as it clears one a streak set.
*/
const tripIfDatabaseDown = async (err: unknown): Promise<void> => {
if (commitBreaker.tripped()) return;
if ((await checkDatabase()) && !commitBreaker.tripped()) commitBreaker.trip(err);
};
const noteCommitRetry = (where: string) =>
(info: { attempt: number; delayMs: number; error: unknown; kind: string }): void => {
if (!debug) return;
const msg = String(info.error);
process.stderr.write(
`\n [${where} retry ${info.attempt}, ${info.kind}] waiting ${Math.round(info.delayMs)} ms: ${msg.length > 200 ? `${msg.slice(0, 200)}…` : msg}\n`,
);
};
let tooLarge = 0;
/** The stat loop's record of empty and oversized files, once it exists; `tooLarge` counts its oversized ones. */
let skippedForSize: ReadonlyMap<string, 'empty' | 'tooLarge'> | undefined;
let minifiedLikely = 0;
let outsideRoot = 0;
let latestRev = 0;
let entitiesParsed = 0;
let currentWorkLabel = '';
let currentWorkStart = performance.now();
let lastSlowWorkNotice = 0;
const setCurrentWork = (label: string): void => {
currentWorkLabel = label;
currentWorkStart = performance.now();
lastSlowWorkNotice = 0;
};
// Phase timing marks
const timings = {
moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,
goIndexMs: 0, // time to build Go global resolution index
resolveMs: 0, buildPatchMs: 0, commitMs: 0,
bulkCommitMs: 0, // time in successful commitPatchBulk() calls
fallbackCommitMs: 0, // time in per-file fallback commitPatch() calls
};
const resolveStats = {
importLookups: 0, transitiveLookups: 0, globalFallbacks: 0,
globalCandidateTotal: 0, resolvedImport: 0, resolvedTransitive: 0,
resolvedGlobal: 0, resolvedQualifier: 0, resolvedType: 0, skippedSameFile: 0, skippedAmbiguous: 0,
};
// Per-language parse timing: language → { ms, files }
const langParseMs = new Map<string, { ms: number; files: number }>();
// Total graph op counts across all committed batches
let totalNodesEmitted = 0;
let totalEdgesEmitted = 0;
let totalClaimsEmitted = 0;
const logSaveTiming = (
files: number,
resolveEdgesMs: number,
buildPatchMs: number,
commitMs: number,
totalSaveMs: number,
nodes: number,
edges: number,
claims: number,
): void => {
if (!debug) return;
process.stderr.write(
`\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs}` +
` commitMs=${commitMs} totalSaveMs=${totalSaveMs} nodes=${nodes} edges=${edges} claims=${claims}\n`
);
};
try {
// Phase: discover files
const langFilter = opts.lang ? parseLangs(opts.lang) : null;
const stat = fs.statSync(scopePath);
// `.ixignore` and `--exclude` are the WORKSPACE's, matched root-relative,
// so a scoped run leaves out exactly what a map of the whole root would.
const excludePatterns = collectExcludePatterns(workspaceRoot, opts.exclude ?? []);
const excludeMatcher = createIgnoreMatcher(excludePatterns);
const exclude = excludeMatcher.size > 0
? { matcher: excludeMatcher, root: workspaceRoot, onSkip: () => { filesExcluded += 1; } }
: undefined;
// Discovery returns canonical paths, so confinement below does not resolve
// every file a second time; only the root is canonicalized there.
const discovery = discoverIngestFilePaths(
stat.isFile()
? (isSupportedSourceFile(scopePath) ? [canonicalizeDiscoveredFilePath(scopePath)] : [])
: discoverSourceFiles(scopePath, { recursive: opts.recursive ?? true, exclude, counts: discoveryCounts }),
stat.isFile() ? undefined : scopePath,
(p) => p,
);
// `--lang` narrows discovery to the named languages. The files it leaves
// out keep their baseline entries (`langExcluded`, below): this run says
// nothing about them, so it must neither drop nor refresh them.
const inLangFilter = (fp: string): boolean => {
const lang = languageFromPath(fp);
return lang !== null && langFilter!.has(lang);
};
const filePaths: string[] = langFilter ? discovery.files.filter(inLangFilter) : discovery.files;
const langExcluded: string[] = langFilter ? discovery.files.filter(fp => !inLangFilter(fp)) : [];
outsideRoot = discovery.outsideRoot;
// The files edges may resolve TO. A scoped run parses only its own files,
// but an import from `src/a.ts` to `lib/b.ts` still names a file the
// workspace holds: resolving against the scope alone would drop that edge
// from every file the run re-sends. Paths only -- nothing outside the
// scope is read or parsed.
const resolutionPaths = scopedRun
? discoverIngestFilePaths(
discoverSourceFiles(workspaceRoot, { exclude: exclude && { ...exclude, onSkip: undefined } }),
workspaceRoot,
(p) => p,
).files
: filePaths;
const resolveOpts = {
...crossRepoResolveOpts,
resolveModuleSpecifier: createTypeScriptModuleResolver(workspaceRoot, resolutionPaths),
packageDirOf: declaredPackageDirs(workspaceRoot, resolutionPaths),
};
filesDiscovered = filePaths.length;
const discovered = performance.now();
timings.discoverMs = Math.round(discovered - start);
progressPhase = 'Parsing';
progressTotal = filePaths.length;
progressCurrent = 0;
progressStart = performance.now();
// Phase: mtime pre-filter — skip readFileSync+sha256 for files whose mtime
// is unchanged since the last successful ingest (common "ix map" re-run case).
// The WORKSPACE's baseline, scoped run or not: it is the one record of what
// the graph holds for this workspace, and a scoped run updates its own
// files in it and carries every other entry forward.
const projectRoot = workspaceRoot;
// A just-migrated workspace (new path-based id) has no nodes under the new id, so
// skip the mtime pre-filter and re-ingest everything, exactly like --force.
const previousBaseline = loadIngestBaseline(projectRoot);
const currentExtractor = extractorName();
// A --lang run, or one scoped to a file or directory below the root,
// re-extracts only its own files, so it must not record the new extractor
// for the rest; keep the old one and the next full run still re-ingests
// them. The same goes for every other whole-workspace claim -- see the
// baseline write and the stitch below.
const partialRun = Boolean(opts.lang) || scopedRun;
const baselineExtractor = partialRun ? previousBaseline?.extractor : currentExtractor;
// Files a full re-ingest for this extractor has committed, recorded as they
// land so a run cut short resumes instead of starting over; null when no
// such re-ingest is under way.
let rebuildProgress: Map<string, number> | null = null;
let resumedRebuild: Map<string, number> | null = null;
if (!opts.force && extractorChanged(previousBaseline, currentExtractor)) {
if (!partialRun) {
const done = workspaceMigrated ? null : loadRebuildProgress(projectRoot, currentExtractor);
if (done && done.size > 0) resumedRebuild = done;
rebuildProgress = new Map(done ?? []);
}
process.stderr.write(
`\n [extractor changed] ${previousBaseline?.extractor ?? 'unrecorded'} -> ${currentExtractor}. ` +
(resumedRebuild
? `Resuming the re-ingest: ${resumedRebuild.size} of ${filePaths.length} files already done.\n`
: partialRun
? `Re-ingesting the ${filePaths.length} file(s) in this run; the next full map re-ingests the rest.\n`
: `Re-ingesting every file once.\n`),
);
// A resumed re-ingest is not forced: the files it already committed are
// skipped by mtime below, and every other file is forced one by one.
if (!resumedRebuild) opts.force = true;
extractorRebuild = true;
clearMapResultCache(projectRoot);
}
// The same-hash orphan sweep (IN-20, `sweepOrphanNodes`) costs one
// stored-patch read per file, and only an extractor change leaves orphans
// behind. So it runs only when the baseline says this workspace's files
// were last written by another extractor -- whether this run was forced
// or not -- and not on a plain --force at the current extractor. The
// baseline is the only signal that costs no read. IX_RECONCILE_ON_FORCE=1
// opts a --force run in, to repair a graph a pre-fix rebuild left
// polluted. On a backend that takes deletes in bulk, the same runs widen
// an edited file's reconcile to the second stored patch; an ordinary edit
// reads only the first, as before IN-20.
const reconcileOrphans = extractorChanged(previousBaseline, currentExtractor)
|| (Boolean(opts.force) && process.env.IX_RECONCILE_ON_FORCE === '1');
// The same-hash sweep reads, per file, the stored patch an earlier
// extractor wrote for the file's hash. On an extractor change that is the
// extractor the baseline recorded. On an opted-in --force at the current
// extractor (a repair) it is whatever an earlier run recorded as left
// unswept (`loadOrphanSweepPending`: a sweep the backend could not run, or
// one that ran out of budget), for only the files it recorded; with
// nothing recorded, the extractor before the current, for every file. An
// extractor change also reads what is recorded, for the files recorded.
const orphanRepair = reconcileOrphans && !extractorChanged(previousBaseline, currentExtractor);
const knownPreviousExtractors: readonly string[] = PREVIOUS_EXTRACTORS ?? [];
const orphanCandidateIndex = (extractor: string): number | null => {
const at = knownPreviousExtractors.indexOf(extractor);
return at < 0 ? null : at + 1;
};
const orphanPending: OrphanSweepPending = reconcileOrphans && !workspaceMigrated
? loadOrphanSweepPending(projectRoot)
: new Map();
for (const extractor of [...orphanPending.keys()]) {
if (extractor === currentExtractor || orphanCandidateIndex(extractor) === null) orphanPending.delete(extractor);
}
const orphanBaseExtractor: string | null = (() => {
if (!reconcileOrphans) return null;
const base = orphanRepair
? (orphanPending.size > 0 ? null : knownPreviousExtractors[0] ?? null)
: previousBaseline?.extractor ?? null;
return base !== null && orphanCandidateIndex(base) !== null ? base : null;
})();
/** The earlier extractors whose stored patch for this file the sweep reads. */
const orphanSourcesFor = (filePath: string): string[] => {
const sources = orphanBaseExtractor === null ? [] : [orphanBaseExtractor];
for (const [extractor, files] of orphanPending) {
if (!sources.includes(extractor) && (files === null || files.has(filePath))) sources.push(extractor);
}
return sources;
};
const orphanSweep = {
blocker: orphanSweepBlocker ?? (reconcileOrphans && !orphanRepair && orphanBaseExtractor === null && orphanPending.size === 0
? `the previous extractor (${previousBaseline?.extractor ?? 'unrecorded'}) is not one this CLI knows`
: null),
budget: orphanSweepBudget(),
noticed: false,
// Only a whole-workspace run records what it left unswept. A partial
// extractor-change run keeps the old extractor in the baseline, so the
// next full map sweeps its files anyway.
track: reconcileOrphans && !partialRun && !workspaceMigrated,
/** Per extractor read this run, the files it did not finish. */
unswept: new Map<string, Set<string>>(),
/** Recorded "every file" entries this run read for every file. */
allAtStart: new Set([...orphanPending].filter(([, files]) => files === null).map(([extractor]) => extractor)),
};
// One line per run when the sweep is wanted and cannot run: the orphans
// stay, and nothing else about the run changes. A whole-workspace
// extractor change records the previous extractor for every file, so a
// repair after a backend upgrade reads the patch that names them.
const noteOrphanSweepBlocked = (): void => {
if (orphanSweep.noticed || orphanSweep.blocker === null) return;
orphanSweep.noticed = true;
if (orphanSweep.track && !orphanRepair && orphanBaseExtractor !== null) {
orphanPending.set(orphanBaseExtractor, null);
saveOrphanSweepPending(projectRoot, orphanPending);
}
process.stderr.write(
`\n [orphans kept] nodes the previous extractor wrote and this one does not were not removed: ` +
`${orphanSweep.blocker}.` +
(orphanSweepBlocker !== null ? ' After upgrading the backend, run IX_RECONCILE_ON_FORCE=1 ix map --force to remove them.' : '') + '\n',
);
};
const previousMtimes = previousBaseline?.files ?? new Map<string, number>();
const {
previousDeletedFiles,
nextDeletedFiles,
forceReingestPaths,
} = planDeletedFileRecovery(
projectRoot,
filePaths,
previousBaseline?.deletedFiles ?? new Map(),
);
const mtimeCache = (opts.force || workspaceMigrated)
? new Map<string, number>()
: new Map(resumedRebuild ?? previousMtimes);
if (resumedRebuild) {
for (const filePath of filePaths) if (!resumedRebuild.has(filePath)) forceReingestPaths.add(filePath);
}
let rebuildSavedAt = 0;
// Committed patches carry the workspace-relative path; progress is keyed
// like the mtime baseline, by absolute path. Built on first use, after the
// stat loop below has filled `currentMtimes`.
let absoluteOf: Map<string, string> | undefined;
const noteRebuilt = rebuildProgress === null ? undefined : (item: { filePath: string }): void => {
absoluteOf ??= new Map([...currentMtimes.keys()].map(abs => [toWorkspaceRelative(abs), abs]));
const abs = absoluteOf.get(item.filePath);
const mtime = abs === undefined ? undefined : currentMtimes.get(abs);
if (abs === undefined || mtime === undefined) return;
rebuildProgress!.set(abs, mtime);
// Throttled: the whole map is rewritten each time, and a cut-short run
// loses at most this much of its work.
if (Date.now() - rebuildSavedAt >= REBUILD_PROGRESS_SAVE_MS) {
rebuildSavedAt = Date.now();
saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress!);
}
};
const currentMtimes = new Map<string, number>();
// DB-reset guard: if the mtime cache has entries but the server returns no hashes
// for a small sample, the DB was wiped (e.g. `ix reset` run from a different dir
// so the cache wasn't cleared). Invalidate the cache so files are re-ingested.
// Every baseline entry was committed (a file that failed keeps its previous
// entry or none), so each sampled file should be known to the backend.
// The sample is spread across the baseline rather than its first five
// entries, and a lookup that FAILED proves nothing: it used to come back
// empty and read as a reset, re-ingesting the whole workspace over one
// timed-out request.
if (!opts.force && mtimeCache.size > 0) {
const keys = [...mtimeCache.keys()];
const step = Math.max(1, Math.floor(keys.length / 5));
const samplePaths = keys.filter((_, i) => i % step === 0).slice(0, 5);
let sampleHashes: Map<string, string> | undefined;
try {
sampleHashes = await loadExistingHashes(client, samplePaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug, true);
} catch (err) {
if (debug) process.stderr.write(`\n DB-reset check skipped: hash lookup failed: ${err}\n`);
}
if (sampleHashes !== undefined && sampleHashes.size === 0) {
mtimeCache.clear();
// What an unfinished re-ingest committed is gone with the rest.
rebuildProgress?.clear();
// And so is the hierarchy a cached map response describes.
dbResetDetected = true;
clearMapResultCache(projectRoot);
if (debug) process.stderr.write(`\n DB reset detected — invalidating mtime cache\n`);
}
}
// Stat all files and partition into mtime-clean (skip) and mtime-changed (need hash check).
const mtimeChangedPaths: string[] = [];
/**
* Paths this loop has already accounted for, and under which counter.
*
* Recorded explicitly rather than inferred from `currentMtimes`. The loop
* leaves a path out of that map in three cases -- empty, oversized, and
* `statSync` THREW -- and only the first two are counted here. So a file
* that errored on the first stat and then read as size 0 on the second (a
* transient lock, an antivirus scan, a file recreated underneath us) was
* treated as already counted and landed in no bucket at all: absent from
* the graph, from `filesSkipped` and from every `skipReasons` entry, which
* is precisely what the check that reads this exists to prevent.
*/
const accountedByStatLoop = new Map<string, 'empty' | 'tooLarge'>();
skippedForSize = accountedByStatLoop;
for (const filePath of filePaths) {
try {
const st = fs.statSync(filePath);
if (st.size === 0) { filesSkipped++; filesSkippedAsEmpty++; accountedByStatLoop.set(filePath, 'empty'); progressCurrent++; continue; }
if (st.size > MAX_FILE_BYTES) { tooLarge++; accountedByStatLoop.set(filePath, 'tooLarge'); progressCurrent++; continue; }
const mtime = st.mtimeMs;
currentMtimes.set(filePath, mtime);
if (!opts.force && !forceReingestPaths.has(filePath) && mtimeCache.get(filePath) === mtime) {
filesSkipped++;
filesSkippedAsUnchanged++;
progressCurrent++; // mtime clean — assume unchanged
} else {
mtimeChangedPaths.push(filePath);
}
} catch (err) {
parseErrors++;
progressCurrent++;
process.stderr.write(`\n [stat error] ${filePath}: ${err}\n`);
}
}
// Only inside the scope: a scoped run says nothing about the rest of the
// workspace, so a file missing there is the next full map's to delete.
const deletedPaths = workspaceMigrated
? []
: [...previousMtimes.keys()].filter(filePath =>
inScope(filePath) && !currentMtimes.has(filePath) && !fs.existsSync(filePath)
);
if (deletedPaths.length > 0) deletionsFound = true;
// Phase: hash lookup — only needed when files changed or were deleted.
let knownHashes: Map<string, string>;
const hashLookupPaths = opts.force
? [...filePaths, ...deletedPaths]
: [...mtimeChangedPaths, ...deletedPaths];
// Deleting from an unreliable baseline is how data gets lost, so the lookup
// fails closed when deletions are pending — but only the *deletions* need
// that guarantee. Letting the throw escape aborted the entire map over one
// transient blip, taking the ordinary ingest of every changed file with it.
// So catch it here: drop the deletions for this run and let the rest
// proceed. `deferredDeletions` carries them into the baseline this run
// writes -- without it they vanished: `currentMtimes` never held a deleted
// file, so the next run's baseline did not name them, they were never
// found deleted again, and their nodes stayed in the graph for good.
const deferredDeletions: string[] = [];
// Whether the lookup ANSWERED. Thrown every time now, rather than only with
// deletions pending, so an empty answer can be told from a failed one; the
// catch below does what `loadExistingHashes` did with a swallowed failure.
let hashLookupAnswered = false;
if (hashLookupPaths.length > 0) {
try {
knownHashes = await loadExistingHashes(
client,
hashLookupPaths,
toWorkspaceRelative,
sourceWorkspaceIdOf,
debug,
true,
);
hashLookupAnswered = true;
} catch (err) {
if (debug && deletedPaths.length > 0) process.stderr.write(`\n [deletion cleanup skipped] hash lookup failed: ${err}\n`);
deferredDeletions.push(...deletedPaths);
deletedPaths.length = 0;
knownHashes = new Map();
}
if (debug) process.stderr.write(`\n Source hash lookup: ${knownHashes.size} known hashes (${mtimeChangedPaths.length} mtime-changed)\n`);
} else {
// All files are mtime-clean — skip server round-trip entirely.
knownHashes = new Map();
if (debug) process.stderr.write(`\n All ${filePaths.length} files mtime-clean — skipping hash lookup\n`);
}
// With no local baseline every file was looked up, so an answer naming
// none of them means the backend holds none. Not under --force, which
// says nothing about the baseline, and not on a failed lookup, which
// says nothing about the backend. Not on a workspace-id migration either:
// the old id's graph is still there until the cleanup after this run.
// And not on a partial run (a file or directory scope, or --lang): its
// lookup covered only its own files, so an empty answer says nothing about
// the rest of the workspace -- `ix ingest <new file>` in a repo mapped from
// another machine has no local baseline but a full graph.
graphStartedEmpty =
dbResetDetected ||
(!opts.force && !workspaceMigrated && !scopedRun && !opts.lang &&
mtimeCache.size === 0 && hashLookupAnswered && knownHashes.size === 0);
const hashed = performance.now();
timings.hashMs = Math.round(hashed - discovered);
// Phase: parse + commit — streaming to bound peak memory.
//
// Files are parsed in PARSE_STREAM_CHUNK-sized batches. After each batch,
// resolveEdges runs on that batch and patches are committed before the next
// batch is parsed. This keeps heap usage at O(chunk) instead of O(total).
//
// Two paths:
// A) Has baseline or mtime-changed files → pre-scan first, skip module load
// entirely if nothing changed (common "ix map" re-run case).
// B) No baseline / force → load modules, parse + stream-commit all files.
type ParsedFile = {
filePath: string; parsed: any; hash: string; previousHash: string | undefined;
/** The hash the backend holds for the file, changed or not. Feeds only reconcile. */
knownHash?: string | undefined;
};
let resolveEdgesFn: Function | null = null;
let buildPatchFn: Function | null = null;
// ── Symbol table ─────────────────────────────────────────────────────
// What every file defines, exports and imports, kept between runs (see
// `ingest-symbols.ts`). The resolution index is built from it, so an edit
// parses only the files that changed, in any language, and still resolves
// against all the others.
// Loaded on first use: a run with nothing to parse never reads it, and on a
// large repository it is tens of megabytes of JSON.
let loadedSymbolTable: Map<string, SymbolEntry> | undefined;
const symbolTable = (): Map<string, SymbolEntry> =>
(loadedSymbolTable ??= loadIngestSymbols(projectRoot, currentExtractor));
/** Paths whose entries this run set, altered or removed: what the save writes. */
const symbolTableChanges = new Set<string>();
/** sha256 of each file this run has read, by absolute path. */
const currentHashes = new Map<string, string>();
const mtimeChangedSet = new Set(mtimeChangedPaths);
let summarize: ((parsed: any) => StoredSummary) | null = null;
/** Record what a parse in this run learned about a file, for the next run. */
const noteSummary = (relFilePath: string, hash: string, parsed: any, mtime: number | undefined): void => {
if (!summarize || !parsed) return;
const table = symbolTable();
const before = table.get(relFilePath);
if (!summaryBeforeRun.has(relFilePath)) summaryBeforeRun.set(relFilePath, before?.summary);
// The resolution hash describes the patch built from these bytes; it is
// kept until that patch is built again (noteResolution).
const res = before?.hash === hash ? before.res : undefined;
const next: SymbolEntry = {
hash, summary: summarize(parsed), ...(mtime !== undefined ? { mtime } : {}), ...(res !== undefined ? { res } : {}),
};
// The same bytes parse to the same summary: a forced or touched-only
// re-parse leaves the entry as it was, and so leaves nothing to write.
if (before !== undefined && JSON.stringify(before) === JSON.stringify(next)) return;
table.set(relFilePath, next);
symbolTableChanges.add(relFilePath);
};
/**
* What each file this run parsed defined before the run, by relative path
* (undefined: a file the table did not have). Compared with its new
* summary to find the names that changed (IN-11).
*/
const summaryBeforeRun = new Map<string, StoredSummary | undefined>();
/** Record what a file's patch resolved to, so a later run can tell whether re-resolving it changes anything. */
const noteResolution = (relFilePath: string, patch: GraphPatchPayload): void => {
const entry = symbolTable().get(relFilePath);
if (!entry) return;
const res = resolutionHash(patch.ops);
if (entry.res !== res) {
entry.res = res;
symbolTableChanges.add(relFilePath);
}
};
/** The file's mtime now: from this run's stat loop, or a stat for a file outside it. */
const mtimeNow = (absFilePath: string): number | undefined => {
const known = currentMtimes.get(absFilePath);
if (known !== undefined) return known;
try { return fs.statSync(absFilePath).mtimeMs; } catch { return undefined; }
};
/**
* An entry still describes its file: the same bytes, or (with a baseline)
* a file that is mtime-clean AND still has the mtime the entry was read at.
* The baseline alone is not enough: the table is saved after it and can be
* older (a failed save, a run killed in between), and a scoped run does not
* stat the files outside its scope, so they never look changed.
*/
const tableEntryValid = (absFilePath: string, entry: SymbolEntry | undefined, trustMtime: boolean): boolean => {
if (!entry) return false;
const hash = currentHashes.get(absFilePath);
if (hash !== undefined) {
if (entry.hash !== hash) return false;
// Same bytes under a new mtime (a touch, a checkout): record the new
// one, or the entry fails the mtime check on every run from now on.
const mtime = currentMtimes.get(absFilePath);
if (mtime !== undefined && entry.mtime !== mtime) {
entry.mtime = mtime;
symbolTableChanges.add(toWorkspaceRelative(absFilePath));
}
return true;
}
if (!trustMtime || mtimeChangedSet.has(absFilePath) || entry.mtime === undefined) return false;
return mtimeNow(absFilePath) === entry.mtime;
};
/**
* The resolution index over every file edges may resolve to, from the
* table where it is still valid. A file it lacks is read and parsed here
* for its summary only -- unless the streaming loop is about to parse it
* in a single chunk, where the batch's own summary takes precedence in
* `resolveEdges` anyway. Across several chunks an earlier chunk would see
* a later one's stale entry, so then every file without a valid entry is
* summarized first (for a first map, that is one extra parse of the repo).
*
* `trustMtime` is false on Path B, where there is no baseline to trust.
*/
const buildResolutionIndex = async (
ingestion: { summarizeParseResult: (parsed: any) => StoredSummary; buildGlobalResolutionIndex: Function },
parsedInStream: Set<string>,
singleChunk: boolean,
trustMtime: boolean,
): Promise<any> => {
summarize = ingestion.summarizeParseResult;
const summaries = new Map<string, StoredSummary>();
const missing: string[] = [];
for (const abs of resolutionPaths) {
const rel = toWorkspaceRelative(abs);
const entry = symbolTable().get(rel);
if (tableEntryValid(abs, entry, trustMtime)) {
summaries.set(rel, entry!.summary);
continue;
}
if (singleChunk && parsedInStream.has(abs)) continue;
// A scoped run reads nothing outside its scope.
if (scopedRun && !inScope(abs)) continue;
missing.push(abs);
}
const SUMMARY_CHUNK = 500;
for (let i = 0; i < missing.length; i += SUMMARY_CHUNK) {
const chunk = missing.slice(i, i + SUMMARY_CHUNK);
// Stat'd before the read, so a file that changes in between is
// recorded with its older mtime and re-read next time.
const mtimes = chunk.map(mtimeNow);
const bytes = await Promise.all(chunk.map(fp => fs.promises.readFile(fp).catch(() => null)));
const parsed = await Promise.all(chunk.map((fp, j) => {
const b = bytes[j];
if (!b || b.length === 0 || b.length > MAX_FILE_BYTES) return null;
// `false`: a loss here is not a file the run lost -- see `ParsePool.parse`.
return ensureParsePool(missing.length).parse(toWorkspaceRelative(fp), b.toString('utf-8'), false).catch(() => null);
}));
for (let j = 0; j < chunk.length; j++) {
const b = bytes[j];
if (!b) continue;
const hash = sha256(b);
currentHashes.set(chunk[j], hash);
const rel = toWorkspaceRelative(chunk[j]);
noteSummary(rel, hash, parsed[j], mtimes[j]);
const entry = symbolTable().get(rel);
if (entry && entry.hash === hash) summaries.set(rel, entry.summary);
}
}
return ingestion.buildGlobalResolutionIndex(resolutionPaths.map(toWorkspaceRelative), undefined, undefined, summaries);
};
let globalIndex: any = undefined;
// Smaller chunks pipeline better: resolve(N) overlaps with commit(N-1) + parse(N+1).
// With 2000-file chunks, resolve takes 68s and dominates. With 500, resolve ~17s
// and overlaps well with commit (~5s) + parse (~5s).
// IX_PARSE_STREAM_CHUNK is for tests: it lets a run of a hundred files
// cross the same batch boundaries as a run of a thousand. Unset, 500.
const PARSE_STREAM_CHUNK = parsePositiveIntEnv('IX_PARSE_STREAM_CHUNK', 500);
const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend
// Ops per HTTP request; see DEFAULT_COMMIT_MAX_OPS.
const COMMIT_MAX_OPS = parsePositiveIntEnv('IX_COMMIT_MAX_OPS', DEFAULT_COMMIT_MAX_OPS);
/**
* The bulk bounds for the rest of the run. A 413 lowers them, so later
* batches start at a size that fits instead of failing and splitting again.
*/
const bulkLimits: BulkLimits = { maxFiles: COMMIT_HTTP_MAX_FILES, maxOps: COMMIT_MAX_OPS };
const opsOf = (item: { patch: GraphPatchPayload }): number => item.patch.ops.length;
/**
* A patch's size on the wire, measured only once a byte bound is in play
* (on a 413 that did not blame the op count, and in the cuts after it).
*/
const patchBytes = new WeakMap<GraphPatchPayload, number>();
const bytesOf = (item: { patch: GraphPatchPayload }): number => {
let n = patchBytes.get(item.patch);
if (n === undefined) {
n = Buffer.byteLength(JSON.stringify(item.patch));
patchBytes.set(item.patch, n);
}
return n;
};
const cutToLimits = <T extends { patch: GraphPatchPayload }>(items: T[]): T[][] =>
cutBulkChunks(items, opsOf, bulkLimits, bytesOf);
/**
* Learn from a group the backend found too large, then cut the group to
* the new bounds.
*
* Only a 413 that names a patch count (the bulk writer's estimate,
* Ix-memory#232) says the group held too many ops. That one lowers the op
* budget: the suggested count at this group's ops per patch, or half the
* group's ops if the suggestion is no smaller than the group.
*
* Any other 413 -- the backend's body cap, a proxy's limit, or one with no
* reason -- is about bytes, or may be, and the op budget is left alone.
* Halving ops on it let one file with a large body but few ops drag the
* run's op budget down with every bisection, until later requests carried
* about one patch each. It lowers a separate byte budget instead, to half
* the refused group's bytes. A group that was refused for its bytes is
* over the cap, so that budget stays above half the cap however many
* times it is lowered.
*
* Ops, not the suggested count itself: the backend sizes its suggestion
* from the group's average patch, and a count cut lets a run of larger
* files through over the budget, to be refused again.
*/
const learnBulkLimits = <T extends { patch: GraphPatchPayload }>(items: T[], err: unknown): T[][] => {
const suggested = parseSuggestedMaxPatches(err);
if (suggested !== undefined) {
const ops = items.reduce((sum, item) => sum + opsOf(item), 0);
const budget = suggested < items.length
? Math.floor((ops * suggested) / items.length)
: Math.floor(ops / 2);
bulkLimits.maxOps = Math.max(1, Math.min(bulkLimits.maxOps, budget));
} else {
const bytes = items.reduce((sum, item) => sum + bytesOf(item), 0);
const budget = Math.max(1, Math.floor(bytes / 2));
bulkLimits.maxBytes = Math.min(bulkLimits.maxBytes ?? Number.POSITIVE_INFINITY, budget);
}
if (debug) {
const byteNote = bulkLimits.maxBytes !== undefined ? ` / ${bulkLimits.maxBytes} bytes` : '';
process.stderr.write(`\n [bulk limits] now ${bulkLimits.maxFiles} files / ${bulkLimits.maxOps} ops${byteNote} per request\n`);
}
return cutToLimits(items);
};
const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requests
// Retries of one commit after a lock conflict or a transport drop. 0: none.
const COMMIT_CONFLICT_RETRIES = parseNonNegativeIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6);
// Total wait for re-sending a BULK the backend answered 502/503/504/429
// (at most three retries, `Retry-After` cut to it). 0: no such retries.
// Its own knob, so turning conflict retries off leaves these alone.
const COMMIT_TRANSIENT_RETRY_MS = parseNonNegativeIntEnv('IX_COMMIT_TRANSIENT_RETRY_MS', TRANSIENT_RETRY_BUDGET_MS);
// Re-sends of a commit that lost the base-rev race to another writer; see retryOnBaseRevRace.
const COMMIT_BASE_REV_RETRIES = parsePositiveIntEnv('IX_COMMIT_BASE_REV_RETRIES', 8);
let baseRevRetries = 0;
const noteBaseRevRetry = (): void => {
baseRevRetries++;
if (debug) process.stderr.write(`\n [base-rev race] another writer moved the rev; re-sending\n`);
};
if (debug) {
process.stderr.write(
`\n Save config: httpBatch=${COMMIT_HTTP_MAX_FILES} maxOps=${COMMIT_MAX_OPS} concurrency=${COMMIT_CONCURRENCY}\n`
);
}
const emptyEdges: any[] = [];
type PreparedPatch = {
patch: GraphPatchPayload;
fileNumber: number;
filePath: string;
/**
* The graph should change when this lands: the backend held a different
* hash for the file, it is a deletion, or it was forced (a restored file
* and its dependents). Such a patch answered `Idempotent` wrote nothing
* the graph needed -- see `settleCommitted`.
*/
changed: boolean;
};
const makePreparedPatch = (
patch: GraphPatchPayload,
fileNumber: number,
filePath: string,
changed: boolean,
): PreparedPatch => ({ patch, fileNumber, filePath, changed });
let forcedRelativePaths: Set<string> | undefined;
/** Whether a workspace-relative path was forced this run. Built on first use, once the set is final. */
const isForced = (relFilePath: string): boolean =>
(forcedRelativePaths ??= new Set([...forceReingestPaths].map(toWorkspaceRelative))).has(relFilePath);
/**
* Record a patch the backend took, or one it only claims to hold.
*
* A changed file answered `Idempotent` is a replay of a patch id the backend
* committed before: a file reverted to earlier bytes, or deleted and
* restored. The backend wrote nothing, so the graph still shows the content
* in between (F-01). Such a patch is counted in `replayedChanges`, named in
* the summary, and kept away from `onCommitted`, and its file keeps its old
* mtime in the baseline so the next run tries again.
*/
const settleCommitted = (
item: PreparedPatch,
rev: number,
replay: boolean,
onCommitted?: (item: PreparedPatch, rev: number) => void,
): void => {
if (replay && item.changed) {
replayedChanges.add(item.filePath);
return;
}
committedRel.add(item.filePath);
onCommitted?.(item, rev);
};
// Commit HTTP batches in parallel (COMMIT_CONCURRENCY workers). Each worker pulls
// the next unsent chunk from a shared queue. JS is single-threaded so the queue
// index increment is atomic across await boundaries — no locking needed.
const commitPreparedPatches = async (
preparedPatches: PreparedPatch[],
totalFiles: number,
opts?: {
updateProgress?: boolean;
onCommitted?: (item: PreparedPatch, rev: number) => void;
},
): Promise<number> => {
if (preparedPatches.length === 0) return 0;
/**
* Patches the cutoff held back rather than sent (Ix#560).
*
* Held, not discarded. Abandoning them outright is what made an earlier
* version able to wedge a workspace permanently: the mtime baseline is
* not written on a run with commit errors, so the next `ix map`
* re-ingests in the same order, trips at the same point, and abandons the
* same patches forever. They get one ordinary pass at the end IF the
* backend has meanwhile proved it accepts writes -- see the drain below.
*/
const deferredByCutoff: PreparedPatch[] = [];
/** Held by the run deadline rather than the cutoff -- see the branch below. */
const deferredByDeadline: PreparedPatch[] = [];
const chunks: PreparedPatch[][] = [];
let bulkRun: PreparedPatch[] = [];
const flushBulkRun = (): void => {
if (bulkRun.length === 0) return;
chunks.push(...cutToLimits(bulkRun));
bulkRun = [];
};
for (const item of preparedPatches) {
if (patchRequiresPerFileCommit(item.patch, bulkDeletes)) {
flushBulkRun();
chunks.push([item]);
} else {
bulkRun.push(item);
}
}
flushBulkRun();
const isBulkChunk = (chunk: PreparedPatch[]): boolean =>
!chunk.some(item => patchRequiresPerFileCommit(item.patch, bulkDeletes));
const commitMsPerChunk = new Array<number>(chunks.length).fill(0);
// The drain belongs to no chunk, but its time is still commit time --
// a no-op sink here dropped it from the reported total.
let drainMs = 0;
const recordDrainMs = (ms: number): void => { drainMs += ms; };
let nextChunk = 0;
// Fallback mutex: serializes per-file commits across all workers so that
// concurrent fallback loops don't race on revisions.current (Error 1200).
let fallbackTail: Promise<void> = Promise.resolve();
/**
* Commit `items` one at a time, serialized against every other per-file
* loop in this run.
*
* Hoisted out of runChunk so the end-of-run drain runs the SAME loop
* rather than a second copy of it -- the counters, the mutex and the
* breaker interaction are exactly the parts that must not drift.
* `recordMs` is the only thing that was chunk-scoped.
*/
const commitItemsSerially = async (
items: PreparedPatch[],
recordMs: (ms: number) => void,
bulkError?: unknown,
fallbackOpts?: {
replay?: boolean;
/**
* Total failures this loop may absorb before giving up, replacing the
* breaker's CONSECUTIVE streak for callers that need to get past a
* run of bad patches rather than stop at one.
*
* A streak cannot: N adjacent patches the backend rejects on their
* own merits look exactly like a dead backend for as long as the run
* of them lasts, so any streak of N or fewer stops inside the
* cluster and strands everything after it -- and the mtime baseline,
* never written on a run with commit errors, makes the next `ix map`
* repeat it exactly. A budget keeps going past the cluster and only
* stops when the FAILURES add up, which is the thing worth bounding.
*/
failureBudget?: number;
},
): Promise<void> => {
if (debug && bulkError !== undefined) {
const first = items[0];
const last = items[items.length - 1];
process.stderr.write(
`\n [bulk failed, falling back to per-file] files ${first.fileNumber}-${last.fileNumber} (${items.length} patches): ${bulkError}\n`
);
}
// Chain this chunk's fallback work onto the shared tail so that only one
// per-file loop runs at a time. This prevents concurrent commitPatch
// transactions from racing on revisions.current (write-write conflict).
const prev = fallbackTail;
let resolveThis!: () => void;
fallbackTail = new Promise<void>(r => { resolveThis = r; });
await prev;
let budgetLeft = fallbackOpts?.failureBudget ?? Number.POSITIVE_INFINITY;
/** Failed re-sends of server-confirmed patches. Bounds the replay, locally. */
let replayFailures = 0;
try {
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const item = items[itemIndex];
const isReplay = fallbackOpts?.replay === true;
if (runDeadlineExpired()) {
// A replay first: these patches are confirmed landed by the
// server's own 409 body, so the clock stops the BOOKKEEPING and
// not a write. Holding them would push up to a thousand
// confirmed-committed patches into the commit-error count,
// report the graph as missing files it holds, and suppress the
// mtime baseline over them -- the exact over-reporting the
// `replay` flag exists to prevent, reintroduced by putting this
// deadline check above it.
if (isReplay) {
patchesApplied += items.length - itemIndex;
break;
}
// Every request from here is rejected by `fetch` on an already
// aborted signal, so the fan-out is pure waste -- and it is not
// otherwise stopped, because `commitFailureIndictsBackend`
// returns false for everything once the deadline has fired.
// That is right for CLASSIFYING (the deadline is our clock, not
// the backend's answer) and it left the cutoff and the drain
// budget both inert for the tail of the run, so a bulk failure
// at that point fanned out to one serialized commit per patch
// with nothing able to stop it.
//
// A SEPARATE list from `deferredByCutoff`, deliberately. These
// patches were abandoned by the clock, not by the backend, and
// feeding them to `recordSkipped` made the cutoff banner blame
// the backend for them -- printing "400 patches were not sent,
// sending them would have added load to a backend that is
// already the reason they fail" directly above "Ingest ran out
// of time: raise IX_MAP_DEADLINE_MS". Two contradictory
// diagnoses, and the wrong count on the first.
// `describeCommitOutcome` already orders the deadline ahead of
// the cutoff for exactly this reason.
for (let k = itemIndex; k < items.length; k++) deferredByDeadline.push(items[k]);
break;
}
const action = perFileAction({
replay: isReplay,
tripped: commitBreaker.tripped(),
budgetLeft: fallbackOpts?.failureBudget === undefined ? undefined : budgetLeft,
});
if (action === "hold") {
for (let k = itemIndex; k < items.length; k++) deferredByCutoff.push(items[k]);
break;
}
if (action === "count-applied") {
// Confirmed landed by the server's 409, and we have given up
// sending. They are still IN THE GRAPH, so dropping them from
// the counters would make the cutoff report "The graph is
// unchanged" for a run that changed it.
//
// This branch adds no commitErrors, and `persistIngestBaseline-
// IfClean` is gated on the RUN-WIDE count, so on a run whose
// only anomaly is this the baseline IS written and the next run
// skips these files. That is correct here and only here --
// the server's own 409 confirmed they landed -- but it is the
// opposite of what an earlier comment claimed, so it is worth
// stating rather than leaving to be re-derived.
patchesApplied += items.length - itemIndex;
break;
}
try {
// Move the label per file. It used to be set once, before the
// bulk attempt, and never again -- so a fallback of several
// hundred serialized commits sat on `commit 1-498 of 498` with
// only the elapsed timer moving, which reads as a hang (Ix#495).
setCurrentWork(
`commit ${item.fileNumber} of ${totalFiles} ${nodePath.basename(item.filePath)} (per-file)`
);
const commitStart = performance.now();
const result = await retryOnBaseRevRace(
() => retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES),
COMMIT_BASE_REV_RETRIES,
noteBaseRevRetry,
);
const chunkMs = Math.round(performance.now() - commitStart);
recordMs(chunkMs);
timings.fallbackCommitMs += chunkMs;
latestRev = advanceRev(latestRev, result.rev);
// See onBulkCommitted: a lost base-rev race wrote nothing and
// must not be reported as applied.
if (result.status === COMMIT_STATUS_BASE_REV_MISMATCH) {
commitErrors++;
continue;
}
patchesApplied++;
patchesTheBackendTook++;
commitBreaker.recordSuccess();
if (result.status === COMMIT_STATUS_IDEMPOTENT) idempotentPatches++;
settleCommitted(item, result.rev, result.status === COMMIT_STATUS_IDEMPOTENT, opts?.onCommitted);
} catch (commitErr) {
// Counted apart from parseErrors: a patch that parsed fine and
// failed to commit means the graph is now behind the working
// tree, which is a different problem from a file we could not
// read. Reporting it as a parse error hid that distinction in
// both the summary and the JSON contract.
// Only failures that indict the BACKEND spend the budget --
// the same filter the breaker uses. A payload-too-large is
// about that one patch, and letting a handful of oversized
// generated files exhaust the drain would strand hundreds of
// perfectly committable patches on a healthy backend.
const indicts = commitFailureIndictsBackend(commitErr, runDeadlineExpired());
if (indicts) budgetLeft--;
// ...but a REPLAY never indicts the backend, whatever the error
// looks like: the server's own 409 confirmed those patches
// landed, so a failure re-sending one says nothing about its
// willingness to write. This counter is the drain gate's
// evidence, and its own doc says exactly that twenty lines
// below; counting replays here contradicted it.
if (indicts && !isReplay) backendIndictingFailures++;
if (isReplay) {
// Confirmed landed by the server's own 409 body. A failed
// re-send loses no write, so counting it as a commit error
// would let the cutoff claim "The graph is unchanged" for a
// run whose patches are all in the graph.
patchesApplied++;
// It still has to STOP, though. The full-landed branch of
// `commitBulkWithPayloadSplit` hands the whole chunk here and
// returns before its `shouldStop` check, so nothing else
// bounds it: an un-tripped breaker would watch up to
// IX_COMMIT_HTTP_MAX_FILES doomed commits go out one at a
// time behind the global fallback mutex, blocking every other
// chunk's fallback.
//
// Bounded LOCALLY, not through the run-wide breaker. Feeding
// it there was round 13's fix and it indicted the whole run
// for something the CLI knows says nothing about the backend's
// willingness to write: the server already confirmed these
// patches landed, so a backend that answers a hard error for
// re-committing one -- a legitimate idempotency
// implementation -- tripped the run-wide cutoff after five
// replays while perfectly healthy.
replayFailures++;
if (commitBreaker.limit > 0 && replayFailures >= commitBreaker.limit) {
patchesApplied += items.length - itemIndex - 1;
break;
}
} else {
commitErrors++;
if (indicts) {
commitBreaker.recordFailure(commitErr);
// Not in the drain (the only caller with a failure
// budget): it ignores `tripped`, so a probe there would
// only add health requests and latency.
if (fallbackOpts?.failureBudget === undefined) await tripIfDatabaseDown(commitErr);
}
}
if (debug) {
const errMsg = String(commitErr);
const truncated = errMsg.length > 200 ? errMsg.slice(0, 200) + '…' : errMsg;
process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${truncated}\n`);
}
}
}
} finally {
resolveThis();
}
};
/**
* Re-send the patches the cutoff held back, in alternating passes.
*
* Not a bypass, but the bound is the BUDGET, not the breaker: with a
* failure budget in play `perFileAction` ignores the streak entirely, so
* saying "the breaker is active, so this stays bounded" -- as this
* comment used to -- described a mechanism that does not run. What each
* pass carries is `failureBudget` doomed requests. There is deliberately
* NO pass cap -- one would hand back committable patches unsent, a real
* loss where this is only slowness -- so the bound is the HELD-SET SIZE:
* a pass only attempts items no earlier pass reached, so the drain sends
* at most one request per held patch, which is what `main` sends anyway.
* A dead backend is stopped after `3 x budget` -- one per sample; one that
* accepts a little and refuses the rest is not stopped at all and pays the
* full size.
*
* The streak is still RESET per pass, for a different reason: five
* adjacent patches the backend rejects on their own merits are
* indistinguishable from a dead backend at the time, and the difference
* only shows when the next patch commits. Carrying the streak in would
* leave the run's banner reporting the hold that caused the drain rather
* than what the drain found.
*
* Without any of this the mtime baseline -- never written on a run with
* commit errors -- makes the next run repeat the same order, stop at the
* same point, and drop the same patches forever.
*/
/** Re-send the held set. What it cannot place is left on `deferredByCutoff`. */
const commitIndividuallyAfterCutoff = async (itemsIn: PreparedPatch[]): Promise<void> => {
let items = itemsIn;
// Re-sent even when the backend has reported its database down. The
// probe may hold patches back during the fan-out, but whether the run
// fails is decided by commits, as on main: a wrong or stale health
// answer must not turn a run whose patches would land into an exit 1.
if (debug) {
process.stderr.write(
`\n [cutoff] re-sending ${items.length} held-back patches with a fresh streak\n`,
);
}
// One bulk first, because the held set is usually not the problem.
//
// `onAbandoned` hands over whole CHUNKS, not just the tail the fan-out
// reached, so on a backend that recovers by the end of the batch this
// would otherwise serialize thousands of single-patch commits where one
// request per chunk would do -- the Ix#495 shape, on the recovery path
// this design exists to serve. A backend that is still refusing costs
// one extra doomed request, the same price the per-chunk bulk already
// pays and deliberately never skips.
//
// DELETIONS are excluded on a backend whose bulk route is not known to
// apply delete ops (`backendAppliesBulkDeletes`): `runChunk` routes
// those to their own single-item `client.commitPatch`. They fall
// through to the passes below, which commit them one at a time as their
// routing requires. On a current backend they ride the bulk like the rest.
//
// CHUNKED at `COMMIT_HTTP_MAX_FILES`, not skipped above it. Skipping
// meant that lowering `IX_COMMIT_HTTP_MAX_FILES` -- which #560's
// reporters explicitly tried -- turned the probe off entirely and gave
// them the serialized fan-out it exists to avoid.
//
// A PARTIAL 409 is handled here rather than left to the passes, because
// the passes cannot: they run with a failure budget and never with
// `replay: true`, so re-sending a patch the server has already
// confirmed would be counted as a commit error -- suppressing the mtime
// baseline and reporting files as missing from a graph that holds them,
// which is the over-reporting `replay` exists to prevent (Ix#495).
const needsPerFile = (item: PreparedPatch): boolean => patchRequiresPerFileCommit(item.patch, bulkDeletes);
const stillToSend: PreparedPatch[] = items.filter(needsPerFile);
let probeQueue = items.filter(item => !needsPerFile(item));
while (probeQueue.length > 1) {
const chunk = cutToLimits(probeQueue)[0];
const rest = probeQueue.slice(chunk.length);
const bulkStart = performance.now();
try {
const result = await retryOnBaseRevRace(
() => retryCommit(
() => client.commitPatchBulk(chunk.map(item => item.patch)),
{
maxRetries: COMMIT_CONFLICT_RETRIES,
transient: COMMIT_TRANSIENT_RETRY_MS > 0,
transientBudgetMs: COMMIT_TRANSIENT_RETRY_MS,
signal: runDeadlineSignal,
// Not the breaker: this bulk exists because the breaker
// tripped, so it would always say give up. Asked, not just
// read: a 503 here may be the database going down.
shouldGiveUp: () => checkDatabase(),
onRetry: noteCommitRetry('cutoff bulk'),
},
),
COMMIT_BASE_REV_RETRIES,
noteBaseRevRetry,
);
const ms = Math.round(performance.now() - bulkStart);
recordDrainMs(ms);
timings.bulkCommitMs += ms;
latestRev = advanceRev(latestRev, result.rev);
// Same base-rev guard the other two commit sites carry, and for the
// same reason. This one was written without it: `BaseRevMismatch`
// means the backend read the latest rev outside the transaction and
// it moved before the commit ran, so NOTHING was written. Counting
// that chunk as applied left `commitErrors` at zero, which is the
// condition `persistIngestBaselineIfClean` requires -- so the run
// wrote an mtime baseline for files the graph never received, and
// every later incremental map skipped them as unchanged. Recoverable
// only by `--force`, and silent until someone noticed the symbols
// missing. `onBulkCommitted` and the per-file path both guard it;
// the drain added for Ix#571 is the odd one out.
if (result.status === COMMIT_STATUS_BASE_REV_MISMATCH) {
commitErrors += chunk.length;
if (debug) {
process.stderr.write(
`\n [cutoff] bulk lost the base-rev race ${COMMIT_BASE_REV_RETRIES + 1} times; ${chunk.length} patches wrote nothing and will be re-sent next run\n`
);
}
probeQueue = rest;
continue;
}
patchesApplied += chunk.length;
patchesTheBackendTook += chunk.length;
commitBreaker.recordSuccess();
// The cutoff drain is a third commit site, added after this counter
// was written (Ix#571). It is a plain bulk commit carrying a status,
// so it counts like the other one -- a run whose held patches were
// all placed here and all deduplicated would otherwise report
// `idempotentPatches: 0` and fall back to the language hypothesis
// this change exists to retire.
if (result.status === COMMIT_STATUS_IDEMPOTENT) idempotentPatches += chunk.length;
for (const item of chunk) {
settleCommitted(item, result.rev, result.status === COMMIT_STATUS_IDEMPOTENT, opts?.onCommitted);
}
if (debug) process.stderr.write(` [cutoff] bulk placed ${chunk.length} of ${items.length} held
`);
probeQueue = rest;
} catch (bulkErr) {
// Record the time either way. The FAILING probe is the expensive
// one -- a saturated backend answers with a per-request timeout,
// multiplied by the conflict retries -- so dropping it hid minutes
// from the reported commit total on exactly the run this targets.
const ms = Math.round(performance.now() - bulkStart);
recordDrainMs(ms);
timings.bulkCommitMs += ms;
let unconfirmed = chunk;
let confirmedAny = false;
if (isBulkPartiallyCommittedError(bulkErr)) {
const landedIds = parseBulkCommittedPatchIds(bulkErr);
if (landedIds !== undefined) {
const landed = chunk.filter(item => landedIds.has(item.patch.patchId));
if (landed.length > 0) {
patchesApplied += landed.length;
// ...and through `onCommitted`, like every other "these
// landed" path. The deletion batch is the one caller that
// supplies it, and it uses it to write `durableDeletedFiles`:
// a deletion patch whose reconcile found nothing to remove
// carries no delete op, so it rides the bulk path, can be held
// by the cutoff, and can be 409-confirmed here -- counted
// applied but never recorded in the deleted-files baseline.
// `latestRev` rather than a rev of its own: the server holds
// these already, so their rev is at or below it, and
// `saveIngestBaseline` floors what it stores anyway.
// A replay by definition: the server already held these ids.
for (const item of landed) settleCommitted(item, latestRev, true, opts?.onCommitted);
// NOT `patchesTheBackendTook`. That counter's whole reason for
// existing is to keep 409-sourced counts from reopening a gate
// closed against a dead backend -- "counted only where a
// request came back OK", as its own doc says. A backend that
// can read but cannot begin a write transaction answers 409
// with ids it already holds, and on a re-run after a failed
// ingest the patch ids are identical, so feeding those here
// would reopen the gate and un-trip the breaker every batch.
confirmedAny = true;
if (debug) {
process.stderr.write(
` [cutoff] bulk partly committed: ${landed.length} of ${chunk.length} already landed
`,
);
}
}
unconfirmed = chunk.filter(item => !landedIds.has(item.patch.patchId));
}
}
// Keep the quoted error current, for the same reason `onAbandoned`
// does -- but NOT for a 409 that confirmed patches. That answer says
// the backend is taking writes, and the banner's `Last error:` is
// what the message and the troubleshooting row both tell the user
// distinguishes a saturated database from a rejected patch. A 409
// says neither, and quoting it would replace the TimeoutError that
// actually caused the cutoff. Not `recordFailure` either -- one bulk
// is one request, not N.
if (!confirmedAny && commitFailureIndictsBackend(bulkErr, runDeadlineExpired())) {
commitBreaker.noteError(bulkErr);
}
if (confirmedAny) {
// Positive proof the backend is taking writes, so KEEP PROBING --
// and re-bulk what it did NOT confirm rather than hand it to the
// passes, which have no bulk path. `commitBulkWithPayloadSplit`
// does exactly this with the same response: the unconfirmed set
// is a different group id, so one more request settles it where
// 497 serialized commits behind the global mutex would otherwise
// be the Ix#495 amplification this probe exists to prevent.
//
// Deliberately NOT `recordSuccess()`. A 409 naming ids the server
// already holds is not evidence it can take a NEW write -- that is
// the same reasoning that keeps those ids out of
// `patchesTheBackendTook` twenty lines above, and calling both
// would have made the two contradict each other. A backend that
// can read but cannot begin a write transaction answers exactly
// this way on a re-run, and un-tripping the run-wide cutoff there
// hands the next batch's failing bulk back to the fan-out.
probeQueue = [...unconfirmed, ...rest];
} else {
for (const item of unconfirmed) stillToSend.push(item);
// One real failure ends the probing: the rest goes to the passes,
// which is where a backend that is actually refusing belongs.
for (const item of rest) stillToSend.push(item);
probeQueue = [];
}
}
}
for (const item of probeQueue) stillToSend.push(item);
if (stillToSend.length === 0) return;
items = stillToSend;
// A budget, not the breaker's streak: the held set can begin with the
// tail of a poison cluster -- the fan-out stopped inside it, so the
// rest of it is the first thing here -- and a streak would stop again
// in the same place and strand everything after.
//
// The budget is the limit itself, not twice it. Twice was chosen to
// walk out of a cluster in ONE pass; `drainInPasses` walks out of it by
// changing direction instead. `drainFailureBudget` is shared with the
// tests, which previously asserted the stranding guarantee at a budget
// the CLI never runs at and so could not see it fail.
const leftover = await drainInPasses(items, async pending => {
// No database probe here (the per-file loop skips it when given a
// failure budget): the drain sends, as on main, so the exit code
// follows what the commits did, never a health answer.
commitBreaker.reset();
const tookBefore = patchesTheBackendTook;
await commitItemsSerially(pending, recordDrainMs, undefined, {
failureBudget: drainFailureBudget(commitBreaker.limit),
});
// Whatever the budget did not reach was pushed straight back onto the
// held list by the per-file loop; take it back out so the next pass
// owns it and the caller does not see it twice.
return {
placed: patchesTheBackendTook > tookBefore,
unreached: deferredByCutoff.splice(0, deferredByCutoff.length),
};
});
// A loop, not a spread. The ordinary path bounds this at
// `PARSE_STREAM_CHUNK` (500), but the reconcile call passes every
// deletion patch in the run unchunked, so the argument limit is not
// obviously out of reach -- and spreading past it throws RangeError from
// inside the commit phase, an uncaught throw that loses the whole
// batch's error accounting. The loop costs nothing and removes the
// question.
for (const item of leftover) deferredByCutoff.push(item);
};
const runChunk = async (ci: number): Promise<void> => {
const chunk = chunks[ci];
const endFile = chunk[chunk.length - 1].fileNumber;
const commitIndividually = async (
items: PreparedPatch[],
bulkError?: unknown,
fallbackOpts?: { replay?: boolean },
): Promise<void> =>
commitItemsSerially(items, ms => { commitMsPerChunk[ci] += ms; }, bulkError, fallbackOpts);
const hasDeletion = chunk.some(item => patchRequiresPerFileCommit(item.patch, bulkDeletes));
if (hasDeletion) {
await commitIndividually(chunk);
} else {
await commitBulkWithPayloadSplit(chunk, {
commitBulk: async items => {
const first = items[0];
const last = items[items.length - 1];
setCurrentWork(
`commit ${first.fileNumber}-${last.fileNumber} of ${totalFiles} ending ${nodePath.basename(last.filePath)}`
);
const commitStart = performance.now();
const result = await retryOnBaseRevRace(
() => retryCommit(
() => client.commitPatchBulk(items.map(item => item.patch)),
{
maxRetries: COMMIT_CONFLICT_RETRIES,
// 502/503/504/429 too, honouring Retry-After up to the
// budget (5 s per bulk by default, requests included; a
// slow bulk is not sent again). A 413 and a 409 are
// thrown at once, to the payload split and the
// partial-commit handling below.
transient: COMMIT_TRANSIENT_RETRY_MS > 0,
transientBudgetMs: COMMIT_TRANSIENT_RETRY_MS,
signal: runDeadlineSignal,
// Asked, not just read. On the first bulk of a run nothing
// has probed yet, and a 503 from a backend whose database
// is down would otherwise spend the whole budget
// before anything asked it. A database reported down trips
// the breaker, so `shouldStop` hands the chunk to the
// cutoff instead of the per-file fallback.
shouldGiveUp: async err => {
if (commitBreaker.tripped()) return true;
await tripIfDatabaseDown(err);
return commitBreaker.tripped() || databaseKnownDown();
},
onRetry: noteCommitRetry('bulk'),
},
),
COMMIT_BASE_REV_RETRIES,
noteBaseRevRetry,
);
const chunkMs = Math.round(performance.now() - commitStart);
commitMsPerChunk[ci] += chunkMs;
timings.bulkCommitMs += chunkMs;
return result;
},
onBulkCommitted: (items, result) => {
latestRev = advanceRev(latestRev, result.rev);
// `BaseRevMismatch` wrote nothing: the backend read the latest rev
// outside the transaction and it moved before the commit ran. It
// is a lost race, not a replay, so it must not count as applied —
// it belongs in commitErrors, and it must not reach
// `settleCommitted`: an unsettled file keeps its previous mtime,
// which is what stops the baseline caching it as unchanged and
// leaving it missing from the graph until the next edit.
if (result.status === COMMIT_STATUS_BASE_REV_MISMATCH) {
commitErrors += items.length;
if (debug) {
process.stderr.write(
`\n [commit lost the base-rev race ${COMMIT_BASE_REV_RETRIES + 1} times] ${items.length} patches wrote nothing; they will be re-sent next run\n`
);
}
return;
}
patchesApplied += items.length;
patchesTheBackendTook += items.length;
commitBreaker.recordSuccess();
// One status covers the whole bulk: the backend commits the chunk
// as a unit, so `Idempotent` means every patch in it was a replay.
if (result.status === COMMIT_STATUS_IDEMPOTENT) idempotentPatches += items.length;
for (const item of items) {
settleCommitted(item, result.rev, result.status === COMMIT_STATUS_IDEMPOTENT, opts?.onCommitted);
}
},
commitIndividually,
shouldStop: () => commitBreaker.tripped(),
onAbandoned: (items, err) => {
// The clock's, if it has fired. `shouldStop` is the breaker, which
// does not consult the deadline, so a chunk whose bulk was aborted
// by the run budget arrived here and was reported as withheld by
// the cutoff -- the same misattribution the two lists exist to
// keep apart, on the one path that was not routing through them.
const held = runDeadlineExpired() ? deferredByDeadline : deferredByCutoff;
for (const item of items) held.push(item);
// Keep the quoted error current. This is the only place a bulk
// failure is seen once the breaker has tripped -- the per-file
// loop it would otherwise reach is exactly what the trip skips --
// and after the first batch trips, every later batch fails here
// and nowhere else. Without it the banner quotes batch 1's error
// for the whole run, which is wrong in the case the message was
// written for: a backend that recovers from saturation and starts
// REJECTING patches instead still reads as "the database is busy".
if (err !== undefined && commitFailureIndictsBackend(err, runDeadlineExpired())) {
commitBreaker.noteError(err);
}
},
patchIdOf: item => item.patch.patchId,
onPartialBulk: (landed, missing, err) => {
if (!debug) return;
process.stderr.write(
`\n [bulk partly committed, resuming] ${landed.length} already landed, re-sending ${missing.length}: ${err}\n`
);
},
splitTooLarge: learnBulkLimits,
recut: cutToLimits,
onSplit: (items, err) => {
if (!debug) return;
const first = items[0];
const last = items[items.length - 1];
process.stderr.write(
`\n [bulk payload too large, splitting] files ${first.fileNumber}-${last.fileNumber} (${items.length} patches): ${err}\n`
);
},
});
}
if (opts?.updateProgress) {
progressCurrent = Math.max(progressCurrent, endFile);
}
};
const worker = async (): Promise<void> => {
while (true) {
const ci = nextChunk++;
if (ci >= chunks.length) break;
await runChunk(ci);
}
};
// Several bulk requests from one batch go one at a time. Each takes the
// backend's exclusive lock, so sending them side by side bought no
// throughput and lost base-rev races (BaseRevMismatch) to each other.
const workers = chunks.filter(isBulkChunk).length > 1 ? 1 : Math.min(COMMIT_CONCURRENCY, chunks.length);
await Promise.all(Array.from({ length: workers }, () => worker()));
// Nothing stays abandoned while the backend is demonstrably accepting
// writes. Five adjacent patches the backend rejects on their own merits
// look exactly like a dead backend until a later chunk commits, so the
// held-back set gets a real attempt rather than being written off.
//
// `drainInPasses` is what decides when to stop -- a pass that REACHED
// THE END, or three passes (tail, head, middle) that between them placed
// nothing at all, and not even that once the backend has proved it is
// alive. What the last pass could not reach is left for the next run,
// which re-ingests it anyway because the mtime baseline is not written on
// a run with commit errors. (A single pass that placed nothing is
// deliberately NOT a stopping condition, and there is no pass cap; its own
// doc says why for both.) The gate below is a coarser judgement across
// BATCHES, so that a genuinely dead backend is not re-probed once per
// batch for the length of the run.
if (deferredByCutoff.length > 0) {
if (!runDeadlineExpired() && drainGate.shouldDrain(patchesTheBackendTook)) {
const held = deferredByCutoff.splice(0, deferredByCutoff.length);
const tookBefore = patchesTheBackendTook;
const indictedBefore = backendIndictingFailures;
await commitIndividuallyAfterCutoff(held);
// A miss means the backend took nothing AND said something that
// indicts it. Both halves are load-bearing, and each was wrong on its
// own in an earlier revision:
//
// - taking nothing alone is not it. `commitFailureIndictsBackend`
// excludes payload-too-large, so a held set of oversized generated
// files is attempted in full and refused in full without the
// backend being at fault; two such batches closed the gate on a
// backend that never refused a write.
// - "nothing was stranded" alone is not it either, which is subtler.
// A held set no larger than one pass's budget -- a trip near the
// end of a chunk, or the deletion path where each deletion is its
// own one-patch chunk -- is attempted in full and refused in full
// by a DEAD backend, and reporting that as progress meant the gate
// could never close and every batch re-probed, which is the whole
// thing the gate exists to stop.
//
// Whether the drain ended tripped says nothing useful either: one
// that commits hundreds and then meets the tail of a bad cluster ends
// tripped too.
const backendMisbehaved = backendIndictingFailures > indictedBefore;
drainGate.record(patchesTheBackendTook > tookBefore || !backendMisbehaved, patchesTheBackendTook);
}
// Held back again, or held back after the retry was already spent.
if (deferredByCutoff.length > 0) {
commitErrors += deferredByCutoff.length;
// Counted as the cutoff's even when the deadline has since fired.
// Everything on this list was held by `perFileAction` BEFORE the
// clock ran out -- the deadline check sits above the hold branch and
// diverts to `deferredByDeadline` once it has -- so they are the
// cutoff's patches, and suppressing the count reported "0 of them
// were withheld by this cutoff" while the deadline message claimed
// all of them. That is the same misattribution the
// `deferredByDeadline` split exists to prevent, pointing the other
// way. The contradiction it was guarding against is handled where it
// belongs: `describeCommitCutoff` drops the "would have added load"
// framing when `deadlineHit`, and names the split instead.
commitBreaker.recordSkipped(deferredByCutoff.length);
deferredByCutoff.length = 0;
}
}
// Missing from the graph either way, but attributed to the clock: the
// deadline branch of `describeCommitOutcome` is what reports these.
if (deferredByDeadline.length > 0) {
commitErrors += deferredByDeadline.length;
// Deliberately not fed to the cutoff's own accounting: these are the
// clock's, and `describeCommitOutcome` reports them under the deadline.
deferredByDeadline.length = 0;
}
return commitMsPerChunk.reduce((a, b) => a + b, 0) + drainMs;
};
/**
* Build each file's patch and, for a file the backend had before, reconcile
* what it removed. Reconcile runs for several files at once: awaited one
* file at a time it was a serial chain of entity lookups per batch. The
* patches come back in file order, without the ones that failed to build.
*/
const noteOrphanSkip = (filePath: string, reason: string): void => {
if (debug) process.stderr.write(`\n [orphan reconcile skipped] ${filePath}: ${reason}\n`);
};
const buildFilePatches = async (
files: ParsedFile[],
edgesByFile: Map<string, any[]>,
): Promise<PreparedPatch[]> => {
const built = await mapLimit(files, RECONCILE_CONCURRENCY, async ({ parsed: p, hash, previousHash, knownHash }, j) => {
try {
const fileWorkspace = fileWorkspaceId(p.filePath);
let patch = buildPatchFn!(p, hash, fileWorkspace, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));
if (previousHash) {
patch = await reconcileRemovedEntities(
client,
patch,
sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),
undefined,
mapMode,
reconcileLookups,
{ onWindowSkipped: reason => noteOrphanSkip(p.filePath, reason), window: reconcileOrphans && orphanSweepBlocker === null ? RECONCILE_MANIFEST_WINDOW : 1 },
);
}
// Same content, new extractor (or an opted-in --force): the
// previous extractor's patch for this hash names what the sweep
// below may retire. Without a backend that takes deletes in bulk
// and serves the batched node read the file is committed exactly as
// built, as before IN-20.
let orphanSources: string[] | undefined;
let previousPatchIds: string[] | undefined;
if (!previousHash && knownHash && reconcileOrphans) {
if (orphanSweep.blocker !== null) noteOrphanSweepBlocked();
else {
orphanSources = orphanSourcesFor(p.filePath);
const candidates = sourcePatchIdCandidates(p.filePath, knownHash, fileWorkspace);
previousPatchIds = orphanSources.map(extractor => candidates[orphanCandidateIndex(extractor)!]!);
if (previousPatchIds.length === 0) previousPatchIds = undefined;
}
}
return { p, hash, j, previousHash, patch: patch as GraphPatchPayload, previousPatchIds, orphanSources };
} catch (err) {
parseErrors++;
process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);
return null;
}
});
// The orphan sweep, for the whole batch at once: one stored-patch read
// per file and one batched node read. Best effort -- whatever fails here
// leaves the patches to commit as built, with no parse error and no
// change to the exit code.
const sweepable = built.filter((b): b is NonNullable<typeof b> & { previousPatchIds: string[]; orphanSources: string[] } =>
b !== null && b.previousPatchIds !== undefined);
if (sweepable.length > 0 && orphanSweep.blocker === null) {
const batchUpserts = new Set<string>();
for (const b of built) {
if (b === null) continue;
for (const op of b.patch.ops) if (op.type === 'UpsertNode' && typeof op['id'] === 'string') batchUpserts.add(op['id']);
}
const unswept = new Set<number>();
const writtenBefore = new Set<number>();
try {
const sweepFiles = sweepable.map(b => ({ patch: b.patch, previousPatchIds: b.previousPatchIds }));
const orphans = await sweepOrphanNodes(
client,
sweepFiles,
{
mapMode,
budget: orphanSweep.budget,
batchUpserts,
onSkip: (file, reason) => noteOrphanSkip(file.patch.source.uri, reason),
onUnswept: file => unswept.add(sweepFiles.indexOf(file)),
onWrittenBefore: file => writtenBefore.add(sweepFiles.indexOf(file)),
},
);
sweepable.forEach((b, i) => {
const ids = orphans[i]!;
if (ids.length === 0) return;
b.patch = { ...b.patch, ops: [...ids.map(id => ({ type: 'DeleteNode', id })), ...b.patch.ops] };
if (orphanRepair || writtenBefore.has(i)) {
// The backend already holds this file's patch id: a repair runs
// at the current extractor, and an extractor-change run may
// follow an earlier one (a partial run, a run cut short) that
// committed the file without the deletes. Sent under that id
// the deletes would be answered Idempotent and dropped, so they
// go under an id salted with them. Every other patch keeps its
// deterministic id, the one a later edit, deletion or sweep
// reads back.
const salt = `orphans-${sha256(Buffer.from([...ids].sort().join('\n'))).slice(0, 12)}`;
const salted = buildPatchFn!(b.p, b.hash, fileWorkspaceId(b.p.filePath), edgesByFile.get(b.p.filePath) ?? emptyEdges, undefined, fileMultiRepo(b.p.filePath), salt);
b.patch = { ...b.patch, patchId: salted.patchId };
}
});
} catch (err) {
if (err instanceof BatchNodeReadsUnsupported) {
orphanSweep.blocker = err.message;
noteOrphanSweepBlocked();
} else {
for (const b of sweepable) noteOrphanSkip(b.p.filePath, err instanceof Error ? err.message : String(err));
}
sweepable.forEach((_, i) => unswept.add(i));
}
// What this batch left unswept stays recorded for a repair run, and
// what it finished comes off the record. Saved per batch, so a run cut
// short keeps what it learned.
if (orphanSweep.track) {
let changed = false;
sweepable.forEach((b, i) => {
for (const extractor of b.orphanSources) {
const files = orphanPending.get(extractor);
if (unswept.has(i)) {
let mine = orphanSweep.unswept.get(extractor);
if (!mine) orphanSweep.unswept.set(extractor, mine = new Set());
mine.add(b.p.filePath);
if (files === undefined) orphanPending.set(extractor, new Set([b.p.filePath]));
else files?.add(b.p.filePath);
changed = true;
} else if (files?.delete(b.p.filePath)) {
changed = true;
}
}
});
if (changed) saveOrphanSweepPending(projectRoot, orphanPending);
}
}
return built.map(b => {
if (b === null) return null;
try {
let patch = b.patch;
if (mapMode) patch = stripMapModeOps(patch);
noteResolution(b.p.filePath, patch);
// source.uri (workspace-relative) and source.workspaceId are set
// inside buildPatch; the backend stores both as opaque attributes.
return makePreparedPatch(patch, b.j + 1, b.p.filePath, b.previousHash !== undefined || isForced(b.p.filePath));
} catch (err) {
parseErrors++;
process.stderr.write(`\n [patch build error] ${b.p.filePath}: ${err}\n`);
return null;
}
}).filter((pp): pp is PreparedPatch => pp !== null);
};
/** Resolve edges within a batch, build patches, and commit in sub-chunks. */
const flushBatch = async (batch: ParsedFile[]): Promise<void> => {
if (batch.length === 0) return;
filesChanged += batch.length;
const saveStart = performance.now();
let resolveEdgesMs = 0;
let buildPatchMs = 0;
let commitMs = 0;
let batchNodes = 0;
let batchEdges = 0;
let batchClaims = 0;
progressPhase = 'Saving';
try {
setCurrentWork(`resolve+commit batch ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);
const resolveStart = performance.now();
const resolvedEdges = resolveEdgesFn!(batch.map(f => f.parsed), resolveStats, globalIndex, resolveOpts);
resolveEdgesMs = Math.round(performance.now() - resolveStart);
if (stitchEnabled) collectStitch(batch.map(f => f.parsed));
const batchEdgesByFile = new Map<string, any[]>();
for (const edge of resolvedEdges) {
let arr = batchEdgesByFile.get(edge.srcFilePath);
if (!arr) { arr = []; batchEdgesByFile.set(edge.srcFilePath, arr); }
arr.push(edge);
}
setCurrentWork(`build commit payloads ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);
const buildStart = performance.now();
const preparedPatches = await buildFilePatches(batch, batchEdgesByFile);
buildPatchMs += Math.round(performance.now() - buildStart);
for (const pp of preparedPatches) {
for (const op of pp.patch.ops) {
if (op.type === 'UpsertNode') batchNodes++;
else if (op.type === 'UpsertEdge') batchEdges++;
else if (op.type === 'AssertClaim') batchClaims++;
}
}
if (preparedPatches.length === 0) return;
commitMs += await commitPreparedPatches(preparedPatches, batch.length, { onCommitted: noteRebuilt });
} finally {
const totalSaveMs = Math.round(performance.now() - saveStart);
timings.resolveMs += resolveEdgesMs;
timings.buildPatchMs += buildPatchMs;
timings.commitMs += commitMs;
totalNodesEmitted += batchNodes;
totalEdgesEmitted += batchEdges;
totalClaimsEmitted += batchClaims;
logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
progressPhase = 'Parsing';
progressStart = performance.now();
}
};
// A local baseline with mtimes is enough to take the incremental path, even
// when the backend knows none of the changed files: adding one new file to a
// mapped repo is exactly that, and it used to re-send the whole repository.
// `mtimeCache` is empty after --force, a workspace migration or the DB-reset
// guard above, so those still take Path B.
//
// A new file in any language takes it: Path A resolves the changed files
// against `globalIndex`, which the persisted symbol table gives every
// language's symbols, so a new Python or Java file binds its cross-file
// calls as a whole-repository pass would.
const hasBaseline = previousBaseline !== null && mtimeCache.size > 0;
const incrementalRun = (hasBaseline || knownHashes.size > 0 || mtimeChangedPaths.length === 0) && !opts.force;
baselineRun = hasBaseline && incrementalRun;
if (incrementalRun) {
// Path A: has baseline or all mtime-clean → pre-scan to detect changes before loading modules.
// If nothing changed, module load is skipped entirely. A file with no
// backend hash is new: it is parsed and sent with no reconcile.
const changedPaths: Array<{ filePath: string; bytes: Buffer; hash: string; previousHash: string | undefined; knownHash: string | undefined }> = [];
for (const filePath of mtimeChangedPaths) {
try {
const bytes = fs.readFileSync(filePath);
const hash = sha256(bytes);
currentHashes.set(filePath, hash);
if (!forceReingestPaths.has(filePath) && knownHashes.get(filePath) === hash) {
filesSkipped++;
filesSkippedAsUnchanged++;
settledAbs.add(filePath);
progressCurrent++;
continue;
}
const previousHash = knownHashes.get(filePath);
changedPaths.push({ filePath, bytes, hash, previousHash: previousHash !== hash ? previousHash : undefined, knownHash: previousHash });
} catch (err) {
parseErrors++;
progressCurrent++;
process.stderr.write(`\n [read error] ${filePath}: ${err}\n`);
}
}
if (changedPaths.length > 0) {
const moduleStart = performance.now();
const [ingestion, patchBuilder] = await loadIngestionModules();
timings.moduleLoadMs = Math.round(performance.now() - moduleStart);
resolveEdgesFn = ingestion.resolveEdges as Function;
buildPatchFn = patchBuilder.buildPatchWithResolution as Function;
// Pre-filter minified files (main thread) then parse in parallel via worker pool.
// filePath here is the workspace-relative path (what goes to parseFile
// and downstream into buildPatch → source_uri). absFilePath is the
// absolute path retained for any on-disk ops (reads already happened).
const parseable: Array<{ filePath: string; absFilePath: string; source: string; hash: string; previousHash: string | undefined; knownHash: string | undefined }> = [];
for (const { filePath, bytes, hash, previousHash, knownHash } of changedPaths) {
const sourceText = bytes.toString('utf-8');
if (isLikelyMinifiedSource(sourceText)) {
minifiedLikely++;
settledAbs.add(filePath);
filesSkipped++;
progressCurrent++;
if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
continue;
}
// parseFile receives the workspace-relative path so every
// deterministic ID derived in buildPatch (node/edge/chunk/patch)
// hashes relative paths. This makes graph IDs portable across
// machines and removes backend dependence on host paths.
parseable.push({ filePath: toWorkspaceRelative(filePath), absFilePath: filePath, source: sourceText, hash, previousHash, knownHash });
}
progressPhase = 'Parsing';
progressTotal = parseable.length;
progressCurrent = 0;
// The resolution index over every file, from the symbol table: only
// the changed files are parsed (below), plus any file the table lacks.
{
const indexStart = performance.now();
globalIndex = await buildResolutionIndex(
ingestion,
new Set(parseable.map(p => p.absFilePath)),
parseable.length <= PARSE_STREAM_CHUNK,
true,
);
timings.goIndexMs = Math.round(performance.now() - indexStart);
}
let pendingFlush: Promise<void> = Promise.resolve();
for (let i = 0; i < parseable.length; i += PARSE_STREAM_CHUNK) {
const chunk = parseable.slice(i, i + PARSE_STREAM_CHUNK);
// parseFile receives the workspace-relative path (f.filePath), so
// every deterministic ID in the resulting patch hashes relative
// paths. f.absFilePath is retained only for debug/error display.
const parseResults = await Promise.all(
chunk.map(f =>
ensureParsePool(parseable.length).parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),
),
);
const batch: ParsedFile[] = [];
for (let j = 0; j < chunk.length; j++) {
const parsed = parseResults[j] as any;
if (!parsed) { filesSkipped++; filesSkippedUnparsed++; unparsedRel.add(chunk[j].filePath); continue; }
entitiesParsed += parsed.entities.length;
batch.push({ filePath: chunk[j].filePath, parsed, hash: chunk[j].hash, previousHash: chunk[j].previousHash, knownHash: chunk[j].knownHash });
noteSummary(chunk[j].filePath, chunk[j].hash, parsed, currentMtimes.get(chunk[j].absFilePath));
if (parsed.hasParseErrors) filesWithParseErrors++;
}
await pendingFlush;
pendingFlush = flushBatch(batch);
}
await pendingFlush;
}
} else {
// Path B: no baseline (first ingest, migrated workspace, wiped backend) or
// --force → load modules, then stream parse + commit.
//
// Ix#568: this branch walks ALL of `filePaths`, re-reading, re-hashing and
// re-parsing every one, and its hash-clean short-circuit cannot fire here
// -- `knownHashes` is empty by construction on the non-force entry, and
// the check is `!opts.force` on the other. So the mtime skips the stat
// loop recorded did not actually happen, and leaving them counted would
// refuse the stitch on a run that parsed every file. (Adding one file to
// a mapped repo used to land here whatever its language; the baseline
// check above keeps additions in every language on Path A now.)
filesSkipped -= filesSkippedAsUnchanged;
filesSkippedAsUnchanged = 0;
const moduleStart = performance.now();
const [ingestion, patchBuilder] = await loadIngestionModules();
timings.moduleLoadMs = Math.round(performance.now() - moduleStart);
resolveEdgesFn = ingestion.resolveEdges as Function;
buildPatchFn = patchBuilder.buildPatchWithResolution as Function;
// Build global resolution index from all repo file paths before the parse loop
// so cross-batch imports resolve correctly even in streaming per-chunk mode.
// Read pre-scan files (Go, R, SAS) in parallel to maximize I/O throughput.
// Index keys are workspace-relative to match the relative paths we pass
// into parseFile below.
{
const indexStart = performance.now();
globalIndex = await buildResolutionIndex(
ingestion,
new Set(filePaths),
filePaths.length <= PARSE_STREAM_CHUNK,
false,
);
timings.goIndexMs = Math.round(performance.now() - indexStart);
}
progressPhase = 'Parsing';
progressTotal = filePaths.length;
progressCurrent = 0;
// Pipeline overlap: read files asynchronously and dispatch to parse pool
// immediately so workers start processing while more files are still being read.
// Commit previous chunk's results while parsing the next chunk.
let pendingFlushB: Promise<void> = Promise.resolve();
for (let i = 0; i < filePaths.length; i += PARSE_STREAM_CHUNK) {
const chunk = filePaths.slice(i, i + PARSE_STREAM_CHUNK);
type FileData = { filePath: string; source: string; hash: string; previousHash: string | undefined; knownHash: string | undefined } | null;
const fileData: FileData[] = new Array(chunk.length).fill(null);
const parsePromises: Promise<unknown>[] = new Array(chunk.length).fill(Promise.resolve(null));
// Read files asynchronously and dispatch to parse pool as each file completes.
// This keeps workers busy while I/O is still in flight for other files.
// fd.filePath is the workspace-relative path, which is what parseFile
// and all downstream ID derivations see. absFilePath is kept only so
// error messages still point at the file on disk.
const readAndDispatch = chunk.map(async (absFilePath, idx) => {
try {
// Open once and fstat the same handle so the size check and the read
// observe the same inode (no stat→read TOCTOU; CodeQL js/file-system-race).
const fh = await fs.promises.open(absFilePath, 'r');
let bytes!: Buffer;
try {
const st = await fh.stat();
// The stat loop above walks the same `filePaths` unconditionally,
// so a file it already rejected is counted there and must not be
// counted again here -- incrementing unconditionally counted every
// empty or oversized file TWICE, invisible until
// `skipReasons.emptyFile` stopped being hardcoded to 0 and
// reported 2 for a repo with one empty `__init__.py`.
//
// `accountedByStatLoop` is exactly that record. It is a set the
// loop writes deliberately, not `currentMtimes` read backwards:
// that map also omits files whose `statSync` THREW, which the loop
// counts as a parse error and not as a skip, so inferring from it
// treated those as already counted. Reaching here for a file the
// loop passed means it changed between the two stats -- truncated,
// or grown past the cap -- and it must be counted, or it vanishes
// from the graph, from `filesSkipped` and from every `skipReasons`
// bucket at once.
//
// The check itself is not optional either way: it is the fstat on
// the handle we are about to read, which is what makes the size cap
// TOCTOU-free (CodeQL js/file-system-race).
const countedByStatLoop = accountedByStatLoop.get(absFilePath);
if (st.size === 0) {
// Re-bucket rather than just skip: the stat loop may have
// recorded this as `tooLarge`, and the file has since been
// truncated. Leaving it there reports an empty file as
// oversized and keeps it out of `filesSkipped` entirely.
if (countedByStatLoop === undefined) { filesSkipped++; filesSkippedAsEmpty++; }
else if (countedByStatLoop === 'tooLarge') { tooLarge--; filesSkipped++; filesSkippedAsEmpty++; }
accountedByStatLoop.set(absFilePath, 'empty');
return;
}
if (st.size > MAX_FILE_BYTES) {
// ...and the mirror image, for a file that has since grown.
if (countedByStatLoop === undefined) tooLarge++;
else if (countedByStatLoop === 'empty') { filesSkipped--; filesSkippedAsEmpty--; tooLarge++; }
accountedByStatLoop.set(absFilePath, 'tooLarge');
return;
}
// The other direction of the same race, and it needs undoing
// rather than skipping: the stat loop saw this file as empty or
// oversized and counted it, and the fstat -- the one we are about
// to READ from, so the authoritative one -- says it is neither.
// The file is ingested, so leaving the earlier count in place
// reports it as skipped in the very number the docs now call the
// real one.
if (countedByStatLoop !== undefined) {
accountedByStatLoop.delete(absFilePath);
// Which counter, from the record rather than a guess.
if (countedByStatLoop === 'empty') { filesSkipped--; filesSkippedAsEmpty--; }
else tooLarge--;
}
bytes = await fh.readFile();
} finally {
await fh.close();
}
const hash = sha256(bytes);
currentHashes.set(absFilePath, hash);
if (!opts.force && !forceReingestPaths.has(absFilePath) && knownHashes.get(absFilePath) === hash) {
filesSkipped++;
filesSkippedAsUnchanged++;
settledAbs.add(absFilePath);
return;
}
const previousHash = knownHashes.get(absFilePath);
const sourceText = bytes.toString('utf-8');
if (isLikelyMinifiedSource(sourceText)) {
minifiedLikely++;
settledAbs.add(absFilePath);
filesSkipped++;
if (debug) process.stderr.write(`\n [skip minified-likely] ${absFilePath}\n`);
return;
}
const relFilePath = toWorkspaceRelative(absFilePath);
const fd: NonNullable<FileData> = { filePath: relFilePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined, knownHash: previousHash };
fileData[idx] = fd;
parsePromises[idx] = ensureParsePool(filePaths.length).parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; });
} catch (err) {
parseErrors++;
process.stderr.write(`\n [read error] ${absFilePath}: ${err}\n`);
}
});
await Promise.all(readAndDispatch);
const parseResults = await Promise.all(parsePromises);
const batch: ParsedFile[] = [];
for (let j = 0; j < fileData.length; j++) {
const f = fileData[j];
if (!f) continue;
const parsed = parseResults[j] as any;
if (!parsed) { filesSkipped++; filesSkippedUnparsed++; unparsedRel.add(f.filePath); continue; }
entitiesParsed += parsed.entities.length;
batch.push({ filePath: f.filePath, parsed, hash: f.hash, previousHash: f.previousHash, knownHash: f.knownHash });
noteSummary(f.filePath, f.hash, parsed, currentMtimes.get(chunk[j]));
if (parsed.hasParseErrors) filesWithParseErrors++;
}
await pendingFlushB;
pendingFlushB = flushBatch(batch);
}
await pendingFlushB;
}
const parsed = performance.now();
timings.parseMs = Math.round(parsed - hashed);
// Clear the parse bar
if (interval) process.stderr.write('\r' + ' '.repeat(PROG_LINE_WIDTH) + '\r');
// A recorded "every file" entry this run swept file by file now holds
// only the files it left.
if (orphanSweep.track && orphanSweep.blocker === null && orphanSweep.allAtStart.size > 0) {
for (const extractor of orphanSweep.allAtStart) {
if (orphanPending.get(extractor) === null) orphanPending.set(extractor, orphanSweep.unswept.get(extractor) ?? new Set());
}
saveOrphanSweepPending(projectRoot, orphanPending);
}
if (orphanSweep.budget.skipped > 0) {
process.stderr.write(
`\n [orphans kept] the orphan cleanup reached its budget (${Math.round(orphanSweep.budget.maxMs / 1000)}s / ` +
`${orphanSweep.budget.maxRequests} requests); ${orphanSweep.budget.skipped} file(s) were committed without it. ` +
(orphanSweep.track
? `Run IX_RECONCILE_ON_FORCE=1 ix map --force to continue: it sweeps only the files left, and each run picks up where the last stopped. ` +
`IX_ORPHAN_SWEEP_MAX_MS and IX_ORPHAN_SWEEP_MAX_REQUESTS raise the budget.\n`
: orphanRepair ? `Run the same command again to retry them.\n` : `The next full ix map sweeps them.\n`),
);
}
if (deletedPaths.length > 0) {
progressPhase = 'Saving';
progressTotal = deletedPaths.length;
progressCurrent = 0;
const deletedPatches: PreparedPatch[] = [];
const durableDeletedFiles = new Map(previousDeletedFiles);
const pendingDeletionRecovery = new Map<
string,
{ absolutePath: string; dependents: string[] }
>();
for (let i = 0; i < deletedPaths.length; i++) {
const absFilePath = deletedPaths[i];
const previousHash = knownHashes.get(absFilePath);
if (!previousHash) {
nextDeletedFiles.set(absFilePath, []);
progressCurrent++;
continue;
}
const relFilePath = toWorkspaceRelative(absFilePath);
const fileWorkspace = sourceWorkspaceIdOf(absFilePath);
const dependentSourceUris = new Set(nextDeletedFiles.get(absFilePath) ?? []);
try {
let patch = buildDeletionPatch(
relFilePath,
previousHash,
previousBaseline?.lastIngestAt ?? String(previousMtimes.get(absFilePath) ?? ''),
fileWorkspace,
[],
fileMultiRepo(relFilePath),
);
patch = await reconcileRemovedEntities(
client,
patch,
sourcePatchIdCandidates(relFilePath, previousHash, fileWorkspace),
dependentSourceUris,
mapMode,
reconcileLookups,
{
onWindowSkipped: reason => noteOrphanSkip(relFilePath, reason),
// The second stored patch (a graph a pre-fix rebuild left
// polluted) is merged only where its extra deletes can go in
// bulk; elsewhere the first-hit walk, as before IN-20.
window: orphanSweepBlocker === null ? RECONCILE_MANIFEST_WINDOW : 1,
},
);
if (mapMode) patch = stripMapModeOps(patch);
deletedPatches.push(makePreparedPatch(patch, i + 1, relFilePath, true));
const dependents = [...dependentSourceUris].sort();
nextDeletedFiles.set(absFilePath, dependents);
pendingDeletionRecovery.set(relFilePath, {
absolutePath: absFilePath,
dependents,
});
} catch (err) {
commitErrors++;
if (debug) process.stderr.write(`\n [deletion error] ${relFilePath}: ${err}\n`);
}
progressCurrent++;
}
if (deletedPatches.length > 0) {
filesChanged += deletedPatches.length;
await commitPreparedPatches(deletedPatches, deletedPaths.length, {
updateProgress: true,
onCommitted: item => {
const recovery = pendingDeletionRecovery.get(item.filePath);
if (!recovery || !previousBaseline) return;
durableDeletedFiles.set(recovery.absolutePath, recovery.dependents);
},
});
// Once, after the deletions -- it was once per deleted file, the whole
// baseline rewritten each time. The run's own baseline write follows;
// this records the restore hints in case the run dies before then.
if (previousBaseline && durableDeletedFiles.size > 0) {
saveIngestBaseline(
projectRoot,
previousMtimes,
previousBaseline.currentRev,
new Date(previousBaseline.lastIngestAt),
durableDeletedFiles,
previousBaseline.extractor,
{
replayedFiles: previousBaseline.replayedFiles,
pendingFiles: previousBaseline.pendingFiles,
parseTimeouts: previousBaseline.parseTimeouts,
},
);
}
}
}
// ── Dependents (IN-11) ──────────────────────────────────────────────
// An unchanged file whose calls resolve to a name that was renamed,
// removed or added elsewhere keeps its old edges until it is re-sent:
// callers of a renamed function point at the old node, and when the name
// comes back -- renamed back, the file restored -- nothing re-binds them.
// The table says which names changed and which files refer to them; those
// files are re-parsed and re-resolved, and sent only when their resolution
// actually differs, under a patch id salted with it (an unsalted re-send of
// unchanged bytes is a replay the backend answers `Idempotent`).
//
// Path A only: Path B parses and resolves every file anyway.
const deletedRel = deletedPaths.map(toWorkspaceRelative);
// Left pending by an earlier run whose re-send did not land.
const pendingRel = incrementalRun ? [...symbolTable()].filter(([, e]) => e.pending).map(([rel]) => rel) : [];
if (incrementalRun && (summaryBeforeRun.size > 0 || deletedRel.length > 0 || pendingRel.length > 0)) {
const table = symbolTable();
const names = new Set<string>();
// Paths whose importers may resolve differently: files that appeared,
// and files that changed what they import (a re-export moved).
const pathsChanged: string[] = [];
for (const [rel, before] of summaryBeforeRun) {
const after = table.get(rel)?.summary;
for (const n of changedNames(before, after)) names.add(n);
if (before === undefined || importsChanged(before, after)) pathsChanged.push(rel);
}
for (const rel of deletedRel) {
for (const n of changedNames(table.get(rel)?.summary, undefined)) names.add(n);
}
const absByRel = new Map(resolutionPaths.map(abs => [toWorkspaceRelative(abs), abs] as const));
const exclude = new Set<string>([...summaryBeforeRun.keys(), ...deletedRel]);
for (const rel of table.keys()) if (!absByRel.has(rel)) exclude.add(rel);
const found = findDependents(table, names, [...pathsChanged, ...deletedRel], exclude);
const dependents = [...new Set([...found, ...pendingRel.filter(rel => !exclude.has(rel))])].sort()
// A scoped run reads nothing outside its scope.
.filter(rel => inScope(absByRel.get(rel)!));
// Until each lands or proves unneeded: a re-send that fails, or a file
// that cannot be read now, is picked up again by the next run.
for (const rel of dependents) {
const entry = table.get(rel);
if (entry && !entry.pending) {
entry.pending = true;
symbolTableChanges.add(rel);
}
}
if (debug && dependents.length > 0) {
process.stderr.write(`\n [dependents] ${names.size} name(s) changed; re-resolving ${dependents.length} file(s)\n`);
}
if (dependents.length > 0) {
const [ingestion, patchBuilder] = await loadIngestionModules();
resolveEdgesFn ??= ingestion.resolveEdges;
buildPatchFn ??= patchBuilder.buildPatchWithResolution;
// Over the files as they are now: the changed files' new summaries in,
// the deleted files out.
globalIndex = await buildResolutionIndex(ingestion, new Set(), false, true);
const reread = await Promise.all(dependents.map(async rel => {
const bytes = await fs.promises.readFile(absByRel.get(rel)!).catch(() => null);
if (!bytes || bytes.length === 0 || bytes.length > MAX_FILE_BYTES) return null;
const parsed = await ensureParsePool(dependents.length).parse(rel, bytes.toString('utf-8'), false).catch(() => null);
return parsed ? { filePath: rel, parsed, hash: sha256(bytes), previousHash: undefined } as ParsedFile : null;
}));
const batch = reread.filter((f): f is ParsedFile => f !== null);
const edges = resolveEdgesFn!(batch.map(f => f.parsed), resolveStats, globalIndex, resolveOpts) as any[];
const edgesByFile = new Map<string, any[]>();
for (const edge of edges) {
let arr = edgesByFile.get(edge.srcFilePath);
if (!arr) { arr = []; edgesByFile.set(edge.srcFilePath, arr); }
arr.push(edge);
}
const resent: PreparedPatch[] = [];
const pendingRes = new Map<string, string>();
for (const { filePath: rel, parsed: p, hash } of batch) {
const entry = table.get(rel);
const build = (salt?: string): GraphPatchPayload => {
const patch = buildPatchFn!(p, hash, fileWorkspaceId(rel), edgesByFile.get(rel) ?? emptyEdges, undefined, fileMultiRepo(rel), salt);
return mapMode ? stripMapModeOps(patch) : patch;
};
const res = resolutionHash(build().ops);
// Equal: re-resolving changes nothing this file says. A different
// hash, or none recorded, and it is re-sent.
if (entry?.res === res && entry.hash === hash) {
delete entry.pending;
continue;
}
resent.push(makePreparedPatch(build(res.slice(0, 12)), resent.length + 1, rel, false));
pendingRes.set(rel, res);
}
if (debug) process.stderr.write(`\n [dependents] ${resent.length} of ${batch.length} resolve differently; re-sending them\n`);
if (resent.length > 0) {
filesChanged += resent.length;
await commitPreparedPatches(resent, resent.length, {
onCommitted: item => {
const entry = table.get(item.filePath);
const res = pendingRes.get(item.filePath);
if (entry && res !== undefined) {
entry.res = res;
delete entry.pending;
symbolTableChanges.add(item.filePath);
}
},
});
}
}
}
const committed = performance.now();
if (debug && baseRevRetries > 0) {
process.stderr.write(`\n [base-rev race] ${baseRevRetries} commit(s) re-sent after another writer moved the rev\n`);
}
// Persist mtime cache so next run can skip unchanged files quickly.
// Only save when no parse errors (avoid poisoning cache on partial failures).
// commitErrors counts here too, and must: it used to be folded into
// parseErrors, so splitting the counters without naming it in each of these
// three guards would start caching mtimes for files whose patches never
// landed. The next run would skip them as unchanged and they would stay
// missing from the graph until a --force.
// A scoped run carries every entry outside its scope forward, and keeps
// the workspace's last full-ingest time: `lastIngestAt` is what a file
// missing from the baseline is judged stale against, and only a run over
// the whole root has looked at those. With no baseline to update it
// writes none -- one written from a part of the workspace would mark the
// whole graph complete.
const outOfScope = scopedRun ? [...previousMtimes.keys()].filter(fp => !inScope(fp)) : [];
// Replayed changes keep their previous mtime, so the next run sends them
// again. Patches name files workspace-relative; the baseline is keyed by
// absolute path, deleted files included.
const absoluteFor = new Map(
[...previousMtimes.keys(), ...currentMtimes.keys()].map(abs => [toWorkspaceRelative(abs), abs]),
);
const replayedAbsolute = [...replayedChanges].flatMap(rel => absoluteFor.get(rel) ?? []);
// A run that did not look at a file has not resolved its replay.
const replayedFiles = [
...replayedChanges,
...(previousBaseline?.replayedFiles ?? []).filter(rel => {
const abs = absoluteFor.get(rel);
return abs !== undefined && (!inScope(abs) || langExcluded.includes(abs));
}),
];
// Per file, not all or nothing. The baseline used to be written only when
// the whole run was clean, so one permanently unreadable file meant no run
// ever recorded anything again: every map re-read the whole workspace and
// deletions were never retracted. Now every file this run settled is
// recorded, and every changed file it did not settle -- a read or build
// error, a commit that failed, a parse lost to a dead worker -- keeps its
// previous mtime (or none) so the next run tries it again. Lost parses
// still count: a run whose worker pool died resolves every later file as
// null, and recording those would skip them as unchanged for good.
const runClean = ingestCompletedCleanly(parseErrors + crashedParses(), commitErrors);
const lostParses = crashedParses() > 0;
// A parse that ran past the budget is not settled either, though it is
// not a crash: the file is not in the graph. It used to be recorded clean,
// and since a slow file times out on every run, it was then skipped as
// unchanged for good while status called the graph current.
const timedOut = new Set(timedOutParses());
const unsettled = mtimeChangedPaths.filter(abs => {
if (settledAbs.has(abs)) return false;
const rel = toWorkspaceRelative(abs);
if (timedOut.has(rel)) return true;
return !committedRel.has(rel) && !(unparsedRel.has(rel) && !lostParses);
});
// Name the unsettled files too. A new one has no previous mtime, so it is
// left out of the baseline, and `ix status` judges a file the baseline
// does not list against `lastIngestAt` -- which this write moves past the
// file's mtime. Without the list, status called the graph current while
// the file was missing from it. A replayed change has its own list. As
// with replays, a run that did not look at a file has not resolved it.
const notLookedAt = (rel: string): boolean => {
const abs = nodePath.resolve(workspaceRoot, rel);
return (!inScope(abs) || langExcluded.includes(abs)) && fs.existsSync(abs);
};
const pendingFiles = [
...unsettled.map(toWorkspaceRelative).filter(rel => !replayedChanges.has(rel) && !timedOut.has(rel)),
...(previousBaseline?.pendingFiles ?? []).filter(notLookedAt),
];
// Timeouts get their own list, which `ix status` warns about by name: the
// fix is a larger IX_PARSE_BUDGET_MS, not another run.
const parseTimeouts = [...timedOut, ...(previousBaseline?.parseTimeouts ?? []).filter(notLookedAt)];
// A deletion whose patch did not land keeps its entry, so it is found
// deleted again next run.
const failedDeletions = deletedPaths.filter(abs => !committedRel.has(toWorkspaceRelative(abs)));
const baselinePersisted = (scopedRun && !previousBaseline) ? false : persistIngestBaseline(
projectRoot,
retryOnNextRun(
carryForwardMtimes(currentMtimes, previousMtimes, [...deferredDeletions, ...failedDeletions, ...langExcluded, ...outOfScope]),
previousMtimes,
[...replayedAbsolute, ...unsettled],
),
latestRev,
scopedRun && previousBaseline ? new Date(previousBaseline.lastIngestAt) : undefined,
nextDeletedFiles,
// An unfinished re-ingest must not vouch for the new extractor: files
// that failed would then be hash-clean next run and keep the old output.
runClean ? baselineExtractor : (previousBaseline?.extractor ?? baselineExtractor),
{ replayedFiles, pendingFiles, parseTimeouts },
);
// The symbol table describes files, not the graph, so it is kept whether or
// not every commit landed -- pruned to the files that still exist. A
// `--lang` run's resolution paths hold its languages only; the files it
// left out still exist and keep their entries, as they keep their baseline.
// Only what changed is written: nothing on a run that changed no entry,
// and on most runs a few journal lines rather than the whole table.
if (symbolTableChanges.size > 0) {
const existing = new Set([...resolutionPaths, ...langExcluded].map(toWorkspaceRelative));
const table = symbolTable();
for (const rel of [...table.keys()]) {
if (!existing.has(rel)) {
table.delete(rel);
symbolTableChanges.add(rel);
}
}
saveIngestSymbols(projectRoot, currentExtractor, table, symbolTableChanges);
}
if (rebuildProgress !== null) {
// Finished: the baseline now records the new extractor. Otherwise keep
// what landed, so the next run resumes from here.
if (baselinePersisted && runClean) clearRebuildProgress(projectRoot);
else saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress);
}
// Migration cleanup (Ix#225 gap 2): the re-ingest under the new path-based id has
// committed, so delete the OLD id's now-orphaned nodes/edges/patches. Best-effort —
// a failure here is non-fatal (orphans are harmless dead storage, cleanable later).
// `+ crashedParses()`, exactly as the baseline guard below. This one
// DELETES the old workspace's graph, so it must be at least as strict: a
// run whose parse pool died resolves every later file as null without
// raising `parseErrors`, so both guards saw a clean run while the new
// workspace held a half-populated graph -- and this one would drop the
// complete old one permanently. The two adjacent guards must not disagree
// about whether the run was clean, and the destructive one certainly must
// not be the laxer of the pair.
if (
workspaceMigrated &&
previousWorkspaceId &&
ingestCompletedCleanly(parseErrors + crashedParses(), commitErrors)
) {
try {
await client.deleteWorkspace(previousWorkspaceId);
if (debug) process.stderr.write(` Cleaned up pre-migration nodes under ${previousWorkspaceId}.\n`);
} catch (err) {
if (debug) process.stderr.write(` [migration cleanup skipped] ${err}\n`);
}
}
// Path-2 stitch (Ix#225 Half A): register this repo's provides/consumes and
// write the cross-repo IMPORTS edges to other separately-ingested repos.
// Best-effort: a failure (or an older backend with no /v1/stitch) never fails
// the ingest. Collection is over the files actually parsed this run, so it is
// only COMPLETE on a full ingest. Gating on filesSkippedAsUnchanged === 0
// means a partial incremental re-map (some files mtime- or hash-skipped)
// leaves the prior full registration intact rather than overwriting it with a
// partial set; a fresh map, `--force`, or a post-reset re-map re-registers.
// (Incremental registry updates that touch only changed files are a future
// refinement.)
//
// Ix#568: the two narrow counts, not `filesSkipped` -- see their
// declarations. The broad total also includes empty and minified-looking
// files, which contribute nothing under any run, so one empty `__init__.py`
// used to disqualify a repo from ever stitching, with or without `--force`.
// A CRASHED parse worker is in that category and is counted here: it
// resolves its in-flight task as null without raising `parseErrors`, and
// stitching then would overwrite a complete registration with one missing
// that file's exports. A file `parseFile` simply returned null for is not --
// that is deterministic, usually an unavailable optional grammar, and
// gating on it blocked stitching forever for any repo containing one.
// And never on a partial run (`--lang`, or a path below the root): it
// parsed a part of the repo by construction, so even with nothing skipped
// its provides/consumes would replace the full registration with a slice.
const registrationIsComplete = !partialRun && filesSkippedAsUnchanged === 0 && crashedParses() === 0;
// Every way the stitch does not happen is REPORTED, not just the guard's.
// `stitch_skipped: null` means "the cross-repo edges are current", so every
// path that leaves it unset while skipping the stitch says something false
// -- and the two below are far commoner than any refusal the guard makes.
// Neither prints a human Note or a `--silent` token: `incomplete` would
// appear on nearly every incremental map, and a run with errors has already
// said so, loudly, in the lines above.
// `lost-parses` FIRST. It is the only one of the three that prints, and a
// dead pool also raises `parseErrors`, so testing `run-errors` first meant a
// `--force` run with one patch-build failure and four thousand lost files
// reported the one and said nothing about the rest -- suppressing the very
// Note this rule was added to show.
if (stitchEnabled && crashedParses() > 0) {
// Its OWN rule, and one that IS printed. `incomplete` is silent because
// it fires on nearly every incremental map -- but on a `--force` run
// nothing is skipped as unchanged, so `incomplete` there could only ever
// mean a crashed worker, and staying silent made the recovery command
// this CLI advertises exit 0 having quietly done nothing.
stitchSkippedRule = "lost-parses";
stitchSkipped =
`a parse worker crashed and ${crashedParses()} file(s) went unparsed, so the ` +
"cross-workspace registration would be missing them";
} else if (stitchEnabled && !ingestCompletedCleanly(parseErrors, commitErrors)) {
stitchSkippedRule = "run-errors";
stitchSkipped =
"this run had parse or commit errors, so its registration would be built from " +
"an incomplete picture of the repo";
} else if (stitchEnabled && !registrationIsComplete) {
// `stitchFiles.length > 0` is deliberately NOT required on any of these
// three. A map where nothing changed parses nothing, so `stitchFiles` is
// empty and the field stayed unset -- reporting "the cross-repo edges are
// current" for the commonest map there is, while a cooldown was refusing
// stitches the whole time. The staleness is identical either way.
stitchSkippedRule = "incomplete";
stitchSkipped = partialRun
? "this run ingested only part of the workspace, so it has no complete " +
"cross-workspace registration to send"
: "this map did not re-parse every file, so it has no complete cross-workspace " +
"registration to send";
}
if (stitchEnabled && registrationIsComplete && stitchFiles.length > 0 && ingestCompletedCleanly(parseErrors, commitErrors)) {
try {
const entry = pickEntryFile(stitchFiles);
const provides = entry
? readPackageNames(workspaceRoot).map(name => ({ name, entryNodeId: fileNodeId(workspaceId, entry), entryUri: entry }))
: [];
// Provider public export aliases: advertise default/`as`-exported symbols by
// the name consumers import them as (`export { impl as format }` -> "format";
// `export default function debug` -> "default"), pointing at the same node.
for (const a of stitchExportAliases) {
const def = stitchDefs.get(a.local);
if (!def) continue;
const nodeId = symbolNodeId(workspaceId, def.filePath, def.qk);
const ek = `${a.pub}\u0000${nodeId}`;
if (stitchExportAliasSeen.has(ek)) continue;
stitchExportAliasSeen.add(ek);
stitchExports.push({ name: a.pub, nodeId });
}
// Symbol-level consumes: a call whose callee resolves to an EXTERNAL package
// symbol. A renamed/default import binds a local name (fmt, dbg) to a public
// symbol (format, "default") in a package — resolve through the binding and
// carry the package so the backend can scope the match precisely. Without a
// binding (namespace member call, or a plain call) keep the old rule: skip
// calls to symbols defined locally (intra-repo, resolved by normal ingest).
const symbolConsumes: Array<{ symbol: string; callerNodeId: string; pkg?: string }> = [];
const symbolConsumeSeen = new Set<string>();
for (const c of stitchCalls) {
const callerDef = stitchDefs.get(c.caller);
if (!callerDef) continue;
const binding = stitchBindings.get(c.callerFilePath)?.get(c.symbol);
let symbol = c.symbol;
let pkg: string | undefined;
if (binding) {
const bpkg = specToPkg(binding.pkg);
if (!bpkg || !stitchProdDeps.has(bpkg)) continue; // only stitch prod-dep packages
symbol = binding.imported;
pkg = bpkg;
} else {
if (stitchDefs.has(c.symbol)) continue;
}
const callerNodeId = symbolNodeId(workspaceId, callerDef.filePath, callerDef.qk);
const sk = `${symbol}\u0000${pkg ?? ''}\u0000${callerNodeId}`;
if (symbolConsumeSeen.has(sk)) continue;
symbolConsumeSeen.add(sk);
symbolConsumes.push({ symbol, callerNodeId, pkg });
}
if (provides.length > 0 || stitchConsumes.length > 0 || stitchExports.length > 0 || symbolConsumes.length > 0) {
// Ix#568: ask before stitching. The join is cross-workspace and
// outlives the HTTP call that starts it, so `ix map`'s per-workspace
// lock does not bound it -- see stitch-guard.ts. A refusal is not a
// failure: the previous registration stands, exactly as it does after
// a stitch that FAILED. It is not picked up by the next map either --
// see describeStitchSkipped for what actually re-registers.
const admission = await admitStitchWaiting(
client.endpoint,
undefined,
undefined,
opts.deadlineSignal,
);
if (!admission.admitted) {
stitchSkipped = admission.reason;
stitchSkippedRule = admission.rule;
if (debug) process.stderr.write(` [stitch not started] ${admission.reason}\n`);
} else {
const stitchStart = performance.now();
let res;
try {
stitchSent = true;
res = await client.stitch({ workspaceId, provides, consumes: stitchConsumes, exports: stitchExports, symbolConsumes });
} catch (err) {
// Settle before rethrowing, so the shared catch below still sees
// the original error and the lock is never held past the request.
admission.settle({
ok: false,
elapsedMs: performance.now() - stitchStart,
// Two proofs that the backend is not running a join, and only
// two: it answered and refused (a status), or we never reached
// it at all. Everything else -- a 5xx, a timeout, an abort, a
// socket dropped after the request went out -- leaves it.
status: stitchFailureStatus(err),
neverConnected: connectionNeverEstablished(err),
});
throw err;
}
admission.settle({ ok: true, elapsedMs: performance.now() - stitchStart });
if (debug) process.stderr.write(` [stitch] provides=${provides.length} consumes=${stitchConsumes.length} exports=${stitchExports.length} symConsumes=${symbolConsumes.length} -> ${res.stitched} edges\n`);
// This call is the one thing that can change whether the workspace
// belongs to a system, and reads cache that answer on disk. Drop it
// here so the next read asks again rather than scoping to the
// pre-stitch workspace for as long as the file survives.
clearStitchScopeCache(workspaceId);
}
}
} catch (err) {
// Unsupported is not failed: an older backend answers 404 here, and
// that is the case this call has always been allowed to shrug off.
if (!isStitchUnsupported(err)) {
stitchErrors++;
stitchError = err;
}
if (debug) process.stderr.write(` [stitch skipped] ${err}\n`);
}
}
} finally {
if (pool) {
await pool.destroy();
}
if (interval) {
clearInterval(interval);
if (progressTotal > 0) {
process.stderr.write('\r' + renderProgressLine(progressPhase, progressTotal, progressTotal));
}
process.stderr.write('\r' + ' '.repeat(PROG_LINE_WIDTH) + '\r');
}
// Outside the interval guard, on the format instead. This line is
// newline-terminated output, not a progress frame, and it is the only
// timing the user ever sees — leaving it keyed to the bar would have made
// silencing the bar silently drop it too, turning a fix for noisy output
// into a loss of the summary that output is for.
if (wantsTextProgress) {
const elapsedSec = ((performance.now() - start) / 1000).toFixed(1);
process.stderr.write(chalk.dim(` Ingested in ${elapsedSec}s\n`));
}
}
const elapsed = ((performance.now() - start) / 1000).toFixed(2);
// A new workspace is registered only now, and only if the backend took
// something: see workspaceStateFor.
if (registrationPending && patchesTheBackendTook > 0 && registerAfterCommit) {
// A repository (or --root) ingest may become the default when none exists, as
// before CL-05; a loose directory never does: an ingest of a stray directory
// must not pick the workspace other commands fall back to.
ensureWorkspaceIdState(workspaceRoot, { setDefault: rootSource !== 'loose' });
if (!opts.suppressOutput && opts.format === 'text') {
process.stderr.write(chalk.dim(`Registered workspace "${workspaceName}" (${workspaceRoot}).\n`));
}
} else if (registrationPending && patchesTheBackendTook > 0) {
if (!opts.suppressOutput && opts.format === 'text') {
process.stderr.write(chalk.dim(`Not registered as a workspace: ${workspaceRoot} (a file outside every workspace and repository). Run ix map ${workspaceRoot} to query and read it.\n`));
}
}
// Reported before the suppressOutput return, and thrown when nothing landed.
//
// `ix map` passes suppressOutput unconditionally, so every commit failure
// used to exit here silently: ingestFiles never threw, never set exitCode,
// and map.ts went on to print its regions. CLAUDE.md RULE 5 has every agent
// run `ix map --silent` after an edit, so a backend that accepted nothing
// left the agent believing the graph was current while `ix search` and
// `ix impact` answered from a stale one. A wrong answer nobody can see is
// worse than a failed command.
// A trip alone is not enough: the database probe can trip the breaker on a
// bulk failure with no `commitErrors++` (a health answer of "unreachable"
// that turns out wrong), and the cutoff and drain may then land every patch.
// Printing the cutoff banner then would say the graph is "partly updated"
// and ask for a re-run when nothing is missing. A skipped patch is counted
// in `commitErrors`, so this still fires whenever the cutoff held anything
// back that did not land.
const cutoffFired = commitBreaker.everTripped() && commitErrors > 0;
const commitReport = describeCommitOutcome(
commitErrors,
patchesApplied,
// ix map has no --debug and never passes one; --verbose is its equivalent.
opts.mapMode === true ? "--verbose" : "--debug",
opts.deadlineSignal?.aborted === true,
commitBreaker.skipped(),
// The same expression the banner is gated on, so this can never say "See
// the cutoff above" with no cutoff above it.
cutoffFired,
graphStartedEmpty === true
);
const summary: IngestFilesSummary = {
filesDiscovered,
patchesApplied,
idempotentPatches,
replayedChanges: [...replayedChanges].sort(),
filesSkippedAsUnchanged,
filesTooLarge: tooLarge,
filesTooLargePaths: listTooLargeFiles(skippedForSize, toWorkspaceRelative),
// `+ crashedParses()`, as the baseline and delete guards already do. Files
// lost to a dead parse pool raise `filesSkippedUnparsed`, never
// `parseErrors`, so without this everything downstream read the run as
// clean: `describeDroppedFiles` stayed silent, and the completed-map checks
// blamed the backend "after a clean local ingest" and cleared the map
// baseline. On a co-ingest workspace the `lost-parses` channel is off too
// (`stitchEnabled` is false there), so the run exited 0 with no signal
// anywhere at all.
parseErrors: parseErrors + crashedParses(),
commitErrors,
stitchErrors,
stitchSkipped,
stitchSkippedRule,
filesWithParseErrors,
parseTimeouts: timedOutParses(),
graphUnchanged:
opts.force !== true &&
!workspaceMigrated &&
!dbResetDetected &&
!extractorRebuild &&
filesChanged === 0 &&
patchesApplied === 0 &&
!deletionsFound &&
!stitchSent &&
parseErrors + crashedParses() === 0 &&
commitErrors === 0 &&
stitchErrors === 0,
incremental:
baselineRun &&
opts.force !== true &&
!workspaceMigrated &&
!dbResetDetected &&
!extractorRebuild &&
(filesChanged > 0 || deletionsFound) &&
parseErrors + crashedParses() + commitErrors === 0,
};
if (cutoffFired) {
// Gated on the cutoff having FIRED, not on what was left unsent. A patch
// that was never sent produces no `[commit error]` line even under
// --verbose, so this has to print for those -- but gating on them alone
// meant a run whose drain eventually placed every held patch printed
// nothing at all. The 12-file reproduction at the default limit is exactly
// that shape: it held 7, the drain sent all 7, and the user got the generic
// "all 12 patches failed to commit, re-run with --verbose" -- the
// unexplained failure this change exists to remove.
//
// Before the stitch line and before the commit report: this is the cause,
// and everything else printed about this run is its consequence.
process.stderr.write(`${describeCommitCutoff(commitBreaker, client.endpoint, patchesApplied, opts.deadlineSignal?.aborted === true, {
graphStartedEmpty: graphStartedEmpty === true,
databaseUnreachable: databaseKnownDown(),
})}\n`);
}
if (stitchErrors > 0) {
process.stderr.write(` ${describeStitchFailure(stitchError)}\n`);
process.exitCode = 1;
} else if (
stitchSkipped !== undefined &&
shouldPrintStitchSkipped(stitchSkippedRule, opts.suppressOutput === true)
) {
// Not an error and not an exit code: nothing failed, and the graph is
// exactly where a failed stitch would have left it. But the sentence a
// user gets otherwise -- cross-repo relationships may be incomplete --
// reads as an unexplained regression without the reason.
process.stderr.write(` ${describeStitchSkipped(stitchSkipped, stitchSkippedRule, debug)}\n`);
}
if (summary.parseTimeouts.length > 0) {
process.stderr.write(` ${describeParseTimeouts(summary.parseTimeouts)}\n`);
}
if (replayedChanges.size > 0) {
// stderr and exit 0: nothing failed on this side, and the next run retries.
process.stderr.write(` ${describeReplayedChanges(summary.replayedChanges)}\n`);
}
if (commitReport.kind === "warn") {
process.stderr.write(` ${commitReport.message}\n`);
// Non-zero even though we do not throw. A partial failure still means the
// graph does not match the working tree, and the caller — usually a hook
// running `ix map --silent` under CLAUDE.md RULE 5 — reads the exit code,
// not stderr. Exiting 0 here made the severity of an identical fault depend
// on how many files happened to change: one file failing was fatal, the
// same failure alongside a file that landed was silent.
process.exitCode = 1;
}
if (opts.suppressOutput === true) {
if (commitReport.kind === "fatal") throw new Error(commitReport.message);
return summary;
}
if (opts.format === 'json') {
printJson({
filesProcessed: filesDiscovered,
...(filesExcluded > 0 ? { filesExcluded } : {}),
filesChanged,
patchesApplied,
filesSkipped,
idempotentPatches,
replayedChanges: summary.replayedChanges,
filesWithParseErrors,
entitiesParsed,
latestRev,
// `unchanged` is the files we ASSUMED unchanged, not every skip. It used
// to be `filesSkipped`, which also counts empty, minified-looking and
// unparseable files -- so a consumer reading this could not tell an
// incremental no-op from a repo full of empty __init__.py. `emptyFile` was
// hardcoded 0 for the same reason and is now the real count.
//
// `unparsed` is its own bucket. Narrowing `unchanged` left the pool's
// nulls in no bucket at all, but folding them into `parseError` was
// worse: that counts every parse-stage failure, including stat, read and
// patch-build errors that were never skips, so one unstat-able file gave
// `filesSkipped: 0` with a bucket claiming 1. `unchanged`, `emptyFile`,
// `minifiedLikely` and `unparsed` are the buckets that are subsets of
// `filesSkipped`; `parseError` and `tooLarge` are counted separately and
// always were.
skipReasons: {
unchanged: filesSkippedAsUnchanged, emptyFile: filesSkippedAsEmpty, parseError: parseErrors + crashedParses(),
unparsed: filesSkippedUnparsed, parseTimeout: summary.parseTimeouts.length, tooLarge, minifiedLikely, outsideRoot,
unreadable: discoveryCounts.unreadable, skippedDirs: discoveryCounts.skippedDirs,
},
parseTimeouts: summary.parseTimeouts,
commitErrors,
stitchErrors,
// Ix#568. Present only when the stitch was refused before it was sent, so
// a consumer reading this body can tell "cross-repo edges are current"
// from "cross-repo edges are as stale as the last successful stitch" --
// a distinction the exit code deliberately does not make.
// `?? null`, not left undefined: JSON.stringify drops an undefined
// value, so a consumer could not tell the field apart from an older CLI
// that never emitted it. `ix map --format json` does the same.
stitchSkipped: stitchSkipped ?? null,
stitchSkippedRule: stitchSkippedRule ?? null,
elapsedSeconds: parseFloat(elapsed),
timings,
emitted: { nodes: totalNodesEmitted, edges: totalEdgesEmitted, claims: totalClaimsEmitted },
langParseMs: Object.fromEntries(
[...langParseMs.entries()].map(([lang, s]) => [lang, { ms: Math.round(s.ms), files: s.files }])
),
resolveStats,
});
} else if (opts.printSummary !== false) {
console.log(chalk.bold('\nIngest summary'));
console.log(` processed: ${patchesApplied} files (${elapsed}s)`);
console.log(` discovered: ${filesDiscovered} files`);
if (filesExcluded > 0) {
console.log(` excluded: ${filesExcluded} paths (--exclude / .ixignore)`);
}
console.log(` changed: ${filesChanged} files`);
// "skipped" without "unchanged": the count includes empty, minified-looking
// and unparseable files, none of which were assumed unchanged. Splitting the
// counter for the stitch gate (Ix#568) made the old label demonstrably
// wrong on a repo with an empty __init__.py and --force.
if (filesSkipped > 0) console.log(` ${chalk.dim('skipped:')} ${filesSkipped}`);
// `+ crashedParses()`, matching the summary field. On a co-ingest workspace
// `stitchEnabled` is false, so the `lost-parses` Note never fires and
// `describeDroppedFiles` is only wired into `ix map` -- which left
// `ix ingest` on such a workspace printing "skipped: 4000", no parse-error
// line at all, and exiting 0 after losing its whole parse pool.
const parseErrorsShown = parseErrors + crashedParses();
if (parseErrorsShown > 0) console.log(` ${chalk.red('parse errors:')} ${parseErrorsShown}`);
if (commitErrors > 0) console.log(` ${chalk.red('commit errors:')} ${commitErrors}`);
if (replayedChanges.size > 0) console.log(` ${chalk.yellow('not applied:')} ${replayedChanges.size} ${chalk.dim('(see the warning above)')}`);
if (tooLarge > 0) console.log(` ${chalk.dim('skipped too large:')} ${tooLarge}`);
if (minifiedLikely > 0) console.log(` ${chalk.dim('skipped minified:')} ${minifiedLikely}`);
if (filesWithParseErrors > 0) console.log(` ${chalk.yellow("parsed with errors:")} ${filesWithParseErrors} ${chalk.dim("(tree-sitter recovered; parts may be missing)")}`);
if (summary.parseTimeouts.length > 0) console.log(` ${chalk.yellow("skipped parse timeout:")} ${summary.parseTimeouts.length} ${chalk.dim(`(${summary.parseTimeouts.join(", ")})`)}`);
// Not dimmed like the others: these were dropped because the repo pointed
// at files outside itself, which is worth a look rather than a shrug.
if (outsideRoot > 0) console.log(` ${chalk.yellow('skipped outside root:')} ${outsideRoot} ${chalk.dim('(symlinks leaving the tree)')}`);
if (discoveryCounts.unreadable > 0) console.log(` ${chalk.yellow("skipped unreadable:")} ${discoveryCounts.unreadable} ${chalk.dim("(listed but could not be read)")}`);
if (discoveryCounts.skippedDirs > 0) console.log(` ${chalk.dim("skipped build/dependency dirs:")} ${discoveryCounts.skippedDirs}`);
console.log(` rev: ${latestRev}`);
if (patchesApplied === 0 && filesDiscovered === 0) {
console.log(chalk.yellow('\n Warning: No files found. Check the path and supported extensions.'));
} else if (patchesApplied === 0 && filesDiscovered > 0 && commitErrors === 0) {
// commitErrors === 0, or this reads "All files unchanged since last
// ingest" directly under "commit errors: 12". Files were not unchanged —
// they were rejected. This branch was unreachable on the fatal path until
// the throw moved below the summary so `--format json` could still emit,
// which is exactly the kind of false statement about the graph this
// change exists to stop making.
console.log(chalk.dim('\n All files unchanged since last ingest.'));
}
if (debug) {
console.log(chalk.dim(`\n Phase timings:`));
console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));
console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));
console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
console.log(chalk.dim(` go index: ${timings.goIndexMs}ms`));
console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));
console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));
console.log(chalk.dim(` commit: ${timings.commitMs}ms`));
console.log(chalk.dim(` bulk: ${timings.bulkCommitMs}ms`));
console.log(chalk.dim(` fallback: ${timings.fallbackCommitMs}ms`));
console.log(chalk.dim(`\n Emitted ops:`));
console.log(chalk.dim(` nodes: ${totalNodesEmitted}`));
console.log(chalk.dim(` edges: ${totalEdgesEmitted}`));
console.log(chalk.dim(` claims: ${totalClaimsEmitted}`));
if (langParseMs.size > 0) {
console.log(chalk.dim(`\n Per-language parse times:`));
const sorted = [...langParseMs.entries()].sort((a, b) => b[1].ms - a[1].ms);
for (const [lang, stat] of sorted) {
console.log(chalk.dim(` ${lang.padEnd(12)} ${Math.round(stat.ms)}ms (${stat.files} files)`));
}
}
console.log(chalk.dim(`\n Resolve stats:`));
console.log(chalk.dim(` import lookups: ${resolveStats.importLookups}`));
console.log(chalk.dim(` transitive lookups: ${resolveStats.transitiveLookups}`));
console.log(chalk.dim(` global fallbacks: ${resolveStats.globalFallbacks}`));
console.log(chalk.dim(` avg candidates: ${resolveStats.globalFallbacks > 0 ? (resolveStats.globalCandidateTotal / resolveStats.globalFallbacks).toFixed(1) : '0'}`));
console.log(chalk.dim(` resolved import: ${resolveStats.resolvedImport}`));
console.log(chalk.dim(` resolved transitive:${resolveStats.resolvedTransitive}`));
console.log(chalk.dim(` resolved global: ${resolveStats.resolvedGlobal}`));
console.log(chalk.dim(` resolved qualifier: ${resolveStats.resolvedQualifier}`));
console.log(chalk.dim(` resolved type: ${resolveStats.resolvedType}`));
console.log(chalk.dim(` skipped same-file: ${resolveStats.skippedSameFile}`));
console.log(chalk.dim(` skipped ambiguous: ${resolveStats.skippedAmbiguous}`));
}
console.log();
}
// Thrown last, so a total failure still emits its report first. Throwing
// before the JSON branch made `ix ingest --format json` print nothing at all
// on the very run a consumer most needs to parse — empty stdout and exit 1,
// where main emitted a valid document carrying patchesApplied: 0. The
// suppressOutput path above throws earlier because it has no report to emit.
if (commitReport.kind === "fatal") throw new Error(commitReport.message);
return summary;
}
// ---------------------------------------------------------------------------
// Load existing hashes from the server for change detection
// ---------------------------------------------------------------------------
// The backend now stores workspace-relative source_uri values. The CLI still
// tracks files internally by absolute path (needed for fs reads), so this
// helper converts to relative before the wire call and maps the server's
// response back onto the caller's absolute keys.
export async function loadExistingHashes(
client: Pick<IxClient, "getSourceHashes">,
filePaths: string[],
toRelative: (absPath: string) => string,
workspaceIdOf: (absPath: string) => string,
debug = false,
throwOnError = false,
): Promise<Map<string, string>> {
try {
// Each file is matched against its OWN workspace's stored hash. The server keys
// source_uri globally, so without the workspace_id two members (or two unrelated
// repos) sharing a relative path + content would collide; pairing (workspace_id,
// uri) makes the baseline collision-free (Ix#225 gap 3).
const entries = filePaths.map(abs => ({ abs, uri: toRelative(abs), ws: workspaceIdOf(abs) }));
const uris = [...new Set(entries.map(e => e.uri))];
const workspaceIds = [...new Set(entries.map(e => e.ws).filter(Boolean))];
const rows = await client.getSourceHashes(uris, workspaceIds);
const sep = "\u0000";
const index = new Map<string, string>();
for (const r of rows) index.set(`${r.workspaceId ?? ""}${sep}${r.uri}`, r.hash);
const out = new Map<string, string>();
for (const e of entries) {
const hash = index.get(`${e.ws}${sep}${e.uri}`);
if (hash !== undefined) out.set(e.abs, hash);
}
return out;
} catch (err) {
if (debug) process.stderr.write(`\n [hash lookup failed] ${err}\n`);
if (throwOnError) throw err;
return new Map();
}
}
// ---------------------------------------------------------------------------
// GitHub ingestion
// ---------------------------------------------------------------------------
/**
* Whether a GitHub ingest's commit landed. `retryOnBaseRevRace` returns the
* last result, and a `BaseRevMismatch` after the retries means the backend
* wrote nothing -- which `ix ingest --github` reported as a summary and exit 0.
*/
export function githubCommitFailure(result: { status?: string }): string | undefined {
if (result.status !== COMMIT_STATUS_BASE_REV_MISMATCH) return undefined;
return "The backend kept rejecting the commit because another write moved the graph revision under it " +
"(BaseRevMismatch), so nothing was ingested. Run the command again once other ingests have finished.";
}
/**
* The `--github` patch id, from what is sent rather than the clock: the same
* issues, PRs and commits give the same id, so re-running an unchanged ingest
* is a no-op on the backend instead of a new revision of identical facts.
* Exported for tests.
*/
export function githubPatchId(repo: { owner: string; repo: string }, ops: unknown[]): string {
return deterministicId(`github://${repo.owner}/${repo.repo}:${sha256(Buffer.from(JSON.stringify(ops)))}`);
}
async function ingestGitHub(opts: {
github?: string; token?: string; since?: string;
limit: string; format: string;
}): Promise<void> {
const repo = parseGitHubRepo(opts.github!);
const token = await resolveGitHubToken(opts.token);
const client = createClient();
const limit = parseInt(opts.limit, 10);
const start = performance.now();
const since = opts.since ?? new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10);
if (opts.format !== 'json') {
process.stderr.write(`Fetching from ${repo.owner}/${repo.repo} (since ${since})...\n`);
}
const data = await fetchGitHubData(repo, token, { since, limit });
const allOps: any[] = [];
for (const issue of data.issues) allOps.push(...transformIssue(repo, issue));
for (const [issueNum, comments] of data.issueComments) {
for (const comment of comments) allOps.push(...transformIssueComment(repo, issueNum, comment));
}
for (const pr of data.pullRequests) allOps.push(...transformPR(repo, pr));
for (const [prNum, comments] of data.prComments) {
for (const comment of comments) allOps.push(...transformPRComment(repo, prNum, comment));
}
for (const commit of data.commits) allOps.push(...transformCommit(repo, commit));
const patch: GraphPatchPayload = {
patchId: githubPatchId(repo, allOps),
// IX_PATCH_ACTOR="" lets a kOS cloud backend stamp the verified principal
// (it 403s a non-empty actor that differs from it); see core-ingestion patchActor().
actor: process.env.IX_PATCH_ACTOR ?? 'ix/github-ingest',
timestamp: new Date().toISOString(),
source: {
uri: `github://${repo.owner}/${repo.repo}`,
extractor: 'github-ingest/1.0',
sourceType: 'comment',
},
baseRev: 0,
ops: allOps,
replaces: [],
intent: `GitHub ingestion: ${repo.owner}/${repo.repo}`,
};
const result = await retryOnBaseRevRace(() => client.commitPatch(patch), 8);
const elapsed = ((performance.now() - start) / 1000).toFixed(2);
const failure = githubCommitFailure(result);
if (failure) {
process.exitCode = 1;
if (opts.format === 'json') {
printJson({ error: 'base_rev_mismatch', message: failure, source: `${repo.owner}/${repo.repo}`, status: result.status });
} else {
process.stderr.write(chalk.red('Error: ') + failure + '\n');
}
return;
}
if (opts.format === 'json') {
printJson({
source: `${repo.owner}/${repo.repo}`,
issues: data.issues.length,
pullRequests: data.pullRequests.length,
commits: data.commits.length,
totalOps: allOps.length,
rev: result.rev,
status: result.status,
elapsedSeconds: parseFloat(elapsed),
});
} else {
console.log(chalk.bold('\nGitHub ingest summary'));
console.log(` repo: ${repo.owner}/${repo.repo}`);
console.log(` issues: ${data.issues.length}`);
console.log(` pull requests: ${data.pullRequests.length}`);
console.log(` commits: ${data.commits.length}`);
console.log(` total ops: ${allOps.length}`);
console.log(` rev: ${result.rev}`);
console.log(` elapsed: ${elapsed}s`);
console.log();
}
}