Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: restore parsing speed improvements from patch branch

Tanner Kocher08d883c9ff2d
diff --git a/core-ingestion/src/__tests__/__snapshots__/parseFile.test.ts.snap b/core-ingestion/src/__tests__/__snapshots__/parseFile.test.ts.snap
index a97c3f17..8187c2de 100644
--- a/core-ingestion/src/__tests__/__snapshots__/parseFile.test.ts.snap
+++ b/core-ingestion/src/__tests__/__snapshots__/parseFile.test.ts.snap
@@ -2,6 +2,272 @@
exports[`parseFile snapshots > snapshots Identifiers.scala 1`] = `
{
+ "chunks": [
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "93b03910ef938562",
+ "endByte": 135,
+ "language": "scala",
+ "lineEnd": 7,
+ "lineStart": 7,
+ "name": "NodeId",
+ "startByte": 84,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "367d1dcecdc55b4f",
+ "endByte": 299,
+ "language": "scala",
+ "lineEnd": 11,
+ "lineStart": 8,
+ "name": "NodeId",
+ "startByte": 136,
+ },
+ {
+ "chunkKind": "function",
+ "container": "NodeId",
+ "contentHash": "651bc99aed4de8b2",
+ "endByte": 226,
+ "language": "scala",
+ "lineEnd": 9,
+ "lineStart": 9,
+ "name": "encoder",
+ "startByte": 154,
+ },
+ {
+ "chunkKind": "function",
+ "container": "NodeId",
+ "contentHash": "3fb1fdd553d01200",
+ "endByte": 297,
+ "language": "scala",
+ "lineEnd": 10,
+ "lineStart": 10,
+ "name": "decoder",
+ "startByte": 229,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "2b6c54c1c4c30580",
+ "endByte": 352,
+ "language": "scala",
+ "lineEnd": 13,
+ "lineStart": 13,
+ "name": "EdgeId",
+ "startByte": 301,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "f9f1c6b8418c30d1",
+ "endByte": 516,
+ "language": "scala",
+ "lineEnd": 17,
+ "lineStart": 14,
+ "name": "EdgeId",
+ "startByte": 353,
+ },
+ {
+ "chunkKind": "function",
+ "container": "EdgeId",
+ "contentHash": "5fed49c0d68e8331",
+ "endByte": 443,
+ "language": "scala",
+ "lineEnd": 15,
+ "lineStart": 15,
+ "name": "encoder",
+ "startByte": 371,
+ },
+ {
+ "chunkKind": "function",
+ "container": "EdgeId",
+ "contentHash": "474b1dabed027cce",
+ "endByte": 514,
+ "language": "scala",
+ "lineEnd": 16,
+ "lineStart": 16,
+ "name": "decoder",
+ "startByte": 446,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "57a567f9f976b17d",
+ "endByte": 570,
+ "language": "scala",
+ "lineEnd": 19,
+ "lineStart": 19,
+ "name": "PatchId",
+ "startByte": 518,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "7a73c31b3eed31da",
+ "endByte": 738,
+ "language": "scala",
+ "lineEnd": 23,
+ "lineStart": 20,
+ "name": "PatchId",
+ "startByte": 571,
+ },
+ {
+ "chunkKind": "function",
+ "container": "PatchId",
+ "contentHash": "8c770dd37c949073",
+ "endByte": 663,
+ "language": "scala",
+ "lineEnd": 21,
+ "lineStart": 21,
+ "name": "encoder",
+ "startByte": 590,
+ },
+ {
+ "chunkKind": "function",
+ "container": "PatchId",
+ "contentHash": "20bb8d635f58176a",
+ "endByte": 736,
+ "language": "scala",
+ "lineEnd": 22,
+ "lineStart": 22,
+ "name": "decoder",
+ "startByte": 666,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "d68c470ad1b11435",
+ "endByte": 792,
+ "language": "scala",
+ "lineEnd": 25,
+ "lineStart": 25,
+ "name": "ClaimId",
+ "startByte": 740,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "85a61be88d64bf63",
+ "endByte": 960,
+ "language": "scala",
+ "lineEnd": 29,
+ "lineStart": 26,
+ "name": "ClaimId",
+ "startByte": 793,
+ },
+ {
+ "chunkKind": "function",
+ "container": "ClaimId",
+ "contentHash": "4db1372b9a8944e3",
+ "endByte": 885,
+ "language": "scala",
+ "lineEnd": 27,
+ "lineStart": 27,
+ "name": "encoder",
+ "startByte": 812,
+ },
+ {
+ "chunkKind": "function",
+ "container": "ClaimId",
+ "contentHash": "8d8ddbe595d64c23",
+ "endByte": 958,
+ "language": "scala",
+ "lineEnd": 28,
+ "lineStart": 28,
+ "name": "decoder",
+ "startByte": 888,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "5712a45ef1b12b31",
+ "endByte": 1017,
+ "language": "scala",
+ "lineEnd": 31,
+ "lineStart": 31,
+ "name": "ConflictId",
+ "startByte": 962,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "e9affa9a7356afa3",
+ "endByte": 1197,
+ "language": "scala",
+ "lineEnd": 35,
+ "lineStart": 32,
+ "name": "ConflictId",
+ "startByte": 1018,
+ },
+ {
+ "chunkKind": "function",
+ "container": "ConflictId",
+ "contentHash": "c80e4f0c73ea6d1d",
+ "endByte": 1116,
+ "language": "scala",
+ "lineEnd": 33,
+ "lineStart": 33,
+ "name": "encoder",
+ "startByte": 1040,
+ },
+ {
+ "chunkKind": "function",
+ "container": "ConflictId",
+ "contentHash": "ef72658fa91ff811",
+ "endByte": 1195,
+ "language": "scala",
+ "lineEnd": 34,
+ "lineStart": 34,
+ "name": "decoder",
+ "startByte": 1119,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "9621ad4828529d2a",
+ "endByte": 1247,
+ "language": "scala",
+ "lineEnd": 37,
+ "lineStart": 37,
+ "name": "Rev",
+ "startByte": 1199,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "27996787914bcd1d",
+ "endByte": 1399,
+ "language": "scala",
+ "lineEnd": 41,
+ "lineStart": 38,
+ "name": "Rev",
+ "startByte": 1248,
+ },
+ {
+ "chunkKind": "function",
+ "container": "Rev",
+ "contentHash": "92ad4d9a4dca16f0",
+ "endByte": 1332,
+ "language": "scala",
+ "lineEnd": 39,
+ "lineStart": 39,
+ "name": "encoder",
+ "startByte": 1263,
+ },
+ {
+ "chunkKind": "function",
+ "container": "Rev",
+ "contentHash": "20c75169352560a4",
+ "endByte": 1397,
+ "language": "scala",
+ "lineEnd": 40,
+ "lineStart": 40,
+ "name": "decoder",
+ "startByte": 1335,
+ },
+ ],
"entities": [
{
"kind": "file",
@@ -548,11 +814,354 @@ exports[`parseFile snapshots > snapshots Identifiers.scala 1`] = `
exports[`parseFile snapshots > snapshots Node.scala 1`] = `
{
+ "chunks": [
+ {
+ "chunkKind": "interface",
+ "container": undefined,
+ "contentHash": "2c3602b7735b0980",
+ "endByte": 178,
+ "language": "scala",
+ "lineEnd": 8,
+ "lineStart": 8,
+ "name": "NodeKind",
+ "startByte": 157,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "30951d9e6d654371",
+ "endByte": 2795,
+ "language": "scala",
+ "lineEnd": 86,
+ "lineStart": 9,
+ "name": "NodeKind",
+ "startByte": 179,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "ce581acd57c5e51f",
+ "endByte": 239,
+ "language": "scala",
+ "lineEnd": 10,
+ "lineStart": 10,
+ "name": "Module",
+ "startByte": 199,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "3894e0f29e12a950",
+ "endByte": 282,
+ "language": "scala",
+ "lineEnd": 11,
+ "lineStart": 11,
+ "name": "File",
+ "startByte": 242,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "4dcb85c557b0aedd",
+ "endByte": 325,
+ "language": "scala",
+ "lineEnd": 12,
+ "lineStart": 12,
+ "name": "Class",
+ "startByte": 285,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "03a112c9600de2f0",
+ "endByte": 368,
+ "language": "scala",
+ "lineEnd": 13,
+ "lineStart": 13,
+ "name": "Function",
+ "startByte": 328,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "1fdadba11c42a39b",
+ "endByte": 411,
+ "language": "scala",
+ "lineEnd": 14,
+ "lineStart": 14,
+ "name": "Variable",
+ "startByte": 371,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "5f274fd3f508d85b",
+ "endByte": 454,
+ "language": "scala",
+ "lineEnd": 15,
+ "lineStart": 15,
+ "name": "Config",
+ "startByte": 414,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "af4c6b74ee8dfdec",
+ "endByte": 497,
+ "language": "scala",
+ "lineEnd": 16,
+ "lineStart": 16,
+ "name": "ConfigEntry",
+ "startByte": 457,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "7ad9b2042370126a",
+ "endByte": 540,
+ "language": "scala",
+ "lineEnd": 17,
+ "lineStart": 17,
+ "name": "Service",
+ "startByte": 500,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "3243234db79f279f",
+ "endByte": 583,
+ "language": "scala",
+ "lineEnd": 18,
+ "lineStart": 18,
+ "name": "Endpoint",
+ "startByte": 543,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "fe900f1a33179972",
+ "endByte": 626,
+ "language": "scala",
+ "lineEnd": 19,
+ "lineStart": 19,
+ "name": "Intent",
+ "startByte": 586,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "cabcf7bc0b832502",
+ "endByte": 669,
+ "language": "scala",
+ "lineEnd": 20,
+ "lineStart": 20,
+ "name": "Decision",
+ "startByte": 629,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "692a1d329f150cb6",
+ "endByte": 712,
+ "language": "scala",
+ "lineEnd": 21,
+ "lineStart": 21,
+ "name": "Doc",
+ "startByte": 672,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "b489d197b9c326f5",
+ "endByte": 755,
+ "language": "scala",
+ "lineEnd": 22,
+ "lineStart": 22,
+ "name": "Object",
+ "startByte": 715,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "93680e321b988261",
+ "endByte": 799,
+ "language": "scala",
+ "lineEnd": 23,
+ "lineStart": 23,
+ "name": "Trait",
+ "startByte": 758,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "83534a6ebc0a194d",
+ "endByte": 842,
+ "language": "scala",
+ "lineEnd": 24,
+ "lineStart": 24,
+ "name": "Interface",
+ "startByte": 802,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "fcbb1b54fb1ed49c",
+ "endByte": 885,
+ "language": "scala",
+ "lineEnd": 25,
+ "lineStart": 25,
+ "name": "Method",
+ "startByte": 845,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "601c3245bb280851",
+ "endByte": 928,
+ "language": "scala",
+ "lineEnd": 26,
+ "lineStart": 26,
+ "name": "Plan",
+ "startByte": 888,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "f4bb8655a119081f",
+ "endByte": 971,
+ "language": "scala",
+ "lineEnd": 27,
+ "lineStart": 27,
+ "name": "Task",
+ "startByte": 931,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "9747366229c0bfdd",
+ "endByte": 1014,
+ "language": "scala",
+ "lineEnd": 28,
+ "lineStart": 28,
+ "name": "Bug",
+ "startByte": 974,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "7ef46a010eea53b9",
+ "endByte": 1057,
+ "language": "scala",
+ "lineEnd": 29,
+ "lineStart": 29,
+ "name": "Goal",
+ "startByte": 1017,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "b5d008578681570b",
+ "endByte": 1100,
+ "language": "scala",
+ "lineEnd": 30,
+ "lineStart": 30,
+ "name": "Region",
+ "startByte": 1060,
+ },
+ {
+ "chunkKind": "class",
+ "container": "NodeKind",
+ "contentHash": "f9150e0d644649d1",
+ "endByte": 1143,
+ "language": "scala",
+ "lineEnd": 31,
+ "lineStart": 31,
+ "name": "Chunk",
+ "startByte": 1103,
+ },
+ {
+ "chunkKind": "function",
+ "container": "NodeKind",
+ "contentHash": "95329b6c5c6903c8",
+ "endByte": 1859,
+ "language": "scala",
+ "lineEnd": 56,
+ "lineStart": 33,
+ "name": "nameMap",
+ "startByte": 1147,
+ },
+ {
+ "chunkKind": "function",
+ "container": "NodeKind",
+ "contentHash": "bf89078ccc0d7613",
+ "endByte": 2664,
+ "language": "scala",
+ "lineEnd": 81,
+ "lineStart": 58,
+ "name": "encoder",
+ "startByte": 1863,
+ },
+ {
+ "chunkKind": "function",
+ "container": "NodeKind",
+ "contentHash": "b8f70f2c26f8f73e",
+ "endByte": 2793,
+ "language": "scala",
+ "lineEnd": 85,
+ "lineStart": 83,
+ "name": "decoder",
+ "startByte": 2668,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "ce7365acee3c48bb",
+ "endByte": 3031,
+ "language": "scala",
+ "lineEnd": 98,
+ "lineStart": 88,
+ "name": "GraphNode",
+ "startByte": 2797,
+ },
+ {
+ "chunkKind": "class",
+ "container": undefined,
+ "contentHash": "a046c9fd1d01daa9",
+ "endByte": 3247,
+ "language": "scala",
+ "lineEnd": 105,
+ "lineStart": 100,
+ "name": "GraphNode",
+ "startByte": 3033,
+ },
+ {
+ "chunkKind": "function",
+ "container": "GraphNode",
+ "contentHash": "8614f0d5d5f8950b",
+ "endByte": 3175,
+ "language": "scala",
+ "lineEnd": 103,
+ "lineStart": 103,
+ "name": "encoder",
+ "startByte": 3108,
+ },
+ {
+ "chunkKind": "function",
+ "container": "GraphNode",
+ "contentHash": "4b9bc3a1d304ef20",
+ "endByte": 3245,
+ "language": "scala",
+ "lineEnd": 104,
+ "lineStart": 104,
+ "name": "decoder",
+ "startByte": 3178,
+ },
+ ],
"entities": [
{
"kind": "file",
"language": "scala",
- "lineEnd": 103,
+ "lineEnd": 106,
"lineStart": 1,
"name": "Node.scala",
},
@@ -568,7 +1177,7 @@ exports[`parseFile snapshots > snapshots Node.scala 1`] = `
"container": undefined,
"kind": "class",
"language": "scala",
- "lineEnd": 83,
+ "lineEnd": 86,
"lineStart": 9,
"name": "NodeKind",
},
@@ -740,60 +1349,68 @@ exports[`parseFile snapshots > snapshots Node.scala 1`] = `
"lineStart": 30,
"name": "Region",
},
+ {
+ "container": "NodeKind",
+ "kind": "class",
+ "language": "scala",
+ "lineEnd": 31,
+ "lineStart": 31,
+ "name": "Chunk",
+ },
{
"container": "NodeKind",
"kind": "function",
"language": "scala",
- "lineEnd": 54,
- "lineStart": 32,
+ "lineEnd": 56,
+ "lineStart": 33,
"name": "nameMap",
},
{
"container": "NodeKind",
"kind": "function",
"language": "scala",
- "lineEnd": 78,
- "lineStart": 56,
+ "lineEnd": 81,
+ "lineStart": 58,
"name": "encoder",
},
{
"container": "NodeKind",
"kind": "function",
"language": "scala",
- "lineEnd": 82,
- "lineStart": 80,
+ "lineEnd": 85,
+ "lineStart": 83,
"name": "decoder",
},
{
"container": undefined,
"kind": "class",
"language": "scala",
- "lineEnd": 95,
- "lineStart": 85,
+ "lineEnd": 98,
+ "lineStart": 88,
"name": "GraphNode",
},
{
"container": undefined,
"kind": "class",
"language": "scala",
- "lineEnd": 102,
- "lineStart": 97,
+ "lineEnd": 105,
+ "lineStart": 100,
"name": "GraphNode",
},
{
"container": "GraphNode",
"kind": "function",
"language": "scala",
- "lineEnd": 100,
- "lineStart": 100,
+ "lineEnd": 103,
+ "lineStart": 103,
"name": "encoder",
},
{
"container": "GraphNode",
"kind": "function",
"language": "scala",
- "lineEnd": 101,
- "lineStart": 101,
+ "lineEnd": 104,
+ "lineStart": 104,
"name": "decoder",
},
{
@@ -841,15 +1458,15 @@ exports[`parseFile snapshots > snapshots Node.scala 1`] = `
{
"kind": "module",
"language": "scala",
- "lineEnd": 98,
- "lineStart": 98,
+ "lineEnd": 101,
+ "lineStart": 101,
"name": "Provenance.instantEncoder",
},
{
"kind": "module",
"language": "scala",
- "lineEnd": 98,
- "lineStart": 98,
+ "lineEnd": 101,
+ "lineStart": 101,
"name": "Provenance.instantDecoder",
},
],
@@ -1081,6 +1698,16 @@ exports[`parseFile snapshots > snapshots Node.scala 1`] = `
"predicate": "EXTENDS",
"srcName": "Region",
},
+ {
+ "dstName": "Chunk",
+ "predicate": "CONTAINS",
+ "srcName": "NodeKind",
+ },
+ {
+ "dstName": "NodeKind",
+ "predicate": "EXTENDS",
+ "srcName": "Chunk",
+ },
{
"dstName": "nameMap",
"predicate": "CONTAINS",
@@ -1227,6 +1854,30 @@ exports[`parseFile snapshots > snapshots Node.scala 1`] = `
exports[`parseFile snapshots > snapshots doctor.ts 1`] = `
{
+ "chunks": [
+ {
+ "chunkKind": "interface",
+ "container": undefined,
+ "contentHash": "2af0710a68e6f595",
+ "endByte": 322,
+ "language": "typescript",
+ "lineEnd": 10,
+ "lineStart": 7,
+ "name": "Check",
+ "startByte": 232,
+ },
+ {
+ "chunkKind": "function",
+ "container": undefined,
+ "contentHash": "a80b54ecd0296c6e",
+ "endByte": 3307,
+ "language": "typescript",
+ "lineEnd": 100,
+ "lineStart": 12,
+ "name": "registerDoctorCommand",
+ "startByte": 331,
+ },
+ ],
"entities": [
{
"kind": "file",
diff --git a/core-ingestion/src/__tests__/parseFile.test.ts b/core-ingestion/src/__tests__/parseFile.test.ts
index 5842b5ae..f0dd3c9c 100644
--- a/core-ingestion/src/__tests__/parseFile.test.ts
+++ b/core-ingestion/src/__tests__/parseFile.test.ts
@@ -59,4 +59,53 @@ describe('mixed-language fixture integration', () => {
resolved.some(edge => edge.srcFilePath.endsWith('.ts') && edge.dstFilePath.endsWith('.scala')),
).toBe(false);
});
+
+ it('captures C++ out-of-line method containers and qualifier-based calls', () => {
+ const source = `
+class VersionSet {
+ public:
+ void Recover();
+};
+
+class Version {
+ public:
+ void Get();
+};
+
+class DBImpl {
+ public:
+ VersionSet* versions_;
+ Version* current_;
+ void Open();
+ void Read();
+};
+
+void DBImpl::Open() {
+ versions_->Recover();
+}
+
+void DBImpl::Read() {
+ Version* current = current_;
+ current->Get();
+}
+`;
+ const result = parseFile('leveldb/db/db_impl.cc', source);
+ expect(result).not.toBeNull();
+
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'Open',
+ kind: 'method',
+ container: 'DBImpl',
+ }));
+ expect(result!.relationships).toContainEqual({
+ srcName: 'DBImpl.Open',
+ dstName: 'VersionSet.Recover',
+ predicate: 'CALLS',
+ });
+ expect(result!.relationships).toContainEqual({
+ srcName: 'DBImpl.Read',
+ dstName: 'Version.Get',
+ predicate: 'CALLS',
+ });
+ });
});
diff --git a/core-ingestion/src/__tests__/resolveEdges.test.ts b/core-ingestion/src/__tests__/resolveEdges.test.ts
index c46ad0fb..58f2adac 100644
--- a/core-ingestion/src/__tests__/resolveEdges.test.ts
+++ b/core-ingestion/src/__tests__/resolveEdges.test.ts
@@ -364,4 +364,31 @@ describe('resolveEdges', () => {
const refEdges = resolved.filter(e => e.predicate === 'REFERENCES');
expect(refEdges).toEqual([]);
});
+
+ it('resolves qualified C++ member calls even when the source file defines a same-named method', () => {
+ const caller = fileResult(
+ '/repo/db_impl.cc',
+ SupportedLanguages.CPlusPlus,
+ [
+ entity('Open', SupportedLanguages.CPlusPlus, 'method', 'DBImpl'),
+ entity('Recover', SupportedLanguages.CPlusPlus, 'method', 'DBImpl'),
+ ],
+ [{ srcName: 'DBImpl.Open', dstName: 'VersionSet.Recover', predicate: 'CALLS' }],
+ );
+ const callee = fileResult(
+ '/repo/version_set.cc',
+ SupportedLanguages.CPlusPlus,
+ [entity('Recover', SupportedLanguages.CPlusPlus, 'method', 'VersionSet')],
+ );
+
+ expect(resolveEdges([caller, callee])).toContainEqual({
+ srcFilePath: '/repo/db_impl.cc',
+ srcName: 'DBImpl.Open',
+ dstFilePath: '/repo/version_set.cc',
+ dstName: 'VersionSet.Recover',
+ dstQualifiedKey: 'VersionSet.Recover',
+ predicate: 'CALLS',
+ confidence: 0.7,
+ });
+ });
});
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index 60dd05b8..d0a141b0 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -333,7 +333,7 @@ function unwrapRustCfgMacros(source: string): string {
// Main parse function
// ---------------------------------------------------------------------------
-export function parseFile(filePath: string, source: string, opts?: { emitChunks?: boolean }): FileParseResult | null {
+export function parseFile(filePath: string, source: string): FileParseResult | null {
const language = detectLanguageForSource(filePath, source);
if (!language) return null;
@@ -374,7 +374,7 @@ export function parseFile(filePath: string, source: string, opts?: { emitChunks?
const matches = query.matches(tree.rootNode);
// Pre-partition matches so each pass only iterates its relevant subset.
- // Pass 1: definitions, heritage, Python typed-param/assign annotations.
+ // Pass 1: definitions, heritage, and language-specific type hints.
// Pass 2: imports, calls, type references.
// Without this, every definition match does 5 useless find() calls in pass 2,
// and every call match does 4 useless find() calls in pass 1.
@@ -385,7 +385,7 @@ export function parseFile(filePath: string, source: string, opts?: { emitChunks?
for (const c of m.captures) {
const n = c.name;
if (n.startsWith('definition.') || n.startsWith('heritage.') ||
- n === '_typed_param_scope' || n === '_assign_scope') {
+ n === '_typed_param_scope' || n === '_assign_scope' || n === '_typed_var_scope') {
isPass1 = true;
break;
}
@@ -425,6 +425,8 @@ export function parseFile(filePath: string, source: string, opts?: { emitChunks?
const paramTypeMap = new Map<string, Map<string, string>>();
// Python: map function name → (variable name → assigned type) for untyped-param qualifier substitution
const assignTypeMap = new Map<string, Map<string, string>>();
+ // C/C++: map enclosing scope → (variable/member name → declared type)
+ const declaredTypeMap = new Map<string, Map<string, string>>();
// --- First pass: collect definitions ---
for (const match of pass1Matches) {
@@ -557,6 +559,24 @@ export function parseFile(filePath: string, source: string, opts?: { emitChunks?
continue;
}
}
+
+ // C/C++ typed declarations: capture local vars, parameters, and fields so that
+ // member calls like current->Get(...) can be rewritten to Version.Get.
+ if (language === SupportedLanguages.C || language === SupportedLanguages.CPlusPlus) {
+ const typedVarScope = match.captures.find((c: any) => c.name === '_typed_var_scope');
+ const typedVarName = match.captures.find((c: any) => c.name === '_typed_var_name');
+ const typedVarType = match.captures.find((c: any) => c.name === '_typed_var_type');
+ if (typedVarScope && typedVarName && typedVarType) {
+ const line = typedVarScope.node.startPosition.row + 1;
+ const scope =
+ findEnclosingFunction(entities, line)
+ ?? findEnclosing(classRanges, line, '')
+ ?? fileName;
+ if (!declaredTypeMap.has(scope)) declaredTypeMap.set(scope, new Map());
+ declaredTypeMap.get(scope)!.set(typedVarName.node.text, typedVarType.node.text);
+ continue;
+ }
+ }
}
// --- Second pass: calls and imports ---
@@ -666,6 +686,15 @@ export function parseFile(filePath: string, source: string, opts?: { emitChunks?
if (typeForParam) qualifier = typeForParam;
else if (typeForAssign) qualifier = typeForAssign;
}
+ } else if (language === SupportedLanguages.C || language === SupportedLanguages.CPlusPlus) {
+ const funcName = findEnclosingFunction(entities, callLine) ?? fileName;
+ const className = findEnclosing(classRanges, callLine, '')
+ ?? (funcName.includes('.') ? funcName.slice(0, funcName.lastIndexOf('.')) : undefined);
+ const typeForDecl =
+ declaredTypeMap.get(funcName)?.get(qualifier)
+ ?? (className ? declaredTypeMap.get(className)?.get(qualifier) : undefined)
+ ?? declaredTypeMap.get(fileName)?.get(qualifier);
+ if (typeForDecl) qualifier = typeForDecl;
}
return `${qualifier}.${callee}`;
})();
@@ -698,30 +727,28 @@ export function parseFile(filePath: string, source: string, opts?: { emitChunks?
}
}
- if (opts?.emitChunks !== false) {
- for (const pendingChunk of pendingChunks) {
- const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
- const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
- chunks.push({
- ...pendingChunk,
- contentHash,
- });
- }
+ for (const pendingChunk of pendingChunks) {
+ const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
+ const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
+ chunks.push({
+ ...pendingChunk,
+ contentHash,
+ });
+ }
- // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
- if (chunks.length === 0) {
- const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
- chunks.push({
- name: null,
- chunkKind: 'file_body',
- lineStart: 1,
- lineEnd: sourceLineCount,
- startByte: 0,
- endByte: source.length,
- contentHash,
- language,
- });
- }
+ // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
+ if (chunks.length === 0) {
+ const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
+ chunks.push({
+ name: null,
+ chunkKind: 'file_body',
+ lineStart: 1,
+ lineEnd: sourceLineCount,
+ startByte: 0,
+ endByte: source.length,
+ contentHash,
+ language,
+ });
}
return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };
@@ -774,8 +801,10 @@ function bestQKey(
fileQKeys: Map<string, Map<string, string[]>>,
filePath: string,
plainName: string,
+ preferredQKey?: string,
): string | null {
const qks = [...new Set(fileQKeys.get(filePath)?.get(plainName) ?? [])];
+ if (preferredQKey && qks.includes(preferredQKey)) return preferredQKey;
return qks.length === 1 ? qks[0] : null;
}
@@ -1076,6 +1105,11 @@ export function resolveEdges(results: FileParseResult[], stats?: {
// ── Main resolution loop ───────────────────────────────────────────
+ function fileDefinesQualifiedMember(filePath: string, qualifierPart: string, memberPart: string): boolean {
+ const qks = fileQKeys.get(filePath)?.get(memberPart) ?? [];
+ return qks.includes(`${qualifierPart}.${memberPart}`);
+ }
+
const resolved: ResolvedEdge[] = [];
for (const result of results) {
@@ -1171,12 +1205,14 @@ export function resolveEdges(results: FileParseResult[], stats?: {
// Try import-scoped qualifier first
const qualImportMatches: string[] = [];
for (const fp of importedFilePaths) {
- if (fileHasSymbol.get(fp)?.has(qualifierPart)) qualImportMatches.push(fp);
+ if (fileDefinesQualifiedMember(fp, qualifierPart, memberPart) || fileHasSymbol.get(fp)?.has(qualifierPart)) {
+ qualImportMatches.push(fp);
+ }
}
if (qualImportMatches.length === 1) {
const qfp = qualImportMatches[0];
if (fileHasSymbol.get(qfp)?.has(memberPart)) {
- const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart);
+ const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart, `${qualifierPart}.${memberPart}`);
if (dstQualifiedKey !== null) {
// dstName must match rel.dstName so buildPatchWithResolution can look it up
resolved.push({ srcFilePath, srcName, dstFilePath: qfp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.9 });
@@ -1184,12 +1220,14 @@ export function resolveEdges(results: FileParseResult[], stats?: {
}
continue;
}
- // Global qualifier fallback — use symbolToFiles index instead of full scan
- const qualGlobalMatches = (symbolToFiles.get(qualifierPart) ?? []).filter(fp => fp !== srcFilePath);
+ // Global qualifier fallback — check files that define the qualified member
+ const qualGlobalMatches = results
+ .map(r => r.filePath)
+ .filter(fp => fp !== srcFilePath && fileDefinesQualifiedMember(fp, qualifierPart, memberPart));
if (qualGlobalMatches.length === 1) {
const qfp = qualGlobalMatches[0];
if (fileHasSymbol.get(qfp)?.has(memberPart)) {
- const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart);
+ const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart, `${qualifierPart}.${memberPart}`);
if (dstQualifiedKey !== null) {
resolved.push({ srcFilePath, srcName, dstFilePath: qfp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.7 });
}
diff --git a/core-ingestion/src/patch-builder.ts b/core-ingestion/src/patch-builder.ts
index 5500cb24..74f674bd 100644
--- a/core-ingestion/src/patch-builder.ts
+++ b/core-ingestion/src/patch-builder.ts
@@ -47,11 +47,11 @@ function sourceType(filePath: string): string {
}
export function extractorName(): string {
- return `tree-sitter/1.18`;
+ return `tree-sitter/1.19`;
}
/** Previous extractor versions — their patches are superseded when re-ingesting. */
-export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];
+export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.18', 'tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];
/** Compute a patchId for a (filePath, sourceHash, extractorVersion) triple. */
function computePatchId(filePath: string, sourceHash: string, extractor: string): string {
@@ -272,7 +272,6 @@ export function buildPatchWithResolution(
sourceHash: string,
resolvedEdges: ResolvedEdge[],
previousSourceHash?: string,
- opts?: { emitClaims?: boolean },
): GraphPatchPayload {
// Build lookup: `${srcName}:${predicate}:${dstName}` → { dstFilePath, dstQualifiedKey }
// Callers should pass only edges for this file (pre-grouped) for best performance,
@@ -421,17 +420,15 @@ export function buildPatchWithResolution(
});
}
- if (opts?.emitClaims !== false) {
- for (const r of relationships) {
- const srcKey = resolveKey(r.srcName);
- ops.push({
- type: 'AssertClaim',
- entityId: nodeId(filePath, srcKey),
- field: `${r.predicate.toLowerCase()}:${r.dstName}`,
- value: r.dstName,
- confidence: null,
- });
- }
+ for (const r of relationships) {
+ const srcKey = resolveKey(r.srcName);
+ ops.push({
+ type: 'AssertClaim',
+ entityId: nodeId(filePath, srcKey),
+ field: `${r.predicate.toLowerCase()}:${r.dstName}`,
+ value: r.dstName,
+ confidence: null,
+ });
}
const extractor = extractorName();
diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.ts
index a65fe764..ef7c2d6e 100644
--- a/core-ingestion/src/queries.ts
+++ b/core-ingestion/src/queries.ts
@@ -515,22 +515,22 @@ export const CPP_QUERIES = `
; Functions & Methods (direct declarator)
(function_definition declarator: (function_declarator declarator: (identifier) @name)) @definition.function
-(function_definition declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name))) @definition.method
+(function_definition declarator: (function_declarator declarator: (qualified_identifier scope: (_) @receiver.type name: (identifier) @name))) @definition.method
; Functions/methods returning pointers (pointer_declarator wraps function_declarator)
(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function
-(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name)))) @definition.method
+(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier scope: (_) @receiver.type name: (identifier) @name)))) @definition.method
; Functions/methods returning double pointers (nested pointer_declarator)
(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name)))) @definition.function
-(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name))))) @definition.method
+(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier scope: (_) @receiver.type name: (identifier) @name))))) @definition.method
; Functions/methods returning references (reference_declarator wraps function_declarator)
(function_definition declarator: (reference_declarator (function_declarator declarator: (identifier) @name))) @definition.function
-(function_definition declarator: (reference_declarator (function_declarator declarator: (qualified_identifier name: (identifier) @name)))) @definition.method
+(function_definition declarator: (reference_declarator (function_declarator declarator: (qualified_identifier scope: (_) @receiver.type name: (identifier) @name)))) @definition.method
; Destructors (destructor_name is distinct from identifier in tree-sitter-cpp)
-(function_definition declarator: (function_declarator declarator: (qualified_identifier name: (destructor_name) @name))) @definition.method
+(function_definition declarator: (function_declarator declarator: (qualified_identifier scope: (_) @receiver.type name: (destructor_name) @name))) @definition.method
; Function declarations / prototypes (common in headers)
(declaration declarator: (function_declarator declarator: (identifier) @name)) @definition.function
@@ -555,6 +555,7 @@ export const CPP_QUERIES = `
; Calls
(call_expression function: (identifier) @call.name) @call
(call_expression function: (field_expression field: (field_identifier) @call.name)) @call
+(call_expression function: (field_expression argument: (identifier) @_qualifier field: (field_identifier) @call.name)) @call
(call_expression function: (qualified_identifier name: (identifier) @call.name)) @call
(call_expression function: (template_function name: (identifier) @call.name)) @call
@@ -570,6 +571,35 @@ export const CPP_QUERIES = `
; Type references — captures types used in declaration/parameter positions
(declaration type: (type_identifier) @reference.type)
(parameter_declaration type: (type_identifier) @reference.type)
+
+; Typed declarations for qualifier substitution: local vars, fields, params
+(declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (identifier) @_typed_var_name) @_typed_var_scope
+(declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (pointer_declarator declarator: (identifier) @_typed_var_name)) @_typed_var_scope
+(declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (init_declarator declarator: (identifier) @_typed_var_name)) @_typed_var_scope
+(declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (init_declarator declarator: (pointer_declarator declarator: (identifier) @_typed_var_name))) @_typed_var_scope
+(declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (reference_declarator (identifier) @_typed_var_name)) @_typed_var_scope
+(field_declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (field_identifier) @_typed_var_name) @_typed_var_scope
+(field_declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (pointer_declarator declarator: (field_identifier) @_typed_var_name)) @_typed_var_scope
+(parameter_declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (identifier) @_typed_var_name) @_typed_var_scope
+(parameter_declaration
+ type: (type_identifier) @_typed_var_type
+ declarator: (pointer_declarator declarator: (identifier) @_typed_var_name)) @_typed_var_scope
`;
// C# queries - works with tree-sitter-c-sharp
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 3f809146..aa71a2fb 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -3,7 +3,9 @@ import * as fs from 'node:fs';
import * as crypto from 'node:crypto';
import * as os from 'node:os';
import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
import type { Command } from 'commander';
+import { ParsePool } from './parse-pool.js';
import chalk from 'chalk';
import { IxClient } from '../../client/api.js';
import type { GraphPatchPayload } from '../../client/types.js';
@@ -345,6 +347,14 @@ export async function ingestFiles(
const client = new IxClient(getEndpoint());
const start = performance.now();
+ // Worker pool for parallel file parsing across CPU cores
+ const workerPath = nodePath.resolve(
+ nodePath.dirname(fileURLToPath(import.meta.url)),
+ '../../../../core-ingestion/dist/parse-worker.js',
+ );
+ const pool = new ParsePool(workerPath, Math.max(1, os.cpus().length - 1));
+ pool.init();
+
let progressPhase = 'Scanning';
let progressCurrent = 0;
let progressTotal = 0;
@@ -663,7 +673,7 @@ export async function ingestFiles(
for (let j = 0; j < batch.length; j++) {
const { parsed: p, hash, previousHash } = batch[j];
try {
- const patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);
+ const patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
} catch (err) {
parseErrors++;
@@ -728,7 +738,7 @@ export async function ingestFiles(
for (let j = 0; j < allParsed.length; j++) {
const { parsed: p, hash, previousHash } = allParsed[j];
try {
- const patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);
+ const patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
} catch (err) {
parseErrors++;
@@ -788,38 +798,44 @@ export async function ingestFiles(
resolveEdgesFn = ingestion.resolveEdges as Function;
buildPatchFn = patchBuilder.buildPatchWithResolution as Function;
- const allParsed: ParsedFile[] = [];
- for (let i = 0; i < changedPaths.length; i++) {
- const { filePath, bytes, hash, previousHash } = changedPaths[i];
- setCurrentWork(`parse ${filePath}`);
- try {
- const sourceText = bytes.toString('utf-8');
- if (isLikelyMinifiedSource(sourceText)) {
- minifiedLikely++;
- filesSkipped++;
- progressCurrent++;
- if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
- continue;
- }
- const p0 = performance.now();
- const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });
- const pfMs = performance.now() - p0;
- if (!parsed) { filesSkipped++; progressCurrent++; continue; }
- timings.parseOnlyMs += pfMs;
- const lang: string = (parsed as any).language ?? 'unknown';
- const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };
- ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);
- entitiesParsed += parsed.entities.length;
- allParsed.push({ filePath, parsed, hash, previousHash });
- progressCurrent++;
- } catch (err) {
- parseErrors++;
+ // Pre-filter minified files (main thread) then parse in parallel via worker pool.
+ const parseable: Array<{ filePath: string; source: string; hash: string; previousHash: string | undefined }> = [];
+ for (const { filePath, bytes, hash, previousHash } of changedPaths) {
+ const sourceText = bytes.toString('utf-8');
+ if (isLikelyMinifiedSource(sourceText)) {
+ minifiedLikely++;
+ filesSkipped++;
progressCurrent++;
- process.stderr.write(`\n [parse error] ${filePath}: ${err}\n`);
+ if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
+ continue;
}
- if ((i + 1) % YIELD_EVERY === 0) await new Promise<void>(resolve => setImmediate(resolve));
+ parseable.push({ filePath, source: sourceText, hash, previousHash });
}
- await flushAll(allParsed);
+
+ progressPhase = 'Parsing';
+ progressTotal = parseable.length;
+ progressCurrent = 0;
+
+ // Pipeline overlap: parse next chunk while committing current chunk
+ let pendingFlush: Promise<void> = Promise.resolve();
+ for (let i = 0; i < parseable.length; i += PARSE_STREAM_CHUNK) {
+ const chunk = parseable.slice(i, i + PARSE_STREAM_CHUNK);
+ const parseResults = await Promise.all(
+ chunk.map(f =>
+ pool.parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),
+ ),
+ );
+ const batch: ParsedFile[] = [];
+ for (let j = 0; j < chunk.length; j++) {
+ const parsed = parseResults[j] as any;
+ if (!parsed) { filesSkipped++; continue; }
+ entitiesParsed += parsed.entities.length;
+ batch.push({ filePath: chunk[j].filePath, parsed, hash: chunk[j].hash, previousHash: chunk[j].previousHash });
+ }
+ await pendingFlush;
+ pendingFlush = flushBatch(batch);
+ }
+ await pendingFlush;
}
} else {
// Path B: no baseline (first ingest) or --force → load modules, then stream parse + commit.
@@ -829,53 +845,60 @@ export async function ingestFiles(
resolveEdgesFn = ingestion.resolveEdges as Function;
buildPatchFn = patchBuilder.buildPatchWithResolution as Function;
- const useParseAllFirst = filePaths.length <= SMALL_REPO_THRESHOLD;
- const allParsedB: ParsedFile[] = useParseAllFirst ? [] : (null as any);
- let batch: ParsedFile[] = useParseAllFirst ? (null as any) : [];
- for (let i = 0; i < filePaths.length; i++) {
- const filePath = filePaths[i];
- setCurrentWork(`parse ${filePath}`);
- try {
- const fileSize = fs.statSync(filePath).size;
- if (fileSize === 0) { filesSkipped++; progressCurrent++; continue; }
- if (fileSize > MAX_FILE_BYTES) { tooLarge++; progressCurrent++; continue; }
- const bytes = fs.readFileSync(filePath);
- const hash = sha256(bytes);
- if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; progressCurrent++; continue; }
- const previousHash = knownHashes.get(filePath);
- const sourceText = bytes.toString('utf-8');
- if (isLikelyMinifiedSource(sourceText)) {
- minifiedLikely++;
- filesSkipped++;
- progressCurrent++;
- if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
- continue;
+ progressPhase = 'Parsing';
+ progressTotal = filePaths.length;
+ progressCurrent = 0;
+
+ // Pipeline overlap: parse next chunk in worker pool while committing current chunk
+ let pendingFlush: Promise<void> = Promise.resolve();
+ for (let i = 0; i < filePaths.length; i += PARSE_STREAM_CHUNK) {
+ const chunk = filePaths.slice(i, i + PARSE_STREAM_CHUNK);
+ type FileData = { filePath: string; source: string; hash: string; previousHash: string | undefined } | null;
+ const fileData: FileData[] = [];
+ for (const filePath of chunk) {
+ try {
+ const fileSize = fs.statSync(filePath).size;
+ if (fileSize === 0) { filesSkipped++; fileData.push(null); continue; }
+ if (fileSize > MAX_FILE_BYTES) { tooLarge++; fileData.push(null); continue; }
+ const bytes = fs.readFileSync(filePath);
+ const hash = sha256(bytes);
+ if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; fileData.push(null); continue; }
+ const previousHash = knownHashes.get(filePath);
+ const sourceText = bytes.toString('utf-8');
+ if (isLikelyMinifiedSource(sourceText)) {
+ minifiedLikely++;
+ filesSkipped++;
+ if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
+ fileData.push(null);
+ continue;
+ }
+ fileData.push({ filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined });
+ } catch (err) {
+ parseErrors++;
+ fileData.push(null);
+ process.stderr.write(`\n [read error] ${filePath}: ${err}\n`);
}
- const p0 = performance.now();
- const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });
- const pfMs = performance.now() - p0;
- if (!parsed) { filesSkipped++; progressCurrent++; continue; }
- timings.parseOnlyMs += pfMs;
- const lang: string = (parsed as any).language ?? 'unknown';
- const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };
- ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);
+ }
+ const parseResults = await Promise.all(
+ fileData.map(f =>
+ f
+ ? pool.parse(f.filePath, f.source).then(r => { progressCurrent++; return r; })
+ : Promise.resolve(null),
+ ),
+ );
+ const batch: ParsedFile[] = [];
+ for (let j = 0; j < fileData.length; j++) {
+ const f = fileData[j];
+ if (!f) continue;
+ const parsed = parseResults[j] as any;
+ if (!parsed) { filesSkipped++; continue; }
entitiesParsed += parsed.entities.length;
- const entry: ParsedFile = { filePath, parsed, hash, previousHash: previousHash !== hash ? previousHash : undefined };
- if (useParseAllFirst) { allParsedB.push(entry); } else { batch.push(entry); }
- progressCurrent++;
- } catch (err) {
- parseErrors++;
- progressCurrent++;
- process.stderr.write(`\n [parse error] ${filePath}: ${err}\n`);
+ batch.push({ filePath: f.filePath, parsed, hash: f.hash, previousHash: f.previousHash });
}
- if ((i + 1) % YIELD_EVERY === 0) await new Promise<void>(resolve => setImmediate(resolve));
- if (!useParseAllFirst && batch.length >= PARSE_STREAM_CHUNK) { await flushBatch(batch); batch = []; }
- }
- if (useParseAllFirst) {
- await flushAll(allParsedB);
- } else {
- await flushBatch(batch);
+ await pendingFlush;
+ pendingFlush = flushBatch(batch);
}
+ await pendingFlush;
}
const parsed = performance.now();
@@ -891,6 +914,7 @@ export async function ingestFiles(
saveMtimeCache(projectRoot, currentMtimes);
}
} finally {
+ await pool.destroy();
if (interval) {
clearInterval(interval);
if (progressTotal > 0) {