Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
Feat/new parser r (#204)
Brandon Garate00ef7a082351
diff --git a/core-ingestion/package-lock.json b/core-ingestion/package-lock.jsonindex 7ad3f728..0ffeba04 100644--- a/core-ingestion/package-lock.json+++ b/core-ingestion/package-lock.json@@ -31,6 +31,7 @@"vitest": "^4.1.0"},"optionalDependencies": {+ "@davisvaughan/tree-sitter-r": "^1.2.0","tree-sitter-elixir": "^0.3.5","tree-sitter-kotlin": "^0.3.8","tree-sitter-make": "^1.1.1",@@ -38,6 +39,26 @@"tree-sitter-swift": "^0.6.0"}},+ "node_modules/@davisvaughan/tree-sitter-r": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/@davisvaughan/tree-sitter-r/-/tree-sitter-r-1.2.0.tgz",+ "integrity": "sha512-KnvUaJJS6h1oCcRaq8fFb/xYyvqodMcE5M48vYmIWW7uOJCFkNwaHdGXuLzTipOWfXGzrKuq4EFRifHphLu9GQ==",+ "hasInstallScript": true,+ "license": "MIT",+ "optional": true,+ "dependencies": {+ "node-addon-api": "^8.2.1",+ "node-gyp-build": "^4.8.2"+ },+ "peerDependencies": {+ "tree-sitter": "^0.21.1"+ },+ "peerDependenciesMeta": {+ "tree-sitter": {+ "optional": true+ }+ }+ },"node_modules/@emnapi/core": {"version": "1.9.2","resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz",diff --git a/core-ingestion/package.json b/core-ingestion/package.jsonindex 410787c5..c78df8d0 100644--- a/core-ingestion/package.json+++ b/core-ingestion/package.json@@ -36,7 +36,8 @@"tree-sitter-sas": "^0.4.2","tree-sitter-swift": "^0.6.0","tree-sitter-elixir": "^0.3.5",- "tree-sitter-make": "^1.1.1"+ "tree-sitter-make": "^1.1.1",+ "@davisvaughan/tree-sitter-r": "^1.2.0"},"devDependencies": {"@emnapi/core": "1.9.2",diff --git a/core-ingestion/src/__tests__/patchBuilder.resolution.test.ts b/core-ingestion/src/__tests__/patchBuilder.resolution.test.tsindex 4f16ec9a..19652fcc 100644--- a/core-ingestion/src/__tests__/patchBuilder.resolution.test.ts+++ b/core-ingestion/src/__tests__/patchBuilder.resolution.test.ts@@ -4,7 +4,7 @@ import { describe, expect, it } from 'vitest';import { parseFile, resolveEdges, type FileParseResult, type ParsedEntity, type ParsedRelationship, type ResolvedEdge } from '../index.js';import { SupportedLanguages } from '../languages.js';-import { buildPatchWithResolution } from '../patch-builder.js';+import { buildPatch, buildPatchWithResolution } from '../patch-builder.js';const defaultFileRole = { role: 'production' as const, role_confidence: 0.5, role_signals: [] };@@ -109,6 +109,122 @@ describe('buildPatchWithResolution', () => {}));});+ it('materialises external stub node for unresolved :: package calls', () => {+ const file = '/repo/model.R';+ const externalNodeId = nodeId('external://dplyr', 'dplyr::filter');+ const result = fileResult(+ file,+ SupportedLanguages.R,+ [entity('fitModel', SupportedLanguages.R)],+ [+ { srcName: 'fitModel', dstName: 'dplyr::filter', predicate: 'CALLS' },+ { srcName: 'fitModel', dstName: 'localHelper', predicate: 'CALLS' },+ ],+ );+ const patch = buildPatchWithResolution(result, 'hash', '', []);+ const callEdges = patch.ops.filter(op => op.type === 'UpsertEdge' && op.predicate === 'CALLS');+ const upsertNodes = patch.ops.filter(op => op.type === 'UpsertNode');++ // dplyr::filter edge must point to the external stub, not to a same-file phantom+ expect(callEdges).toContainEqual(+ expect.objectContaining({ src: nodeId(file, 'fitModel'), dst: externalNodeId }),+ );+ expect(callEdges).not.toContainEqual(+ expect.objectContaining({ dst: nodeId(file, 'dplyr::filter') }),+ );+ // External stub node must be materialised with correct attrs+ expect(upsertNodes).toContainEqual(+ expect.objectContaining({ id: externalNodeId, kind: 'function', name: 'filter' }),+ );+ // Unqualified same-file call still resolves within the file+ expect(callEdges).toContainEqual(+ expect.objectContaining({ src: nodeId(file, 'fitModel'), dst: nodeId(file, 'localHelper') }),+ );+ });++ it('splits multi-dot :: members on the namespace, not the last dot', () => {+ // utils::write.csv must externalise as pkg "utils" / func "write.csv",+ // not pkg "utils.write" / func "csv" (the old lastIndexOf('.') bug).+ const file = '/repo/io.R';+ const result = fileResult(+ file,+ SupportedLanguages.R,+ [entity('writeOut', SupportedLanguages.R)],+ [{ srcName: 'writeOut', dstName: 'utils::write.csv', predicate: 'CALLS' }],+ );+ const patch = buildPatchWithResolution(result, 'hash', '', []);+ const upsertNodes = patch.ops.filter(op => op.type === 'UpsertNode');++ expect(upsertNodes).toContainEqual(+ expect.objectContaining({+ id: nodeId('external://utils', 'utils::write.csv'),+ kind: 'function',+ name: 'write.csv',+ }),+ );+ });++ it('does NOT externalise plain dotted names (base-R / Go) — only :: namespace calls', () => {+ // A base-R dotted call (is.null) and a Go dotted call must keep main's+ // behavior: edge to a local nodeId, no external:// stub. Only genuine+ // :: namespace calls are externalised.+ const rFile = '/repo/check.R';+ const rResult = fileResult(+ rFile,+ SupportedLanguages.R,+ [entity('validate', SupportedLanguages.R)],+ [{ srcName: 'validate', dstName: 'is.null', predicate: 'CALLS' }],+ );+ const rPatch = buildPatchWithResolution(rResult, 'hash', '', []);+ expect(rPatch.ops.filter(op => op.type === 'UpsertNode' && String((op as any).id).startsWith('external://'))).toEqual([]);+ expect(rPatch.ops).toContainEqual(+ expect.objectContaining({ type: 'UpsertEdge', predicate: 'CALLS', dst: nodeId(rFile, 'is.null') }),+ );++ const goFile = '/repo/main.go';+ const goResult = fileResult(+ goFile,+ SupportedLanguages.Go,+ [entity('run', SupportedLanguages.Go)],+ [{ srcName: 'run', dstName: 'fmt.Println', predicate: 'CALLS' }],+ );+ const goPatch = buildPatchWithResolution(goResult, 'hash', '', []);+ expect(goPatch.ops.filter(op => op.type === 'UpsertNode' && String((op as any).id).startsWith('external://'))).toEqual([]);+ expect(goPatch.ops).toContainEqual(+ expect.objectContaining({ type: 'UpsertEdge', predicate: 'CALLS', dst: nodeId(goFile, 'fmt.Println') }),+ );+ });++ it('keeps resolved qualified CALLS that cross-file resolution found', () => {+ const file = '/repo/utils.R';+ const result = fileResult(+ file,+ SupportedLanguages.R,+ [entity('myFunc', SupportedLanguages.R)],+ [{ srcName: 'myFunc', dstName: 'pkg.helper', predicate: 'CALLS' }],+ );+ const resolvedEdges: ResolvedEdge[] = [+ {+ srcFilePath: file,+ srcName: 'myFunc',+ dstFilePath: '/repo/pkg.R',+ dstName: 'pkg.helper',+ dstQualifiedKey: 'helper',+ predicate: 'CALLS',+ confidence: 0.9,+ },+ ];+ const patch = buildPatchWithResolution(result, 'hash', '', resolvedEdges);+ const callEdges = patch.ops.filter(op => op.type === 'UpsertEdge' && op.predicate === 'CALLS');++ expect(callEdges).toContainEqual(+ expect.objectContaining({+ src: nodeId(file, 'myFunc'),+ dst: nodeId('/repo/pkg.R', 'helper'),+ }),+ );+ });+it('rewrites resolved C++ imports to the imported file node', () => {const importer = parseFile('/repo/include/KimeraRPGO/outlier/Pcm.h',@@ -150,3 +266,58 @@ struct Trajectory {};}));});});++describe('buildPatch', () => {+ it('materialises external stub node for unresolved :: package calls', () => {+ const file = '/repo/model.R';+ const externalNodeId = nodeId('external://dplyr', 'dplyr::filter');+ const result = fileResult(+ file,+ SupportedLanguages.R,+ [entity('fitModel', SupportedLanguages.R)],+ [+ { srcName: 'fitModel', dstName: 'dplyr::filter', predicate: 'CALLS' },+ { srcName: 'fitModel', dstName: 'localHelper', predicate: 'CALLS' },+ ],+ );+ const patch = buildPatch(result, 'hash');+ const callEdges = patch.ops.filter(op => op.type === 'UpsertEdge' && op.predicate === 'CALLS');+ const upsertNodes = patch.ops.filter(op => op.type === 'UpsertNode');++ expect(callEdges).toContainEqual(+ expect.objectContaining({ src: nodeId(file, 'fitModel'), dst: externalNodeId }),+ );+ expect(callEdges).not.toContainEqual(+ expect.objectContaining({ dst: nodeId(file, 'dplyr::filter') }),+ );+ expect(upsertNodes).toContainEqual(+ expect.objectContaining({ id: externalNodeId, kind: 'function', name: 'filter' }),+ );+ expect(callEdges).toContainEqual(+ expect.objectContaining({ src: nodeId(file, 'fitModel'), dst: nodeId(file, 'localHelper') }),+ );+ });++ it('keeps same-file qualified CALLS where qualifier is a defined class', () => {+ const file = '/repo/models.py';+ const result = fileResult(+ file,+ SupportedLanguages.Python,+ [+ entity('MyClass', SupportedLanguages.Python, 'class'),+ entity('process', SupportedLanguages.Python, 'function', 'MyClass'),+ entity('run', SupportedLanguages.Python, 'function'),+ ],+ [{ srcName: 'run', dstName: 'MyClass.process', predicate: 'CALLS' }],+ );+ const patch = buildPatch(result, 'hash');+ const callEdges = patch.ops.filter(op => op.type === 'UpsertEdge' && op.predicate === 'CALLS');++ expect(callEdges).toContainEqual(+ expect.objectContaining({+ src: nodeId(file, 'run'),+ dst: nodeId(file, 'MyClass.process'),+ }),+ );+ });+});diff --git a/core-ingestion/src/__tests__/queries.r.test.ts b/core-ingestion/src/__tests__/queries.r.test.tsnew file mode 100644index 00000000..f5c492fd--- /dev/null+++ b/core-ingestion/src/__tests__/queries.r.test.ts@@ -0,0 +1,173 @@+import { describe, it, expect } from 'vitest';+import { parseFile } from '../index.js';+import { SupportedLanguages } from '../languages.js';++describe('R queries', () => {+ it('detects language as R', () => {+ const result = parseFile('/repo/analysis.R', `+x <- 1+`);+ expect(result).not.toBeNull();+ expect(result!.language).toBe(SupportedLanguages.R);+ });++ it('captures function definition via <- assignment', () => {+ const result = parseFile('/repo/utils.R', `+cleanData <- function(df, threshold = 0.5) {+ df[df$score > threshold, ]+}+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'cleanData', kind: 'function' }),+ );+ });++ it('captures multiple function definitions', () => {+ const result = parseFile('/repo/helpers.R', `+normalize <- function(x) {+ (x - min(x)) / (max(x) - min(x))+}++summarize <- function(df) {+ list(n = nrow(df), mean = mean(df$value))+}+`);+ expect(result).not.toBeNull();+ expect(result!.entities.map(e => e.name)).toEqual(+ expect.arrayContaining(['normalize', 'summarize']),+ );+ });++ it('emits CALLS relationship for direct function calls', () => {+ const result = parseFile('/repo/main.R', `+run <- function(data) {+ cleaned <- cleanData(data)+ cleaned+}+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ dstName: 'cleanData', predicate: 'CALLS' }),+ );+ });++ it('emits CALLS relationship for package-qualified calls with qualifier prefix', () => {+ const result = parseFile('/repo/model.R', `+fitModel <- function(df) {+ dplyr::filter(df, value > 0)+}+`);+ expect(result).not.toBeNull();+ // Package-qualified calls keep the :: separator (dplyr::filter) so the patch+ // builder can externalise genuine package calls without confusing them for+ // base-R dotted names like is.null.+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ dstName: 'dplyr::filter', predicate: 'CALLS' }),+ );+ });++ it('keeps backticks on backtick-named function defs (only string LHS is unquoted)', () => {+ const result = parseFile('/repo/backtick.r', '`my fn` <- function(x) x\n');+ expect(result).not.toBeNull();+ // Backtick identifiers are part of the token text; stripping them on the def+ // path (but not the call path) would desync resolution. The strip is reserved+ // for string-keyed S3 names only.+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: '`my fn`', kind: 'function' }),+ );+ expect(result!.entities.map(e => e.name)).not.toContain('my fn');+ });++ it('emits IMPORTS for library() with quoted string', () => {+ const result = parseFile('/repo/setup.R', `+library("dplyr")+library("ggplot2")+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'dplyr' }),+ );+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'ggplot2' }),+ );+ });++ it('emits IMPORTS for library() with unquoted symbol', () => {+ const result = parseFile('/repo/setup.R', `+library(tidyr)+require(stringr)+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'tidyr' }),+ );+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'stringr' }),+ );+ });++ it('emits IMPORTS for source() calls', () => {+ const result = parseFile('/repo/main.R', `+source("./helpers.R")+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'helpers.R' }),+ );+ });++ it('captures nested function definitions', () => {+ const result = parseFile('/repo/factory.R', `+makeCounter <- function(start = 0) {+ increment <- function() {+ start <<- start + 1+ start+ }+ increment+}+`);+ expect(result).not.toBeNull();+ // Both outer and inner functions are captured; R has no class-based containment+ expect(result!.entities.map(e => e.name)).toEqual(+ expect.arrayContaining(['makeCounter', 'increment']),+ );+ });++ it('captures function definition via = assignment', () => {+ const result = parseFile('/repo/utils.R', `+foo = function(x) x * 2+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'foo', kind: 'function' }),+ );+ });++ it('captures string-keyed function definition (S3 method name)', () => {+ const result = parseFile('/repo/methods.R', `+"print.myClass" <- function(x, ...) invisible(x)+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'print.myClass', kind: 'function' }),+ );+ });++ it('emits CALLS relationship for method calls via $', () => {+ const result = parseFile('/repo/main.R', `+run <- function(obj) {+ obj$render(obj$data)+}+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ dstName: 'render', predicate: 'CALLS' }),+ );+ });++ it('returns null for non-R extensions', () => {+ const result = parseFile('/repo/script.unknown', 'x <- 1');+ expect(result).toBeNull();+ });+});diff --git a/core-ingestion/src/__tests__/resolveEdges.test.ts b/core-ingestion/src/__tests__/resolveEdges.test.tsindex 0b9513f4..649694a7 100644--- a/core-ingestion/src/__tests__/resolveEdges.test.ts+++ b/core-ingestion/src/__tests__/resolveEdges.test.ts@@ -660,6 +660,23 @@ describe('resolveEdges', () => {expect(resolved.filter(e => e.predicate === 'IMPORTS' && e.dstName === 'HttpClient')).toEqual([]);});+ // The R cross-batch index is parser-derived (not regex), so it captures every+ // definition form the parser does — including `= function` and string-keyed S3+ // method names that the old `<- function(` regex missed.+ it('R: cross-batch index captures <-, = and string-keyed S3 function defs', () => {+ const src = [+ 'clean_data <- function(x) x',+ 'fit_model = function(y) y',+ '"print.myClass" <- function(z) z',+ ].join('\n');+ const sources = new Map([['/repo/funcs.r', src]]);+ const index = buildGlobalResolutionIndex(['/repo/funcs.r'], sources);++ expect(index.symbolToFiles.get('clean_data')).toContain('/repo/funcs.r'); // <- form+ expect(index.symbolToFiles.get('fit_model')).toContain('/repo/funcs.r'); // = form (regex missed)+ expect(index.symbolToFiles.get('print.myClass')).toContain('/repo/funcs.r'); // string-keyed S3 (regex missed)+ });+// SAS macro libraries define the same %macro name in many files. With no// %include to scope the call (which would resolve at Tier-2), the closest// definer by path prefix is preferred rather than dropping the edge.diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex ba2ff60b..1e2aad3a 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -33,6 +33,7 @@ function tryLoadGrammar(pkg: string): any {}const Kotlin = tryLoadGrammar('tree-sitter-kotlin');const Swift = tryLoadGrammar('tree-sitter-swift');+const R = tryLoadGrammar('@davisvaughan/tree-sitter-r');const Elixir = tryLoadGrammar('tree-sitter-elixir');const Make = tryLoadGrammar('tree-sitter-make');// tree-sitter-sas uses ESM bindings with top-level await — incompatible with tryLoadGrammar (CJS require)@@ -127,6 +128,7 @@ const GRAMMAR_MAP: Partial<Record<SupportedLanguages, any>> = {[SupportedLanguages.Scala]: Scala,...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),+ ...(R ? { [SupportedLanguages.R]: R } : {}),...(SAS ? { [SupportedLanguages.SAS]: SAS } : {}),...(Elixir ? { [SupportedLanguages.Elixir]: Elixir } : {}),...(Make ? { [SupportedLanguages.Makefile]: Make } : {}),@@ -220,6 +222,43 @@ const JS_BUILTINS = new Set(['require', 'fetch', 'parseInt', 'parseFloat', 'isNaN', 'isFinite',]);+const R_BUILTINS = new Set([+ ...SHARED_BUILTINS,+ 'library', 'require', 'source',+ 'print', 'cat', 'message', 'warning', 'stop', 'sprintf', 'paste', 'paste0',+ 'c', 'list', 'vector', 'matrix', 'array', 'data.frame',+ 'length', 'nrow', 'ncol', 'dim', 'names', 'colnames', 'rownames',+ 'head', 'tail', 'subset', 'merge', 'rbind', 'cbind', 'order', 'sort',+ 'apply', 'lapply', 'sapply', 'vapply', 'tapply', 'mapply',+ 'sum', 'mean', 'median', 'sd', 'var', 'min', 'max', 'abs',+ 'round', 'floor', 'ceiling', 'sqrt', 'exp', 'log', 'log10',+ 'as.character', 'as.numeric', 'as.integer', 'as.logical', 'as.factor',+ 'is.character', 'is.numeric', 'is.integer', 'is.logical', 'is.null', 'is.na',+ 'seq', 'seq_len', 'seq_along', 'rep', 'which', 'match', 'any', 'all', 'ifelse',+ 'read.csv', 'write.csv', 'readRDS', 'saveRDS', 'load', 'save',+ 'attr', 'attributes', 'class', 'inherits', 'structure',+ 'nzchar', 'nchar', 'trimws', 'toupper', 'tolower', 'chartr', 'strsplit',+ 'grep', 'grepl', 'regexpr', 'regmatches', 'sub', 'gsub',+ 'isFALSE', 'isTRUE', 'missing', 'exists', 'get', 'assign', 'rm',+ 'srcfile', 'srcref', 'getSrcref', 'getSrcFilename',+ 'tryCatch', 'withCallingHandlers', 'try', 'simpleError', 'simpleCondition',+ 'environment', 'parent.env', 'new.env', 'ls', 'rm',+ 'Recall', 'do.call', 'match.arg', 'match.call', 'sys.call',+ 'identical', 'stopifnot', 'switch', 'invisible', 'force',+ 'numeric', 'character', 'logical', 'integer', 'complex', 'raw',+ 'unlist', 'setdiff', 'union', 'intersect', 'unique', 'duplicated', 'table',+ 'format', 'formatC', 'strtoi', 'chartr', 'startsWith', 'endsWith',+ 'Reduce', 'Filter', 'Find', 'Map', 'Position',+ // base R functions whose names contain dots (not package::function calls)+ 'on.exit', 'sys.call', 'sys.function', 'sys.frame', 'sys.nframe', 'sys.source',+ 'proc.time', 'date.time',+ 'file.path', 'file.exists', 'file.copy', 'file.remove', 'file.rename',+ 'file.create', 'file.info', 'file.size', 'file.show',+ 'dir.create', 'dir.exists',+ 'Sys.time', 'Sys.sleep', 'Sys.getenv', 'Sys.setenv', 'Sys.getpid',+ 'Sys.info', 'Sys.chmod', 'Sys.umask',+]);+// SAS-specific builtins: common PROC names and macro keywords that generate// noise as CALLS edges (user-defined macros are what we want to track).const SAS_BUILTINS = new Set([@@ -239,6 +278,8 @@ function builtinsForLanguage(lang: SupportedLanguages): Set<string> {case SupportedLanguages.JavaScript:case SupportedLanguages.TypeScript:return JS_BUILTINS;+ case SupportedLanguages.R:+ return R_BUILTINS;case SupportedLanguages.SAS:return SAS_BUILTINS;default:@@ -1362,8 +1403,15 @@ export function parseFile(filePath: string, source: string): FileParseResult | nif (defCapture) {const kind = DEFINITION_KIND_MAP[defCapture.name] ?? 'function';- let name = nameCapture?.node.text+ const rawName = nameCapture?.node.text?? (defCapture.name === 'definition.constructor' ? 'init' : '');+ // Strip surrounding quotes only for string-keyed definitions (R S3 method+ // names like "print.myClass"). Leave backtick identifiers intact (Scala+ // `type`, R `my fn`): the call/heritage paths don't strip, so stripping+ // here would desync the def name from its call captures.+ let name = nameCapture?.node.type === 'string'+ ? rawName.replace(/^(['"])(.*)\1$/, '$2')+ : rawName;if (!name || name.length === 0) continue;// SAS module entities (DATA/PROC steps, PROC SQL CREATE) need extra@@ -1654,7 +1702,10 @@ export function parseFile(filePath: string, source: string): FileParseResult | n// `filter` or `map` are Python builtins when called bare, but are valid// user-defined method calls when invoked as `query.filter(...)`.const qualifierCapture = match.captures.find((c: any) => c.name === '_qualifier');- if (builtinsForLanguage(language).has(callee) && !qualifierCapture) continue;+ // R package-qualified calls (pkg::func) carry a distinct @_namespace+ // capture so the dstName can keep the :: separator (vs a plain dotted name).+ const namespaceCapture = match.captures.find((c: any) => c.name === '_namespace');+ if (builtinsForLanguage(language).has(callee) && !qualifierCapture && !namespaceCapture) continue;// For Python: skip calls that are decorator applications.// When a method is decorated (e.g. @util.deprecated(...)), the call sits at the@@ -1706,6 +1757,12 @@ export function parseFile(filePath: string, source: string): FileParseResult | nconst seen = seenCalls.get(scope)!;const effectiveCallee = (() => {+ // R package-qualified call (pkg::func / pkg:::func): keep the :: so the+ // patch builder can externalize it under external://<pkg> and split it+ // correctly even for multi-dot members (utils::write.csv).+ if (namespaceCapture) {+ return `${namespaceCapture.node.text}::${callee}`;+ }if (!qualifierCapture) {// Go: bare-function calls (call.name is an `identifier` — i.e.// pattern 1: `Foo()`) resolve to the caller's package, so prepend@@ -2051,6 +2108,29 @@ export function buildGlobalResolutionIndex(}}+ // R: index function definitions for cross-batch Tier-3 resolution. Derive+ // them from the parser's definition.function entities (not a regex) so this+ // never drifts from what the in-batch path extracts in resolveEdges — the+ // parser also captures forms a regex misses (`= function`, string-keyed S3+ // method names like "print.myClass"). Mirrors the qkMap construction at+ // resolveEdges (key by name, value qkey). Same pattern as the SAS index.+ for (const [fp, src] of sources) {+ if (nodePath.extname(fp).toLowerCase() !== '.r') continue;+ const parsed = parseFile(fp, src);+ if (!parsed) continue;+ const qkMap = new Map<string, string[]>();+ for (const e of parsed.entities) {+ if (e.kind !== 'function') continue;+ const list = qkMap.get(e.name) ?? [];+ list.push(qualifiedKey(e));+ qkMap.set(e.name, list);+ }+ if (qkMap.size > 0) {+ fileQKeys.set(fp, qkMap);+ fileHasSymbol.set(fp, new Set(qkMap.keys()));+ }+ }+// SAS: index macro definitions for cross-batch Tier-3 resolution. Derive// them from the parser's definition.macro entities (not a regex) so this// never drifts from what the in-batch path extracts in resolveEdges — thediff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.tsindex 350d10e6..c7819547 100644--- a/core-ingestion/src/languages.ts+++ b/core-ingestion/src/languages.ts@@ -19,6 +19,7 @@ export enum SupportedLanguages {JSON = 'json',TOML = 'toml',Markdown = 'markdown',+ R = 'r',SAS = 'sas',Elixir = 'elixir',Makefile = 'makefile',@@ -57,6 +58,7 @@ const EXT_MAP: Record<string, SupportedLanguages> = {'.toml': SupportedLanguages.TOML,'.md': SupportedLanguages.Markdown,'.markdown': SupportedLanguages.Markdown,+ '.r': SupportedLanguages.R,'.sas': SupportedLanguages.SAS,'.ex': SupportedLanguages.Elixir,'.exs': SupportedLanguages.Elixir,diff --git a/core-ingestion/src/patch-builder.ts b/core-ingestion/src/patch-builder.tsindex 1cbfefd9..5b64501f 100644--- a/core-ingestion/src/patch-builder.ts+++ b/core-ingestion/src/patch-builder.ts@@ -69,11 +69,11 @@ function sourceType(filePath: string): string {}export function extractorName(): string {- return `tree-sitter/1.21`;+ return `tree-sitter/1.24`;}/** Previous extractor versions — their patches are superseded when re-ingesting. */-export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.20', 'tree-sitter/1.19', 'tree-sitter/1.18', 'tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];+export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.23', 'tree-sitter/1.22', 'tree-sitter/1.21', 'tree-sitter/1.20', 'tree-sitter/1.19', 'tree-sitter/1.18', 'tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];// nodeId / edgeId / chunkId / computePatchId / legacyPatchId are produced by// makeIds(workspaceId) above so every id is scoped to its workspace.@@ -108,6 +108,11 @@ export function buildPatch(nameToQKeys.set(e.name, list);}+ // Set of all qualified keys defined in this file — used to distinguish same-file+ // dotted calls (e.g. "Foo.bar" where Foo is defined here) from external qualified+ // calls (e.g. "dplyr.filter" from R's pkg::func syntax) that would produce phantoms.+ const allQKeys = new Set<string>(entityQKey.values());+// Resolve a relationship endpoint to the best qualified key.// For unambiguous names (appear once), returns the single qualified key.// For ambiguous names (appear multiple times), falls back to the plain name@@ -223,6 +228,7 @@ export function buildPatch(}// UpsertEdge for each relationship+ const seenExternalNodes = new Set<string>();for (const r of relationships) {// For CONTAINS edges, srcName is the container of dstName — use that to disambiguate.const srcKey = resolveKey(r.srcName);@@ -230,11 +236,32 @@ export function buildPatch(? resolveKey(r.dstName, r.srcName): resolveKey(r.dstName);+ let dstNid: string;+ if (r.predicate === 'CALLS' && dstKey.includes('::') && !allQKeys.has(dstKey)) {+ const sep = dstKey.indexOf('::');+ const pkgName = dstKey.slice(0, sep);+ const funcName = dstKey.slice(sep + 2);+ const extPath = `external://${pkgName}`;+ dstNid = nodeId(extPath, dstKey);+ if (!seenExternalNodes.has(dstNid)) {+ seenExternalNodes.add(dstNid);+ ops.push({+ type: 'UpsertNode',+ id: dstNid,+ kind: 'function',+ name: funcName,+ attrs: { package: pkgName, external: true, language: result.language },+ });+ }+ } else {+ dstNid = nodeId(filePath, dstKey);+ }+ops.push({type: 'UpsertEdge',id: edgeId(filePath, srcKey, dstKey, r.predicate),src: nodeId(filePath, srcKey),- dst: nodeId(filePath, dstKey),+ dst: dstNid,predicate: r.predicate,attrs: {},});@@ -321,6 +348,8 @@ export function buildPatchWithResolution(nameToQKeys.set(e.name, list);}+ const allQKeys2 = new Set<string>(entityQKey.values());+function resolveKey(name: string, container?: string): string {const rawKeys = nameToQKeys.get(name);if (!rawKeys) return name;@@ -416,18 +445,36 @@ export function buildPatchWithResolution(}}+ const seenExternalNodes2 = new Set<string>();for (const r of relationships) {const srcKey = resolveKey(r.srcName);const dstKey = r.predicate === 'CONTAINS'? resolveKey(r.dstName, r.srcName): resolveKey(r.dstName);- // For cross-file resolved edges (CALLS, EXTENDS), use the defining file's nodeIdlet dstNodeId: string;const resolutionKey = `${r.srcName}:${r.predicate}:${r.dstName}`;+if (edgeResolution.has(resolutionKey)) {+ // Cross-file resolved — use the defining file's nodeId.const { dstFilePath, dstQualifiedKey } = edgeResolution.get(resolutionKey)!;dstNodeId = nodeId(dstFilePath, dstQualifiedKey);+ } else if (r.predicate === 'CALLS' && dstKey.includes('::') && !allQKeys2.has(dstKey)) {+ const sep = dstKey.indexOf('::');+ const pkgName = dstKey.slice(0, sep);+ const funcName = dstKey.slice(sep + 2);+ const extPath = `external://${pkgName}`;+ dstNodeId = nodeId(extPath, dstKey);+ if (!seenExternalNodes2.has(dstNodeId)) {+ seenExternalNodes2.add(dstNodeId);+ ops.push({+ type: 'UpsertNode',+ id: dstNodeId,+ kind: 'function',+ name: funcName,+ attrs: { package: pkgName, external: true, language: result.language },+ });+ }} else {dstNodeId = nodeId(filePath, dstKey);}diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.tsindex 896a8c4a..f292433c 100644--- a/core-ingestion/src/queries.ts+++ b/core-ingestion/src/queries.ts@@ -1553,6 +1553,58 @@ export const SAS_QUERIES = `(dataset_name) @import.source)) @import`;+// R queries — works with @davisvaughan/tree-sitter-r+export const R_QUERIES = `+; Function definitions — mirrors the official tags.scm bundled with the grammar.+; Covers <- and = operators, identifier and string LHS (S3/R5 method names).+(binary_operator+ lhs: (identifier) @name+ operator: ["<-" "="]+ rhs: (function_definition)) @definition.function++(binary_operator+ lhs: (string) @name+ operator: ["<-" "="]+ rhs: (function_definition)) @definition.function++; Direct calls: foo(args)+(call+ function: (identifier) @call.name) @call++; Package-qualified calls: pkg::func(args) and pkg:::func(args)+; @_namespace (not @_qualifier) marks a real :: namespace call so the dstName+; keeps the :: separator. That lets the patch builder externalize only genuine+; package calls, not base-R dotted names like is.null.+(call+ function: (namespace_operator+ lhs: (identifier) @_namespace+ rhs: (identifier) @call.name)) @call++; Method calls: obj$method(args)+(call+ function: (extract_operator+ rhs: (identifier) @call.name)) @call++; Imports: library(pkg) / require(pkg) — unquoted symbol form+; In the R grammar, each positional argument is wrapped in an (argument) node with a value: field.+(call+ function: (identifier) @_import_fn+ arguments: (arguments (argument value: (identifier) @import.source))+ (#match? @_import_fn "^(library|require)$")) @import++; Imports: library("pkg") / require("pkg") — quoted string form+(call+ function: (identifier) @_import_fn+ arguments: (arguments (argument value: (string) @import.source))+ (#match? @_import_fn "^(library|require)$")) @import++; Source file imports: source("path/to/file.R")+(call+ function: (identifier) @_src+ arguments: (arguments (argument value: (string) @import.source))+ (#eq? @_src "source")) @import+`;+export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {[SupportedLanguages.TypeScript]: TYPESCRIPT_QUERIES,[SupportedLanguages.JavaScript]: JAVASCRIPT_QUERIES,@@ -1574,6 +1626,7 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {[SupportedLanguages.JSON]: '',[SupportedLanguages.TOML]: '',[SupportedLanguages.Markdown]: '',+ [SupportedLanguages.R]: R_QUERIES,[SupportedLanguages.SAS]: SAS_QUERIES,[SupportedLanguages.Elixir]: ELIXIR_QUERIES,[SupportedLanguages.Makefile]: MAKEFILE_QUERIES,diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 501f28c1..c03a9954 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -40,6 +40,7 @@ const SUPPORTED_EXTENSIONS = new Set(['.json','.toml','.md', '.markdown',+ '.r','.sas','.ex', '.exs','.mk', '.makefile',@@ -56,8 +57,9 @@ export function isSupportedSourceFile(filePath: string): boolean {// Extensions whose source text must be pre-read so buildGlobalResolutionIndex can// extract cross-batch symbols before the streaming parse loop. Go uses a fast-// regex scan; SAS is parsed (its macro index is derived from definition.macro).-const INDEX_PRESCAN_EXTENSIONS = new Set(['.go', '.sas']);+// regex scan; SAS and R are parsed (their cross-batch indexes are derived from+// the parser's definition entities).+const INDEX_PRESCAN_EXTENSIONS = new Set(['.go', '.r', '.sas']);function needsIndexPrescan(filePath: string): boolean {return INDEX_PRESCAN_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());@@ -959,7 +961,7 @@ export async function ingestFiles(// Build global resolution index from all repo file paths (not just changed files)// so cross-batch imports resolve correctly even in streaming per-chunk mode.// Reuse bytes already read into changedPaths (on first install = all files, zero extra reads).- // Only async-read Go files that were mtime-clean and therefore not in changedPaths.+ // Only async-read Go/R files that were mtime-clean and therefore not in changedPaths.{const sources = new Map<string, string>();const changedSet = new Set(changedPaths.map(c => c.filePath));@@ -1019,7 +1021,7 @@ export async function ingestFiles(// Build global resolution index from all repo file paths before the parse loop// so cross-batch imports resolve correctly even in streaming per-chunk mode.- // Read pre-scan files (Go, SAS) in parallel to maximize I/O throughput.+ // Read pre-scan files (Go, R, SAS) in parallel to maximize I/O throughput.// Index keys are workspace-relative to match the relative paths we pass// into parseFile below.{