Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat/sas-parser (#208)
Brandon Garate873cc5c845f4
diff --git a/core-ingestion/package-lock.json b/core-ingestion/package-lock.jsonindex 5b6935c3..7ad3f728 100644--- a/core-ingestion/package-lock.json+++ b/core-ingestion/package-lock.json@@ -34,6 +34,7 @@"tree-sitter-elixir": "^0.3.5","tree-sitter-kotlin": "^0.3.8","tree-sitter-make": "^1.1.1",+ "tree-sitter-sas": "^0.4.2","tree-sitter-swift": "^0.6.0"}},@@ -1458,6 +1459,26 @@"integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==","license": "MIT"},+ "node_modules/tree-sitter-sas": {+ "version": "0.4.2",+ "resolved": "https://registry.npmjs.org/tree-sitter-sas/-/tree-sitter-sas-0.4.2.tgz",+ "integrity": "sha512-nDKmkVipL8i85T7qcEtW0tzFR8JWzfqJbH40uw9nRaQxdF03wGjK8GutkRP5tqdmi2kZSDZPDFTEDhGj5TIoBg==",+ "hasInstallScript": true,+ "license": "MIT",+ "optional": true,+ "dependencies": {+ "node-addon-api": "^8.5.0",+ "node-gyp-build": "^4.8.4"+ },+ "peerDependencies": {+ "tree-sitter": ">=0.21.0"+ },+ "peerDependenciesMeta": {+ "tree-sitter": {+ "optional": true+ }+ }+ },"node_modules/tree-sitter-scala": {"version": "0.24.0","resolved": "https://registry.npmjs.org/tree-sitter-scala/-/tree-sitter-scala-0.24.0.tgz",diff --git a/core-ingestion/package.json b/core-ingestion/package.jsonindex 56be4a7d..410787c5 100644--- a/core-ingestion/package.json+++ b/core-ingestion/package.json@@ -33,6 +33,7 @@},"optionalDependencies": {"tree-sitter-kotlin": "^0.3.8",+ "tree-sitter-sas": "^0.4.2","tree-sitter-swift": "^0.6.0","tree-sitter-elixir": "^0.3.5","tree-sitter-make": "^1.1.1"diff --git a/core-ingestion/src/__tests__/queries.sas.test.ts b/core-ingestion/src/__tests__/queries.sas.test.tsnew file mode 100644index 00000000..d2fa02d0--- /dev/null+++ b/core-ingestion/src/__tests__/queries.sas.test.ts@@ -0,0 +1,460 @@+import { describe, it, expect } from 'vitest';+import { parseFile, buildGlobalResolutionIndex } from '../index.js';+import { SupportedLanguages } from '../languages.js';++describe('SAS queries', () => {+ it('detects language as SAS', () => {+ const result = parseFile('/repo/analysis.sas', `+data mydata; run;+`);+ expect(result).not.toBeNull();+ expect(result!.language).toBe(SupportedLanguages.SAS);+ });++ it('captures macro definition', () => {+ const result = parseFile('/repo/macros.sas', `+%macro greet(name);+ %put Hello &name.;+%mend greet;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'greet', kind: 'macro' }),+ );+ });++ it('captures macro definition with no parameters', () => {+ const result = parseFile('/repo/macros.sas', `+%macro setup;+ libname mylib '/data';+%mend setup;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'setup', kind: 'macro' }),+ );+ });++ it('captures multiple macro definitions', () => {+ const result = parseFile('/repo/utils.sas', `+%macro clean(ds);+ proc sort data=&ds.; by id; run;+%mend clean;++%macro summarize(ds, var);+ proc means data=&ds.; var &var.; run;+%mend summarize;+`);+ expect(result).not.toBeNull();+ expect(result!.entities.map(e => e.name)).toEqual(+ expect.arrayContaining(['clean', 'summarize']),+ );+ });++ it('emits CALLS for macro call statement', () => {+ const result = parseFile('/repo/main.sas', `+%macro run_all;+ %clean(rawdata);+%mend run_all;+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ dstName: 'clean', predicate: 'CALLS' }),+ );+ });++ it('emits CALLS for inline macro call inside DATA step', () => {+ const result = parseFile('/repo/main.sas', `+data out;+ set %getdata();+run;+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ dstName: 'getdata', predicate: 'CALLS' }),+ );+ });++ it('captures DATA step as module entity', () => {+ const result = parseFile('/repo/datastep.sas', `+data myoutput;+ set myinput;+ keep id name;+run;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'myoutput', kind: 'module' }),+ );+ });++ // Finding B — a PROC step is keyed on its out= output dataset, not the+ // (too-generic) procedure name. proc sort across many files used to emit a+ // node literally named "sort" once per file.+ it('keys PROC step module on its out= output dataset', () => {+ const result = parseFile('/repo/procs.sas', `+proc sort data=raw out=clean;+ by id;+run;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'clean', kind: 'module' }),+ );+ // the procedure name itself is never materialized as a module+ expect(result!.entities.map(e => e.name)).not.toContain('sort');+ });++ it('drops PROC step that produces no output dataset', () => {+ const result = parseFile('/repo/procs.sas', `+proc means data=mydata;+ var age weight;+run;+`);+ expect(result).not.toBeNull();+ expect(result!.entities.filter(e => e.kind === 'module')).toHaveLength(0);+ expect(result!.entities.map(e => e.name)).not.toContain('means');+ });++ // Finding B (extended) — out= often lives in a body OUTPUT statement, not the+ // PROC header (proc means/summary/univariate). Those output datasets are real+ // data artifacts and must be captured too, not dropped with the procedure name.+ it('keys PROC step module on an out= dataset in a body OUTPUT statement', () => {+ const result = parseFile('/repo/procs.sas', `+proc means data=mydata noprint;+ var age;+ output out=age_summary mean=avg_age;+run;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'age_summary', kind: 'module' }),+ );+ expect(result!.entities.map(e => e.name)).not.toContain('means');+ });++ // out= can also appear in a TABLES clause (proc freq); still a real dataset.+ it('keys PROC FREQ module on a TABLES out= dataset', () => {+ const result = parseFile('/repo/procs.sas', `+proc freq data=mydata;+ tables race*sex / out=freq_counts;+run;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'freq_counts', kind: 'module' }),+ );+ expect(result!.entities.map(e => e.name)).not.toContain('freq');+ });++ it('emits IMPORTS for %INCLUDE', () => {+ const result = parseFile('/repo/main.sas', `+%include '/shared/macros/utils.sas';+`);+ expect(result).not.toBeNull();+ // paths are normalized to basename by normalizeCapturedImport+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'utils.sas' }),+ );+ });++ it('emits IMPORTS for LIBNAME path', () => {+ const result = parseFile('/repo/setup.sas', `+libname mylib '/data/project';+`);+ expect(result).not.toBeNull();+ // paths are normalized to basename by normalizeCapturedImport+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'project' }),+ );+ });++ it('captures nested macro definitions', () => {+ const result = parseFile('/repo/nested.sas', `+%macro outer;+ %macro inner;+ %put inner called;+ %mend inner;+ %inner;+%mend outer;+`);+ expect(result).not.toBeNull();+ expect(result!.entities.map(e => e.name)).toEqual(+ expect.arrayContaining(['outer', 'inner']),+ );+ });++ it('returns null for non-SAS extensions', () => {+ const result = parseFile('/repo/script.unknown', '%macro foo; %mend foo;');+ expect(result).toBeNull();+ });++ // SPF production style — inline block comments in parameter lists+ it('captures macro with trailing inline block comment on parameter (Mode 1)', () => {+ const result = parseFile('/repo/loadpackage.sas', `+%macro loadPackage(+ packageName /* name of a package */+, path = %sysfunc(pathname(packages)) /* location of a package */+, options = %str(LOWCASE_MEMNAME) /* possible options */+);+%mend loadPackage;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'loadPackage', kind: 'macro' }),+ );+ });++ it('captures macro with multi-line inline block comment on parameter (Mode 1 multi-line)', () => {+ const result = parseFile('/repo/spf.sas', `+%macro installPackage(+ packagesNames /* space separated list of packages names,+ without the zip extension */+, sourcePath = /* location of the package,+ e.g. "www.some.page/", mind the "/" at the end */+);+%mend installPackage;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'installPackage', kind: 'macro' }),+ );+ });++ it('captures macro with block comment as default value (Mode 2)', () => {+ const result = parseFile('/repo/spf.sas', `+%macro loadPackage(+ source2 = /*source2*/ /* option to print out details */+);+%mend loadPackage;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'loadPackage', kind: 'macro' }),+ );+ });++ it('captures macro with standalone block comment between parameters (Mode 3)', () => {+ const result = parseFile('/repo/generatepackage.sas', `+%macro generatePackage(+ filesLocation /* location of package files */+, buildLocation= /* location of package ZIP file */+/* testing options: */+, testPackage=Y /* indicator if tests should be executed */+, packages= /* location of other packages */+);+%mend generatePackage;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'generatePackage', kind: 'macro' }),+ );+ });++ it('captures macro with /options and des= clause (Mode 4)', () => {+ const result = parseFile('/repo/spfinit.sas', `+%macro SPFinit_intrnl_forceV7DSname(+ mcParam /* name of a macro parameter */+)/secure minoperator+des='SAS Packages Framework internal macro.';+%mend SPFinit_intrnl_forceV7DSname;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'SPFinit_intrnl_forceV7DSname', kind: 'macro' }),+ );+ });++ // Bug 1 regression — fileref %INCLUDE forms (82% of production SAS uses these, not string literals)+ it('emits IMPORTS for %INCLUDE with fileref(member) syntax', () => {+ const result = parseFile('/repo/main.sas', `+%include PROGRMS(IQI_HOSP_FORMATS.SAS);+`);+ expect(result).not.toBeNull();+ // normalizeCapturedImport extracts the member name from FILEREF(member)+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'IQI_HOSP_FORMATS.SAS' }),+ );+ });++ it('emits IMPORTS for %INCLUDE with bare fileref syntax', () => {+ const result = parseFile('/repo/main.sas', `+%include MacLib;+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'MacLib' }),+ );+ });++ it('emits IMPORTS for %INCLUDE fileref(member) with lowercase extension', () => {+ const result = parseFile('/repo/main.sas', `+%include MacLib(MHI_MEASURES_macro.sas);+`);+ expect(result).not.toBeNull();+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'MHI_MEASURES_macro.sas' }),+ );+ });++ // Bug 2 regression — indented %MACRO definitions must be indexed for cross-file resolution+ it('indexes indented %MACRO definitions for cross-file macro resolution', () => {+ const sources = new Map([+ ['/repo/macros.sas', ' %MACRO MDX(FMT);\n %put &FMT;\n %MEND MDX;\n\n %MACRO MDX1(FMT);\n %MEND MDX1;\n'],+ ]);+ const index = buildGlobalResolutionIndex(['/repo/macros.sas'], sources);+ expect(index.symbolToFiles.get('MDX')).toContain('/repo/macros.sas');+ expect(index.symbolToFiles.get('MDX1')).toContain('/repo/macros.sas');+ });++ // Finding E1 — the cross-batch index is parser-derived, so it captures forms+ // the old regex missed, e.g. a block comment between %macro and the name.+ it('indexes %macro with a comment between keyword and name (regex→parser)', () => {+ const sources = new Map([+ ['/repo/macros.sas', '%macro /* helper */ withComment(arg);\n %put &arg;\n%mend withComment;\n'],+ ]);+ const index = buildGlobalResolutionIndex(['/repo/macros.sas'], sources);+ expect(index.symbolToFiles.get('withComment')).toContain('/repo/macros.sas');+ });++ // v0.3.6 fix #1 — &var.suffix macro references parse as one token; previously+ // &dsname. stopped at the dot leaving a stray identifier that could corrupt+ // surrounding nodes. Finding A — such macro-var dataset names have no stable+ // identity, so they are dropped rather than materialized as noise nodes/edges.+ it('parses macro-var dataset names without corruption and drops them as nodes', () => {+ const result = parseFile('/repo/datastep.sas', `+%macro build;+data &filesWithCodes.addCnt;+ set &dsname.base;+run;+%mend build;+`);+ expect(result).not.toBeNull();+ // the surrounding macro still parses cleanly (no cascade failure)+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'build', kind: 'macro' }),+ );+ // the &-prefixed dataset name is neither a module entity nor an IMPORTS target+ expect(result!.entities.some(e => e.name.includes('&'))).toBe(false);+ expect(result!.relationships.some(r => r.predicate === 'IMPORTS' && r.dstName.includes('&'))).toBe(false);+ });++ // Finding A — a two-part dataset_name (work.foo) is one entity, not two+ // (work + foo). The old unanchored capture double-emitted under dataset_name.+ it('captures a two-part dataset name as a single module entity', () => {+ const result = parseFile('/repo/datastep.sas', `+data work.foo;+ set lib.bar;+run;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'work.foo', kind: 'module' }),+ );+ expect(result!.entities.map(e => e.name)).not.toContain('foo');+ // SET input is likewise a single IMPORTS edge, not lib + bar+ expect(result!.relationships).toContainEqual(+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'lib.bar' }),+ );+ expect(result!.relationships.filter(r => r.predicate === 'IMPORTS' && r.dstName === 'bar')).toHaveLength(0);+ });++ // Finding A — side-effect-only DATA steps (data _null_;) produce no dataset.+ it('drops _null_/_data_/_last_ sentinel DATA steps', () => {+ const result = parseFile('/repo/datastep.sas', `+data _null_;+ set real_input;+run;+`);+ expect(result).not.toBeNull();+ expect(result!.entities.map(e => e.name.toLowerCase())).not.toContain('_null_');+ });++ // Finding C — Windows single-backslash LIBNAME paths with unexpanded macro+ // vars (libname X "&P.\SASData") must not leak a node/edge.+ it('drops LIBNAME targets that are unexpanded macro vars / backslash paths', () => {+ const result = parseFile('/repo/setup.sas', `+libname OUTMSR "&PATHNAME.\\SASData";+`);+ expect(result).not.toBeNull();+ expect(result!.entities.some(e => e.name.includes('&') || e.name.includes('\\'))).toBe(false);+ expect(result!.relationships.some(r => r.predicate === 'IMPORTS' && (r.dstName.includes('&') || r.dstName.includes('\\')))).toBe(false);+ });++ // v0.3.6 fix #2 — %" inside double-quoted strings does not close the string early+ it('parses double-quoted string containing %" without breaking', () => {+ const result = parseFile('/repo/labels.sas', `+%macro addLabel;+ title "value is %str(%'%")";+%mend addLabel;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'addLabel', kind: 'macro' }),+ );+ });++ // v0.3.6 fix #3 — '' escape inside single-quoted strings not consumed by %' pattern+ it('parses single-quoted string with escaped quote without cascade failure', () => {+ const result = parseFile('/repo/generatepackage.sas', `+%macro generatePackage(+ filesLocation+, buildLocation=+, testPackage=Y+, packages=+);+ data _null_;+ x = 'it''s fine';+ y = 'another ''quoted'' value';+ run;+%mend generatePackage;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'generatePackage', kind: 'macro' }),+ );+ });++ // v0.4.2 fix #1 — parenthesized macro-expression body without a trailing semicolon+ it('captures macro definition with paren-led expression body', () => {+ const result = parseFile('/repo/iqi.sas', `+%MACRO MDX1(FMT);+ ((put(DX1,&FMT.) = '1'))+%MEND;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'MDX1', kind: 'macro' }),+ );+ });++ // v0.4.2 fix #2 — %DO/%END used inside a parenthesized expression generator+ it('captures macro definition with %DO loop inside parenthesized expression body', () => {+ const result = parseFile('/repo/iqi.sas', `+%MACRO MDX(FMT);+ (%DO I = 1 %TO &NDX.-1;+ (put(DX&I.,&FMT.) = '1') or+ %END;+ (put(DX&NDX.,&FMT.) = '1'))+%MEND;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'MDX', kind: 'macro' }),+ );+ });++ // v0.4.2 fix #3 — final body item can end immediately before %MEND+ it('captures macro definition whose body ends without semicolon before %MEND', () => {+ const result = parseFile('/repo/iqi.sas', `+%MACRO MDX2Q2(FMT);+ result = 0;+ if result = 1+%MEND;+`);+ expect(result).not.toBeNull();+ expect(result!.entities).toContainEqual(+ expect.objectContaining({ name: 'MDX2Q2', kind: 'macro' }),+ );+ });+});diff --git a/core-ingestion/src/__tests__/resolveEdges.test.ts b/core-ingestion/src/__tests__/resolveEdges.test.tsindex fe631ccc..0b9513f4 100644--- a/core-ingestion/src/__tests__/resolveEdges.test.ts+++ b/core-ingestion/src/__tests__/resolveEdges.test.ts@@ -659,4 +659,45 @@ describe('resolveEdges', () => {const resolved = resolveEdges([consumer, lib]);expect(resolved.filter(e => e.predicate === 'IMPORTS' && e.dstName === 'HttpClient')).toEqual([]);});++ // SAS macro libraries define the same %macro name in many files. With no+ // %include to scope the call (which would resolve at Tier-2), the closest+ // definer by path prefix is preferred rather than dropping the edge.+ it('SAS: resolves a multiply-defined macro call to the closest definer by path proximity', () => {+ const caller = fileResult(+ '/repo/qis/a/call.sas',+ SupportedLanguages.SAS,+ [entity('driver', SupportedLanguages.SAS, 'macro')],+ [{ srcName: 'driver', dstName: 'mdx', predicate: 'CALLS' }],+ );+ const near = fileResult('/repo/qis/a/mdx_local.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);+ const far = fileResult('/repo/other/mdx_lib.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);++ expect(resolveEdges([caller, near, far])).toEqual([+ {+ srcFilePath: '/repo/qis/a/call.sas',+ srcName: 'driver',+ dstFilePath: '/repo/qis/a/mdx_local.sas',+ dstName: 'mdx',+ dstQualifiedKey: 'mdx',+ predicate: 'CALLS',+ confidence: 0.5,+ },+ ]);+ });++ it('SAS: still drops a multiply-defined macro call when definers are equidistant', () => {+ const caller = fileResult(+ '/repo/qis/a/call.sas',+ SupportedLanguages.SAS,+ [entity('driver', SupportedLanguages.SAS, 'macro')],+ [{ srcName: 'driver', dstName: 'mdx', predicate: 'CALLS' }],+ );+ // Both definers share the same prefix length with the caller (/repo/...),+ // so proximity can't disambiguate — conservative behavior is to emit nothing.+ const b = fileResult('/repo/qis/b/mdx.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);+ const c = fileResult('/repo/qis/c/mdx.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);++ expect(resolveEdges([caller, b, c])).toEqual([]);+ });});diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex d6eddd79..ba2ff60b 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -32,9 +32,20 @@ function tryLoadGrammar(pkg: string): any {try { return _require(pkg); } catch { return null; }}const Kotlin = tryLoadGrammar('tree-sitter-kotlin');-const Swift = tryLoadGrammar('tree-sitter-swift');+const Swift = tryLoadGrammar('tree-sitter-swift');const Elixir = tryLoadGrammar('tree-sitter-elixir');const Make = tryLoadGrammar('tree-sitter-make');+// tree-sitter-sas uses ESM bindings with top-level await — incompatible with tryLoadGrammar (CJS require)+let SAS: any = null;+// @ts-ignore+try { SAS = (await import('tree-sitter-sas')).default; }+catch (e: any) {+ // A genuinely-missing optional dep is expected; surface anything else (e.g. a+ // broken native build / ABI mismatch) so .sas files aren't silently dropped.+ if (e?.code !== 'ERR_MODULE_NOT_FOUND' && e?.code !== 'MODULE_NOT_FOUND') {+ process.stderr.write('[tree-sitter-sas load failed] ' + e + '\n');+ }+}import { SupportedLanguages, languageFromPath } from './languages.js';import { LANGUAGE_QUERIES } from './queries.js';@@ -116,6 +127,7 @@ const GRAMMAR_MAP: Partial<Record<SupportedLanguages, any>> = {[SupportedLanguages.Scala]: Scala,...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),+ ...(SAS ? { [SupportedLanguages.SAS]: SAS } : {}),...(Elixir ? { [SupportedLanguages.Elixir]: Elixir } : {}),...(Make ? { [SupportedLanguages.Makefile]: Make } : {}),};@@ -208,6 +220,16 @@ const JS_BUILTINS = new Set(['require', 'fetch', 'parseInt', 'parseFloat', 'isNaN', 'isFinite',]);+// SAS-specific builtins: common PROC names and macro keywords that generate+// noise as CALLS edges (user-defined macros are what we want to track).+const SAS_BUILTINS = new Set([+ ...SHARED_BUILTINS,+ 'print', 'means', 'freq', 'sort', 'import', 'export', 'contents', 'datasets',+ 'sql', 'reg', 'logistic', 'glm', 'anova', 'ttest', 'format', 'report',+ 'univariate', 'corr', 'mixed', 'phreg', 'lifetest', 'transpose', 'append',+ 'put', 'let', 'do', 'end', 'goto', 'abort',+]);+// Per-language BUILTINS lookup — falls back to shared for languages without a// specific set (e.g. Java, Go, Rust) so they only skip obvious non-calls.function builtinsForLanguage(lang: SupportedLanguages): Set<string> {@@ -217,6 +239,8 @@ function builtinsForLanguage(lang: SupportedLanguages): Set<string> {case SupportedLanguages.JavaScript:case SupportedLanguages.TypeScript:return JS_BUILTINS;+ case SupportedLanguages.SAS:+ return SAS_BUILTINS;default:return SHARED_BUILTINS;}@@ -270,6 +294,25 @@ function normalizeCapturedImport(rawValue: string, language: SupportedLanguages)return unwrapped.replace(/^\.+/, '');}+ if (language === SupportedLanguages.SAS) {+ // Real SAS uses single backslashes in Windows paths (libname "&P.\SASData").+ // The general pre-normalization only collapses double backslashes, so do it+ // here before any path inspection.+ const slashed = unwrapped.replace(/\\/g, '/');+ // Unexpanded macro-var targets (&OUTFILE, &P.\dir) have no stable identity+ // until macro expansion — drop them so they create no bogus node/edge.+ if (slashed.includes('&')) return '';+ if (!slashed.includes('/')) {+ // fileref_source text: FILEREF or FILEREF(member.sas) — no path separators.+ const memberMatch = slashed.match(/^[A-Za-z_][A-Za-z0-9_]*\(([^)]+)\)$/);+ if (memberMatch) return memberMatch[1];+ return slashed;+ }+ // String-literal %include and LIBNAME paths → basename.+ const base = slashed.split('/').filter((s: string) => s !== '*').pop() ?? slashed;+ return base.replace(/^\.+/, '');+ }+const rawMod = unwrapped.split('/').filter((s: string) => s !== '*').pop() ?? unwrapped;return rawMod.replace(/^\.+/, '');}@@ -1319,10 +1362,32 @@ export function parseFile(filePath: string, source: string): FileParseResult | nif (defCapture) {const kind = DEFINITION_KIND_MAP[defCapture.name] ?? 'function';- const name = nameCapture?.node.text+ let name = nameCapture?.node.text?? (defCapture.name === 'definition.constructor' ? 'init' : '');if (!name || name.length === 0) continue;+ // SAS module entities (DATA/PROC steps, PROC SQL CREATE) need extra+ // shaping so they reflect real data artifacts rather than parser noise.+ if (language === SupportedLanguages.SAS && kind === 'module') {+ if (defCapture.node.type === 'proc_step') {+ // The procedure name (means/sort/freq…) is too generic to be a node.+ // Re-key on the PROC's out= output dataset; drop the step entirely+ // when it produces none (e.g. proc print, proc means with no out=).+ // out= may sit on the header (proc sort out=) OR in a body statement+ // (proc means; output out=x; / proc freq; tables a / out=y;), so fall+ // back to scanning the full step text when the header has none.+ const header = defCapture.node.namedChild(0);+ const headerText = header?.type === 'proc_step_header' ? header.text : defCapture.node.text;+ const outRe = /\bout\s*=\s*([A-Za-z_][\w.]*)/i;+ const outMatch = headerText.match(outRe) ?? defCapture.node.text.match(outRe);+ if (!outMatch) continue;+ name = outMatch[1];+ }+ // Drop side-effect-only sentinels (_null_/_data_/_last_) and unexpanded+ // macro-var names (&ds) — neither denotes a stable dataset identity.+ if (/^_(null|data|last)_$/i.test(name) || name.includes('&')) continue;+ }+const defNode = defCapture.node;const lineStart = defNode.startPosition.row + 1;const lineEnd = defNode.endPosition.row + 1;@@ -1986,6 +2051,28 @@ export function buildGlobalResolutionIndex(}}+ // SAS: index macro definitions for cross-batch Tier-3 resolution. Derive+ // them from the parser's definition.macro entities (not a regex) so this+ // never drifts from what the in-batch path extracts in resolveEdges — the+ // parser also captures forms a regex misses (e.g. %macro /*comment*/ name).+ // Mirrors the qkMap construction at resolveEdges (key by name, value qkey).+ for (const [fp, src] of sources) {+ if (nodePath.extname(fp).toLowerCase() !== '.sas') continue;+ const parsed = parseFile(fp, src);+ if (!parsed) continue;+ const qkMap = new Map<string, string[]>();+ for (const e of parsed.entities) {+ if (e.kind !== 'macro') continue;+ const list = qkMap.get(e.name) ?? [];+ list.push(qualifiedKey(e));+ qkMap.set(e.name, list);+ }+ if (qkMap.size > 0) {+ fileQKeys.set(fp, qkMap);+ fileHasSymbol.set(fp, new Set(qkMap.keys()));+ }+ }+for (const [fp, symbols] of fileHasSymbol) {for (const sym of symbols) {const list = symbolToFiles.get(sym) ?? [];@@ -2177,7 +2264,10 @@ export function resolveEdges(if (stripped && stripped !== normalizedModName) candidates.push(stripped);// Strip file extensions so "explain.js" resolves to the "explain" stem// (TS/JS ESM imports use .js extensions that map to .ts source files)- const noExt = normalizedModName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx)$/, '');+ // Only the SAS extension is case-insensitive (real SAS uses .sas/.SAS+ // interchangeably). Keeping the rest case-sensitive avoids stripping+ // uppercase .C/.H (legitimate C++ source) or .TS/.JAVA across languages.+ const noExt = normalizedModName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx|[Ss][Aa][Ss])$/, '');if (noExt !== normalizedModName && noExt && !candidates.includes(noExt)) candidates.push(noExt);const pathBasename = nodePath.posix.basename(normalizedModName);if (pathBasename && !candidates.includes(pathBasename)) candidates.push(pathBasename);@@ -2375,7 +2465,7 @@ export function resolveEdges(if (canonicalMatches.length === 1) return canonicalMatches;}- if (srcLanguage === SupportedLanguages.Java) {+ if (srcLanguage === SupportedLanguages.Java || srcLanguage === SupportedLanguages.SAS) {const srcParts = srcFilePath.replace(/\\/g, '/').split('/');const withProximity = matches.map(fp => {const fpParts = fp.replace(/\\/g, '/').split('/');diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.tsindex 1141eb70..350d10e6 100644--- a/core-ingestion/src/languages.ts+++ b/core-ingestion/src/languages.ts@@ -19,6 +19,7 @@ export enum SupportedLanguages {JSON = 'json',TOML = 'toml',Markdown = 'markdown',+ SAS = 'sas',Elixir = 'elixir',Makefile = 'makefile',}@@ -56,6 +57,7 @@ const EXT_MAP: Record<string, SupportedLanguages> = {'.toml': SupportedLanguages.TOML,'.md': SupportedLanguages.Markdown,'.markdown': SupportedLanguages.Markdown,+ '.sas': SupportedLanguages.SAS,'.ex': SupportedLanguages.Elixir,'.exs': SupportedLanguages.Elixir,'.mk': SupportedLanguages.Makefile,diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.tsindex 39f3923b..896a8c4a 100644--- a/core-ingestion/src/queries.ts+++ b/core-ingestion/src/queries.ts@@ -1483,6 +1483,76 @@ export const ELIXIR_QUERIES = `(#match? @_defmacrog "^(defmacro|defmacrop)$")) @definition.macro`;+export const SAS_QUERIES = `+; Macro definitions — %MACRO name ... %MEND+(macro_definition+ name: (macro_name) @name) @definition.macro++; Macro call statements — %name(args);+(macro_call_statement+ name: (macro_name) @call.name) @call++; Inline macro calls — %name(args) inside DATA/PROC/etc.+(macro_call+ name: (macro_name) @call.name) @call++; DATA step — capture the whole dataset_name node so two-part names (work.foo)+; stay one entity. Sentinels (_null_) and unexpanded macro vars (&ds) are+; filtered downstream in index.ts (they have no stable identity).+(data_step+ (data_step_header+ (dataset_name) @name)) @definition.module++; PROC step — the procedure name itself is too generic to be a graph node+; (means/sort/freq repeat once per file). index.ts re-keys this on the PROC's+; out= output dataset, or drops it when the step produces no dataset.+(proc_step+ (proc_step_header+ name: (identifier) @name)) @definition.module++; %INCLUDE — string literal form: %include 'path/to/file.sas';+(include_statement+ source: (string_literal) @import.source) @import++; %INCLUDE — fileref form: %include FILEREF; or %include FILEREF(member.sas);+(include_statement+ source: (fileref_source) @import.source) @import++; LIBNAME — library path import+(libname_statement+ (string_literal) @import.source) @import++; PROC SQL — CREATE TABLE AS output+(sql_create_statement+ output: (dataset_name) @name) @definition.module++; PROC SQL — FROM table inputs+(sql_select_statement+ (table_reference+ (dataset_name) @import.source)) @import++; PROC SQL — JOIN table inputs+(sql_select_statement+ (sql_join_clause+ (table_reference+ (dataset_name) @import.source))) @import++; DATA step SET inputs+(data_step+ (set_statement+ (dataset_name) @import.source)) @import++; DATA step MERGE inputs+(data_step+ (merge_statement+ (dataset_name) @import.source)) @import++; DATA step UPDATE inputs+(data_step+ (update_statement+ (dataset_name) @import.source)) @import+`;+export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {[SupportedLanguages.TypeScript]: TYPESCRIPT_QUERIES,[SupportedLanguages.JavaScript]: JAVASCRIPT_QUERIES,@@ -1504,6 +1574,7 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {[SupportedLanguages.JSON]: '',[SupportedLanguages.TOML]: '',[SupportedLanguages.Markdown]: '',+ [SupportedLanguages.SAS]: SAS_QUERIES,[SupportedLanguages.Elixir]: ELIXIR_QUERIES,[SupportedLanguages.Makefile]: MAKEFILE_QUERIES,};diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 1d31c8ff..501f28c1 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -40,6 +40,7 @@ const SUPPORTED_EXTENSIONS = new Set(['.json','.toml','.md', '.markdown',+ '.sas','.ex', '.exs','.mk', '.makefile',]);@@ -53,6 +54,15 @@ export function isSupportedSourceFile(filePath: string): boolean {|| SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());}+// Extensions whose source text must be pre-read so buildGlobalResolutionIndex can+// extract cross-batch symbols before the streaming parse loop. Go uses a fast+// regex scan; SAS is parsed (its macro index is derived from definition.macro).+const INDEX_PRESCAN_EXTENSIONS = new Set(['.go', '.sas']);++function needsIndexPrescan(filePath: string): boolean {+ return INDEX_PRESCAN_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());+}+// ---------------------------------------------------------------------------// Language filter helpers// ---------------------------------------------------------------------------@@ -954,12 +964,12 @@ export async function ingestFiles(const sources = new Map<string, string>();const changedSet = new Set(changedPaths.map(c => c.filePath));for (const { filePath, bytes } of changedPaths) {- if (nodePath.extname(filePath) === '.go') sources.set(filePath, bytes.toString('utf-8'));+ if (needsIndexPrescan(filePath)) sources.set(filePath, bytes.toString('utf-8'));}- const remainingGoFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go' && !changedSet.has(fp));- const GO_READ_CONCURRENCY = 2000;- for (let i = 0; i < remainingGoFiles.length; i += GO_READ_CONCURRENCY) {- const batch = remainingGoFiles.slice(i, i + GO_READ_CONCURRENCY);+ const remainingIndexFiles = filePaths.filter(fp => needsIndexPrescan(fp) && !changedSet.has(fp));+ const INDEX_READ_CONCURRENCY = 2000;+ for (let i = 0; i < remainingIndexFiles.length; i += INDEX_READ_CONCURRENCY) {+ const batch = remainingIndexFiles.slice(i, i + INDEX_READ_CONCURRENCY);const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));for (let j = 0; j < batch.length; j++) {if (texts[j] != null) sources.set(batch[j], texts[j]!);@@ -1009,16 +1019,16 @@ export async function ingestFiles(// Build global resolution index from all repo file paths before the parse loop// so cross-batch imports resolve correctly even in streaming per-chunk mode.- // Read all Go files in parallel to maximize I/O throughput.+ // Read pre-scan files (Go, SAS) in parallel to maximize I/O throughput.// Index keys are workspace-relative to match the relative paths we pass// into parseFile below.{const goIndexStart = performance.now();const sources = new Map<string, string>();- const goFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go');- const GO_READ_CONCURRENCY = 2000;- for (let i = 0; i < goFiles.length; i += GO_READ_CONCURRENCY) {- const batch = goFiles.slice(i, i + GO_READ_CONCURRENCY);+ const indexFiles = filePaths.filter(fp => needsIndexPrescan(fp));+ const INDEX_READ_CONCURRENCY = 2000;+ for (let i = 0; i < indexFiles.length; i += INDEX_READ_CONCURRENCY) {+ const batch = indexFiles.slice(i, i + INDEX_READ_CONCURRENCY);const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));for (let j = 0; j < batch.length; j++) {if (texts[j] != null) sources.set(toWorkspaceRelative(batch[j]), texts[j]!);