Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat/sas-parser (#208)

Brandon Garate873cc5c845f4
diff --git a/core-ingestion/package-lock.json b/core-ingestion/package-lock.json
index 5b6935c3..7ad3f728 100644
--- a/core-ingestion/package-lock.json
+++ b/core-ingestion/package-lock.json
@@ -34,6 +34,7 @@
"tree-sitter-elixir": "^0.3.5",
"tree-sitter-kotlin": "^0.3.8",
"tree-sitter-make": "^1.1.1",
+ "tree-sitter-sas": "^0.4.2",
"tree-sitter-swift": "^0.6.0"
}
},
@@ -1458,6 +1459,26 @@
"integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==",
"license": "MIT"
},
+ "node_modules/tree-sitter-sas": {
+ "version": "0.4.2",
+ "resolved": "https://registry.npmjs.org/tree-sitter-sas/-/tree-sitter-sas-0.4.2.tgz",
+ "integrity": "sha512-nDKmkVipL8i85T7qcEtW0tzFR8JWzfqJbH40uw9nRaQxdF03wGjK8GutkRP5tqdmi2kZSDZPDFTEDhGj5TIoBg==",
+ "hasInstallScript": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "node-addon-api": "^8.5.0",
+ "node-gyp-build": "^4.8.4"
+ },
+ "peerDependencies": {
+ "tree-sitter": ">=0.21.0"
+ },
+ "peerDependenciesMeta": {
+ "tree-sitter": {
+ "optional": true
+ }
+ }
+ },
"node_modules/tree-sitter-scala": {
"version": "0.24.0",
"resolved": "https://registry.npmjs.org/tree-sitter-scala/-/tree-sitter-scala-0.24.0.tgz",
diff --git a/core-ingestion/package.json b/core-ingestion/package.json
index 56be4a7d..410787c5 100644
--- a/core-ingestion/package.json
+++ b/core-ingestion/package.json
@@ -33,6 +33,7 @@
},
"optionalDependencies": {
"tree-sitter-kotlin": "^0.3.8",
+ "tree-sitter-sas": "^0.4.2",
"tree-sitter-swift": "^0.6.0",
"tree-sitter-elixir": "^0.3.5",
"tree-sitter-make": "^1.1.1"
diff --git a/core-ingestion/src/__tests__/queries.sas.test.ts b/core-ingestion/src/__tests__/queries.sas.test.ts
new file mode 100644
index 00000000..d2fa02d0
--- /dev/null
+++ b/core-ingestion/src/__tests__/queries.sas.test.ts
@@ -0,0 +1,460 @@
+import { describe, it, expect } from 'vitest';
+import { parseFile, buildGlobalResolutionIndex } from '../index.js';
+import { SupportedLanguages } from '../languages.js';
+
+describe('SAS queries', () => {
+ it('detects language as SAS', () => {
+ const result = parseFile('/repo/analysis.sas', `
+data mydata; run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.language).toBe(SupportedLanguages.SAS);
+ });
+
+ it('captures macro definition', () => {
+ const result = parseFile('/repo/macros.sas', `
+%macro greet(name);
+ %put Hello &name.;
+%mend greet;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'greet', kind: 'macro' }),
+ );
+ });
+
+ it('captures macro definition with no parameters', () => {
+ const result = parseFile('/repo/macros.sas', `
+%macro setup;
+ libname mylib '/data';
+%mend setup;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'setup', kind: 'macro' }),
+ );
+ });
+
+ it('captures multiple macro definitions', () => {
+ const result = parseFile('/repo/utils.sas', `
+%macro clean(ds);
+ proc sort data=&ds.; by id; run;
+%mend clean;
+
+%macro summarize(ds, var);
+ proc means data=&ds.; var &var.; run;
+%mend summarize;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities.map(e => e.name)).toEqual(
+ expect.arrayContaining(['clean', 'summarize']),
+ );
+ });
+
+ it('emits CALLS for macro call statement', () => {
+ const result = parseFile('/repo/main.sas', `
+%macro run_all;
+ %clean(rawdata);
+%mend run_all;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ dstName: 'clean', predicate: 'CALLS' }),
+ );
+ });
+
+ it('emits CALLS for inline macro call inside DATA step', () => {
+ const result = parseFile('/repo/main.sas', `
+data out;
+ set %getdata();
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ dstName: 'getdata', predicate: 'CALLS' }),
+ );
+ });
+
+ it('captures DATA step as module entity', () => {
+ const result = parseFile('/repo/datastep.sas', `
+data myoutput;
+ set myinput;
+ keep id name;
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'myoutput', kind: 'module' }),
+ );
+ });
+
+ // Finding B — a PROC step is keyed on its out= output dataset, not the
+ // (too-generic) procedure name. proc sort across many files used to emit a
+ // node literally named "sort" once per file.
+ it('keys PROC step module on its out= output dataset', () => {
+ const result = parseFile('/repo/procs.sas', `
+proc sort data=raw out=clean;
+ by id;
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'clean', kind: 'module' }),
+ );
+ // the procedure name itself is never materialized as a module
+ expect(result!.entities.map(e => e.name)).not.toContain('sort');
+ });
+
+ it('drops PROC step that produces no output dataset', () => {
+ const result = parseFile('/repo/procs.sas', `
+proc means data=mydata;
+ var age weight;
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities.filter(e => e.kind === 'module')).toHaveLength(0);
+ expect(result!.entities.map(e => e.name)).not.toContain('means');
+ });
+
+ // Finding B (extended) — out= often lives in a body OUTPUT statement, not the
+ // PROC header (proc means/summary/univariate). Those output datasets are real
+ // data artifacts and must be captured too, not dropped with the procedure name.
+ it('keys PROC step module on an out= dataset in a body OUTPUT statement', () => {
+ const result = parseFile('/repo/procs.sas', `
+proc means data=mydata noprint;
+ var age;
+ output out=age_summary mean=avg_age;
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'age_summary', kind: 'module' }),
+ );
+ expect(result!.entities.map(e => e.name)).not.toContain('means');
+ });
+
+ // out= can also appear in a TABLES clause (proc freq); still a real dataset.
+ it('keys PROC FREQ module on a TABLES out= dataset', () => {
+ const result = parseFile('/repo/procs.sas', `
+proc freq data=mydata;
+ tables race*sex / out=freq_counts;
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'freq_counts', kind: 'module' }),
+ );
+ expect(result!.entities.map(e => e.name)).not.toContain('freq');
+ });
+
+ it('emits IMPORTS for %INCLUDE', () => {
+ const result = parseFile('/repo/main.sas', `
+%include '/shared/macros/utils.sas';
+`);
+ expect(result).not.toBeNull();
+ // paths are normalized to basename by normalizeCapturedImport
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'utils.sas' }),
+ );
+ });
+
+ it('emits IMPORTS for LIBNAME path', () => {
+ const result = parseFile('/repo/setup.sas', `
+libname mylib '/data/project';
+`);
+ expect(result).not.toBeNull();
+ // paths are normalized to basename by normalizeCapturedImport
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'project' }),
+ );
+ });
+
+ it('captures nested macro definitions', () => {
+ const result = parseFile('/repo/nested.sas', `
+%macro outer;
+ %macro inner;
+ %put inner called;
+ %mend inner;
+ %inner;
+%mend outer;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities.map(e => e.name)).toEqual(
+ expect.arrayContaining(['outer', 'inner']),
+ );
+ });
+
+ it('returns null for non-SAS extensions', () => {
+ const result = parseFile('/repo/script.unknown', '%macro foo; %mend foo;');
+ expect(result).toBeNull();
+ });
+
+ // SPF production style — inline block comments in parameter lists
+ it('captures macro with trailing inline block comment on parameter (Mode 1)', () => {
+ const result = parseFile('/repo/loadpackage.sas', `
+%macro loadPackage(
+ packageName /* name of a package */
+, path = %sysfunc(pathname(packages)) /* location of a package */
+, options = %str(LOWCASE_MEMNAME) /* possible options */
+);
+%mend loadPackage;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'loadPackage', kind: 'macro' }),
+ );
+ });
+
+ it('captures macro with multi-line inline block comment on parameter (Mode 1 multi-line)', () => {
+ const result = parseFile('/repo/spf.sas', `
+%macro installPackage(
+ packagesNames /* space separated list of packages names,
+ without the zip extension */
+, sourcePath = /* location of the package,
+ e.g. "www.some.page/", mind the "/" at the end */
+);
+%mend installPackage;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'installPackage', kind: 'macro' }),
+ );
+ });
+
+ it('captures macro with block comment as default value (Mode 2)', () => {
+ const result = parseFile('/repo/spf.sas', `
+%macro loadPackage(
+ source2 = /*source2*/ /* option to print out details */
+);
+%mend loadPackage;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'loadPackage', kind: 'macro' }),
+ );
+ });
+
+ it('captures macro with standalone block comment between parameters (Mode 3)', () => {
+ const result = parseFile('/repo/generatepackage.sas', `
+%macro generatePackage(
+ filesLocation /* location of package files */
+, buildLocation= /* location of package ZIP file */
+/* testing options: */
+, testPackage=Y /* indicator if tests should be executed */
+, packages= /* location of other packages */
+);
+%mend generatePackage;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'generatePackage', kind: 'macro' }),
+ );
+ });
+
+ it('captures macro with /options and des= clause (Mode 4)', () => {
+ const result = parseFile('/repo/spfinit.sas', `
+%macro SPFinit_intrnl_forceV7DSname(
+ mcParam /* name of a macro parameter */
+)/secure minoperator
+des='SAS Packages Framework internal macro.';
+%mend SPFinit_intrnl_forceV7DSname;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'SPFinit_intrnl_forceV7DSname', kind: 'macro' }),
+ );
+ });
+
+ // Bug 1 regression — fileref %INCLUDE forms (82% of production SAS uses these, not string literals)
+ it('emits IMPORTS for %INCLUDE with fileref(member) syntax', () => {
+ const result = parseFile('/repo/main.sas', `
+%include PROGRMS(IQI_HOSP_FORMATS.SAS);
+`);
+ expect(result).not.toBeNull();
+ // normalizeCapturedImport extracts the member name from FILEREF(member)
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'IQI_HOSP_FORMATS.SAS' }),
+ );
+ });
+
+ it('emits IMPORTS for %INCLUDE with bare fileref syntax', () => {
+ const result = parseFile('/repo/main.sas', `
+%include MacLib;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'MacLib' }),
+ );
+ });
+
+ it('emits IMPORTS for %INCLUDE fileref(member) with lowercase extension', () => {
+ const result = parseFile('/repo/main.sas', `
+%include MacLib(MHI_MEASURES_macro.sas);
+`);
+ expect(result).not.toBeNull();
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'MHI_MEASURES_macro.sas' }),
+ );
+ });
+
+ // Bug 2 regression — indented %MACRO definitions must be indexed for cross-file resolution
+ it('indexes indented %MACRO definitions for cross-file macro resolution', () => {
+ const sources = new Map([
+ ['/repo/macros.sas', ' %MACRO MDX(FMT);\n %put &FMT;\n %MEND MDX;\n\n %MACRO MDX1(FMT);\n %MEND MDX1;\n'],
+ ]);
+ const index = buildGlobalResolutionIndex(['/repo/macros.sas'], sources);
+ expect(index.symbolToFiles.get('MDX')).toContain('/repo/macros.sas');
+ expect(index.symbolToFiles.get('MDX1')).toContain('/repo/macros.sas');
+ });
+
+ // Finding E1 — the cross-batch index is parser-derived, so it captures forms
+ // the old regex missed, e.g. a block comment between %macro and the name.
+ it('indexes %macro with a comment between keyword and name (regex→parser)', () => {
+ const sources = new Map([
+ ['/repo/macros.sas', '%macro /* helper */ withComment(arg);\n %put &arg;\n%mend withComment;\n'],
+ ]);
+ const index = buildGlobalResolutionIndex(['/repo/macros.sas'], sources);
+ expect(index.symbolToFiles.get('withComment')).toContain('/repo/macros.sas');
+ });
+
+ // v0.3.6 fix #1 — &var.suffix macro references parse as one token; previously
+ // &dsname. stopped at the dot leaving a stray identifier that could corrupt
+ // surrounding nodes. Finding A — such macro-var dataset names have no stable
+ // identity, so they are dropped rather than materialized as noise nodes/edges.
+ it('parses macro-var dataset names without corruption and drops them as nodes', () => {
+ const result = parseFile('/repo/datastep.sas', `
+%macro build;
+data &filesWithCodes.addCnt;
+ set &dsname.base;
+run;
+%mend build;
+`);
+ expect(result).not.toBeNull();
+ // the surrounding macro still parses cleanly (no cascade failure)
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'build', kind: 'macro' }),
+ );
+ // the &-prefixed dataset name is neither a module entity nor an IMPORTS target
+ expect(result!.entities.some(e => e.name.includes('&'))).toBe(false);
+ expect(result!.relationships.some(r => r.predicate === 'IMPORTS' && r.dstName.includes('&'))).toBe(false);
+ });
+
+ // Finding A — a two-part dataset_name (work.foo) is one entity, not two
+ // (work + foo). The old unanchored capture double-emitted under dataset_name.
+ it('captures a two-part dataset name as a single module entity', () => {
+ const result = parseFile('/repo/datastep.sas', `
+data work.foo;
+ set lib.bar;
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'work.foo', kind: 'module' }),
+ );
+ expect(result!.entities.map(e => e.name)).not.toContain('foo');
+ // SET input is likewise a single IMPORTS edge, not lib + bar
+ expect(result!.relationships).toContainEqual(
+ expect.objectContaining({ predicate: 'IMPORTS', dstName: 'lib.bar' }),
+ );
+ expect(result!.relationships.filter(r => r.predicate === 'IMPORTS' && r.dstName === 'bar')).toHaveLength(0);
+ });
+
+ // Finding A — side-effect-only DATA steps (data _null_;) produce no dataset.
+ it('drops _null_/_data_/_last_ sentinel DATA steps', () => {
+ const result = parseFile('/repo/datastep.sas', `
+data _null_;
+ set real_input;
+run;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities.map(e => e.name.toLowerCase())).not.toContain('_null_');
+ });
+
+ // Finding C — Windows single-backslash LIBNAME paths with unexpanded macro
+ // vars (libname X "&P.\SASData") must not leak a node/edge.
+ it('drops LIBNAME targets that are unexpanded macro vars / backslash paths', () => {
+ const result = parseFile('/repo/setup.sas', `
+libname OUTMSR "&PATHNAME.\\SASData";
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities.some(e => e.name.includes('&') || e.name.includes('\\'))).toBe(false);
+ expect(result!.relationships.some(r => r.predicate === 'IMPORTS' && (r.dstName.includes('&') || r.dstName.includes('\\')))).toBe(false);
+ });
+
+ // v0.3.6 fix #2 — %" inside double-quoted strings does not close the string early
+ it('parses double-quoted string containing %" without breaking', () => {
+ const result = parseFile('/repo/labels.sas', `
+%macro addLabel;
+ title "value is %str(%'%")";
+%mend addLabel;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'addLabel', kind: 'macro' }),
+ );
+ });
+
+ // v0.3.6 fix #3 — '' escape inside single-quoted strings not consumed by %' pattern
+ it('parses single-quoted string with escaped quote without cascade failure', () => {
+ const result = parseFile('/repo/generatepackage.sas', `
+%macro generatePackage(
+ filesLocation
+, buildLocation=
+, testPackage=Y
+, packages=
+);
+ data _null_;
+ x = 'it''s fine';
+ y = 'another ''quoted'' value';
+ run;
+%mend generatePackage;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'generatePackage', kind: 'macro' }),
+ );
+ });
+
+ // v0.4.2 fix #1 — parenthesized macro-expression body without a trailing semicolon
+ it('captures macro definition with paren-led expression body', () => {
+ const result = parseFile('/repo/iqi.sas', `
+%MACRO MDX1(FMT);
+ ((put(DX1,&FMT.) = '1'))
+%MEND;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'MDX1', kind: 'macro' }),
+ );
+ });
+
+ // v0.4.2 fix #2 — %DO/%END used inside a parenthesized expression generator
+ it('captures macro definition with %DO loop inside parenthesized expression body', () => {
+ const result = parseFile('/repo/iqi.sas', `
+%MACRO MDX(FMT);
+ (%DO I = 1 %TO &NDX.-1;
+ (put(DX&I.,&FMT.) = '1') or
+ %END;
+ (put(DX&NDX.,&FMT.) = '1'))
+%MEND;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'MDX', kind: 'macro' }),
+ );
+ });
+
+ // v0.4.2 fix #3 — final body item can end immediately before %MEND
+ it('captures macro definition whose body ends without semicolon before %MEND', () => {
+ const result = parseFile('/repo/iqi.sas', `
+%MACRO MDX2Q2(FMT);
+ result = 0;
+ if result = 1
+%MEND;
+`);
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(
+ expect.objectContaining({ name: 'MDX2Q2', kind: 'macro' }),
+ );
+ });
+});
diff --git a/core-ingestion/src/__tests__/resolveEdges.test.ts b/core-ingestion/src/__tests__/resolveEdges.test.ts
index fe631ccc..0b9513f4 100644
--- a/core-ingestion/src/__tests__/resolveEdges.test.ts
+++ b/core-ingestion/src/__tests__/resolveEdges.test.ts
@@ -659,4 +659,45 @@ describe('resolveEdges', () => {
const resolved = resolveEdges([consumer, lib]);
expect(resolved.filter(e => e.predicate === 'IMPORTS' && e.dstName === 'HttpClient')).toEqual([]);
});
+
+ // SAS macro libraries define the same %macro name in many files. With no
+ // %include to scope the call (which would resolve at Tier-2), the closest
+ // definer by path prefix is preferred rather than dropping the edge.
+ it('SAS: resolves a multiply-defined macro call to the closest definer by path proximity', () => {
+ const caller = fileResult(
+ '/repo/qis/a/call.sas',
+ SupportedLanguages.SAS,
+ [entity('driver', SupportedLanguages.SAS, 'macro')],
+ [{ srcName: 'driver', dstName: 'mdx', predicate: 'CALLS' }],
+ );
+ const near = fileResult('/repo/qis/a/mdx_local.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);
+ const far = fileResult('/repo/other/mdx_lib.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);
+
+ expect(resolveEdges([caller, near, far])).toEqual([
+ {
+ srcFilePath: '/repo/qis/a/call.sas',
+ srcName: 'driver',
+ dstFilePath: '/repo/qis/a/mdx_local.sas',
+ dstName: 'mdx',
+ dstQualifiedKey: 'mdx',
+ predicate: 'CALLS',
+ confidence: 0.5,
+ },
+ ]);
+ });
+
+ it('SAS: still drops a multiply-defined macro call when definers are equidistant', () => {
+ const caller = fileResult(
+ '/repo/qis/a/call.sas',
+ SupportedLanguages.SAS,
+ [entity('driver', SupportedLanguages.SAS, 'macro')],
+ [{ srcName: 'driver', dstName: 'mdx', predicate: 'CALLS' }],
+ );
+ // Both definers share the same prefix length with the caller (/repo/...),
+ // so proximity can't disambiguate — conservative behavior is to emit nothing.
+ const b = fileResult('/repo/qis/b/mdx.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);
+ const c = fileResult('/repo/qis/c/mdx.sas', SupportedLanguages.SAS, [entity('mdx', SupportedLanguages.SAS, 'macro')]);
+
+ expect(resolveEdges([caller, b, c])).toEqual([]);
+ });
});
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index d6eddd79..ba2ff60b 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -32,9 +32,20 @@ function tryLoadGrammar(pkg: string): any {
try { return _require(pkg); } catch { return null; }
}
const Kotlin = tryLoadGrammar('tree-sitter-kotlin');
-const Swift = tryLoadGrammar('tree-sitter-swift');
+const Swift = tryLoadGrammar('tree-sitter-swift');
const Elixir = tryLoadGrammar('tree-sitter-elixir');
const Make = tryLoadGrammar('tree-sitter-make');
+// tree-sitter-sas uses ESM bindings with top-level await — incompatible with tryLoadGrammar (CJS require)
+let SAS: any = null;
+// @ts-ignore
+try { SAS = (await import('tree-sitter-sas')).default; }
+catch (e: any) {
+ // A genuinely-missing optional dep is expected; surface anything else (e.g. a
+ // broken native build / ABI mismatch) so .sas files aren't silently dropped.
+ if (e?.code !== 'ERR_MODULE_NOT_FOUND' && e?.code !== 'MODULE_NOT_FOUND') {
+ process.stderr.write('[tree-sitter-sas load failed] ' + e + '\n');
+ }
+}
import { SupportedLanguages, languageFromPath } from './languages.js';
import { LANGUAGE_QUERIES } from './queries.js';
@@ -116,6 +127,7 @@ const GRAMMAR_MAP: Partial<Record<SupportedLanguages, any>> = {
[SupportedLanguages.Scala]: Scala,
...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),
...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),
+ ...(SAS ? { [SupportedLanguages.SAS]: SAS } : {}),
...(Elixir ? { [SupportedLanguages.Elixir]: Elixir } : {}),
...(Make ? { [SupportedLanguages.Makefile]: Make } : {}),
};
@@ -208,6 +220,16 @@ const JS_BUILTINS = new Set([
'require', 'fetch', 'parseInt', 'parseFloat', 'isNaN', 'isFinite',
]);
+// SAS-specific builtins: common PROC names and macro keywords that generate
+// noise as CALLS edges (user-defined macros are what we want to track).
+const SAS_BUILTINS = new Set([
+ ...SHARED_BUILTINS,
+ 'print', 'means', 'freq', 'sort', 'import', 'export', 'contents', 'datasets',
+ 'sql', 'reg', 'logistic', 'glm', 'anova', 'ttest', 'format', 'report',
+ 'univariate', 'corr', 'mixed', 'phreg', 'lifetest', 'transpose', 'append',
+ 'put', 'let', 'do', 'end', 'goto', 'abort',
+]);
+
// Per-language BUILTINS lookup — falls back to shared for languages without a
// specific set (e.g. Java, Go, Rust) so they only skip obvious non-calls.
function builtinsForLanguage(lang: SupportedLanguages): Set<string> {
@@ -217,6 +239,8 @@ function builtinsForLanguage(lang: SupportedLanguages): Set<string> {
case SupportedLanguages.JavaScript:
case SupportedLanguages.TypeScript:
return JS_BUILTINS;
+ case SupportedLanguages.SAS:
+ return SAS_BUILTINS;
default:
return SHARED_BUILTINS;
}
@@ -270,6 +294,25 @@ function normalizeCapturedImport(rawValue: string, language: SupportedLanguages)
return unwrapped.replace(/^\.+/, '');
}
+ if (language === SupportedLanguages.SAS) {
+ // Real SAS uses single backslashes in Windows paths (libname "&P.\SASData").
+ // The general pre-normalization only collapses double backslashes, so do it
+ // here before any path inspection.
+ const slashed = unwrapped.replace(/\\/g, '/');
+ // Unexpanded macro-var targets (&OUTFILE, &P.\dir) have no stable identity
+ // until macro expansion — drop them so they create no bogus node/edge.
+ if (slashed.includes('&')) return '';
+ if (!slashed.includes('/')) {
+ // fileref_source text: FILEREF or FILEREF(member.sas) — no path separators.
+ const memberMatch = slashed.match(/^[A-Za-z_][A-Za-z0-9_]*\(([^)]+)\)$/);
+ if (memberMatch) return memberMatch[1];
+ return slashed;
+ }
+ // String-literal %include and LIBNAME paths → basename.
+ const base = slashed.split('/').filter((s: string) => s !== '*').pop() ?? slashed;
+ return base.replace(/^\.+/, '');
+ }
+
const rawMod = unwrapped.split('/').filter((s: string) => s !== '*').pop() ?? unwrapped;
return rawMod.replace(/^\.+/, '');
}
@@ -1319,10 +1362,32 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
if (defCapture) {
const kind = DEFINITION_KIND_MAP[defCapture.name] ?? 'function';
- const name = nameCapture?.node.text
+ let name = nameCapture?.node.text
?? (defCapture.name === 'definition.constructor' ? 'init' : '');
if (!name || name.length === 0) continue;
+ // SAS module entities (DATA/PROC steps, PROC SQL CREATE) need extra
+ // shaping so they reflect real data artifacts rather than parser noise.
+ if (language === SupportedLanguages.SAS && kind === 'module') {
+ if (defCapture.node.type === 'proc_step') {
+ // The procedure name (means/sort/freq…) is too generic to be a node.
+ // Re-key on the PROC's out= output dataset; drop the step entirely
+ // when it produces none (e.g. proc print, proc means with no out=).
+ // out= may sit on the header (proc sort out=) OR in a body statement
+ // (proc means; output out=x; / proc freq; tables a / out=y;), so fall
+ // back to scanning the full step text when the header has none.
+ const header = defCapture.node.namedChild(0);
+ const headerText = header?.type === 'proc_step_header' ? header.text : defCapture.node.text;
+ const outRe = /\bout\s*=\s*([A-Za-z_][\w.]*)/i;
+ const outMatch = headerText.match(outRe) ?? defCapture.node.text.match(outRe);
+ if (!outMatch) continue;
+ name = outMatch[1];
+ }
+ // Drop side-effect-only sentinels (_null_/_data_/_last_) and unexpanded
+ // macro-var names (&ds) — neither denotes a stable dataset identity.
+ if (/^_(null|data|last)_$/i.test(name) || name.includes('&')) continue;
+ }
+
const defNode = defCapture.node;
const lineStart = defNode.startPosition.row + 1;
const lineEnd = defNode.endPosition.row + 1;
@@ -1986,6 +2051,28 @@ export function buildGlobalResolutionIndex(
}
}
+ // SAS: index macro definitions for cross-batch Tier-3 resolution. Derive
+ // them from the parser's definition.macro entities (not a regex) so this
+ // never drifts from what the in-batch path extracts in resolveEdges — the
+ // parser also captures forms a regex misses (e.g. %macro /*comment*/ name).
+ // Mirrors the qkMap construction at resolveEdges (key by name, value qkey).
+ for (const [fp, src] of sources) {
+ if (nodePath.extname(fp).toLowerCase() !== '.sas') continue;
+ const parsed = parseFile(fp, src);
+ if (!parsed) continue;
+ const qkMap = new Map<string, string[]>();
+ for (const e of parsed.entities) {
+ if (e.kind !== 'macro') continue;
+ const list = qkMap.get(e.name) ?? [];
+ list.push(qualifiedKey(e));
+ qkMap.set(e.name, list);
+ }
+ if (qkMap.size > 0) {
+ fileQKeys.set(fp, qkMap);
+ fileHasSymbol.set(fp, new Set(qkMap.keys()));
+ }
+ }
+
for (const [fp, symbols] of fileHasSymbol) {
for (const sym of symbols) {
const list = symbolToFiles.get(sym) ?? [];
@@ -2177,7 +2264,10 @@ export function resolveEdges(
if (stripped && stripped !== normalizedModName) candidates.push(stripped);
// Strip file extensions so "explain.js" resolves to the "explain" stem
// (TS/JS ESM imports use .js extensions that map to .ts source files)
- const noExt = normalizedModName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx)$/, '');
+ // Only the SAS extension is case-insensitive (real SAS uses .sas/.SAS
+ // interchangeably). Keeping the rest case-sensitive avoids stripping
+ // uppercase .C/.H (legitimate C++ source) or .TS/.JAVA across languages.
+ const noExt = normalizedModName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx|[Ss][Aa][Ss])$/, '');
if (noExt !== normalizedModName && noExt && !candidates.includes(noExt)) candidates.push(noExt);
const pathBasename = nodePath.posix.basename(normalizedModName);
if (pathBasename && !candidates.includes(pathBasename)) candidates.push(pathBasename);
@@ -2375,7 +2465,7 @@ export function resolveEdges(
if (canonicalMatches.length === 1) return canonicalMatches;
}
- if (srcLanguage === SupportedLanguages.Java) {
+ if (srcLanguage === SupportedLanguages.Java || srcLanguage === SupportedLanguages.SAS) {
const srcParts = srcFilePath.replace(/\\/g, '/').split('/');
const withProximity = matches.map(fp => {
const fpParts = fp.replace(/\\/g, '/').split('/');
diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.ts
index 1141eb70..350d10e6 100644
--- a/core-ingestion/src/languages.ts
+++ b/core-ingestion/src/languages.ts
@@ -19,6 +19,7 @@ export enum SupportedLanguages {
JSON = 'json',
TOML = 'toml',
Markdown = 'markdown',
+ SAS = 'sas',
Elixir = 'elixir',
Makefile = 'makefile',
}
@@ -56,6 +57,7 @@ const EXT_MAP: Record<string, SupportedLanguages> = {
'.toml': SupportedLanguages.TOML,
'.md': SupportedLanguages.Markdown,
'.markdown': SupportedLanguages.Markdown,
+ '.sas': SupportedLanguages.SAS,
'.ex': SupportedLanguages.Elixir,
'.exs': SupportedLanguages.Elixir,
'.mk': SupportedLanguages.Makefile,
diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.ts
index 39f3923b..896a8c4a 100644
--- a/core-ingestion/src/queries.ts
+++ b/core-ingestion/src/queries.ts
@@ -1483,6 +1483,76 @@ export const ELIXIR_QUERIES = `
(#match? @_defmacrog "^(defmacro|defmacrop)$")) @definition.macro
`;
+export const SAS_QUERIES = `
+; Macro definitions — %MACRO name ... %MEND
+(macro_definition
+ name: (macro_name) @name) @definition.macro
+
+; Macro call statements — %name(args);
+(macro_call_statement
+ name: (macro_name) @call.name) @call
+
+; Inline macro calls — %name(args) inside DATA/PROC/etc.
+(macro_call
+ name: (macro_name) @call.name) @call
+
+; DATA step — capture the whole dataset_name node so two-part names (work.foo)
+; stay one entity. Sentinels (_null_) and unexpanded macro vars (&ds) are
+; filtered downstream in index.ts (they have no stable identity).
+(data_step
+ (data_step_header
+ (dataset_name) @name)) @definition.module
+
+; PROC step — the procedure name itself is too generic to be a graph node
+; (means/sort/freq repeat once per file). index.ts re-keys this on the PROC's
+; out= output dataset, or drops it when the step produces no dataset.
+(proc_step
+ (proc_step_header
+ name: (identifier) @name)) @definition.module
+
+; %INCLUDE — string literal form: %include 'path/to/file.sas';
+(include_statement
+ source: (string_literal) @import.source) @import
+
+; %INCLUDE — fileref form: %include FILEREF; or %include FILEREF(member.sas);
+(include_statement
+ source: (fileref_source) @import.source) @import
+
+; LIBNAME — library path import
+(libname_statement
+ (string_literal) @import.source) @import
+
+; PROC SQL — CREATE TABLE AS output
+(sql_create_statement
+ output: (dataset_name) @name) @definition.module
+
+; PROC SQL — FROM table inputs
+(sql_select_statement
+ (table_reference
+ (dataset_name) @import.source)) @import
+
+; PROC SQL — JOIN table inputs
+(sql_select_statement
+ (sql_join_clause
+ (table_reference
+ (dataset_name) @import.source))) @import
+
+; DATA step SET inputs
+(data_step
+ (set_statement
+ (dataset_name) @import.source)) @import
+
+; DATA step MERGE inputs
+(data_step
+ (merge_statement
+ (dataset_name) @import.source)) @import
+
+; DATA step UPDATE inputs
+(data_step
+ (update_statement
+ (dataset_name) @import.source)) @import
+`;
+
export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {
[SupportedLanguages.TypeScript]: TYPESCRIPT_QUERIES,
[SupportedLanguages.JavaScript]: JAVASCRIPT_QUERIES,
@@ -1504,6 +1574,7 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {
[SupportedLanguages.JSON]: '',
[SupportedLanguages.TOML]: '',
[SupportedLanguages.Markdown]: '',
+ [SupportedLanguages.SAS]: SAS_QUERIES,
[SupportedLanguages.Elixir]: ELIXIR_QUERIES,
[SupportedLanguages.Makefile]: MAKEFILE_QUERIES,
};
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 1d31c8ff..501f28c1 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -40,6 +40,7 @@ const SUPPORTED_EXTENSIONS = new Set([
'.json',
'.toml',
'.md', '.markdown',
+ '.sas',
'.ex', '.exs',
'.mk', '.makefile',
]);
@@ -53,6 +54,15 @@ export function isSupportedSourceFile(filePath: string): boolean {
|| SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());
}
+// Extensions whose source text must be pre-read so buildGlobalResolutionIndex can
+// extract cross-batch symbols before the streaming parse loop. Go uses a fast
+// regex scan; SAS is parsed (its macro index is derived from definition.macro).
+const INDEX_PRESCAN_EXTENSIONS = new Set(['.go', '.sas']);
+
+function needsIndexPrescan(filePath: string): boolean {
+ return INDEX_PRESCAN_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());
+}
+
// ---------------------------------------------------------------------------
// Language filter helpers
// ---------------------------------------------------------------------------
@@ -954,12 +964,12 @@ export async function ingestFiles(
const sources = new Map<string, string>();
const changedSet = new Set(changedPaths.map(c => c.filePath));
for (const { filePath, bytes } of changedPaths) {
- if (nodePath.extname(filePath) === '.go') sources.set(filePath, bytes.toString('utf-8'));
+ if (needsIndexPrescan(filePath)) sources.set(filePath, bytes.toString('utf-8'));
}
- const remainingGoFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go' && !changedSet.has(fp));
- const GO_READ_CONCURRENCY = 2000;
- for (let i = 0; i < remainingGoFiles.length; i += GO_READ_CONCURRENCY) {
- const batch = remainingGoFiles.slice(i, i + GO_READ_CONCURRENCY);
+ const remainingIndexFiles = filePaths.filter(fp => needsIndexPrescan(fp) && !changedSet.has(fp));
+ const INDEX_READ_CONCURRENCY = 2000;
+ for (let i = 0; i < remainingIndexFiles.length; i += INDEX_READ_CONCURRENCY) {
+ const batch = remainingIndexFiles.slice(i, i + INDEX_READ_CONCURRENCY);
const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));
for (let j = 0; j < batch.length; j++) {
if (texts[j] != null) sources.set(batch[j], texts[j]!);
@@ -1009,16 +1019,16 @@ export async function ingestFiles(
// Build global resolution index from all repo file paths before the parse loop
// so cross-batch imports resolve correctly even in streaming per-chunk mode.
- // Read all Go files in parallel to maximize I/O throughput.
+ // Read pre-scan files (Go, SAS) in parallel to maximize I/O throughput.
// Index keys are workspace-relative to match the relative paths we pass
// into parseFile below.
{
const goIndexStart = performance.now();
const sources = new Map<string, string>();
- const goFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go');
- const GO_READ_CONCURRENCY = 2000;
- for (let i = 0; i < goFiles.length; i += GO_READ_CONCURRENCY) {
- const batch = goFiles.slice(i, i + GO_READ_CONCURRENCY);
+ const indexFiles = filePaths.filter(fp => needsIndexPrescan(fp));
+ const INDEX_READ_CONCURRENCY = 2000;
+ for (let i = 0; i < indexFiles.length; i += INDEX_READ_CONCURRENCY) {
+ const batch = indexFiles.slice(i, i + INDEX_READ_CONCURRENCY);
const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));
for (let j = 0; j < batch.length; j++) {
if (texts[j] != null) sources.set(toWorkspaceRelative(batch[j]), texts[j]!);