Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat(parser): add TeX/LaTeX support (.tex/.sty/.cls/.ltx/.latex) (#289) (#291)
Joseph Mikhaildbb517b0680b
diff --git a/core-ingestion/src/__tests__/queries.latex.test.ts b/core-ingestion/src/__tests__/queries.latex.test.tsnew file mode 100644index 00000000..f211265b--- /dev/null+++ b/core-ingestion/src/__tests__/queries.latex.test.ts@@ -0,0 +1,253 @@+import { describe, it, expect } from 'vitest';+import { parseFile } from '../index.js';+import { SupportedLanguages } from '../languages.js';++const probe = parseFile('/repo/a.tex', '\\documentclass{article}\n');+const describeFn = probe && probe.language === SupportedLanguages.LaTeX ? describe : describe.skip;++const rels = (r: ReturnType<typeof parseFile>, p: string) =>+ r!.relationships.filter(x => x.predicate === p);++describeFn('LaTeX parser', () => {+ it('detects LaTeX by extension (.tex/.sty/.cls/.ltx/.latex)', () => {+ for (const ext of ['tex', 'sty', 'cls', 'ltx', 'latex']) {+ expect(parseFile(`/r/m.${ext}`, '\\documentclass{article}')!.language).toBe(SupportedLanguages.LaTeX);+ }+ });++ it('builds a nested sectioning hierarchy via CONTAINS', () => {+ const r = parseFile('/r/main.tex', `+\\part{One}+\\chapter{Intro}+\\section{Background}+\\subsection{Prior work}+\\section{Methods}+`);+ const sections = r!.entities.filter(e => e.kind === 'section').map(e => e.name);+ expect(sections).toEqual(['One', 'Intro', 'Background', 'Prior work', 'Methods']);+ const contains = rels(r, 'CONTAINS').map(e => `${e.srcName}>${e.dstName}`);+ expect(contains).toContain('main.tex>One');+ expect(contains).toContain('One>Intro');+ expect(contains).toContain('Intro>Background');+ expect(contains).toContain('Background>Prior work');+ // Methods resets back to chapter scope (sibling of Background's parent section)+ expect(contains).toContain('Intro>Methods');+ });++ it('captures starred sections and cleans nested markup in titles', () => {+ const r = parseFile('/r/m.tex', '\\section*{The \\textbf{Bold} Title}\n');+ const sections = r!.entities.filter(e => e.kind === 'section').map(e => e.name);+ expect(sections).toEqual(['The Bold Title']);+ });++ it('captures \\newcommand / \\renewcommand / \\providecommand / \\DeclareRobustCommand', () => {+ const r = parseFile('/r/m.tex', `+\\newcommand{\\foo}{x}+\\renewcommand{\\bar}[1]{#1}+\\providecommand{\\baz}{y}+\\DeclareRobustCommand{\\qux}{z}+\\newcommand\\unbraced{w}+`);+ const fns = r!.entities.filter(e => e.kind === 'function').map(e => e.name).sort();+ expect(fns).toEqual(['bar', 'baz', 'foo', 'qux', 'unbraced']);+ });++ it('captures \\def and \\let control-sequence definitions', () => {+ const r = parseFile('/r/m.tex', '\\def\\mymacro{hello}\n\\let\\other=\\relax\n');+ const fns = r!.entities.filter(e => e.kind === 'function').map(e => e.name).sort();+ expect(fns).toEqual(['mymacro', 'other']);+ });++ it('does not emit a bogus "csname" macro for \\def\\csname...\\endcsname', () => {+ const r = parseFile('/r/m.tex', '\\expandafter\\def\\csname my@dynamic\\endcsname{body}\n\\def\\real{x}\n');+ const fns = r!.entities.filter(e => e.kind === 'function').map(e => e.name);+ expect(fns).toEqual(['real']);+ });++ it('strips inline math/markup from section titles', () => {+ const r = parseFile('/r/m.tex', '\\section{Convergence of \\texorpdfstring{$\\sigma$}{[sigma]}}\n');+ expect(r!.entities.filter(e => e.kind === 'section').map(e => e.name)).toEqual(['Convergence of [sigma]']);+ });++ it('captures \\newenvironment and \\newtheorem as definitions', () => {+ const r = parseFile('/r/m.tex', `+\\newenvironment{myenv}{\\begin{center}}{\\end{center}}+\\newtheorem{thm}{Theorem}+\\newtheorem*{lem}{Lemma}+`);+ const classes = r!.entities.filter(e => e.kind === 'class').map(e => e.name).sort();+ expect(classes).toEqual(['lem', 'myenv', 'thm']);+ });++ it('emits IMPORTS for \\documentclass / \\usepackage / \\RequirePackage', () => {+ const r = parseFile('/r/m.tex', `+\\documentclass[12pt]{article}+\\usepackage{amsmath}+\\usepackage[utf8]{inputenc}+\\usepackage{tikz,pgfplots}+\\RequirePackage{xcolor}+`);+ const imports = rels(r, 'IMPORTS').map(e => e.dstName).sort();+ expect(imports).toEqual(['amsmath', 'article', 'inputenc', 'pgfplots', 'tikz', 'xcolor']);+ // package imports preserve a bare (non-path) importRaw+ expect(rels(r, 'IMPORTS').every(e => e.importRaw && !e.importRaw.includes('/'))).toBe(true);+ });++ it('resolves \\input / \\include / \\subfile to .tex targets', () => {+ const r = parseFile('/r/main.tex', `+\\input{header}+\\include{chapters/intro}+\\subfile{sections/methods.tex}+\\import{parts/}{appendix}+`);+ const imports = rels(r, 'IMPORTS');+ const dst = imports.map(e => e.dstName).sort();+ expect(dst).toEqual(['chapters/intro.tex', 'header.tex', 'parts/appendix.tex', 'sections/methods.tex']);+ // raw specifier retained for path resolution / multi-repo gate+ expect(imports.find(e => e.dstName === 'header.tex')!.importRaw).toBe('header');+ });++ it('emits \\label anchors and resolves \\ref/\\eqref/\\cref as REFERENCES', () => {+ const r = parseFile('/r/m.tex', `+\\section{Results}+\\label{sec:results}+\\begin{equation}\\label{eq:main}\\end{equation}+See \\ref{sec:results} and \\eqref{eq:main}.+Also \\cref{sec:results,eq:main}.+`);+ const labels = r!.entities.filter(e => e.kind === 'label').map(e => e.name).sort();+ expect(labels).toEqual(['eq:main', 'sec:results']);+ const refs = rels(r, 'REFERENCES').map(e => e.dstName).sort();+ expect(refs).toEqual(['eq:main', 'eq:main', 'sec:results', 'sec:results']);+ });++ it('emits \\cite/\\citep/\\citet REFERENCES to bib keys (incl. comma lists)', () => {+ const r = parseFile('/r/m.tex', `+\\cite{knuth1984}+\\citep{lamport1994,goossens1993}+\\citet{wilson2020}+`);+ const refs = rels(r, 'REFERENCES').map(e => e.dstName).sort();+ expect(refs).toEqual(['goossens1993', 'knuth1984', 'lamport1994', 'wilson2020']);+ });++ it('records environments and their content under the enclosing section', () => {+ const r = parseFile('/r/m.tex', `+\\section{Figures}+\\begin{figure}+\\caption{A figure}+\\label{fig:a}+\\end{figure}+\\begin{table}+\\end{table}+`);+ const envs = r!.entities.filter(e => e.kind === 'environment');+ expect(envs.map(e => e.name).sort()).toEqual(['figure', 'table']);+ const contains = rels(r, 'CONTAINS').map(e => `${e.srcName}>${e.dstName}`);+ expect(contains).toContain('Figures>figure');+ expect(contains).toContain('Figures>table');+ // Content attaches to the section, not the (non-unique) environment name —+ // an environment name is never used as a CONTAINS source (it would dangle:+ // patch-builder resolves edge sources by bare name with no container hint).+ expect(contains).toContain('Figures>fig:a');+ expect(rels(r, 'CONTAINS').some(e => e.srcName === 'figure' || e.srcName === 'table')).toBe(false);+ });++ it('keeps labels resolvable when the same environment type repeats across sections', () => {+ const r = parseFile('/r/m.tex', `+\\section{Results}+\\begin{figure}\\label{fig:a}\\end{figure}+\\section{Discussion}+\\begin{figure}\\label{fig:b}\\end{figure}+`);+ const contains = rels(r, 'CONTAINS').map(e => `${e.srcName}>${e.dstName}`);+ // Each label is contained by its own (uniquely named) section, so no edge+ // is sourced from the ambiguous "figure" node.+ expect(contains).toContain('Results>fig:a');+ expect(contains).toContain('Discussion>fig:b');+ expect(rels(r, 'CONTAINS').some(e => e.srcName === 'figure')).toBe(false);+ });++ it('does not prefix file-level definitions with the file name (container undefined)', () => {+ const r = parseFile('/r/m.tex', '\\newcommand{\\toplevel}{x}\n\\label{top}\n');+ const defs = r!.entities.filter(e => e.kind === 'function' || e.kind === 'label');+ expect(defs.every(e => e.container === undefined)).toBe(true);+ });++ it('skips a verbatim body closed with whitespace (\\end {verbatim})', () => {+ const r = parseFile('/r/m.tex', `+\\begin{verbatim}+\\newcommand{\\fake}{nope}+\\end {verbatim}+\\section{Real}+`);+ expect(r!.entities.filter(e => e.kind === 'function')).toHaveLength(0);+ expect(r!.entities.filter(e => e.kind === 'section').map(e => e.name)).toEqual(['Real']);+ });++ it('does not scan command bodies inside verbatim/lstlisting environments', () => {+ const r = parseFile('/r/m.tex', `+\\begin{verbatim}+\\newcommand{\\fake}{should not be captured}+\\section{not a real section}+\\end{verbatim}+\\section{Real}+`);+ expect(r!.entities.filter(e => e.kind === 'function')).toHaveLength(0);+ expect(r!.entities.filter(e => e.kind === 'section').map(e => e.name)).toEqual(['Real']);+ });++ it('ignores comments (unescaped %) but keeps escaped \\%', () => {+ const r = parseFile('/r/m.tex', `+% \\section{Commented out}+\\section{Live} % \\usepackage{nope}+50\\% done+`);+ expect(r!.entities.filter(e => e.kind === 'section').map(e => e.name)).toEqual(['Live']);+ expect(rels(r, 'IMPORTS')).toHaveLength(0);+ });++ it('captures cross-references nested inside unknown command arguments', () => {+ const r = parseFile('/r/m.tex', '\\caption{See \\ref{fig:x} and \\cite{paper}}\n');+ const refs = rels(r, 'REFERENCES').map(e => e.dstName).sort();+ expect(refs).toEqual(['fig:x', 'paper']);+ });++ it('handles malformed / unbalanced input without crashing or hanging', () => {+ const samples = [+ '\\begin{figure}\n\\section{Orphan}\n', // unclosed environment+ '\\end{figure}\n', // stray \end+ '\\newcommand{\\foo', // unterminated brace+ '\\section{', // unterminated title+ '\\usepackage{a,b,', // unterminated list+ '\\\\ \\% \\{ \\} \\$ text', // control symbols only+ '%'.repeat(10000), // long comment+ '{'.repeat(5000), // deep open braces+ ];+ for (const s of samples) {+ const r = parseFile('/r/x.tex', s);+ expect(r).not.toBeNull();+ expect(r!.language).toBe(SupportedLanguages.LaTeX);+ }+ });++ it('is deterministic (byte-identical entities/relationships on re-parse)', () => {+ const src = `+\\documentclass{book}+\\usepackage{amsmath}+\\begin{document}+\\chapter{One}\\label{ch:one}+\\section{Alpha}+\\newcommand{\\x}{1}+\\begin{theorem}\\label{thm:1}\\end{theorem}+See \\ref{ch:one} and \\cite{a,b}.+\\input{more}+\\end{document}+`;+ const a = parseFile('/r/book.tex', src)!;+ const b = parseFile('/r/book.tex', src)!;+ expect(JSON.stringify(a.entities)).toBe(JSON.stringify(b.entities));+ expect(JSON.stringify(a.relationships)).toBe(JSON.stringify(b.relationships));+ expect(JSON.stringify(a.chunks)).toBe(JSON.stringify(b.chunks));+ });+});diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex ee35bc04..8dcc0eca 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -356,7 +356,7 @@ function builtinsForLanguage(lang: SupportedLanguages): Set<string> {export function isGrammarSupported(filePath: string): boolean {const language = languageFromPath(filePath);if (!language) return false;- if (language === SupportedLanguages.YAML || language === SupportedLanguages.Dockerfile || language === SupportedLanguages.SQL || language === SupportedLanguages.JSON || language === SupportedLanguages.TOML || language === SupportedLanguages.Markdown) return true;+ if (language === SupportedLanguages.YAML || language === SupportedLanguages.Dockerfile || language === SupportedLanguages.SQL || language === SupportedLanguages.JSON || language === SupportedLanguages.TOML || language === SupportedLanguages.Markdown || language === SupportedLanguages.LaTeX) return true;if (filePath.endsWith('.tsx')) return true; // TSX uses TypeScript.tsx, always availablereturn GRAMMAR_MAP[language] !== undefined;}@@ -995,6 +995,352 @@ function parseMarkdownFile(filePath: string, source: string): FileParseResult {return { filePath, language, entities, chunks, relationships, fileRole };}+// ---------------------------------------------------------------------------+// LaTeX / TeX (.tex, .sty, .cls, .ltx, .latex)+//+// Hand-rolled scanner rather than tree-sitter: there is no maintained Node+// tree-sitter-latex binding for this ABI, and TeX is a macro-expansion language+// whose custom-macro arities a static grammar cannot know — real grammars emit+// ERROR nodes on ordinary documents. A targeted single-pass scanner extracts the+// constructs Ix models (sectioning hierarchy, macro/environment/theorem defs,+// labels, package/file dependencies, ref/cite cross-references) and degrades+// gracefully on malformed input. O(n), deterministic.+// ---------------------------------------------------------------------------++const LATEX_SECTION_LEVELS: Record<string, number> = {+ part: 0, chapter: 1, section: 2, subsection: 3,+ subsubsection: 4, paragraph: 5, subparagraph: 6,+};+const LATEX_MACRO_DEFS = new Set([+ 'newcommand', 'renewcommand', 'providecommand', 'DeclareRobustCommand',+ 'DeclareMathOperator', 'newcommandx', 'renewcommandx', 'providecommandx',+]);+const LATEX_CS_DEFS = new Set(['def', 'gdef', 'edef', 'xdef', 'let']);+// When a definition's target is one of these primitives the real name is built+// at expansion time (e.g. \def\csname foo\endcsname{...} defines "foo", not+// "csname"), so it is not statically knowable — skip rather than emit noise.+const LATEX_DYNAMIC_CS = new Set(['csname', 'endcsname']);+const LATEX_ENV_DEFS = new Set(['newenvironment', 'renewenvironment', 'newenvironmentx']);+const LATEX_THEOREM_DEFS = new Set(['newtheorem']);+const LATEX_PACKAGE_IMPORTS = new Set([+ 'usepackage', 'RequirePackage', 'RequirePackageWithOptions',+ 'documentclass', 'LoadClass', 'LoadClassWithOptions', 'documentstyle',+]);+const LATEX_FILE_INCLUDES = new Set([+ 'input', 'include', 'subfile', 'subfileinclude', 'includeonly',+]);+const LATEX_BIB_INCLUDES = new Set(['bibliography', 'addbibresource']);+const LATEX_REF_CMDS = new Set([+ 'ref', 'eqref', 'pageref', 'cref', 'Cref', 'autoref', 'nameref',+ 'vref', 'vpageref', 'labelcref', 'cpageref', 'Autoref', 'fref', 'thref',+]);+const LATEX_CITE_CMDS = new Set([+ 'cite', 'citep', 'citet', 'citealt', 'citealp', 'citeauthor', 'citeyear',+ 'citeyearpar', 'Citep', 'Citet', 'Cite', 'textcite', 'parencite', 'footcite',+ 'autocite', 'smartcite', 'citenum', 'fullcite', 'nocite', 'citealp',+]);+// Inside these environments backslashes are literal text, not commands; skip+// scanning their body so listings/verbatim content never produces spurious nodes.+const LATEX_VERBATIM_ENVS = new Set([+ 'verbatim', 'verbatim*', 'Verbatim', 'BVerbatim', 'LVerbatim',+ 'lstlisting', 'minted', 'comment', 'alltt', 'filecontents', 'filecontents*',+]);++function parseLatexFile(filePath: string, source: string): FileParseResult {+ const language = SupportedLanguages.LaTeX;+ const fileName = nodePath.basename(filePath);+ const sourceLineCount = countSourceLines(source);+ const fileRole = classifyFileRole(filePath);+ const entities: ParsedEntity[] = [+ { name: fileName, kind: 'file', lineStart: 1, lineEnd: sourceLineCount, language },+ ];+ const chunks: ParsedChunk[] = [];+ const relationships: ParsedRelationship[] = [];+ const lineStarts = computeLineStarts(source);+ const n = source.length;++ const lineAt = (idx: number): number => {+ let lo = 0, hi = lineStarts.length - 1, ans = 0;+ while (lo <= hi) {+ const mid = (lo + hi) >> 1;+ if (lineStarts[mid] <= idx) { ans = mid; lo = mid + 1; } else hi = mid - 1;+ }+ return ans + 1;+ };++ // Read a balanced { ... } group starting at `open` (source[open] === '{').+ // Honors \{ and \} escapes. On imbalance, returns the rest of the source.+ const readBraceGroup = (open: number): { text: string; end: number } => {+ let depth = 0;+ for (let q = open; q < n; q++) {+ const cc = source[q];+ if (cc === '\\') { q++; continue; }+ if (cc === '{') depth++;+ else if (cc === '}') { depth--; if (depth === 0) return { text: source.slice(open + 1, q), end: q + 1 }; }+ }+ return { text: source.slice(open + 1, n), end: n };+ };++ // Read an optional [ ... ] group; stops at the first unescaped ] not nested in {}.+ const readOptGroup = (open: number): { text: string; end: number } => {+ let braceDepth = 0;+ for (let q = open + 1; q < n; q++) {+ const cc = source[q];+ if (cc === '\\') { q++; continue; }+ if (cc === '{') braceDepth++;+ else if (cc === '}') { if (braceDepth > 0) braceDepth--; }+ else if (cc === ']' && braceDepth === 0) return { text: source.slice(open + 1, q), end: q + 1 };+ }+ return { text: source.slice(open + 1, n), end: n };+ };++ // From `start`, greedily collect immediately-following { } (required) and [ ]+ // (optional) argument groups. Only spaces/tabs may separate them — never a+ // newline — so an unrelated brace group on a later line is not misattributed.+ const collectArgs = (start: number): { args: string[]; opts: string[]; end: number } => {+ const args: string[] = [];+ const opts: string[] = [];+ let p = start;+ for (;;) {+ let q = p;+ while (q < n && (source[q] === ' ' || source[q] === '\t')) q++;+ if (source[q] === '{') { const g = readBraceGroup(q); args.push(g.text); p = g.end; }+ else if (source[q] === '[') { const g = readOptGroup(q); opts.push(g.text); p = g.end; }+ else break;+ }+ return { args, opts, end: p };+ };++ // Strip nested commands/braces from a sectioning/environment title for a clean name.+ const cleanTitle = (raw: string): string =>+ raw.replace(/\\[a-zA-Z@]+\*?/g, '').replace(/[{}$]/g, '').replace(/\s+/g, ' ').trim();+ const csName = (raw: string): string => {+ const m = /\\([a-zA-Z@]+)/.exec(raw);+ return (m ? m[1] : raw.trim().replace(/^\\/, '')).trim();+ };+ const splitList = (raw: string): string[] =>+ raw.split(',').map(s => s.trim()).filter(Boolean);+ const ensureExt = (raw: string, ext: string): string =>+ /\.[a-zA-Z0-9]+$/.test(raw) ? raw : `${raw}${ext}`;++ // Sectioning hierarchy (levels 0..6); deepest active section is the live container.+ const sectionStack: (string | null)[] = [null, null, null, null, null, null, null];+ const sectionMarks: { name: string; level: number; line: number }[] = [];+ const envStack: { name: string }[] = [];+ const seenEnv = new Set<string>();+ const currentSection = (): string | null => {+ for (let l = 6; l >= 0; l--) if (sectionStack[l]) return sectionStack[l];+ return null;+ };+ // The containment parent for definitions, labels, environments and references+ // is the nearest enclosing SECTION (or the file). It deliberately does NOT use+ // the open-environment name: environment type names ("figure", "table") are not+ // unique within a file, and patch-builder resolves a CONTAINS edge's *source*+ // by bare name with no container hint — so an environment used as a parent would+ // be ambiguous and its children's edges would dangle. Sections (unique titles)+ // are safe parents. Environments are still recorded as section members.+ const container = (): string => currentSection() ?? fileName;++ const addImport = (rawSpec: string, dst: string, line: number): void => {+ if (!dst) return;+ entities.push({ name: dst, kind: 'module', lineStart: line, lineEnd: line, language });+ relationships.push({ srcName: fileName, dstName: dst, predicate: 'IMPORTS', importRaw: rawSpec });+ };+ const addDef = (name: string, kind: string, line: number): void => {+ if (!name) return;+ const cont = container();+ entities.push({ name, kind, lineStart: line, lineEnd: line, language, container: cont === fileName ? undefined : cont });+ relationships.push({ srcName: cont, dstName: name, predicate: 'CONTAINS' });+ };++ let i = 0;+ while (i < n) {+ const ch = source[i];+ if (ch === '%') { while (i < n && source[i] !== '\n') i++; continue; }+ if (ch !== '\\') { i++; continue; }++ // Read the control-sequence name. A single non-letter after '\' (\\, \%, \{,+ // \,, \$) is an escape/control symbol — consume and skip.+ let j = i + 1;+ if (j >= n) { i = j; continue; }+ const c0 = source[j];+ if (!/[a-zA-Z@]/.test(c0)) { i = j + 1; continue; }+ let k = j;+ while (k < n && /[a-zA-Z@]/.test(source[k])) k++;+ const name = source.slice(j, k);+ let after = k;+ if (source[after] === '*') after++; // starred form (\section*, \newtheorem*)+ const line = lineAt(i);++ if (LATEX_SECTION_LEVELS[name] !== undefined) {+ const { args, opts, end } = collectArgs(after);+ const title = cleanTitle(args[0] ?? opts[0] ?? '');+ if (title) {+ const level = LATEX_SECTION_LEVELS[name];+ let cont: string = fileName;+ for (let l = level - 1; l >= 0; l--) if (sectionStack[l]) { cont = sectionStack[l]!; break; }+ for (let l = level; l <= 6; l++) sectionStack[l] = null;+ sectionStack[level] = title;+ entities.push({ name: title, kind: 'section', lineStart: line, lineEnd: line, language, container: cont === fileName ? undefined : cont });+ relationships.push({ srcName: cont, dstName: title, predicate: 'CONTAINS' });+ sectionMarks.push({ name: title, level, line });+ }+ i = end; continue;+ }++ if (name === 'begin') {+ const { args, end } = collectArgs(after);+ const envName = (args[0] ?? '').trim();+ if (envName) {+ const cont = container();+ const key = `${cont} ${envName}`;+ if (!seenEnv.has(key)) {+ seenEnv.add(key);+ entities.push({ name: envName, kind: 'environment', lineStart: line, lineEnd: line, language, container: cont === fileName ? undefined : cont });+ relationships.push({ srcName: cont, dstName: envName, predicate: 'CONTAINS' });+ }+ envStack.push({ name: envName });+ if (LATEX_VERBATIM_ENVS.has(envName)) {+ // Jump to the matching \end so it pops normally. Tolerate whitespace+ // (\end {verbatim}, \end{ verbatim }) — all legal TeX.+ const esc = envName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ const m = new RegExp(`\\\\end\\s*\\{\\s*${esc}\\s*\\}`).exec(source.slice(end));+ i = m ? end + m.index : n;+ continue;+ }+ }+ i = end; continue;+ }++ if (name === 'end') {+ const { args, end } = collectArgs(after);+ const envName = (args[0] ?? '').trim();+ // Pop to the nearest matching open environment; ignore stray \end gracefully.+ for (let s = envStack.length - 1; s >= 0; s--) {+ if (envStack[s].name === envName) { envStack.length = s; break; }+ }+ i = end; continue;+ }++ if (LATEX_MACRO_DEFS.has(name)) {+ let p = after;+ while (p < n && (source[p] === ' ' || source[p] === '\t')) p++;+ let macro: string | null = null;+ if (source[p] === '\\') { const m = /^\\([a-zA-Z@]+)/.exec(source.slice(p, p + 80)); if (m) { macro = m[1]; p += m[0].length; } }+ else if (source[p] === '{') { const g = readBraceGroup(p); macro = csName(g.text); p = g.end; }+ if (macro && !LATEX_DYNAMIC_CS.has(macro)) addDef(macro, 'function', line);+ i = p; continue;+ }++ if (LATEX_CS_DEFS.has(name)) {+ let p = after;+ while (p < n && (source[p] === ' ' || source[p] === '\t')) p++;+ if (source[p] === '\\') { const m = /^\\([a-zA-Z@]+)/.exec(source.slice(p, p + 80)); if (m) { if (!LATEX_DYNAMIC_CS.has(m[1])) addDef(m[1], 'function', line); p += m[0].length; } }+ i = p; continue;+ }++ if (LATEX_ENV_DEFS.has(name)) {+ const { args, end } = collectArgs(after);+ addDef((args[0] ?? '').trim(), 'class', line);+ i = end; continue;+ }+ if (LATEX_THEOREM_DEFS.has(name)) {+ const { args, end } = collectArgs(after);+ addDef((args[0] ?? '').trim(), 'class', line);+ i = end; continue;+ }++ if (LATEX_PACKAGE_IMPORTS.has(name)) {+ const { args, end } = collectArgs(after);+ for (const pkg of splitList(args[0] ?? '')) addImport(pkg, pkg, line);+ i = end; continue;+ }++ if (LATEX_FILE_INCLUDES.has(name)) {+ const { args, end } = collectArgs(after);+ for (const raw of splitList(args[0] ?? '')) addImport(raw, ensureExt(raw, '.tex'), line);+ i = end; continue;+ }+ // \import{dir}{file} / \subimport{dir}{file}: path is dir + file+ if (name === 'import' || name === 'subimport' || name === 'inputfrom' || name === 'subinputfrom') {+ const { args, end } = collectArgs(after);+ if (args.length >= 2 && args[1].trim()) {+ const dir = args[0].trim().replace(/\/?$/, '/');+ const raw = `${dir}${args[1].trim()}`;+ addImport(raw, ensureExt(raw, '.tex'), line);+ }+ i = end; continue;+ }+ if (LATEX_BIB_INCLUDES.has(name)) {+ const { args, end } = collectArgs(after);+ for (const raw of splitList(args[0] ?? '')) addImport(raw, ensureExt(raw, '.bib'), line);+ i = end; continue;+ }++ if (name === 'label') {+ const { args, end } = collectArgs(after);+ addDef((args[0] ?? '').trim(), 'label', line);+ i = end; continue;+ }+ if (name === 'bibitem') {+ const { args, end } = collectArgs(after);+ addDef((args[0] ?? '').trim(), 'label', line);+ i = end; continue;+ }++ if (LATEX_REF_CMDS.has(name) || LATEX_CITE_CMDS.has(name)) {+ const { args, end } = collectArgs(after);+ const src = container();+ for (const target of splitList(args[0] ?? '')) {+ relationships.push({ srcName: src, dstName: target, predicate: 'REFERENCES' });+ }+ i = end; continue;+ }++ // Unknown command: advance past the name only, so any cross-references nested+ // in its argument groups (e.g. \caption{... \ref{x}}, \footnote{\cite{y}}) are+ // still scanned as ordinary content on the next iterations.+ i = after;+ }++ // Section chunks: each spans to the line before the next section at the same or+ // shallower level, so a parent section's chunk includes its sub-sections.+ for (let h = 0; h < sectionMarks.length; h++) {+ const { name, level, line } = sectionMarks[h];+ let endLine = sourceLineCount;+ for (let m = h + 1; m < sectionMarks.length; m++) {+ if (sectionMarks[m].level <= level) { endLine = sectionMarks[m].line - 1; break; }+ }+ const startByte = lineStarts[line - 1] ?? 0;+ const endByte = endLine < lineStarts.length ? (lineStarts[endLine] ?? source.length) : source.length;+ const content = source.slice(startByte, endByte);+ chunks.push({+ name,+ chunkKind: 'section',+ lineStart: line,+ lineEnd: endLine,+ startByte,+ endByte,+ contentHash: crypto.createHash('sha256').update(content).digest('hex'),+ language,+ });+ }+ if (sectionMarks.length === 0) {+ chunks.push({+ name: null,+ chunkKind: 'file_body',+ lineStart: 1,+ lineEnd: Math.max(sourceLineCount, 1),+ startByte: 0,+ endByte: source.length,+ contentHash: crypto.createHash('sha256').update(source).digest('hex'),+ language,+ });+ }++ return { filePath, language, entities, chunks, relationships, fileRole };+}+function parseDockerfileFile(filePath: string, source: string): FileParseResult {const language = SupportedLanguages.Dockerfile;const fileName = nodePath.basename(filePath);@@ -1460,6 +1806,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | nif (language === SupportedLanguages.JSON) return parseJsonFile(filePath, source);if (language === SupportedLanguages.TOML) return parseTomlFile(filePath, source);if (language === SupportedLanguages.Markdown) return parseMarkdownFile(filePath, source);+ if (language === SupportedLanguages.LaTeX) return parseLatexFile(filePath, source);// TypeScript TSX uses a separate grammarconst isTsx = filePath.endsWith('.tsx');diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.tsindex 66c8b1d8..e34d1382 100644--- a/core-ingestion/src/languages.ts+++ b/core-ingestion/src/languages.ts@@ -31,6 +31,7 @@ export enum SupportedLanguages {XML = 'xml',HCL = 'hcl',CSS = 'css',+ LaTeX = 'latex',}const EXT_MAP: Record<string, SupportedLanguages> = {@@ -101,6 +102,11 @@ const EXT_MAP: Record<string, SupportedLanguages> = {'.scss': SupportedLanguages.CSS,'.sass': SupportedLanguages.CSS,'.less': SupportedLanguages.CSS,+ '.tex': SupportedLanguages.LaTeX,+ '.sty': SupportedLanguages.LaTeX,+ '.cls': SupportedLanguages.LaTeX,+ '.ltx': SupportedLanguages.LaTeX,+ '.latex':SupportedLanguages.LaTeX,};export function languageFromPath(filePath: string): SupportedLanguages | null {diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.tsindex ef1e61f8..52af40bb 100644--- a/core-ingestion/src/queries.ts+++ b/core-ingestion/src/queries.ts@@ -1783,5 +1783,8 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {[SupportedLanguages.XML]: XML_QUERIES,[SupportedLanguages.HCL]: HCL_QUERIES,[SupportedLanguages.CSS]: CSS_QUERIES,+ // LaTeX/TeX has no tree-sitter grammar in this stack; it is parsed by the+ // hand-rolled scanner in parseLatexFile (index.ts), like Markdown/YAML/etc.+ [SupportedLanguages.LaTeX]: '',};diff --git a/ix-cli/src/cli/__tests__/ingest-discovery.test.ts b/ix-cli/src/cli/__tests__/ingest-discovery.test.tsindex 596901f2..f1874ecc 100644--- a/ix-cli/src/cli/__tests__/ingest-discovery.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-discovery.test.ts@@ -35,4 +35,23 @@ describe('dedupeDiscoveredFilePaths', () => {expect(isSupportedSourceFile('build/common.mk')).toBe(true);expect(isSupportedSourceFile('README.txt')).toBe(false);});++ it('discovers TeX/LaTeX source files', () => {+ expect(isSupportedSourceFile('paper/main.tex')).toBe(true);+ expect(isSupportedSourceFile('pkg/mystyle.sty')).toBe(true);+ expect(isSupportedSourceFile('cls/thesis.cls')).toBe(true);+ expect(isSupportedSourceFile('legacy/doc.ltx')).toBe(true);+ expect(isSupportedSourceFile('notes.latex')).toBe(true);+ });++ it('discovers the grammar-based parsers that ship in core-ingestion', () => {+ // These extensions are parsed by core-ingestion but were missing from the+ // discovery allowlist, so their files were never walked. Guard against regress.+ for (const f of [+ 'init.lua', 'deploy.sh', 'run.bash', 'Main.hs', 'build.zig',+ 'index.html', 'pom.xml', 'app.csproj', 'main.tf', 'theme.css', 'styles.scss',+ ]) {+ expect(isSupportedSourceFile(f)).toBe(true);+ }+ });});diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 10a42031..d4c29778 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -17,6 +17,7 @@ import { loadIngestionModules } from './ingestion-loader.js';import { ensureWorkspaceIdState } from '../bootstrap.js';import { detectSystem, repoWorkspaceIdFor, lookupPackage, readPackageNames, readPackageDeps } from '../system.js';import { CLIENT_EXPECTED_SCHEMA_VERSION } from '../backend-status.js';+import { SUPPORTED_EXTENSIONS } from '../supported-extensions.js';import {deterministicId,transformIssue,@@ -29,25 +30,6 @@ import {// File discovery// ----------------------------------------------------------------------------// Inline extension set — mirrors core-ingestion/dist/languages.js EXT_MAP.-// Kept here so file discovery does NOT require loading tree-sitter grammars.-const SUPPORTED_EXTENSIONS = new Set([- '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',- '.py', '.java', '.c', '.h', '.cpp', '.cc', '.cxx', '.hpp',- '.cs', '.go', '.rb', '.rs', '.php', '.kt', '.kts', '.swift',- '.scala', '.sc',- '.yaml', '.yml',- '.dockerfile',- '.sql',- '.json',- '.toml',- '.md', '.markdown',- '.r',- '.sas',- '.ex', '.exs',- '.mk', '.makefile',-]);-export function isSupportedSourceFile(filePath: string): boolean {const fileName = nodePath.basename(filePath).toLowerCase();return fileName === 'dockerfile'diff --git a/ix-cli/src/cli/commands/text.ts b/ix-cli/src/cli/commands/text.tsindex 8886b7e2..b0e7491f 100644--- a/ix-cli/src/cli/commands/text.ts+++ b/ix-cli/src/cli/commands/text.ts@@ -97,6 +97,8 @@ function inferLanguage(filePath: string): string | undefined {if (filePath.endsWith(".md")) return "markdown";if (filePath.endsWith(".json")) return "json";if (filePath.endsWith(".yaml") || filePath.endsWith(".yml")) return "yaml";+ if (filePath.endsWith(".tex") || filePath.endsWith(".sty") || filePath.endsWith(".cls")+ || filePath.endsWith(".ltx") || filePath.endsWith(".latex")) return "latex";return undefined;}@@ -111,6 +113,7 @@ function languageGlobs(lang: string): string[] {case "rust": return ["*.rs"];case "ruby": return ["*.rb"];case "markdown": return ["*.md", "*.mdx"];+ case "latex": case "tex": return ["*.tex", "*.sty", "*.cls", "*.ltx", "*.latex"];case "config": return ["*.json", "*.yaml", "*.yml", "*.toml"];default: return [`*.${lang}`];}diff --git a/ix-cli/src/cli/commands/watch.ts b/ix-cli/src/cli/commands/watch.tsindex 5300a9df..d0710807 100644--- a/ix-cli/src/cli/commands/watch.ts+++ b/ix-cli/src/cli/commands/watch.ts@@ -9,13 +9,7 @@ import { getEndpoint, resolveWorkspaceRoot, clearIngestMtimeCache } from "../conimport { bootstrap, ensureWorkspaceId, ensureWorkspaceIdState } from "../bootstrap.js";import { loadWatchIngestionModules } from "./ingestion-loader.js";import { readFileContent } from "./watch-utils.js";-const SUPPORTED_EXTENSIONS = new Set([- ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",- ".scala", ".sc", ".java",- ".py", ".rb", ".go", ".rs", ".kt", ".kts", ".cs", ".php", ".swift",- ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",- ".yaml", ".yml",-]);+import { SUPPORTED_EXTENSIONS } from "../supported-extensions.js";const SUPPORTED_NAMES = new Set([".gitignore", ".gitattributes", ".editorconfig", ".env",diff --git a/ix-cli/src/cli/stale.ts b/ix-cli/src/cli/stale.tsindex a79a0239..d4a27bb7 100644--- a/ix-cli/src/cli/stale.ts+++ b/ix-cli/src/cli/stale.ts@@ -1,6 +1,7 @@import * as fs from "node:fs";import * as path from "node:path";import { IxClient } from "../client/api.js";+import { SUPPORTED_EXTENSIONS } from "./supported-extensions.js";export interface StaleInfo {lastIngestAt: string | null;@@ -9,15 +10,6 @@ export interface StaleInfo {sampleChangedFiles: string[];}-const SUPPORTED_EXTENSIONS = new Set([- ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",- ".scala", ".sc", ".java",- ".py", ".rb", ".go", ".rs",- ".md", ".mdx",- ".json", ".yaml", ".yml", ".toml",- ".sql", ".graphql", ".gql",-]);-const SUPPORTED_NAMES = new Set([".gitignore", ".gitattributes", ".editorconfig", ".env",".eslintrc", ".prettierrc", ".babelrc",diff --git a/ix-cli/src/cli/supported-extensions.ts b/ix-cli/src/cli/supported-extensions.tsnew file mode 100644index 00000000..61142cc3--- /dev/null+++ b/ix-cli/src/cli/supported-extensions.ts@@ -0,0 +1,34 @@+// Single source of truth for which file extensions ix discovers and ingests.+//+// This MUST stay in sync with core-ingestion's EXT_MAP (languages.ts): every+// extension a parser handles belongs here, or those files are never walked by+// `ix map` / `ix watch` / stale detection. It previously lived as three drifting+// copies (ingest/watch/stale) that fell behind every new parser — hence one+// shared set. Dockerfile/Makefile and other extensionless files are matched by+// name in the individual commands, not here.+export const SUPPORTED_EXTENSIONS = new Set<string>([+ ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",+ ".py", ".java", ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",+ ".cs", ".go", ".rb", ".rs", ".php", ".kt", ".kts", ".swift",+ ".scala", ".sc",+ ".yaml", ".yml",+ ".dockerfile",+ ".sql",+ ".json",+ ".toml",+ ".md", ".markdown",+ ".r",+ ".sas",+ ".ex", ".exs",+ ".mk", ".makefile",+ ".lua",+ ".sh", ".bash", ".zsh", ".ksh",+ ".hs", ".lhs",+ ".zig",+ ".html", ".htm", ".xhtml",+ ".xml", ".xsd", ".xsl", ".xslt", ".wsdl",+ ".csproj", ".vbproj", ".fsproj", ".props", ".targets", ".plist",+ ".tf", ".tfvars", ".hcl",+ ".css", ".scss", ".sass", ".less",+ ".tex", ".sty", ".cls", ".ltx", ".latex",+]);