Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat(parser): add HTML parser support (.html, .htm, .xhtml) (closes #129) (#252)
Joseph Mikhail4239e65d48d8
diff --git a/core-ingestion/package.json b/core-ingestion/package.jsonindex 852d70e9..cba097cf 100644--- a/core-ingestion/package.json+++ b/core-ingestion/package.json@@ -38,6 +38,7 @@"tree-sitter-bash": "^0.21.0","tree-sitter-elixir": "^0.3.5","tree-sitter-haskell": "^0.23.1",+ "tree-sitter-html": "^0.23.2","tree-sitter-kotlin": "^0.3.8","tree-sitter-make": "^1.1.1","tree-sitter-sas": "^0.4.2",diff --git a/core-ingestion/src/__tests__/queries.html.test.ts b/core-ingestion/src/__tests__/queries.html.test.tsnew file mode 100644index 00000000..f7ed3eb0--- /dev/null+++ b/core-ingestion/src/__tests__/queries.html.test.ts@@ -0,0 +1,50 @@+import { describe, it, expect } from 'vitest';+import { parseFile } from '../index.js';+import { SupportedLanguages } from '../languages.js';++const h = parseFile('/repo/i.html', '<html></html>\n');+const describeFn = h && h.language === SupportedLanguages.HTML ? describe : describe.skip;++describeFn('HTML queries', () => {+ it('detects language as HTML', () => {+ expect(parseFile('/repo/index.html', '<!DOCTYPE html><html></html>')!.language)+ .toBe(SupportedLanguages.HTML);+ });++ it('emits IMPORTS for script src, link href and anchor/img resources', () => {+ const result = parseFile('/repo/index.html', `+<!DOCTYPE html>+<html>+<head>+ <link rel="stylesheet" href="styles/main.css">+ <script src="js/app.js"></script>+</head>+<body>+ <a href="about.html">About</a>+ <img src="logo.png">+</body>+</html>+`);+ expect(result).not.toBeNull();+ const imports = result!.relationships+ .filter(r => r.predicate === 'IMPORTS')+ .map(r => r.importRaw);+ expect(imports).toEqual(+ expect.arrayContaining(['styles/main.css', 'js/app.js', 'about.html', 'logo.png']),+ );+ });++ it('captures custom elements (web components) as definitions', () => {+ const result = parseFile('/repo/app.html', `+<body>+ <app-root></app-root>+ <my-button label="go"></my-button>+ <div class="plain"></div>+</body>+`);+ expect(result).not.toBeNull();+ const names = result!.entities.map(e => e.name);+ expect(names).toEqual(expect.arrayContaining(['app-root', 'my-button']));+ expect(names).not.toContain('div');+ });+});diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex d08e4215..cd0b1fb6 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -39,6 +39,7 @@ const Make = tryLoadGrammar('tree-sitter-make');const Bash = tryLoadGrammar('tree-sitter-bash');const Haskell = tryLoadGrammar('tree-sitter-haskell');const Zig = tryLoadGrammar('@tree-sitter-grammars/tree-sitter-zig');+const Html = tryLoadGrammar('tree-sitter-html');// tree-sitter-sas uses ESM bindings with top-level await — incompatible with tryLoadGrammar (CJS require)let SAS: any = null;// @ts-ignore@@ -183,6 +184,7 @@ const GRAMMAR_MAP: Partial<Record<SupportedLanguages, any>> = {...(Bash ? { [SupportedLanguages.Bash]: Bash } : {}),...(Haskell ? { [SupportedLanguages.Haskell]: Haskell } : {}),...(Zig ? { [SupportedLanguages.Zig]: Zig } : {}),+ ...(Html ? { [SupportedLanguages.HTML]: Html } : {}),};// Capture key prefix → NodeKind stringdiff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.tsindex 51587066..69f0b618 100644--- a/core-ingestion/src/languages.ts+++ b/core-ingestion/src/languages.ts@@ -27,6 +27,7 @@ export enum SupportedLanguages {Bash = 'bash',Haskell = 'haskell',Zig = 'zig',+ HTML = 'html',}const EXT_MAP: Record<string, SupportedLanguages> = {@@ -76,6 +77,9 @@ const EXT_MAP: Record<string, SupportedLanguages> = {'.hs': SupportedLanguages.Haskell,'.lhs': SupportedLanguages.Haskell,'.zig': SupportedLanguages.Zig,+ '.html': SupportedLanguages.HTML,+ '.htm': SupportedLanguages.HTML,+ '.xhtml': SupportedLanguages.HTML,};export function languageFromPath(filePath: string): SupportedLanguages | null {diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.tsindex 7e126a36..4478087f 100644--- a/core-ingestion/src/queries.ts+++ b/core-ingestion/src/queries.ts@@ -1605,6 +1605,24 @@ export const R_QUERIES = `(#eq? @_src "source")) @import`;+// HTML. Resource dependencies as imports (<script src>, <link href>, and+// <a/img/iframe/source/use href|src>), and custom elements (hyphenated tag names+// = web components) as definitions.+export const HTML_QUERIES = `+((script_element+ (start_tag+ (attribute (attribute_name) @_a (quoted_attribute_value (attribute_value) @import.source))))+ (#eq? @_a "src")) @import++((element+ (start_tag (tag_name) @_t+ (attribute (attribute_name) @_a (quoted_attribute_value (attribute_value) @import.source))))+ (#match? @_t "^(link|a|img|iframe|source|use)$")+ (#match? @_a "^(href|src)$")) @import++((element (start_tag (tag_name) @name) (#match? @name "-")) @definition.class)+`;+// Zig. Functions (top-level + struct/union methods are all function_declaration),// const-bound struct/enum/union types, calls (bare + field-access), and the// @import("path") builtin as an import.@@ -1721,5 +1739,6 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {[SupportedLanguages.Bash]: BASH_QUERIES,[SupportedLanguages.Haskell]: HASKELL_QUERIES,[SupportedLanguages.Zig]: ZIG_QUERIES,+ [SupportedLanguages.HTML]: HTML_QUERIES,};