Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): index HTML partials and templates, not junk XML (#2740) (#2743)

Jake Elbirt89cfb0dfbba6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0231b48fe..d2f682989 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -258,6 +258,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
reading to the table reading. The file is set aside with a rebuild message
(exit 3) and every other file is still indexed. It used to stop the whole
run with "frozen assignment has 0 matches for group" (#2720).
+- `xerj autoindex` indexes HTML partials and template files instead of
+ junk-filing them as malformed XML. A file that opens with markup and has
+ closing tags was read as XML, so a folder of 16 or more such files became an
+ XML dataset and every file was junked at its first `<%`, `&&` or unclosed
+ tag, with the text after that point lost. `.html` and `.htm` are now read as
+ HTML, and `.hbs`, `.handlebars`, `.mustache`, `.ejs`, `.erb`, `.jsp`,
+ `.aspx`, `.cshtml`, `.razor`, `.twig`, `.jinja`, `.jinja2`, `.j2`, `.njk`,
+ `.liquid`, `.astro`, `.marko` and `.riot` as text, also when gzipped. Real
+ XML, including `.xml`, `.xaml` and `.svg`, is unchanged. An index built
+ before this fix keeps its junked files until each one changes or the folder
+ is indexed again with `--fresh` (#2740).
## [1.0.0-rc.100] - 2026-10-26
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 2ff782b65..abbb577d6 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -7624,3 +7624,55 @@ fn a_new_sheet_in_an_indexed_workbook_is_set_aside_not_fatal() {
];
sheet_edit_is_set_aside(&[("Sales", sales_rows(false)), ("Q4", q4)], "Q4");
}
+
+/// #2740: a views folder holds dozens of similar templates. Sixteen or more
+/// form an XML dataset of their own (`dataset::MIN_DATASET_RECORDS`), where
+/// the strict XML reader junked every one at its first `<%` or unclosed tag
+/// and dropped the text after it. Twenty ERB views and twenty HTML partials
+/// must index cleanly, end of file included.
+#[test]
+fn a_folder_of_templates_and_html_fragments_is_indexed_not_junked() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::create_dir_all(corpus.path().join("views")).unwrap();
+ fs::create_dir_all(corpus.path().join("partials")).unwrap();
+ for i in 1..=20 {
+ fs::write(
+ corpus.path().join(format!("views/view{i}.erb")),
+ format!(
+ "<div class=\"profile\">\n <% if @user && @user.admin? %>\n \
+ <%= link_to \"Admin\", admin_path %>\n <% end %>\n</div>\n\
+ <p>lateerb{i}</p>\n"
+ ),
+ )
+ .unwrap();
+ fs::write(
+ corpus.path().join(format!("partials/card{i}.html")),
+ format!(
+ "<div class=\"card\" *ngIf=\"items.length > 0 && !loading\">\n \
+ <input [(ngModel)]=\"query\">\n <span>{{{{ label }}}}</span>\n</div>\n\
+ <p>latehtml{i}</p>\n"
+ ),
+ )
+ .unwrap();
+ }
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+
+ let (code, summary) = run_index_report(config).unwrap();
+ let summary = summary.expect("the run commits a generation");
+ assert_eq!(summary["junk_records_total"], 0, "{summary}");
+ assert_eq!(summary["files_junk"], 0, "{summary}");
+ assert_eq!(code, 0, "{summary}");
+ let docs: Vec<String> = endpoint.data_docs().iter().map(Value::to_string).collect();
+ for needle in ["lateerb7", "latehtml7"] {
+ assert!(
+ docs.iter().any(|d| d.contains(needle)),
+ "{needle} is not in any indexed record"
+ );
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rs
index 51220e9b2..2fb2be005 100644
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@ -507,9 +507,40 @@ fn sniff_bytes(
if is_sql_filename(logical_path) && matches!(out.family, Family::TxtLines | Family::TxtProse) {
out.family = Family::Code;
}
+ // #2740: an HTML partial or a server/JS template opens with `<` and has
+ // closing tags, so the content check calls it XML, and the XML extractor
+ // junks it at the first `<%`, `&&` or unclosed `<br>`, losing the text
+ // after it. The name settles it, as it does for `.sql` above. Only an XML
+ // verdict is replaced: a full page stays HTML, and a template that reads
+ // as anything else keeps that family.
+ if out.family == Family::Xml {
+ if let Some(family) = markup_template_family(logical_path, &nonblank) {
+ out.family = family;
+ }
+ }
Ok(out)
}
+/// #2740: the family for a name that is never standalone XML. `.html` and
+/// `.htm` are HTML, read by its tolerant tokenizer. A template is text, so the
+/// code inside `<% %>`, `{{ }}` and `{% %}` stays searchable. `.vue` and
+/// `.svelte` are not here: #2732 tracks them as code. A trailing `.gz` is
+/// looked through, as [`is_sql_filename`] does, since the sniff reads the
+/// decompressed bytes.
+fn markup_template_family(path: &Path, nonblank: &[&str]) -> Option<Family> {
+ let name = path.file_name()?.to_str()?.to_ascii_lowercase();
+ let name = name.strip_suffix(".gz").unwrap_or(&name);
+ let (_, ext) = name.rsplit_once('.')?;
+ match ext {
+ "html" | "htm" => Some(Family::Html),
+ "hbs" | "handlebars" | "mustache" | "ejs" | "erb" | "jsp" | "aspx" | "cshtml" | "razor"
+ | "twig" | "jinja" | "jinja2" | "j2" | "njk" | "liquid" | "astro" | "marko" | "riot" => {
+ Some(txt_kind(nonblank))
+ }
+ _ => None,
+ }
+}
+
/// `.sql`, or a gzipped `.sql.gz`. Case-insensitive. The dump-vs-source
/// decision is content; this only says the name is SQL.
pub(crate) fn is_sql_filename(path: &Path) -> bool {
@@ -5861,3 +5892,101 @@ mod ole2_sniff_tests {
assert_eq!(fam, Family::Xml);
}
}
+
+#[cfg(test)]
+mod template_sniff_tests {
+ use super::*;
+
+ fn family(body: &str, name: &str) -> Family {
+ sniff_bytes(body.as_bytes(), Path::new(name), Path::new(name), false)
+ .unwrap()
+ .family
+ }
+
+ /// #2740: an Angular-style partial has no `<html>` or `<body>`, so the
+ /// content check calls it XML; a `.html` or `.htm` name makes it HTML.
+ #[test]
+ fn an_html_fragment_is_html_not_xml() {
+ let fragment = "<div class=\"widget\" *ngIf=\"items.length > 0 && !loading\">\n\
+ \x20 <input [(ngModel)]=\"query\" placeholder=\"search\">\n\
+ \x20 <span>{{ label }}</span>\n\
+ </div>\n";
+ assert_eq!(family(fragment, "widget.component.html"), Family::Html);
+ assert_eq!(family(fragment, "WIDGET.HTM"), Family::Html);
+ assert_eq!(family(fragment, "widget.html.gz"), Family::Html);
+ assert_eq!(family(fragment, "widget.xml"), Family::Xml);
+ let page =
+ "<!DOCTYPE html>\n<html><head><title>t</title></head><body><p>x</p></body></html>\n";
+ assert_eq!(family(page, "index.html"), Family::Html);
+ }
+
+ /// #2740: a markup-first template is text under its own name, and the
+ /// same bytes under a `.xml` name are still XML.
+ #[test]
+ fn a_markup_first_template_is_text_not_xml() {
+ let template = "<div class=\"profile\">\n\
+ \x20 <% if @user && @user.admin? %>\n\
+ \x20 <%= link_to \"Admin\", admin_path %>\n\
+ \x20 <% end %>\n\
+ \x20 {{#if author}}<h1>{{ name }}</h1>{{/if}}\n\
+ \x20 {% for item in menu %}<a href=\"{{ item.url }}\">x</a><br>{% endfor %}\n\
+ </div>\n";
+ assert_eq!(family(template, "profile.xml"), Family::Xml);
+ for ext in [
+ "hbs",
+ "handlebars",
+ "mustache",
+ "ejs",
+ "erb",
+ "jsp",
+ "aspx",
+ "cshtml",
+ "razor",
+ "twig",
+ "jinja",
+ "jinja2",
+ "j2",
+ "njk",
+ "liquid",
+ "astro",
+ "marko",
+ "riot",
+ "ERB",
+ ] {
+ let fam = family(template, &format!("profile.{ext}"));
+ assert!(
+ matches!(fam, Family::TxtLines | Family::TxtProse),
+ ".{ext} sniffed as {fam:?}"
+ );
+ }
+ let fam = family(template, "profile.html.erb.GZ");
+ assert!(
+ matches!(fam, Family::TxtLines | Family::TxtProse),
+ ".erb.gz sniffed as {fam:?}"
+ );
+ }
+
+ /// The name only replaces an XML verdict. A template that is a full page
+ /// keeps HTML, and one that opens with a directive keeps what it had.
+ #[test]
+ fn a_template_name_only_overrides_xml() {
+ let page = "<!DOCTYPE html>\n<html><body>{{ title }}</body></html>\n";
+ assert_eq!(family(page, "layout.hbs"), Family::Html);
+ let json = "{\n \"name\": \"{{ app_name }}\",\n \"port\": 8080\n}\n";
+ assert_eq!(family(json, "config.json.j2"), family(json, "config.json"));
+ }
+
+ /// Real XML keeps the XML path: data files, XAML, plain SVG. XHTML with
+ /// its `<html>` root is HTML, as before.
+ #[test]
+ fn real_xml_keeps_its_family() {
+ let xml = "<?xml version=\"1.0\"?>\n<settings>\n <feature name=\"a\">on</feature>\n</settings>\n";
+ assert_eq!(family(xml, "settings.xml"), Family::Xml);
+ let xaml = "<Window x:Class=\"App.Main\"\n xmlns=\"http://schemas.microsoft.com/winfx/2006/xaml/presentation\">\n <Grid><Button Content=\"OK\"/></Grid>\n</Window>\n";
+ assert_eq!(family(xaml, "Main.xaml"), Family::Xml);
+ let svg = "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 24 24\">\n <path d=\"M12 2L2 7z\"/>\n <title>logo</title>\n</svg>\n";
+ assert_eq!(family(svg, "logo.svg"), Family::Xml);
+ let xhtml = "<?xml version=\"1.0\"?>\n<html xmlns=\"http://www.w3.org/1999/xhtml\">\n<body><p>x</p></body>\n</html>\n";
+ assert_eq!(family(xhtml, "page.xhtml"), Family::Html);
+ }
+}