AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): stop sniffing markdown-with-frontmatter as YAML
snmsoodan735ea46ecf2c
diff --git a/engine/crates/xerj-autoindex/src/extract/txt.rs b/engine/crates/xerj-autoindex/src/extract/txt.rsindex a3be67acd..44d410ebf 100644--- a/engine/crates/xerj-autoindex/src/extract/txt.rs+++ b/engine/crates/xerj-autoindex/src/extract/txt.rs@@ -39,6 +39,35 @@ const CHUNK_OVERLAP: usize = 10;/// multi-megabyte record that would blow up `_source` on every hit.const CHUNK_MAX_BYTES: usize = 8 << 10;+/// Byte offset just past a leading `---` … `---` frontmatter block, or `None`+/// when the text does not open with one.+///+/// A closing fence must exist: an unterminated opening `---` is an ordinary+/// horizontal rule, and a document that merely starts with one keeps its first+/// line as the title.+fn frontmatter_end(text: &str) -> Option<usize> {+ let mut offset = 0usize;+ let mut opened = false;+ for line in text.split_inclusive('\n') {+ offset += line.len();+ let trimmed = line.trim();+ if !opened {+ if trimmed.is_empty() {+ continue;+ }+ if trimmed != "---" {+ return None;+ }+ opened = true;+ continue;+ }+ if trimmed == "---" {+ return Some(offset);+ }+ }+ None+}+pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {let mut stats = ExtractStats::default();let Some(bytes) = super::read_whole(path, gzip, TXT_CAP)? else {@@ -46,7 +75,16 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStatsreturn Ok(stats);};let (text, _) = crate::sniff::decode_text(&bytes);- let title = text+ // #2556: a markdown file that opens with a `---` frontmatter block took its+ // title from the fence itself, so every such page was titled `---` in the+ // index and in search results. The frontmatter is metadata, not the+ // document: skip it when looking for the title. It stays in the indexed body+ // below so `description:`/`tags:` remain searchable.+ let title_source = match frontmatter_end(&text) {+ Some(end) => &text[end..],+ None => text.as_str(),+ };+ let title = title_source.lines().find(|l| !l.trim().is_empty()).map(|l| {@@ -338,4 +376,38 @@ mod chunk_tests {);}}++ /// #2556: a markdown page that opens with a `---` frontmatter block was+ /// titled `---` — the fence itself was the first non-blank line. The+ /// frontmatter is metadata, so the title has to come from the body.+ #[test]+ fn a_frontmatter_markdown_page_is_titled_from_its_body() {+ let dir = tempfile::tempdir().unwrap();+ let p = dir.path().join("post.md");+ std::fs::write(+ &p,+ "---\ntitle: A page\ntags: [a, b]\n---\n\n# Outage postmortem\n\nThe pool was exhausted.\n",+ )+ .unwrap();+ let mut titles = Vec::new();+ {+ let mut sink = |r: RawRecord| {+ titles.push(r.fields["title"].as_str().unwrap_or("").to_string());+ true+ };+ extract_prose(&p, false, &mut sink).unwrap();+ }+ assert_eq!(titles, vec!["# Outage postmortem".to_string()]);+ }++ /// An unterminated opening `---` is a horizontal rule, not a fence, so the+ /// title stays the first line.+ #[test]+ fn an_unterminated_fence_does_not_hide_the_title() {+ assert_eq!(frontmatter_end("---\nJust prose.\n"), None);+ assert_eq!(+ frontmatter_end("---\ntitle: x\n---\nbody\n"),+ Some("---\ntitle: x\n---\n".len())+ );+ }}diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex 477ca15cf..51220e9b2 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -488,7 +488,11 @@ fn sniff_bytes(.filter(|l| !l.trim().is_empty()).collect();- let mut out = mk(classify_text(&text, &nonblank));+ let md_named = logical_path+ .extension()+ .and_then(|e| e.to_str())+ .is_some_and(is_markdown_ext);+ let mut out = mk(classify_text(&text, &nonblank, md_named));out.encoding = encoding;if out.family == Family::Csv {out.csv = sniff_csv_dialect(&nonblank);@@ -1400,7 +1404,20 @@ pub fn decode_text(bytes: &[u8]) -> (String, &'static str) {decode(bytes)}-fn classify_text(text: &str, nonblank: &[&str]) -> Family {+/// Markdown extensions that conventionally carry a `---` frontmatter block.+/// Used to settle the one case the line-shape tier cannot: `- item` bullets and+/// `Key: value` prose are valid YAML too, so content alone cannot tell a short+/// markdown body from a YAML document (#2447). Extension is the same signal the+/// code branch above already trusts for "code vs prose is not reliably+/// content-sniffable".+fn is_markdown_ext(ext: &str) -> bool {+ matches!(+ ext.to_ascii_lowercase().as_str(),+ "md" | "markdown" | "mdx" | "mdown" | "mkd" | "mkdn" | "mdwn"+ )+}++fn classify_text(text: &str, nonblank: &[&str], md_named: bool) -> Family {// Email wins first: an RFC 5322 message opens with a header block whose lines// (`From:`/`Date:`/`Message-ID:`/…) would otherwise be misread as prose or a// key:value log. Gated on a STRONG header plus >=3 canonical headers so an@@ -1444,31 +1461,50 @@ fn classify_text(text: &str, nonblank: &[&str]) -> Family {// frontmatter file with a single non-blank body line still went to// the YAML extractor, which keeps the frontmatter mapping and drops// that line. One line is enough to judge on its shape.- if !body.is_empty() && yaml_like_count(body) * 2 < body.len() {+ if !body.is_empty() {let body_text = body.join("\n");- // #587 item 1: classify the body by its real structured family- // (JSON/CSV/HTML/logs/SQL), not prose-only, so a- // frontmatter-over-CSV/JSON body matches its frontmatter-less- // twin. `classify_structured` never re-enters this- // YAML/frontmatter branch, so a recursive frontmatter parse- // cannot occur. It runs before the parse tier below because- // JSON *is* valid YAML — measured over 36,936 real- // frontmatter-shaped files, putting the parse first pulled 341- // JSON bodies and one XML body back into `Yaml` and undid #587- // item 1 for them.- if let Some(family) = classify_structured(&body_text, body) {- return family;- }- // #587 item 2: the line-shape test above is not enough on its- // own — a genuine multi-document YAML stream whose later- // document is dominated by block scalars, comments and- // continuation lines fails it and was re-classified as prose,- // losing its per-document key structure. Parse the body: a real- // stream parses, a markdown body does not (`parses_as_yaml_stream`).- if !parses_as_yaml_stream(&body_text) {- return txt_kind(body);+ // #2447: a markdown *name* settles the question the line tier+ // cannot. `- item` bullets and `Key: value` prose are valid YAML+ // too, so a short markdown body under frontmatter failed+ // `yaml_like_count(body) * 2 < body.len()` and skipped the parse+ // tier entirely — falling through to `leads_with_marker` below,+ // which sent the whole file to the YAML extractor and dropped the+ // body. Measured on six realistic short-post shapes (bullet list,+ // link list, `Note:`/`Warning:` prose, heading + list, one-line+ // body, plain paragraph), four classified `Yaml`; the frontmatter+ // that every Jekyll/Hugo/Docusaurus/MkDocs/Astro page carries was+ // enough to make a docs corpus unsearchable. A `.md`/`.mdx` file+ // that opens with `---` has frontmatter there, not a YAML document+ // separator, so its body decides. `classify_structured` still runs+ // first, so a frontmatter-over-CSV/JSON body keeps its structured+ // family (#587 item 1).+ let body_decides = md_named || yaml_like_count(body) * 2 < body.len();+ if body_decides {+ // #587 item 1: classify the body by its real structured family+ // (JSON/CSV/HTML/logs/SQL), not prose-only, so a+ // frontmatter-over-CSV/JSON body matches its frontmatter-less+ // twin. `classify_structured` never re-enters this+ // YAML/frontmatter branch, so a recursive frontmatter parse+ // cannot occur. It runs before the parse tier below because+ // JSON *is* valid YAML — measured over 36,936 real+ // frontmatter-shaped files, putting the parse first pulled 341+ // JSON bodies and one XML body back into `Yaml` and undid #587+ // item 1 for them.+ if let Some(family) = classify_structured(&body_text, body) {+ return family;+ }+ // #587 item 2: the line-shape test above is not enough on its+ // own — a genuine multi-document YAML stream whose later+ // document is dominated by block scalars, comments and+ // continuation lines fails it and was re-classified as prose,+ // losing its per-document key structure. Parse the body: a real+ // stream parses, a markdown body does not+ // (`parses_as_yaml_stream`).+ if md_named || !parses_as_yaml_stream(&body_text) {+ return txt_kind(body);+ }+ // else: a real YAML stream — fall through to `Family::Yaml`.}- // else: a real YAML stream — fall through to `Family::Yaml`.}}}@@ -4273,7 +4309,7 @@ mod tests {fn classify(s: &str) -> Family {let lines: Vec<&str> = s.lines().filter(|l| !l.trim().is_empty()).collect();- classify_text(s, &lines)+ classify_text(s, &lines, false)}#[test]@@ -4868,7 +4904,64 @@ mod text_family_tests {/// through the real entry point rather than against `txt_kind` directly.fn classify_full(text: &str) -> Family {let nonblank: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();- classify_text(text, &nonblank)+ classify_text(text, &nonblank, false)+ }++ /// Same, as a file whose name says markdown (#2447).+ fn classify_full_named(text: &str, md_named: bool) -> Family {+ let nonblank: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();+ classify_text(text, &nonblank, md_named)+ }++ /// #2447: a short markdown file with YAML frontmatter was still sniffed as+ /// YAML, so the YAML extractor kept the frontmatter mapping and dropped the+ /// body — a docs corpus that is mostly short pages indexed nothing searchable.+ ///+ /// The line tier cannot decide this on its own: `- item` bullets and+ /// `Key: value` prose are valid YAML. All four shapes below were `Yaml`+ /// before this change, and a frontmatter-less twin of each was prose. The+ /// name is what settles it, exactly as the code branch settles "code vs+ /// prose" by extension.+ #[test]+ fn short_markdown_with_frontmatter_is_not_yaml() {+ let bodies = [+ "- npm\n- pnpm\n- yarn\n",+ "- [Install](install.md)\n- [Usage](usage.md)\n",+ "Note: read this first\nWarning: mind the gap\n",+ "# Steps\n\n- one\n- two\n",+ ];+ for body in bodies {+ let with_frontmatter = format!("---\ntitle: A page\ntags: [a, b]\n---\n\n{body}");+ // The content-only decision is the bug, stated here so the test+ // carries its own before/after: bullets and `Key: value` prose are+ // valid YAML, so nothing in the text says markdown.+ assert_eq!(+ classify_full_named(&with_frontmatter, false),+ Family::Yaml,+ "content alone reads this as YAML, body was {body:?}"+ );+ assert_eq!(+ classify_full_named(&with_frontmatter, true),+ Family::TxtProse,+ "a .md file with frontmatter must index its body, body was {body:?}"+ );+ }+ }++ /// The markdown name must not swallow a genuinely structured body: the+ /// frontmatter-over-CSV case from #587 item 1 still resolves by the body.+ #[test]+ fn a_markdown_name_does_not_override_a_structured_body() {+ let csv = "---\ntitle: Team roster\n---\n\nname,role\nalice,eng\nbob,design\n";+ assert_eq!(classify_full_named(csv, true), Family::Csv);+ }++ /// A name that does not say markdown keeps the old content-only decision, so+ /// a real YAML stream under `---` is still YAML.+ #[test]+ fn a_yaml_stream_keeps_yaml_when_the_name_does_not_say_markdown() {+ let stream = "---\napiVersion: v1\nkind: List\n---\napiVersion: v1\nkind: Pod\n";+ assert_eq!(classify_full_named(stream, false), Family::Yaml);}/// #1504: `journalctl -o short-iso` output and vsftpd `xferlog` transfer