AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): title frontmatter prose from its title key (#2556) (#2560)
Jake Elbirtbdfb329eb2eb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex cd9508c89..de99fcf42 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -30,6 +30,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0restarts the patience clock and the backoff; rounds that settle nothingstill count, so a server that answers 429 forever fails at the same limitsas before.+- `xerj autoindex` titles a prose file that opens with YAML frontmatter from+ its `title:` key, or from the first line after the closing `---` when there+ is none (#2556). Every such file (Jekyll, Hugo, Obsidian) used to be titled+ `---`, its opening fence. A leading `---` block that is not parseable+ frontmatter (unclosed, not a YAML mapping, over 64 KiB) takes the first line+ that is not a bare `---`; the body, the frontmatter text in it and+ `start_line` are unchanged. An unchanged file is not re-extracted, so re-index+ with `xerj autoindex --fresh` to retitle documents already in an index.## [1.0.0-rc.100] - 2026-10-26diff --git a/engine/crates/xerj-autoindex/src/extract/txt.rs b/engine/crates/xerj-autoindex/src/extract/txt.rsindex a3be67acd..ca119b1c8 100644--- a/engine/crates/xerj-autoindex/src/extract/txt.rs+++ b/engine/crates/xerj-autoindex/src/extract/txt.rs@@ -1,6 +1,6 @@-//! Plain text: prose files become one document record (title = first line,-//! body split into ~32KB sections); line-oriented files become overlapping-//! line-window chunks.+//! Plain text: prose files become one document record (title = first line, or+//! a frontmatter `title:`; body split into ~32KB sections); line-oriented+//! files become overlapping line-window chunks.//!//! ## Why line-oriented text is chunked, not emitted per line//!@@ -38,6 +38,10 @@ const CHUNK_OVERLAP: usize = 10;/// Byte ceiling per chunk, so a file of very long lines cannot produce a/// multi-megabyte record that would blow up `_source` on every hit.const CHUNK_MAX_BYTES: usize = 8 << 10;+/// Largest frontmatter block `split_frontmatter` will hand to the YAML parser.+/// The sniffer only routes frontmatter here when it closes inside its 8 KiB+/// prefix, so this bounds the parse locally rather than by that invariant.+const FRONTMATTER_MAX_BYTES: usize = 64 << 10;pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {let mut stats = ExtractStats::default();@@ -46,11 +50,8 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStatsreturn Ok(stats);};let (text, _) = crate::sniff::decode_text(&bytes);- let title = text- .lines()- .find(|l| !l.trim().is_empty())- .map(|l| {- let t = l.trim();+ let title = prose_title(&text)+ .map(|t| {let mut s: String = t.chars().take(200).collect();if s.len() < t.len() {s.push('…');@@ -79,6 +80,78 @@ pub fn extract_prose(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStatsOk(stats)}+/// The title of a prose document: the first non-blank line, unless the text+/// opens with a frontmatter block.+///+/// #2556: the sniffer routes frontmatter over a prose body here (#551), so for+/// every Jekyll/Hugo/Obsidian page the first line is the opening `---` and+/// every page was titled `---`. A frontmatter `title:` wins; without one, the+/// first line after the block. A bare `---` rule is never the title, even when+/// the leading block is not frontmatter we can parse. `None` when there is no+/// line to use.+fn prose_title(text: &str) -> Option<String> {+ let mut rest = text;+ if let Some((title, after)) = split_frontmatter(text) {+ if title.is_some() {+ return title;+ }+ rest = after;+ }+ rest.lines()+ .map(str::trim)+ .find(|l| !l.is_empty() && *l != "---")+ .map(str::to_string)+}++/// Splits a leading frontmatter block off `text`: its `title:` (whitespace+/// collapsed, `None` when absent or blank) and the text after the closing+/// fence. The fences follow the sniffer's #551 lookahead (first non-blank line+/// `---`, next `---` line closes it); the block must be at most+/// `FRONTMATTER_MAX_BYTES` and parse as a YAML mapping, so prose between two+/// `---` rules is not frontmatter.+fn split_frontmatter(text: &str) -> Option<(Option<String>, &str)> {+ let mut lines = text.split_inclusive('\n');+ let mut offset = 0;+ loop {+ let line = lines.next()?;+ offset += line.len();+ if !line.trim().is_empty() {+ if line.trim() != "---" {+ return None;+ }+ break;+ }+ }+ let block_start = offset;+ for line in lines {+ let line_start = offset;+ if line_start - block_start > FRONTMATTER_MAX_BYTES {+ return None;+ }+ offset += line.len();+ if line.trim() == "---" {+ let serde_yaml::Value::Mapping(map) =+ serde_yaml::from_str(&text[block_start..line_start]).ok()?+ else {+ return None;+ };+ // `title: 1984` is a YAML number, still a title.+ let title = map+ .get("title")+ .and_then(|v| match v {+ serde_yaml::Value::String(s) => Some(s.clone()),+ serde_yaml::Value::Number(n) => Some(n.to_string()),+ serde_yaml::Value::Bool(b) => Some(b.to_string()),+ _ => None,+ })+ .map(|t| t.split_whitespace().collect::<Vec<_>>().join(" "))+ .filter(|t| !t.is_empty());+ return Some((title, &text[offset..]));+ }+ }+ None+}+pub fn extract_lines(path: &Path,gzip: bool,@@ -338,4 +411,139 @@ mod chunk_tests {);}}++ /// Runs `extract_prose` over `text` and returns each record's+ /// (title, start_line, body).+ fn prose(name: &str, text: &str) -> Vec<(String, u64, String)> {+ let dir = std::env::temp_dir().join(format!("xerj-prose-title-{}", std::process::id()));+ std::fs::create_dir_all(&dir).unwrap();+ let p = dir.join(name);+ std::fs::write(&p, text).unwrap();+ let mut got = Vec::new();+ {+ let mut sink = |r: RawRecord| {+ got.push((+ r.fields["title"].as_str().unwrap().to_string(),+ r.fields["start_line"].as_u64().unwrap(),+ r.fields["body"].as_str().unwrap().to_string(),+ ));+ true+ };+ extract_prose(&p, false, &mut sink).unwrap();+ }+ let _ = std::fs::remove_file(&p);+ got+ }++ /// #2556: the sniffer routes frontmatter over a prose body here (#551), so+ /// the title must come from the frontmatter, not its opening `---` fence.+ /// The body and its start line are unchanged: the frontmatter stays+ /// searchable.+ #[test]+ fn frontmatter_title_key_titles_the_document() {+ let text = "---\ntitle: Deploy runbook\ndate: 2026-10-01\n---\n\n# Deploy runbook\n\nRoll the canary first.\n";+ let got = prose("deploy.md", text);+ assert_eq!(got.len(), 1);+ assert_eq!(got[0].0, "Deploy runbook");+ assert_eq!(got[0].1, 1, "the body still starts at the fence");+ assert_eq!(got[0].2, text.trim(), "the frontmatter stays in the body");+ }++ #[test]+ fn quoted_frontmatter_title_is_unquoted() {+ let got = prose(+ "q.md",+ "---\ntitle: \"Deploy: the runbook\"\n---\nRoll the canary first.\n",+ );+ assert_eq!(got[0].0, "Deploy: the runbook");+ let got = prose(+ "s.md",+ "---\ntitle: 'It''s live'\n---\nRoll the canary first.\n",+ );+ assert_eq!(got[0].0, "It's live");+ let got = prose(+ "f.md",+ "---\ntitle: >\n Deploy\n runbook\n---\nRoll the canary first.\n",+ );+ assert_eq!(got[0].0, "Deploy runbook", "a folded title is one line");+ }++ /// No `title:` key: the first line after the closing fence, as a file+ /// without frontmatter would get.+ #[test]+ fn frontmatter_without_a_title_uses_the_first_body_line() {+ let got = prose(+ "n.md",+ "\n---\ndate: 2026-10-01\ntags: [ops]\n---\n\nRoll the canary first.\nThen promote.\n",+ );+ assert_eq!(got[0].0, "Roll the canary first.");+ let got = prose("e.md", "---\ntitle: \"\"\n---\nRoll the canary first.\n");+ assert_eq!(+ got[0].0, "Roll the canary first.",+ "an empty title is no title"+ );+ }++ #[test]+ fn crlf_frontmatter_and_a_number_title() {+ let got = prose(+ "c.md",+ "---\r\ntitle: Deploy runbook\r\n---\r\nRoll the canary first.\r\n",+ );+ assert_eq!(got[0].0, "Deploy runbook");+ let got = prose("y.md", "---\ntitle: 1984\n---\nRoll the canary first.\n");+ assert_eq!(got[0].0, "1984", "a YAML number is still a title");+ }++ /// Only a closed, bounded block that parses as a YAML mapping is+ /// frontmatter. Anything else (an unclosed fence, prose between two `---`+ /// rules, YAML the parser refuses, an oversized block) takes the first+ /// line that is not a bare `---` rule.+ #[test]+ fn a_leading_rule_that_is_not_frontmatter_never_titles_the_page() {+ let got = prose("u.md", "---\ntitle: never closed\nRoll the canary first.\n");+ assert_eq!(got[0].0, "title: never closed");+ let got = prose(+ "r.md",+ "---\nRoll the canary first, then promote.\n---\nCheck the dashboards.\n",+ );+ assert_eq!(got[0].0, "Roll the canary first, then promote.");+ let got = prose(+ "d.md",+ "---\ntitle: First\ntitle: Second\n---\nRoll the canary first.\n",+ );+ assert_eq!(got[0].0, "title: First", "duplicate keys do not parse");+ let pad = "x".repeat(FRONTMATTER_MAX_BYTES);+ let got = prose(+ "big.md",+ &format!("---\ntitle: Big\npad: {pad}\n---\nRoll the canary first.\n"),+ );+ assert_eq!(got[0].0, "title: Big", "an oversized block is not parsed");+ }++ /// The route the fix depends on: a Jekyll-shaped page sniffs as prose+ /// (#551) and then takes its frontmatter title.+ #[test]+ fn a_sniffed_frontmatter_page_takes_its_frontmatter_title() {+ let dir = std::env::temp_dir().join(format!("xerj-prose-sniff-{}", std::process::id()));+ std::fs::create_dir_all(&dir).unwrap();+ let p = dir.join("deploy.md");+ std::fs::write(+ &p,+ "---\ntitle: Deploy runbook\ndate: 2026-10-01\n---\n\n# Deploy runbook\n\nRoll the canary first, then promote the release.\n",+ )+ .unwrap();+ let family = crate::sniff::sniff(&p).unwrap().family;+ assert_eq!(family, crate::sniff::Family::TxtProse);+ let mut titles = Vec::new();+ {+ let mut sink = |r: RawRecord| {+ titles.push(r.fields["title"].as_str().unwrap().to_string());+ true+ };+ extract_prose(&p, false, &mut sink).unwrap();+ }+ let _ = std::fs::remove_file(&p);+ assert_eq!(titles, ["Deploy runbook"]);+ }}