AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): don't sniff a badge-line README as JSON (#1988)
xerj-orgdb189209cd8a
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex 00692f4cd..a3a91f54e 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -1902,9 +1902,58 @@ fn skip_ws_back(win: &[u8], mut end: usize) -> usize {}fn looks_like_json_start(t: &str) -> bool {- // starts with { or [ and the first ~200 chars look like JSON tokens- let head: String = t.chars().take(200).collect();- head.contains(':') || head.contains('[') || head.contains('{')+ // Pretty-printed JSON, and a JSON value cut off by the 8 KiB sniff+ // prefix, parse as one value or fail because the input ran out. Either+ // one is JSON. A `{`/`[` opener that is prose fails with a syntax error+ // instead and must not take this family.+ //+ // The previous test ("the first ~200 chars contain `:`, `[` or `{`") was+ // true for every opener — the opener itself is one of those bytes — so a+ // README whose first line is a badge or link row+ // (`[](https://…/actions)`, `[Docs](https://x) | [Discord](https://y)`)+ // sniffed as JSON and the extractor junked it (#1983). `](` outside a+ // JSON string is a markdown link, including the link texts that happen+ // to be JSON values (`[true](https://…)`, `[1](https://…)`). Valid JSON+ // can contain that byte pair only inside a string.+ if first_line_has_markdown_link(t) {+ return false;+ }+ use serde::Deserialize as _;+ let mut de = serde_json::Deserializer::from_str(t);+ match serde_json::Value::deserialize(&mut de) {+ Ok(_) => true,+ Err(e) => e.is_eof(),+ }+}++/// `](` outside a JSON string on the first line: a markdown link or badge.+fn first_line_has_markdown_link(t: &str) -> bool {+ let line = t.lines().next().unwrap_or("").trim();+ if !line.starts_with('[') {+ return false;+ }+ let b = line.as_bytes();+ let mut in_string = false;+ let mut escape = false;+ let mut i = 0;+ while i < b.len() {+ let c = b[i];+ if in_string {+ if escape {+ escape = false;+ } else if c == b'\\' {+ escape = true;+ } else if c == b'"' {+ in_string = false;+ }+ } else if c == b'"' {+ in_string = true;+ } else if c == b']' && b.get(i + 1) == Some(&b'(') {+ return true;+ }+ i += 1;+ }+ false}fn yaml_line_ratio(nonblank: &[&str]) -> f64 {@@ -4270,6 +4319,118 @@ mod tests {);}+ /// #1983: a README that opens with a badge or link row starts with `[`+ /// and contains `:`, so the old JSON-start test sniffed it as JSON and+ /// the extractor junked the file. The name is irrelevant — sniffing+ /// never trusts the extension — so the same shape must be prose with+ /// and without `.md`. A pretty-printed JSON array, a JSON value cut off+ /// by the sniff prefix, and a JSON string that merely *contains* a+ /// badge stay JSON.+ #[test]+ fn readme_opening_with_a_badge_or_link_row_is_prose_not_json() {+ let badge = "[](https://example.com/actions)\n\n\+ # Widget\n\n\+ Widget is a reference implementation of the widget protocol.\n\+ Agents should be able to quote this sentence from the index.\n";+ let links = "[Docs](https://x) | [Discord](https://y)\n\n\+ # Widget\n\n\+ Widget is a reference implementation of the widget protocol.\n\+ Agents should be able to quote this sentence from the index.\n";+ // `{` opener that is a note, not an object. Same failure mode.+ let brace = "{this is a note, not a JSON object\n\n\+ # Widget\n\n\+ The opening brace is prose punctuation, not a document.\n";+ // Same badge text under both names: the extension is not what decides.+ for (name, text) in [+ ("README.md", badge),+ ("README", badge),+ ("LINKS", links),+ ("NOTE", brace),+ ] {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join(name);+ std::fs::write(&path, text).unwrap();+ let sn = sniff(&path).unwrap();+ assert!(+ matches!(sn.family, Family::TxtProse | Family::TxtLines),+ "{name} classified as {:?}, want prose",+ sn.family+ );+ let mut recs = Vec::new();+ let stats = crate::extract::extract(&path, &sn, None, &mut |r| {+ recs.push(r);+ true+ })+ .unwrap();+ assert_eq!(stats.junk, 0, "{name} was junked");+ assert!(!recs.is_empty(), "{name} produced no records");+ let indexed = recs+ .iter()+ .filter_map(|r| {+ r.fields+ .get("body")+ .or_else(|| r.fields.get("text"))+ .and_then(|v| v.as_str())+ })+ .collect::<Vec<_>>()+ .join("\n");+ assert!(+ indexed.contains("Widget"),+ "{name} records lost the document text: {indexed}"+ );+ }++ let pretty =+ "[\n {\"id\": 1, \"name\": \"alpha\"},\n {\"id\": 2, \"name\": \"beta\"}\n]\n";+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("array.txt");+ std::fs::write(&path, pretty).unwrap();+ let sn = sniff(&path).unwrap();+ assert_eq!(sn.family, Family::Json, "pretty-printed JSON array");+ let mut n = 0usize;+ let stats = crate::extract::extract(&path, &sn, None, &mut |_r| {+ n += 1;+ true+ })+ .unwrap();+ assert_eq!(stats.junk, 0);+ assert_eq!(n, 2, "one record per array element");++ // A badge stored *inside* a JSON string is still a JSON array.+ assert_eq!(+ classify(+ "[\n \"[](https://example.com/actions)\"\n]\n"+ ),+ Family::Json+ );+ // Sniff prefix cut mid-value (the long-first-line JSONL shape).+ assert_eq!(+ classify("{\"id\":1,\"name\":\"alpha\",\"bio\":\"this value is cut off mid-string"),+ Family::Json+ );+ // Under the JSONL line-ratio, but the file still opens with a JSON+ // value, so it stays Json and the extractor's line fallback runs.+ assert_eq!(+ classify("{\"a\":1}\n{\"a\":2}\nnot json\nnot json\nnot json\n"),+ Family::Json+ );+ // Link text that is itself a JSON value (`true`, `1`) is still a link.+ assert!(+ matches!(+ classify("[true](https://example.com)\n\n# Widget\n\nA link, not an array.\n"),+ Family::TxtProse | Family::TxtLines+ ),+ "keyword link text"+ );+ assert!(+ matches!(+ classify("[1](https://example.com)\n\n# Widget\n\nA link, not an array.\n"),+ Family::TxtProse | Family::TxtLines+ ),+ "numeric link text"+ );+ }+#[test]fn csv_dialect_semicolon_decimal_comma() {let lines = vec![