AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): JSONL with a fat first line indexes per line, not junk
xerj-orgfbfc736bb1ca
diff --git a/engine/crates/xerj-autoindex/src/extract/json.rs b/engine/crates/xerj-autoindex/src/extract/json.rsindex 969fd77ac..52fb169d1 100644--- a/engine/crates/xerj-autoindex/src/extract/json.rs+++ b/engine/crates/xerj-autoindex/src/extract/json.rs@@ -8,7 +8,7 @@use super::{emit_document_with_fields, flatten_object, ExtractStats, FieldOrigin, RawRecord, Sink,- MAX_WHOLE_FILE,+ MAX_LINE, MAX_WHOLE_FILE,};use anyhow::Result;use serde_json::{Map, Value};@@ -23,8 +23,19 @@ pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {let v: Value = match serde_json::from_slice(&bytes) {Ok(v) => v,Err(_) => {- stats.junk += 1;- return Ok(stats);+ // Not one JSON value. Before junking, try the file as JSONL:+ // a JSONL export whose FIRST line is longer than the sniff+ // prefix (8 KB) sniffs as `Json` — the sample holds zero+ // COMPLETE lines, so `classify_structured`'s per-line test+ // never runs and `looks_like_json_start` routes the whole+ // file here. Measured on the published rust-vulns pack:+ // osv/records.jsonl (735 records, first line ≥ 8 KB) was+ // junked as "no records extracted (json candidate family)"+ // and `xerj corpus index` accepted 1,230 of 1,963 records as+ // a success. A file that is neither one JSON value nor a+ // sequence of JSON lines junks exactly as before — every+ // line fails the per-line parse too.+ return jsonl_fallback(&bytes, sink);}};match v {@@ -245,6 +256,47 @@ pub(crate) fn is_keep_note_file(path: &Path) -> bool {}}+/// Whole-file parse failed — re-parse the same in-memory bytes line by+/// line under the JSONL contract (see the fallback note in [`extract`]).+/// Mirrors `jsonl::extract`'s per-line semantics: blank lines skip, a line+/// at the 16 MB cap junks, an object becomes one flattened record with a+/// byte-offset locator, anything else junks. No `limit_bytes` exists here+/// (the family reads the whole file), so there is no truncated-tail+/// exemption to carry over.+fn jsonl_fallback(bytes: &[u8], sink: Sink) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let mut offset: u64 = 0;+ for line in bytes.split(|&b| b == b'\n') {+ let start = offset;+ offset += line.len() as u64 + 1;+ let trimmed = super::jsonl::trim_ws(line);+ if trimmed.is_empty() {+ continue;+ }+ if line.len() >= MAX_LINE {+ stats.junk += 1;+ continue;+ }+ match serde_json::from_slice::<Value>(trimmed) {+ Ok(Value::Object(m)) => {+ stats.records += 1;+ if !sink(RawRecord {+ fields: flatten_object(m),+ locator: format!("b{start}"),+ group: None,+ origin: FieldOrigin::Data,+ }) {+ return Ok(stats);+ }+ }+ Ok(_) | Err(_) => {+ stats.junk += 1;+ }+ }+ }+ Ok(stats)+}+fn emit(v: Value, locator: &str, sink: Sink, stats: &mut ExtractStats) -> bool {let fields = match v {Value::Object(m) => flatten_object(m),@@ -482,6 +534,34 @@ mod tests {assert!(recs.is_empty());}+ /// The published rust-vulns pack's osv half is a JSONL file whose first+ /// line (an OSV record with a large `affected` array) exceeds the sniff+ /// prefix — zero complete lines in the sample, so the file sniffs as+ /// `Json` and arrives HERE, not at the JSONL extractor. It must index+ /// line by line, not junk: `corpus index` accepted 1,230 of 1,963+ /// records as a success before this fallback existed.+ #[test]+ fn a_jsonl_file_whose_first_line_beats_the_sniff_prefix_still_indexes_per_line() {+ let fat = format!(+ r#"{{"id":"OSV-1","affected":{:?}}}"#,+ vec!["x".repeat(9_000); 2usize]+ );+ assert!(+ fat.len() > 8_192,+ "the first line must beat the 8 KB sample"+ );+ let text = format!("{fat}\n{{\"id\":\"OSV-2\"}}\n");+ let (stats, recs) = run(&text);+ assert_eq!((stats.records, stats.junk), (2, 0));+ assert_eq!(recs[0].locator, "b0", "byte-offset locator, JSONL contract");+ assert_eq!(recs[1].fields["id"], serde_json::json!("OSV-2"));++ // A mixed file — some parseable lines, one corrupt — keeps the+ // JSONL semantics: good lines index, the corrupt one junks.+ let (stats, _) = run("{\"id\":\"a\"}\n{broken\n{\"id\":\"c\"}\n");+ assert_eq!((stats.records, stats.junk), (2, 1));+ }+#[test]fn an_empty_array_yields_nothing_at_all_without_being_junk() {let (stats, recs) = run("[]");diff --git a/engine/crates/xerj-autoindex/src/extract/jsonl.rs b/engine/crates/xerj-autoindex/src/extract/jsonl.rsindex fb1bcd0a6..71d10e118 100644--- a/engine/crates/xerj-autoindex/src/extract/jsonl.rs+++ b/engine/crates/xerj-autoindex/src/extract/jsonl.rs@@ -84,7 +84,7 @@ pub fn read_capped_line(r: &mut dyn std::io::BufRead, out: &mut Vec<u8>) -> Resu}}-fn trim_ws(b: &[u8]) -> &[u8] {+pub(super) fn trim_ws(b: &[u8]) -> &[u8] {let start = b.iter().position(|c| !c.is_ascii_whitespace())diff --git a/engine/crates/xerj-autoindex/src/xc.rs b/engine/crates/xerj-autoindex/src/xc.rsindex 98dfab477..583528af8 100644--- a/engine/crates/xerj-autoindex/src/xc.rs+++ b/engine/crates/xerj-autoindex/src/xc.rs@@ -1200,6 +1200,61 @@ fn run_corpus_index(args: &[String]) -> i32 {)}+/// A pack-materialized corpus carries its record count on disk: one JSON+/// line per record under `corpora/<name>/*/records.jsonl`. The verify gate+/// in [`corpus_index_flow`] only checks the node answered MORE THAN ZERO —+/// which let a build that silently junked a whole source file pass as a+/// success (the published rust-vulns pack indexed 1,230 of 1,963 records:+/// `osv/records.jsonl`'s first line beats the 8 KB sniff prefix, the file+/// sniffed as a single JSON document and was junked as "json candidate+/// family" — fixed by the JSONL fallback in `extract::json`, warned here in+/// case any other path loses records). Warn loudly when the node holds+/// fewer records than the pack does. Git-clone corpora (code sources, where+/// junk is normal) have no `records.jsonl` and are exempt by construction.+fn warn_short_of_pack_records(name: &str, root: &Path, docs: Option<u64>) {+ let Some((lines, files)) = pack_record_gap(name, root, docs) else {+ return;+ };+ eprintln!("xerj corpus index: WARNING — the pack holds {lines} records across {files} records.jsonl file(s),");+ eprintln!("xerj corpus index: but the node answered only {} records for this corpus. A source file was", docs.unwrap_or(0));+ eprintln!("xerj corpus index: silently junked — check the junk report (autoindex-catalog, doc_kind=junk)");+ eprintln!("xerj corpus index: before trusting `xerj code {name}` coverage.");+}++/// The on-disk pack record count versus what the node answered:+/// `Some((lines, files))` when a pack corpus's `records.jsonl` files hold+/// MORE lines than the node has records — the shape of a silently junked+/// source file. `None` when there is nothing to compare against (no corpus+/// dir, a git-clone corpus with no records.jsonl, an unanswerable count) or+/// nothing missing.+fn pack_record_gap(name: &str, root: &Path, docs: Option<u64>) -> Option<(u64, u64)> {+ let docs = docs?;+ let entries = std::fs::read_dir(root.join("corpora").join(name)).ok()?;+ let mut lines = 0u64;+ let mut files = 0u64;+ for e in entries.flatten() {+ use std::io::Read;+ let Ok(mut r) = std::fs::File::open(e.path().join("records.jsonl")) else {+ continue;+ };+ let mut buf = [0u8; 65536];+ let mut file_lines = 0u64;+ loop {+ match r.read(&mut buf) {+ Ok(0) => break,+ Ok(n) => file_lines += buf[..n].iter().filter(|&&b| b == b'\n').count() as u64,+ Err(_) => return None,+ }+ }+ if file_lines == 0 {+ continue;+ }+ lines += file_lines;+ files += 1;+ }+ (files > 0 && docs < lines).then_some((lines, files))+}+/// The build/verify/swap flow over an injected node, autoindex runner and/// clock (#1004) — the destructive-operation contracts are pinned by the/// tests below against a [`FakeNode`](tests::FakeNode), which is why none of@@ -1382,6 +1437,7 @@ fn corpus_index_flow(}// Verified. Switch readers first, retire second: a crash between// the two leaves a duplicate, never a gap.+ warn_short_of_pack_records(name, root, Some(docs));let _ = state::write_state(root,name,@@ -1431,7 +1487,9 @@ fn corpus_index_flow(eprintln!("xerj corpus index: stays live until the replacement verifies).");return rc;}- let docs = count_under(node, &prefix)+ let live = count_under(node, &prefix);+ warn_short_of_pack_records(name, root, live);+ let docs = live.map(|n| n.to_string()).unwrap_or_else(|| "?".to_string());let _ = state::write_state(@@ -2049,6 +2107,49 @@ mod tests {assert!(!node.live(&format!("xc-kv-b{T0}-000")));}+ #[test]+ fn a_build_short_of_the_packs_records_is_named_not_swallowed() {+ // the pack-materialized shape: two records.jsonl files, 5 + 3 lines+ let root = corpus_root("kv");+ std::fs::create_dir_all(root.join("corpora/kv/a")).unwrap();+ std::fs::write(+ root.join("corpora/kv/a/records.jsonl"),+ "{}\n{}\n{}\n{}\n{}\n",+ )+ .unwrap();+ std::fs::create_dir_all(root.join("corpora/kv/b")).unwrap();+ std::fs::write(root.join("corpora/kv/b/records.jsonl"), "{}\n{}\n{}\n").unwrap();++ // the node answered fewer records than the pack holds → the gap+ let gap = pack_record_gap("kv", &root, Some(6));+ assert_eq!(+ gap,+ Some((8, 2)),+ "1,230-of-1,963 must be named, not verified"+ );++ // full coverage and over-coverage (corpus.json rides along as extra+ // docs) → no gap+ assert_eq!(pack_record_gap("kv", &root, Some(8)), None);+ assert_eq!(pack_record_gap("kv", &root, Some(10)), None);++ // a git-clone corpus (source repos, junk is normal there) has no+ // records.jsonl at all → exempt by construction+ let clone_root = corpus_root("git");+ std::fs::create_dir_all(clone_root.join("corpora/git/crate-src")).unwrap();+ std::fs::write(+ clone_root.join("corpora/git/crate-src/Cargo.toml"),+ "[package]\n",+ )+ .unwrap();+ assert_eq!(pack_record_gap("git", &clone_root, Some(0)), None);++ // an unanswerable count says nothing+ assert_eq!(pack_record_gap("kv", &root, None), None);+ // so does a corpus this root does not hold+ assert_eq!(pack_record_gap("ghost", &root, Some(0)), None);+ }+#[test]fn a_failed_first_build_that_wrote_records_is_kept_and_marked_salvaged() {let root = corpus_root("kv");