Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): JSONL with a fat first line indexes per line, not junk

xerj-orgfbfc736bb1ca
diff --git a/engine/crates/xerj-autoindex/src/extract/json.rs b/engine/crates/xerj-autoindex/src/extract/json.rs
index 969fd77ac..52fb169d1 100644
--- a/engine/crates/xerj-autoindex/src/extract/json.rs
+++ b/engine/crates/xerj-autoindex/src/extract/json.rs
@@ -8,7 +8,7 @@
use super::{
emit_document_with_fields, flatten_object, ExtractStats, FieldOrigin, RawRecord, Sink,
- MAX_WHOLE_FILE,
+ MAX_LINE, MAX_WHOLE_FILE,
};
use anyhow::Result;
use serde_json::{Map, Value};
@@ -23,8 +23,19 @@ pub fn extract(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {
let v: Value = match serde_json::from_slice(&bytes) {
Ok(v) => v,
Err(_) => {
- stats.junk += 1;
- return Ok(stats);
+ // Not one JSON value. Before junking, try the file as JSONL:
+ // a JSONL export whose FIRST line is longer than the sniff
+ // prefix (8 KB) sniffs as `Json` — the sample holds zero
+ // COMPLETE lines, so `classify_structured`'s per-line test
+ // never runs and `looks_like_json_start` routes the whole
+ // file here. Measured on the published rust-vulns pack:
+ // osv/records.jsonl (735 records, first line ≥ 8 KB) was
+ // junked as "no records extracted (json candidate family)"
+ // and `xerj corpus index` accepted 1,230 of 1,963 records as
+ // a success. A file that is neither one JSON value nor a
+ // sequence of JSON lines junks exactly as before — every
+ // line fails the per-line parse too.
+ return jsonl_fallback(&bytes, sink);
}
};
match v {
@@ -245,6 +256,47 @@ pub(crate) fn is_keep_note_file(path: &Path) -> bool {
}
}
+/// Whole-file parse failed — re-parse the same in-memory bytes line by
+/// line under the JSONL contract (see the fallback note in [`extract`]).
+/// Mirrors `jsonl::extract`'s per-line semantics: blank lines skip, a line
+/// at the 16 MB cap junks, an object becomes one flattened record with a
+/// byte-offset locator, anything else junks. No `limit_bytes` exists here
+/// (the family reads the whole file), so there is no truncated-tail
+/// exemption to carry over.
+fn jsonl_fallback(bytes: &[u8], sink: Sink) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let mut offset: u64 = 0;
+ for line in bytes.split(|&b| b == b'\n') {
+ let start = offset;
+ offset += line.len() as u64 + 1;
+ let trimmed = super::jsonl::trim_ws(line);
+ if trimmed.is_empty() {
+ continue;
+ }
+ if line.len() >= MAX_LINE {
+ stats.junk += 1;
+ continue;
+ }
+ match serde_json::from_slice::<Value>(trimmed) {
+ Ok(Value::Object(m)) => {
+ stats.records += 1;
+ if !sink(RawRecord {
+ fields: flatten_object(m),
+ locator: format!("b{start}"),
+ group: None,
+ origin: FieldOrigin::Data,
+ }) {
+ return Ok(stats);
+ }
+ }
+ Ok(_) | Err(_) => {
+ stats.junk += 1;
+ }
+ }
+ }
+ Ok(stats)
+}
+
fn emit(v: Value, locator: &str, sink: Sink, stats: &mut ExtractStats) -> bool {
let fields = match v {
Value::Object(m) => flatten_object(m),
@@ -482,6 +534,34 @@ mod tests {
assert!(recs.is_empty());
}
+ /// The published rust-vulns pack's osv half is a JSONL file whose first
+ /// line (an OSV record with a large `affected` array) exceeds the sniff
+ /// prefix — zero complete lines in the sample, so the file sniffs as
+ /// `Json` and arrives HERE, not at the JSONL extractor. It must index
+ /// line by line, not junk: `corpus index` accepted 1,230 of 1,963
+ /// records as a success before this fallback existed.
+ #[test]
+ fn a_jsonl_file_whose_first_line_beats_the_sniff_prefix_still_indexes_per_line() {
+ let fat = format!(
+ r#"{{"id":"OSV-1","affected":{:?}}}"#,
+ vec!["x".repeat(9_000); 2usize]
+ );
+ assert!(
+ fat.len() > 8_192,
+ "the first line must beat the 8 KB sample"
+ );
+ let text = format!("{fat}\n{{\"id\":\"OSV-2\"}}\n");
+ let (stats, recs) = run(&text);
+ assert_eq!((stats.records, stats.junk), (2, 0));
+ assert_eq!(recs[0].locator, "b0", "byte-offset locator, JSONL contract");
+ assert_eq!(recs[1].fields["id"], serde_json::json!("OSV-2"));
+
+ // A mixed file — some parseable lines, one corrupt — keeps the
+ // JSONL semantics: good lines index, the corrupt one junks.
+ let (stats, _) = run("{\"id\":\"a\"}\n{broken\n{\"id\":\"c\"}\n");
+ assert_eq!((stats.records, stats.junk), (2, 1));
+ }
+
#[test]
fn an_empty_array_yields_nothing_at_all_without_being_junk() {
let (stats, recs) = run("[]");
diff --git a/engine/crates/xerj-autoindex/src/extract/jsonl.rs b/engine/crates/xerj-autoindex/src/extract/jsonl.rs
index fb1bcd0a6..71d10e118 100644
--- a/engine/crates/xerj-autoindex/src/extract/jsonl.rs
+++ b/engine/crates/xerj-autoindex/src/extract/jsonl.rs
@@ -84,7 +84,7 @@ pub fn read_capped_line(r: &mut dyn std::io::BufRead, out: &mut Vec<u8>) -> Resu
}
}
-fn trim_ws(b: &[u8]) -> &[u8] {
+pub(super) fn trim_ws(b: &[u8]) -> &[u8] {
let start = b
.iter()
.position(|c| !c.is_ascii_whitespace())
diff --git a/engine/crates/xerj-autoindex/src/xc.rs b/engine/crates/xerj-autoindex/src/xc.rs
index 98dfab477..583528af8 100644
--- a/engine/crates/xerj-autoindex/src/xc.rs
+++ b/engine/crates/xerj-autoindex/src/xc.rs
@@ -1200,6 +1200,61 @@ fn run_corpus_index(args: &[String]) -> i32 {
)
}
+/// A pack-materialized corpus carries its record count on disk: one JSON
+/// line per record under `corpora/<name>/*/records.jsonl`. The verify gate
+/// in [`corpus_index_flow`] only checks the node answered MORE THAN ZERO —
+/// which let a build that silently junked a whole source file pass as a
+/// success (the published rust-vulns pack indexed 1,230 of 1,963 records:
+/// `osv/records.jsonl`'s first line beats the 8 KB sniff prefix, the file
+/// sniffed as a single JSON document and was junked as "json candidate
+/// family" — fixed by the JSONL fallback in `extract::json`, warned here in
+/// case any other path loses records). Warn loudly when the node holds
+/// fewer records than the pack does. Git-clone corpora (code sources, where
+/// junk is normal) have no `records.jsonl` and are exempt by construction.
+fn warn_short_of_pack_records(name: &str, root: &Path, docs: Option<u64>) {
+ let Some((lines, files)) = pack_record_gap(name, root, docs) else {
+ return;
+ };
+ eprintln!("xerj corpus index: WARNING — the pack holds {lines} records across {files} records.jsonl file(s),");
+ eprintln!("xerj corpus index: but the node answered only {} records for this corpus. A source file was", docs.unwrap_or(0));
+ eprintln!("xerj corpus index: silently junked — check the junk report (autoindex-catalog, doc_kind=junk)");
+ eprintln!("xerj corpus index: before trusting `xerj code {name}` coverage.");
+}
+
+/// The on-disk pack record count versus what the node answered:
+/// `Some((lines, files))` when a pack corpus's `records.jsonl` files hold
+/// MORE lines than the node has records — the shape of a silently junked
+/// source file. `None` when there is nothing to compare against (no corpus
+/// dir, a git-clone corpus with no records.jsonl, an unanswerable count) or
+/// nothing missing.
+fn pack_record_gap(name: &str, root: &Path, docs: Option<u64>) -> Option<(u64, u64)> {
+ let docs = docs?;
+ let entries = std::fs::read_dir(root.join("corpora").join(name)).ok()?;
+ let mut lines = 0u64;
+ let mut files = 0u64;
+ for e in entries.flatten() {
+ use std::io::Read;
+ let Ok(mut r) = std::fs::File::open(e.path().join("records.jsonl")) else {
+ continue;
+ };
+ let mut buf = [0u8; 65536];
+ let mut file_lines = 0u64;
+ loop {
+ match r.read(&mut buf) {
+ Ok(0) => break,
+ Ok(n) => file_lines += buf[..n].iter().filter(|&&b| b == b'\n').count() as u64,
+ Err(_) => return None,
+ }
+ }
+ if file_lines == 0 {
+ continue;
+ }
+ lines += file_lines;
+ files += 1;
+ }
+ (files > 0 && docs < lines).then_some((lines, files))
+}
+
/// The build/verify/swap flow over an injected node, autoindex runner and
/// clock (#1004) — the destructive-operation contracts are pinned by the
/// tests below against a [`FakeNode`](tests::FakeNode), which is why none of
@@ -1382,6 +1437,7 @@ fn corpus_index_flow(
}
// Verified. Switch readers first, retire second: a crash between
// the two leaves a duplicate, never a gap.
+ warn_short_of_pack_records(name, root, Some(docs));
let _ = state::write_state(
root,
name,
@@ -1431,7 +1487,9 @@ fn corpus_index_flow(
eprintln!("xerj corpus index: stays live until the replacement verifies).");
return rc;
}
- let docs = count_under(node, &prefix)
+ let live = count_under(node, &prefix);
+ warn_short_of_pack_records(name, root, live);
+ let docs = live
.map(|n| n.to_string())
.unwrap_or_else(|| "?".to_string());
let _ = state::write_state(
@@ -2049,6 +2107,49 @@ mod tests {
assert!(!node.live(&format!("xc-kv-b{T0}-000")));
}
+ #[test]
+ fn a_build_short_of_the_packs_records_is_named_not_swallowed() {
+ // the pack-materialized shape: two records.jsonl files, 5 + 3 lines
+ let root = corpus_root("kv");
+ std::fs::create_dir_all(root.join("corpora/kv/a")).unwrap();
+ std::fs::write(
+ root.join("corpora/kv/a/records.jsonl"),
+ "{}\n{}\n{}\n{}\n{}\n",
+ )
+ .unwrap();
+ std::fs::create_dir_all(root.join("corpora/kv/b")).unwrap();
+ std::fs::write(root.join("corpora/kv/b/records.jsonl"), "{}\n{}\n{}\n").unwrap();
+
+ // the node answered fewer records than the pack holds → the gap
+ let gap = pack_record_gap("kv", &root, Some(6));
+ assert_eq!(
+ gap,
+ Some((8, 2)),
+ "1,230-of-1,963 must be named, not verified"
+ );
+
+ // full coverage and over-coverage (corpus.json rides along as extra
+ // docs) → no gap
+ assert_eq!(pack_record_gap("kv", &root, Some(8)), None);
+ assert_eq!(pack_record_gap("kv", &root, Some(10)), None);
+
+ // a git-clone corpus (source repos, junk is normal there) has no
+ // records.jsonl at all → exempt by construction
+ let clone_root = corpus_root("git");
+ std::fs::create_dir_all(clone_root.join("corpora/git/crate-src")).unwrap();
+ std::fs::write(
+ clone_root.join("corpora/git/crate-src/Cargo.toml"),
+ "[package]\n",
+ )
+ .unwrap();
+ assert_eq!(pack_record_gap("git", &clone_root, Some(0)), None);
+
+ // an unanswerable count says nothing
+ assert_eq!(pack_record_gap("kv", &root, None), None);
+ // so does a corpus this root does not hold
+ assert_eq!(pack_record_gap("ghost", &root, Some(0)), None);
+ }
+
#[test]
fn a_failed_first_build_that_wrote_records_is_kept_and_marked_salvaged() {
let root = corpus_root("kv");