AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): #722 - the --no-graph generated pipeline never consumed as_document
Vincenzo Lombardo5639874071e9
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex 2a0e9c4f7..0cf80b0d5 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -125,8 +125,24 @@ pub fn read_whole(path: &Path, gzip: bool, cap: u64) -> Result<Option<Vec<u8>>>/// the text itself. Emits the same `title`/`body`/`section` vocabulary as/// every other document extractor (`FieldOrigin::Extractor`).pub fn extract_as_document(path: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {+ extract_as_document_with_name(path, path, gzip, sink)+}++/// [`extract_as_document`], but the title comes from `logical_path` rather+/// than `content_path`. The `--no-graph` generated pipeline reads a+/// SEALED SNAPSHOT — a real file on disk, but under its own ordinal name+/// (`prepared/00000000`, …), not the source's — so deriving the title from+/// `content_path` there produced `"00000000"` instead of the file's own+/// name (#722). Mirrors `sniff::sniff_with_name`'s split for exactly the+/// same reason.+pub fn extract_as_document_with_name(+ content_path: &Path,+ logical_path: &Path,+ gzip: bool,+ sink: Sink,+) -> Result<ExtractStats> {let mut stats = ExtractStats::default();- let Some(bytes) = read_whole(path, gzip, MAX_WHOLE_FILE)? else {+ let Some(bytes) = read_whole(content_path, gzip, MAX_WHOLE_FILE)? else {stats.junk += 1;return Ok(stats);};@@ -135,7 +151,7 @@ pub fn extract_as_document(path: &Path, gzip: bool, sink: Sink) -> Result<Extracstats.junk += 1;return Ok(stats);}- let title = path+ let title = logical_path.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_else(|| "untitled".into());diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex df3eeaa1f..0ca1a3b47 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -1602,8 +1602,36 @@ fn prepare_artifact(// generation because one line of one log file did not parse would make a// realistic corpus unindexable. The count is sealed into the artifact so// the generation, and then the catalog, can report it.- let stats = crate::extract::extract(snapshot_blob, &sniffed, None, &mut sink)- .with_context(|| format!("extract {} into durable preparation", source.rel))?;+ //+ // #722: `assignment.as_document` (a demoted one-off config file, #173)+ // decides the record *shape*, the same precedence the legacy path's own+ // phase-B dispatch gives it (`lib.rs`, `if fa.as_document { … }`) — the+ // frozen dataset's mapping was built by re-sampling through+ // `extract_as_document`, so that is the only extractor whose output may+ // be compared against it. Before this, the durable/generated pipeline+ // never checked the flag at all: a demoted file was correctly ROUTED+ // into the docs dataset (`reconcile_plan`/`dataset::cluster` agree it+ // belongs there) but its published fields still came from its raw+ // family extractor, silently disagreeing with the mapping. `sniffed` is+ // already sniffed from `snapshot_blob`, a real path on the sealed+ // snapshot (not a byte blob), so the branch is exactly the legacy+ // path's — no format change needed.+ let stats = if assignment.as_document {+ // `extract_as_document_with_name`, not `extract_as_document`:+ // `snapshot_blob` is a real path, but to the SEALED SNAPSHOT under+ // its own ordinal name, not the source file's — the title must come+ // from `source.path` (the same split `sniff_with_name` above makes+ // for the same reason).+ crate::extract::extract_as_document_with_name(+ snapshot_blob,+ &source.path,+ sniffed.gzip,+ &mut sink,+ )+ } else {+ crate::extract::extract(snapshot_blob, &sniffed, None, &mut sink)+ }+ .with_context(|| format!("extract {} into durable preparation", source.rel))?;if let Some(error) = sink_error {return Err(error);}@@ -1919,6 +1947,67 @@ mod tests {assert_eq!(groups[0].expected_records_for("absent"), Some(0));}+ /// #722: `assignment.as_document` was never read here at all — a demoted+ /// one-off config file (#173) was correctly ROUTED into the docs dataset+ /// (`reconcile_plan`/`dataset::cluster` agree) but published through its+ /// raw family extractor, not `extract_as_document`, silently disagreeing+ /// with the docs mapping the run itself built. A first, file-path-only+ /// fix used `extract_as_document(snapshot_blob, …)` directly and derived+ /// the title from `snapshot_blob`'s own name — the SEALED SNAPSHOT's+ /// ordinal filename (`prepared/00000000`), not the source file's, so+ /// every demoted document titled itself `"00000000"`. Pins both halves:+ /// the record is document-shaped (`title`/`body`, not the source's raw+ /// JSON keys) AND the title is the real source name.+ #[test]+ fn preparation_of_a_demoted_file_publishes_a_real_document_with_the_source_title() {+ let _guard = SNAPSHOT_TEST_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state = tempfile::tempdir().unwrap();+ std::fs::write(+ corpus.path().join("config.json"),+ br#"{"host": "example.com", "port": 8080}"#,+ )+ .unwrap();+ let inventory = inventory(corpus.path());+ let mut plan = plan_for(&inventory);+ let key = inventory.keys[0].clone();+ plan.files.get_mut(&key).unwrap().as_document = true;++ let snapshot = create_prepared_snapshot(+ state.path(),+ "tx-demoted-config",+ &inventory,+ &plan,+ "test-preparation-v1",+ u64::MAX,+ )+ .unwrap();++ let prepared = snapshot.files[0].prepared.as_ref().unwrap();+ assert_eq!(prepared.records, 1);+ let root = state.path().join("sync-snapshots/tx-demoted-config");+ let ndjson = std::fs::read_to_string(root.join(&prepared.relative_ndjson)).unwrap();+ let document: Value = serde_json::from_str(ndjson.lines().nth(1).unwrap()).unwrap();+ assert_eq!(+ document["title"], "config",+ "title must be the source file's own stem, not the sealed snapshot's ordinal \+ filename: {document}"+ );+ assert!(+ document["body"]+ .as_str()+ .unwrap()+ .contains(r#""host": "example.com""#),+ "body must be the decoded document text, not raw extracted JSON fields: {document}"+ );+ assert!(+ document.get("host").is_none() && document.get("port").is_none(),+ "a demoted file must not publish its raw family-extractor fields: {document}"+ );+ }+/// `snapshot_digest` covers the serialized `PreparedArtifact` and/// `open_snapshot` re-verifies it on every resume, so a snapshot sealed/// before the per-dataset ledger existed has to keep hashing to the value