Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): reconcile generation file catalogs

buger40d4b84c6674
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index 1c1ab4541..06423deb4 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -173,6 +173,100 @@ impl<'a> EsSyncBackend<'a> {
}
Ok(total)
}
+
+ fn exact_semantic_count(&self, group: &ManifestGroup, plan: &Plan) -> Result<u64> {
+ let by_slug: HashMap<&str, &crate::state::PlanDataset> = plan
+ .datasets
+ .iter()
+ .map(|dataset| (dataset.slug.as_str(), dataset))
+ .collect();
+ let mut total = 0u64;
+ for slug in &group.dataset_slugs {
+ let dataset = by_slug
+ .get(slug.as_str())
+ .copied()
+ .with_context(|| format!("group references absent dataset {slug}"))?;
+ let Some(field) = &dataset.semantic_field else {
+ continue;
+ };
+ let response = self.es.search(
+ &dataset.index,
+ &serde_json::json!({
+ "size": 0,
+ "track_total_hits": true,
+ "query": {"bool": {"must": [
+ {"term": {"ax_file": &group.content_id}},
+ {"exists": {"field": field}}
+ ]}}
+ }),
+ )?;
+ total = total
+ .checked_add(
+ response
+ .pointer("/hits/total/value")
+ .and_then(Value::as_u64)
+ .context("semantic validation response has no total hit count")?,
+ )
+ .context("semantic validation count overflow")?;
+ }
+ Ok(total)
+ }
+
+ fn reconcile_file_catalog(
+ &self,
+ old: Option<&ManifestGroup>,
+ new: Option<&ManifestGroup>,
+ desired: &GenerationManifest,
+ ) -> Result<()> {
+ for content_id in old
+ .into_iter()
+ .map(|group| group.content_id.as_str())
+ .chain(new.into_iter().map(|group| group.content_id.as_str()))
+ {
+ self.es.delete_by_query(
+ crate::catalog::CATALOG_INDEX,
+ &serde_json::json!({"term": {"file_key": content_id}}),
+ )?;
+ }
+ let Some(group) = new else {
+ return Ok(());
+ };
+ let assignment = desired
+ .plan
+ .files
+ .get(&group.content_id)
+ .context("catalog group has no desired file assignment")?;
+ let format = if assignment.gzip {
+ format!("{}(gzip)", assignment.family)
+ } else {
+ assignment.family.clone()
+ };
+ let (id, doc) = crate::catalog::file_doc(
+ &group.content_id,
+ &group.canonical.rel,
+ &format,
+ "indexed",
+ None,
+ group.expected_records,
+ 0,
+ group.content_size,
+ &format!("generation-{}", desired.generation),
+ );
+ let mut body = Vec::new();
+ append_index_action(crate::catalog::CATALOG_INDEX, &id, &doc, &mut body)?;
+ for alias in &group.aliases {
+ let (id, doc) = crate::catalog::duplicate_file_doc(
+ &group.content_id,
+ &alias.rel,
+ &alias.path_id,
+ &group.canonical.rel,
+ group.content_size,
+ &format!("generation-{}", desired.generation),
+ );
+ append_index_action(crate::catalog::CATALOG_INDEX, &id, &doc, &mut body)?;
+ }
+ checked_bulk(self.es, body)
+ }
}
impl SyncOperationBackend for EsSyncBackend<'_> {
@@ -202,7 +296,7 @@ impl SyncOperationBackend for EsSyncBackend<'_> {
crate::sync::SyncOperationKind::Delete => self.delete_group(
old.context("delete operation has no committed group")?,
&base.plan,
- ),
+ )?,
crate::sync::SyncOperationKind::Upsert => {
if let Some(old) = old {
self.delete_group(old, &base.plan)?;
@@ -210,13 +304,16 @@ impl SyncOperationBackend for EsSyncBackend<'_> {
let new = new.context("upsert operation has no desired group")?;
// Remove a partial prior retry of the desired identity too.
self.delete_group(new, &desired.plan)?;
- self.replay_prepared(snapshot, &new.content_id)
+ self.replay_prepared(snapshot, &new.content_id)?;
+ }
+ crate::sync::SyncOperationKind::Metadata => {
+ self.replay_metadata(
+ base,
+ new.context("metadata operation has no desired group")?,
+ )?;
}
- crate::sync::SyncOperationKind::Metadata => self.replay_metadata(
- base,
- new.context("metadata operation has no desired group")?,
- ),
}
+ self.reconcile_file_catalog(old, new, desired)
}
fn validate(
@@ -228,12 +325,33 @@ impl SyncOperationBackend for EsSyncBackend<'_> {
for dataset in &desired.plan.datasets {
self.es.refresh(&dataset.index)?;
}
+ self.es.refresh(crate::catalog::CATALOG_INDEX)?;
for group in &desired.groups {
anyhow::ensure!(
self.exact_group_count(group, &desired.plan)? == group.expected_records,
"live record count disagrees with desired group {}",
group.group_id
);
+ let semantic = self.exact_semantic_count(group, &desired.plan)?;
+ anyhow::ensure!(
+ semantic == group.expected_passages && semantic == group.expected_vectors,
+ "live semantic count disagrees with desired group {}",
+ group.group_id
+ );
+ let catalog = self.es.search(
+ crate::catalog::CATALOG_INDEX,
+ &serde_json::json!({
+ "size": 0,
+ "track_total_hits": true,
+ "query": {"term": {"file_key": &group.content_id}}
+ }),
+ )?;
+ anyhow::ensure!(
+ catalog.pointer("/hits/total/value").and_then(Value::as_u64)
+ == Some(1 + group.aliases.len() as u64),
+ "catalog canonical/alias count disagrees with desired group {}",
+ group.group_id
+ );
}
let desired_ids: std::collections::HashSet<&str> = desired
.groups
@@ -421,6 +539,17 @@ fn checked_bulk(es: &crate::esclient::Es, body: Vec<u8>) -> Result<()> {
Ok(())
}
+fn append_index_action(index: &str, id: &str, doc: &Value, body: &mut Vec<u8>) -> Result<()> {
+ serde_json::to_writer(
+ &mut *body,
+ &serde_json::json!({"index": {"_index": index, "_id": id}}),
+ )?;
+ body.push(b'\n');
+ serde_json::to_writer(&mut *body, doc)?;
+ body.push(b'\n');
+ Ok(())
+}
+
fn stream_ndjson_pairs(
mut reader: impl std::io::BufRead,
bulk_bytes: usize,