Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Merge remote-tracking branch 'origin/main' into rb258

xerj-org9df614c96b6d
diff --cc engine/crates/xerj-api/src/es_compat.rs
index 18dd66967,963bbc7fb..da51b2e49
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@@ -26977,81 -27546,26 +28311,88 @@@ async fn clone_index_to(state: &AppStat
.get_index(target)
.map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;
- // Copy all documents.
- let search_req = xerj_query::parse_request(&json!({
- "query": { "match_all": {} },
- "size": 10000,
- "from": 0,
- // Cloning copies documents verbatim: always read the COMPLETE `_source`,
- // including engine-generated embedding companions, rather than inheriting
- // the search default which hides them from ordinary API responses.
- "_source": true,
- }))
- .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;
+ // Copy EVERY document, and fail loudly if any copy fails.
+ //
+ // Issue #204, the "a partial export instead of a full one" case, verbatim.
+ // This used to be a single `size: 10000` search whose per-document writes
+ // were swallowed with `let _ =`, and then answered `{"acknowledged": true,
+ // "shards_acknowledged": true}` — so cloning an index with 10,001
+ // documents silently produced a 10,000-document copy, and a clone in which
+ // every single write failed was reported as a success. `_clone`, `_shrink`
+ // and `_split` all land here, on user data.
+ //
+ // Same keyset (`search_after` on `_id`) paging the `_reindex` handler uses,
+ // for the same reason: `from`-offset paging truncates at
+ // `max_result_window`. The source is flushed first because the in-memory
+ // memtable's sort path does not order by `_id` reliably, so un-flushed
+ // writes would be skipped or duplicated across pages.
+ //
+ // That makes the flush a CORRECTNESS precondition of the paging below, not
+ // a best-effort hint — so it is propagated rather than swallowed. A
+ // `let _ =` here would have produced exactly the defect this function was
+ // rewritten to remove: a silently partial (or duplicated) copy answering
+ // `{"acknowledged": true, "shards_acknowledged": true}`.
+ src_idx.flush().await.map_err(|e| {
+ ApiError::new(xerj_common::XerjError::internal(format!(
+ "clone of [{source}] into [{target}] was not attempted: flushing the source \
+ failed ({e}); the copy needs a flushed source to page over it without \
+ skipping or duplicating documents"
+ )))
+ })?;
- let results = src_idx
- .search(&search_req)
- .await
- .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;
+ const CLONE_PAGE_SIZE: usize = 1_000;
+ let mut search_after: Option<Value> = None;
+ let mut copied = 0usize;
- for hit in results.hits {
- if !hit.source.is_null() {
- let _ = dest_idx.index_document(Some(hit.id), hit.source).await;
+ loop {
+ let mut search_body = json!({
+ "query": { "match_all": {} },
+ "size": CLONE_PAGE_SIZE,
+ "sort": [{ "_id": "asc" }],
++ // Cloning copies documents verbatim: always read the COMPLETE
++ // `_source`, including engine-generated embedding companions,
++ // rather than inheriting the search default which hides them from
++ // ordinary API responses. Dropping this would make the clone a
++ // silently lossy copy — the same defect class the paging below
++ // exists to remove.
++ "_source": true,
+ });
+ if let Some(sa) = &search_after {
+ search_body["search_after"] = sa.clone();
+ }
+ let search_req = xerj_query::parse_request(&search_body)
+ .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;
+
+ let results = src_idx
+ .search(&search_req)
+ .await
+ .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;
+
+ let batch = results.hits.len();
+ if batch == 0 {
+ break;
+ }
+ search_after = results.hits.last().map(|h| json!([h.id.clone()]));
+
+ for hit in results.hits {
+ if hit.source.is_null() {
+ continue;
+ }
+ let id = hit.id.clone();
+ dest_idx
+ .index_document(Some(hit.id), hit.source)
+ .await
+ .map_err(|e| {
+ ApiError::new(xerj_common::XerjError::internal(format!(
+ "clone of [{source}] into [{target}] failed after {copied} \
+ document(s): document [{id}] could not be written: {e}"
+ )))
+ })?;
+ copied += 1;
+ }
+
+ if batch < CLONE_PAGE_SIZE {
+ break;
}
}
Ok(())
diff --cc engine/crates/xerj-engine/src/index.rs
index 081914e0a,1936e2435..c7b5462e1
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@@ -18050,25 -18771,42 +18795,42 @@@ impl Index
return;
}
let mut schema_changed = false;
- for (_, fc) in new_fields {
- if !schema.schema.has_field(&fc.name) {
- if schema.schema.field_count() as u32 >= self.max_fields_per_index {
- tracing::warn!(
- field = %fc.name,
- limit = self.max_fields_per_index,
- "rejecting new field: field limit reached"
- );
- break;
- }
- if matches!(fc.field_type, FieldType::Date) {
- self.has_date_fields.store(true, Ordering::Relaxed);
+ let mut field_count = schema.schema.field_count() as u32;
+ for (key, val) in obj {
+ if key.starts_with(PASSAGE_METADATA_PREFIX) {
+ continue;
+ }
+ if let Some(existing) = schema.schema.field_mut(key) {
+ if merge_dynamic_children_into(
+ existing,
+ val,
+ date_detection,
+ 1,
+ &mut field_count,
+ self.max_fields_per_index,
+ ) {
+ schema_changed = true;
}
- let _ = schema.schema.add_field(fc);
- schema_changed = true;
+ continue;
}
+ if field_count >= self.max_fields_per_index {
+ tracing::warn!(
+ field = %key,
+ limit = self.max_fields_per_index,
+ "rejecting new field: field limit reached"
+ );
+ break;
+ }
+ let fc = dynamic_field_config(key, val, date_detection);
+ if matches!(fc.field_type, FieldType::Date) {
+ self.has_date_fields.store(true, Ordering::Relaxed);
+ }
+ field_count += 1;
+ let _ = schema.schema.add_field(fc);
+ schema_changed = true;
}
if schema_changed {
- let _ = self.save_schema(&schema).await;
+ self.persist_evolved_schema(&schema).await;
}
}