AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Merge remote-tracking branch 'origin/main' into rb258
xerj-org9df614c96b6d
diff --cc engine/crates/xerj-api/src/es_compat.rsindex 18dd66967,963bbc7fb..da51b2e49--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@@ -26977,81 -27546,26 +28311,88 @@@ async fn clone_index_to(state: &AppStat.get_index(target).map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;- // Copy all documents.- let search_req = xerj_query::parse_request(&json!({- "query": { "match_all": {} },- "size": 10000,- "from": 0,- // Cloning copies documents verbatim: always read the COMPLETE `_source`,- // including engine-generated embedding companions, rather than inheriting- // the search default which hides them from ordinary API responses.- "_source": true,- }))- .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;+ // Copy EVERY document, and fail loudly if any copy fails.+ //+ // Issue #204, the "a partial export instead of a full one" case, verbatim.+ // This used to be a single `size: 10000` search whose per-document writes+ // were swallowed with `let _ =`, and then answered `{"acknowledged": true,+ // "shards_acknowledged": true}` — so cloning an index with 10,001+ // documents silently produced a 10,000-document copy, and a clone in which+ // every single write failed was reported as a success. `_clone`, `_shrink`+ // and `_split` all land here, on user data.+ //+ // Same keyset (`search_after` on `_id`) paging the `_reindex` handler uses,+ // for the same reason: `from`-offset paging truncates at+ // `max_result_window`. The source is flushed first because the in-memory+ // memtable's sort path does not order by `_id` reliably, so un-flushed+ // writes would be skipped or duplicated across pages.+ //+ // That makes the flush a CORRECTNESS precondition of the paging below, not+ // a best-effort hint — so it is propagated rather than swallowed. A+ // `let _ =` here would have produced exactly the defect this function was+ // rewritten to remove: a silently partial (or duplicated) copy answering+ // `{"acknowledged": true, "shards_acknowledged": true}`.+ src_idx.flush().await.map_err(|e| {+ ApiError::new(xerj_common::XerjError::internal(format!(+ "clone of [{source}] into [{target}] was not attempted: flushing the source \+ failed ({e}); the copy needs a flushed source to page over it without \+ skipping or duplicating documents"+ )))+ })?;- let results = src_idx- .search(&search_req)- .await- .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;+ const CLONE_PAGE_SIZE: usize = 1_000;+ let mut search_after: Option<Value> = None;+ let mut copied = 0usize;- for hit in results.hits {- if !hit.source.is_null() {- let _ = dest_idx.index_document(Some(hit.id), hit.source).await;+ loop {+ let mut search_body = json!({+ "query": { "match_all": {} },+ "size": CLONE_PAGE_SIZE,+ "sort": [{ "_id": "asc" }],++ // Cloning copies documents verbatim: always read the COMPLETE++ // `_source`, including engine-generated embedding companions,++ // rather than inheriting the search default which hides them from++ // ordinary API responses. Dropping this would make the clone a++ // silently lossy copy — the same defect class the paging below++ // exists to remove.++ "_source": true,+ });+ if let Some(sa) = &search_after {+ search_body["search_after"] = sa.clone();+ }+ let search_req = xerj_query::parse_request(&search_body)+ .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;++ let results = src_idx+ .search(&search_req)+ .await+ .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;++ let batch = results.hits.len();+ if batch == 0 {+ break;+ }+ search_after = results.hits.last().map(|h| json!([h.id.clone()]));++ for hit in results.hits {+ if hit.source.is_null() {+ continue;+ }+ let id = hit.id.clone();+ dest_idx+ .index_document(Some(hit.id), hit.source)+ .await+ .map_err(|e| {+ ApiError::new(xerj_common::XerjError::internal(format!(+ "clone of [{source}] into [{target}] failed after {copied} \+ document(s): document [{id}] could not be written: {e}"+ )))+ })?;+ copied += 1;+ }++ if batch < CLONE_PAGE_SIZE {+ break;}}Ok(())diff --cc engine/crates/xerj-engine/src/index.rsindex 081914e0a,1936e2435..c7b5462e1--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@@ -18050,25 -18771,42 +18795,42 @@@ impl Indexreturn;}let mut schema_changed = false;- for (_, fc) in new_fields {- if !schema.schema.has_field(&fc.name) {- if schema.schema.field_count() as u32 >= self.max_fields_per_index {- tracing::warn!(- field = %fc.name,- limit = self.max_fields_per_index,- "rejecting new field: field limit reached"- );- break;- }- if matches!(fc.field_type, FieldType::Date) {- self.has_date_fields.store(true, Ordering::Relaxed);+ let mut field_count = schema.schema.field_count() as u32;+ for (key, val) in obj {+ if key.starts_with(PASSAGE_METADATA_PREFIX) {+ continue;+ }+ if let Some(existing) = schema.schema.field_mut(key) {+ if merge_dynamic_children_into(+ existing,+ val,+ date_detection,+ 1,+ &mut field_count,+ self.max_fields_per_index,+ ) {+ schema_changed = true;}- let _ = schema.schema.add_field(fc);- schema_changed = true;+ continue;}+ if field_count >= self.max_fields_per_index {+ tracing::warn!(+ field = %key,+ limit = self.max_fields_per_index,+ "rejecting new field: field limit reached"+ );+ break;+ }+ let fc = dynamic_field_config(key, val, date_detection);+ if matches!(fc.field_type, FieldType::Date) {+ self.has_date_fields.store(true, Ordering::Relaxed);+ }+ field_count += 1;+ let _ = schema.schema.add_field(fc);+ schema_changed = true;}if schema_changed {- let _ = self.save_schema(&schema).await;+ self.persist_evolved_schema(&schema).await;}}