AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): keep a resumed catalog read-back on the changed set (#1267) (#2231)
xerj-orgad657c7c2e03
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 7db4f8af4..9e2807884 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -2158,6 +2158,33 @@ impl Es {Ok(v)}+ /// `POST /{index}/_mget` with the `{"ids":[...]}` short form.+ ///+ /// The catalog read-back uses this instead of a `run_id` + `search_after`+ /// walk (#1267). `_id` lookup stays O(ids) when the catalog's version map+ /// carries deletes; the sorted walk re-parsed every matching stored+ /// document on every page.+ pub fn mget(&self, index: &str, ids: &[String]) -> Result<Value> {+ self.with_retry(+ "mget",+ || {+ self.req(reqwest::Method::POST, &format!("/{index}/_mget"))+ .json(&serde_json::json!({"ids": ids}))+ .send()+ .map_err(|e| anyhow!("mget: {e}"))+ },+ |resp| {+ let status = resp.status();+ let body: Value = resp.json().unwrap_or(Value::Null);+ if status.is_success() {+ Ok(body)+ } else {+ Err(anyhow!("mget /{index} HTTP {status}: {body}"))+ }+ },+ )+ }+/// How often [`Es::search_for_verify`] re-checks that its target still/// exists while the window search is in flight (#2222). A fast window/// returns before the first check, so the common path adds no request.diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex dd56df071..233a18680 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -303,6 +303,8 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {} else if method == "POST" && path.contains("/_delete_by_query") {let deleted = delete_http(path, &body, state);(200, json!({"deleted": deleted, "failures": []}))+ } else if method == "POST" && path.ends_with("/_mget") {+ (200, mget_http(path, &body, state))} else if method == "POST" && path.ends_with("/_search") {if let Ok(query) = serde_json::from_slice::<Value>(&body) {state.lock().unwrap().search_bodies.push(query);@@ -824,6 +826,40 @@ fn delete_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> usizebefore - locked.docs.len()}+/// `POST /{index}/_mget` `{"ids":[...]}` — the catalog read-back (#1267).+/// A missing id is `found: false`, not an error, matching the engine.+fn mget_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value {+ let index = path+ .trim_start_matches('/')+ .trim_end_matches("/_mget")+ .to_owned();+ let query: Value = serde_json::from_slice(body).unwrap_or(Value::Null);+ let ids = query+ .get("ids")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default();+ let locked = state.lock().unwrap();+ let docs: Vec<Value> = ids+ .iter()+ .map(|id| {+ let Some(id) = id.as_str() else {+ return json!({"_index": index, "found": false});+ };+ match locked.docs.get(&(index.clone(), id.to_owned())) {+ Some(source) => json!({+ "_index": index,+ "_id": id,+ "found": true,+ "_source": source+ }),+ None => json!({"_index": index, "_id": id, "found": false}),+ }+ })+ .collect();+ json!({"docs": docs})+}+fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value {let indices = path.trim_start_matches('/')@@ -5272,10 +5308,9 @@ fn the_generated_path_reports_indexing_as_indexing_not_as_a_stalled_scan() {"each long step of the generated route is its own phase, in order:\n{stream}");- // #1267: that read-back phase opens with the engine's own match total as- // its denominator — a moving bar with an ETA from the first page, not a- // count with no total. The two-dataset generation writes at least its run- // document, so the denominator is non-zero.+ // #1267: that read-back phase opens at zero with a real denominator — the+ // number of catalog documents this generation publishes (at least the run+ // document), read by id. A count with no total is the frozen ticker.let readback_opened = lines.iter().find(|line| line["phase"] == "finalize-catalog-readback")diff --git a/engine/crates/xerj-autoindex/src/label_http_tests.rs b/engine/crates/xerj-autoindex/src/label_http_tests.rsindex be8413077..a3d599b30 100644--- a/engine/crates/xerj-autoindex/src/label_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/label_http_tests.rs@@ -140,6 +140,8 @@ fn serve(mut stream: TcpStream, state: &Mutex<NodeState>) {}} else if method == "POST" && path == "/_bulk" {bulk_http(&body, state)+ } else if method == "POST" && path.ends_with("/_mget") {+ (200, mget_http(&path, &body, state))} else if method == "POST" && path.ends_with("/_search") {(200, search_http(&path, &body, state))} else if method == "POST" && path.ends_with("/_delete_by_query") {@@ -183,6 +185,40 @@ fn serve(mut stream: TcpStream, state: &Mutex<NodeState>) {let _ = stream.flush();}+/// `POST /{index}/_mget`. The latest bulk of an id wins: a rewrite appends+/// another row, and the catalog read-back has to see that one.+fn mget_http(path: &str, body: &[u8], state: &Mutex<NodeState>) -> Value {+ let index = path+ .trim_start_matches('/')+ .trim_end_matches("/_mget")+ .to_owned();+ let query: Value = serde_json::from_slice(body).unwrap_or(Value::Null);+ let ids = query+ .get("ids")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default();+ let docs = state.lock().unwrap().docs.clone();+ let out: Vec<Value> = ids+ .iter()+ .map(|id| {+ let Some(id) = id.as_str() else {+ return json!({"_index": index, "found": false});+ };+ if let Some((_, _, doc)) = docs+ .iter()+ .rev()+ .find(|(doc_index, doc_id, _)| doc_index == &index && doc_id == id)+ {+ json!({"_index": index, "_id": id, "found": true, "_source": doc})+ } else {+ json!({"_index": index, "_id": id, "found": false})+ }+ })+ .collect();+ json!({"docs": out})+}+/// A small-but-real `_search` over the captured documents. The generated/// run's exact read-back verifies its sealed record counts with/// `size: 0, query: bool.filter [terms ax_file, term ax_dataset]` and readsdiff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex 8f27abc0d..4fd876db1 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -537,6 +537,177 @@ impl ReplayLookup {}}+/// How many per-digest verify windows to issue for one semantic field, and+/// how many to tick without asking (#1267).+///+/// A field whose `exists` total is zero cannot match any digest. Issuing the+/// `term` + `exists` window anyway is the pre-#1260 brute scan: one window+/// per [`EsSyncBackend::VERIFY_WINDOW`] digests, each walking the index, each+/// returning no hits. `window == 0` asks for nothing.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+struct ExistsProbePlan {+ issue_windows: u64,+ skip_windows: u64,+}++fn plan_exists_probes(values: usize, window: usize, exists_total: u64) -> ExistsProbePlan {+ if window == 0 {+ return ExistsProbePlan {+ issue_windows: 0,+ skip_windows: 0,+ };+ }+ let windows = values.div_ceil(window) as u64;+ if exists_total == 0 {+ ExistsProbePlan {+ issue_windows: 0,+ skip_windows: windows,+ }+ } else {+ ExistsProbePlan {+ issue_windows: windows,+ skip_windows: 0,+ }+ }+}++/// Catalog documents whose live `_source` is not already the sealed projection.+///+/// A resume of the same generation re-enters publication with the same+/// `run_id` and the same bytes (#1267). Those ids are not rewritten: the bulk+/// that used to send every projection document is what made the following+/// `run_id` walk match the whole catalog.+fn catalog_docs_to_write(+ expected: &BTreeMap<String, Value>,+ live: &BTreeMap<String, Value>,+) -> BTreeMap<String, Value> {+ expected+ .iter()+ .filter(|(id, document)| live.get(*id) != Some(*document))+ .map(|(id, document)| ((*id).clone(), (*document).clone()))+ .collect()+}++/// How many of the prior generation's published ids this generation kept.+///+/// Kept documents still carry the prior `run_id`. The live count of that+/// `run_id` equals this number when nothing stray remains and every kept+/// document is still there. A smaller count is the documented cross-journal+/// collision (a kept document was overwritten elsewhere). A larger count is a+/// stray. Equal is the fast path that does not walk.+fn prior_run_expected_remainder(prior_written: &BTreeSet<String>, kept: &BTreeSet<String>) -> u64 {+ prior_written.intersection(kept).count() as u64+}++/// Ids carrying one generation's `run_id`, one per line, inside that+/// generation's already-sealed snapshot directory.+///+/// The manifest digest does not cover this file. Missing means "published by+/// an older client": the next generation walks that `run_id` once. Present+/// means the next generation's stray check is a `size: 0` count.+const CATALOG_RUN_IDS_FILE: &str = "catalog-run.ids";++fn catalog_run_ids_path(state_dir: &Path, run_id: &str) -> Result<std::path::PathBuf> {+ validate_tx_id(run_id)?;+ Ok(state_dir+ .join("sync-snapshots")+ .join(run_id)+ .join(CATALOG_RUN_IDS_FILE))+}++fn write_catalog_run_ids<'a>(+ state_dir: &Path,+ run_id: &str,+ ids: impl IntoIterator<Item = &'a String>,+) -> Result<()> {+ let path = catalog_run_ids_path(state_dir, run_id)?;+ let mut lines: Vec<&str> = Vec::new();+ for id in ids {+ anyhow::ensure!(+ !id.contains('\n') && !id.contains('\r'),+ "catalog document id {id:?} contains a newline and cannot be stored one-per-line"+ );+ lines.push(id.as_str());+ }+ lines.sort_unstable();+ let parent = path+ .parent()+ .context("catalog run id file has no parent directory")?;+ let tmp = parent.join(format!("{CATALOG_RUN_IDS_FILE}.tmp"));+ {+ let mut file = OpenOptions::new()+ .create(true)+ .write(true)+ .truncate(true)+ .open(&tmp)+ .with_context(|| format!("create {}", tmp.display()))?;+ for id in &lines {+ writeln!(file, "{id}")?;+ }+ file.sync_all()?;+ }+ std::fs::rename(&tmp, &path)+ .with_context(|| format!("rename {} to {}", tmp.display(), path.display()))?;+ sync_dir(parent)?;+ Ok(())+}++fn read_catalog_run_ids(state_dir: &Path, run_id: &str) -> Result<Option<BTreeSet<String>>> {+ let path = catalog_run_ids_path(state_dir, run_id)?;+ let text = match std::fs::read_to_string(&path) {+ Ok(text) => text,+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),+ Err(error) => {+ return Err(error).with_context(|| format!("read {}", path.display()));+ }+ };+ let mut ids = BTreeSet::new();+ for line in text.lines() {+ if !line.is_empty() {+ ids.insert(line.to_owned());+ }+ }+ Ok(Some(ids))+}++/// `_id -> _source` for every `found: true` entry of an `_mget` response.+/// `found: false` is an absent id, not an error. A duplicate id is.+fn mget_sources(response: &Value) -> Result<BTreeMap<String, Value>> {+ let docs = response+ .pointer("/docs")+ .and_then(Value::as_array)+ .context("catalog mget response has no docs")?;+ let mut sources = BTreeMap::new();+ for doc in docs {+ let id = doc+ .get("_id")+ .and_then(Value::as_str)+ .context("catalog mget doc has no _id")?;+ if doc.get("found").and_then(Value::as_bool) != Some(true) {+ continue;+ }+ let source = doc+ .get("_source")+ .cloned()+ .with_context(|| format!("catalog mget doc {id} is found but has no _source"))?;+ anyhow::ensure!(+ sources.insert(id.to_owned(), source).is_none(),+ "catalog mget returned duplicate ID {id}"+ );+ }+ Ok(sources)+}++/// Advance the current phase by `n` items. Finalize-verify and the catalog+/// read-back call this once per window (or once per skipped window) so the+/// ticker moves while a request is in flight only for that one window, not+/// for the whole phase (#1267).+fn credit_verify_items(pr: &crate::progress::Progress, n: u64) {+ for _ in 0..n {+ pr.item_done(0);+ }+}+/// Production ES-compatible operation backend for graph-disabled generations.////// Upserts stream sealed index actions after removing both replaced and@@ -778,12 +949,18 @@ impl<'a> EsSyncBackend<'a> {/// `extra_filters` are appended verbatim to the window query's/// `bool.filter`: the semantic leg's `exists`, the catalog leg's/// `run_id` term.+ ///+ /// `credit` ticks the current phase once per window after that window's+ /// answer is in. The capability probe passes `false`: it is one sample,+ /// not one of the phase's windows, and a tick there would walk the bar+ /// past its denominator.fn windowed_value_counts(&self,index: &str,field: &str,values: &[String],extra_filters: &[Value],+ credit: bool,) -> Result<HashMap<String, u64>> {let mut counts: HashMap<String, u64> =values.iter().map(|value| (value.clone(), 0u64)).collect();@@ -820,6 +997,9 @@ impl<'a> EsSyncBackend<'a> {.context("batched validation bucket has no doc_count")?;counts.insert(key.to_owned(), doc_count);}+ if credit {+ credit_verify_items(self.pr, 1);+ }}Ok(counts)}@@ -908,6 +1088,9 @@ impl<'a> EsSyncBackend<'a> {"catalog read-back counted {counted} of {} matched documents in a window",window_total.unwrap_or(0));+ // One tick per window, after the window's pages have all come+ // back. The phase denominator is windows, not pages (#1267).+ credit_verify_items(self.pr, 1);}Ok(counts)}@@ -973,17 +1156,17 @@ impl<'a> EsSyncBackend<'a> {/// reaching the end of the generation's documents.const CATALOG_WALK_PAGE: usize = 1000;- /// The phase [`Self::catalog_generation`] surfaces itself under. A resume- /// that rewrote the whole catalog gives this walk a `run_id` matching every- /// document in the index (402,814 on the cve-records node), so the- /// "O(changed) by construction" invariant degenerates to O(corpus): 403- /// sorted pages at ~5.2 s each ≈ 35 min, during which the surface used to- /// show the FINISHED `finalize-catalog` ticker (every dataset already- /// counted done) with a `since_progress_s` that only climbed — which reads- /// exactly like a hang, and cost an hour of "is it deadlocked?" diagnosis- /// on the #1267 resume. The walk is its own phase now, with the engine's- /// own match total as a real denominator, so the long read-back shows a- /// moving bar and an ETA from its first page.+ /// The phase the catalog read-back surfaces itself under.+ ///+ /// Publication used to rewrite every projection document and then walk+ /// `term run_id` + `sort _id` + `search_after`. A resume of the same+ /// generation rewrote the catalog, so that term matched every document+ /// (402,814 on the cve-records node): 403 sorted pages at ~5.2 s each,+ /// behind a finished `finalize-catalog` ticker (#1267). The happy path+ /// now opens this phase on the projection's own length and reads those+ /// ids with `_mget`. [`Self::catalog_generation`] still opens it when a+ /// count does not match and the sorted walk is the way to name the+ /// difference.const CATALOG_READBACK_PHASE: &'static str = "finalize-catalog-readback";fn catalog_generation(&self, run_id: &str) -> Result<BTreeMap<String, Value>> {@@ -1171,6 +1354,156 @@ impl<'a> EsSyncBackend<'a> {.map(str::to_owned)}+ /// Ids fetched per `_mget`. Under the engine's default+ /// `limits.max_mget_docs` (10_000), so one window is one request.+ const MGET_WINDOW: usize = 1000;++ /// Read catalog `_source`s by id. `tick` credits one phase item per id+ /// after that id's window returns, so a multi-window read-back moves the+ /// bar between requests instead of once at the end.+ fn mget_catalog(&self, ids: &[String], tick: bool) -> Result<BTreeMap<String, Value>> {+ let mut observed = BTreeMap::new();+ if ids.is_empty() {+ return Ok(observed);+ }+ let _watching = self.pr.watch_file(crate::catalog::CATALOG_INDEX, 0);+ for window in ids.chunks(Self::MGET_WINDOW) {+ let response = self.es.mget(crate::catalog::CATALOG_INDEX, window)?;+ observed.extend(mget_sources(&response)?);+ if tick {+ credit_verify_items(self.pr, window.len() as u64);+ }+ }+ Ok(observed)+ }++ /// `hits.total` of `exists` on `field`. One query per semantic+ /// `(index, field)`, in place of a per-digest `term` + `exists` window+ /// that cannot match when this total is zero (#1267).+ fn field_exists_total(&self, index: &str, field: &str) -> Result<u64> {+ let response = self.es.search_for_verify(+ index,+ &serde_json::json!({+ "size": 0,+ "track_total_hits": true,+ "query": {"bool": {"filter": [{"exists": {"field": field}}]}}+ }),+ )?;+ response+ .pointer("/hits/total/value")+ .and_then(Value::as_u64)+ .context("exists preflight has no total hit count")+ }++ /// How many live catalog documents carry `run_id`. `size: 0`, so the+ /// answer is a count, not a page of stored sources.+ fn catalog_run_count(&self, run_id: &str) -> Result<u64> {+ let response = self.es.search(+ crate::catalog::CATALOG_INDEX,+ &serde_json::json!({+ "size": 0,+ "track_total_hits": true,+ "query": {"bool": {"filter": [{"term": {"run_id": run_id}}]}}+ }),+ )?;+ response+ .pointer("/hits/total/value")+ .and_then(Value::as_u64)+ .context("catalog run_id count has no total hit count")+ }++ /// Live documents still carrying the prior generation's `run_id` must be+ /// a subset of the ids this generation kept.+ ///+ /// When this process published that generation, `catalog-run.ids` is the+ /// exact id set that carried the `run_id`. The live count equals the+ /// intersection of that set with the ids kept now, or the sorted walk+ /// names the strays. A missing file (a generation published before this+ /// file existed) takes the walk once.+ fn assert_no_prior_strays(+ &self,+ prior_run_id: &str,+ projection: &crate::generation_catalog::CatalogProjection,+ ) -> Result<()> {+ let written: BTreeSet<String> = projection.documents.keys().cloned().collect();+ let kept: BTreeSet<String> = projection+ .managed_ids+ .difference(&written)+ .cloned()+ .collect();+ match read_catalog_run_ids(self.state_dir, prior_run_id)? {+ Some(prior_written) => {+ let expected = prior_run_expected_remainder(&prior_written, &kept);+ let live = self.catalog_run_count(prior_run_id)?;+ if live == expected {+ return Ok(());+ }+ self.pr.note(&format!(+ "autoindex: prior catalog generation {prior_run_id} still counts {live} \+ document(s) under its run_id against {expected} kept; walking them to name \+ any stray (#1267)"+ ));+ }+ None => {+ self.pr.note(&format!(+ "autoindex: prior catalog generation {prior_run_id} has no saved id list; \+ walking documents that still carry its run_id (#1267)"+ ));+ }+ }+ let remaining = self.catalog_generation(prior_run_id)?;+ let strays: Vec<&String> = remaining.keys().filter(|id| !kept.contains(*id)).collect();+ anyhow::ensure!(+ strays.is_empty(),+ "prior catalog generation {prior_run_id} still has {} document(s) this generation \+ neither rewrote, deleted nor kept: {:?}",+ strays.len(),+ strays+ );+ Ok(())+ }++ /// `exists` total for one semantic field.+ ///+ /// The first field of the phase is the capability probe (#1267): a slow+ /// answer is re-asked once, and a slow warm answer warns, but only when+ /// the field actually matches documents — those are the windows about to+ /// brute-scan. A zero total skips the windows, so a slow empty answer+ /// must not project that one query across the whole phase. Later fields+ /// ask once and do not warn again.+ fn semantic_exists_total(+ &self,+ index: &str,+ field: &str,+ probe: bool,+ windows_if_matched: u64,+ ) -> Result<u64> {+ let started = std::time::Instant::now();+ let mut total = self.field_exists_total(index, field)?;+ let first = started.elapsed();+ if !probe || total == 0 || first <= Self::SERVER_PROBE_SLOW {+ return Ok(total);+ }+ let started = std::time::Instant::now();+ total = self.field_exists_total(index, field)?;+ let warm = started.elapsed();+ if total > 0 && warm > Self::SERVER_PROBE_SLOW {+ let samples = std::cell::RefCell::new(vec![first, warm]);+ Self::probe_server_generation(+ self.pr,+ windows_if_matched,+ Self::SERVER_PROBE_SLOW,+ || {+ let mut samples = samples.borrow_mut();+ anyhow::ensure!(!samples.is_empty(), "probe timing exhausted");+ Ok(samples.remove(0))+ },+ || self.server_xerj_version(),+ );+ }+ Ok(total)+ }+fn exact_dataset_catalog_stats(&self,desired: &GenerationManifest,@@ -1530,6 +1863,17 @@ impl SyncOperationBackend for EsSyncBackend<'_> {&BTreeMap::new(),&BTreeSet::new(),)?;+ // #1267: read the projection's ids before writing. A resume of this+ // same generation already stored these bytes under this `run_id`;+ // sending them again is what made the old `run_id` walk match the+ // whole catalog. The phase denominator is the projection, which is+ // the changed set (#971) plus the documents a same-generation resume+ // still has to confirm.+ let ids: Vec<String> = projection.documents.keys().cloned().collect();+ self.pr+ .phase(Self::CATALOG_READBACK_PHASE, ids.len() as u64, 0);+ let mut observed = self.mget_catalog(&ids, true)?;+ let to_write = catalog_docs_to_write(&projection.documents, &observed);let mut body = Vec::new();for id in &projection.stale_ids {let action =@@ -1537,7 +1881,7 @@ impl SyncOperationBackend for EsSyncBackend<'_> {body.extend_from_slice(serde_json::to_string(&action)?.as_bytes());body.push(b'\n');}- for (id, document) in &projection.documents {+ for (id, document) in &to_write {append_index_action(crate::catalog::CATALOG_INDEX, id, document, &mut body)?;}// #955: one document per file — this body grows with the corpus. On@@ -1547,45 +1891,49 @@ impl SyncOperationBackend for EsSyncBackend<'_> {// been applied. Windowed like every other bulk; deletes stay first.// #971: the projection is now incremental — a one-file change sends// that file's document (plus datasets and the run document), not one- // document per file in the corpus.- checked_bulk_windowed(self.es, body, self.bulk_bytes)?;- self.es.refresh(crate::catalog::CATALOG_INDEX)?;- // Exactly the documents this generation wrote carry its run_id (kept- // documents keep the run_id of the generation that last touched them),- // so the run_id-scoped read-back is O(changed) by construction.- // Re-walked while short of the projection: an unflagged truncated page- // (#1212) is intermittent, and validate_observed's exact equality is- // what should judge a COMPLETE walk, not a starved one.- projection.validate_observed(- &self.catalog_generation_complete(&snapshot.tx_id, projection.documents.len())?,- )?;+ // document per file in the corpus. #1267: a document whose live+ // source already equals that projection is not in `to_write`.+ if !body.is_empty() {+ checked_bulk_windowed(self.es, body, self.bulk_bytes)?;+ self.es.refresh(crate::catalog::CATALOG_INDEX)?;+ }+ if !to_write.is_empty() {+ let written_ids: Vec<String> = to_write.keys().cloned().collect();+ for (id, source) in self.mget_catalog(&written_ids, false)? {+ observed.insert(id, source);+ }+ }+ projection.validate_observed(&observed)?;+ // The mget above sees only the projection's ids. A document this+ // generation did not project that still carries its `run_id` would+ // pass that check and inflate the next generation's kept set. The+ // count is that extra. When it disagrees, the sorted walk — retried+ // while short (#1212) — is what names the missing and unexpected ids.+ // A walk that then matches the projection is the publish: the count+ // was the optimization, and a capped total must not fail a complete+ // read-back.+ let live = self.catalog_run_count(&snapshot.tx_id)?;+ if live != projection.documents.len() as u64 {+ self.pr.note(&format!(+ "autoindex: catalog run_id {} counts {live} live documents against {} in the \+ projection; reading them back by the sorted walk (#1267)",+ snapshot.tx_id,+ projection.documents.len()+ ));+ projection.validate_observed(+ &self.catalog_generation_complete(&snapshot.tx_id, projection.documents.len())?,+ )?;+ }+ write_catalog_run_ids(self.state_dir, &snapshot.tx_id, projection.documents.keys())?;if let Some(prior_run_id) = prior_run_id {// #971 sweep. Documents still carrying the prior generation's// run_id are legitimate — they are the intentionally kept- // (unchanged) ones — but NOTHING ELSE may: every id the prior- // generation published was either rewritten (new run_id), deleted- // (stale), or kept, and the prior generation's own exact- // read-back proved its published set matched its projection. A- // document surviving under the prior run_id that this projection- // did not keep is a stray the publication failed to account for.- // A kept document may legitimately be absent here (a same-prefix- // journal on another state-dir may have overwritten it — the- // documented cross-journal collision), so the check is a subset- // check, not equality. Deliberately NOT re-walked when short: this- // is a subset check, and an unflagged truncated page (#1212) can- // only UNDER-report strays — a false pass is impossible, and a- // re-walk here would only re-run the sweep's deletes.- let remaining = self.catalog_generation(prior_run_id)?;- let written: BTreeSet<String> = projection.documents.keys().cloned().collect();- let kept: BTreeSet<&String> = projection.managed_ids.difference(&written).collect();- let strays: Vec<&String> = remaining.keys().filter(|id| !kept.contains(id)).collect();- anyhow::ensure!(- strays.is_empty(),- "prior catalog generation {prior_run_id} still has {} document(s) this generation \- neither rewrote, deleted nor kept: {:?}",- strays.len(),- strays- );+ // (unchanged) ones — but NOTHING ELSE may. The check itself is+ // unchanged (subset, not equality; a kept document may have been+ // overwritten by another journal). What changed (#1267) is that+ // the common case is a count against the ids the prior generation+ // saved, not another sorted walk of the catalog.+ self.assert_no_prior_strays(prior_run_id, &projection)?;}Ok(())}@@ -1718,7 +2066,10 @@ impl SyncOperationBackend for EsSyncBackend<'_> {readbacks += windows(values.len());}for values in semantic_wanted.values() {- readbacks += windows(values.len());+ // One exists preflight per field, then its digest windows — issued+ // when the field matches anything, ticked and skipped when it+ // matches nothing (#1267).+ readbacks += 1 + windows(values.len());}let catalog_wanted: Vec<String> = changed_groups.iter()@@ -1729,55 +2080,71 @@ impl SyncOperationBackend for EsSyncBackend<'_> {}self.pr.phase("finalize-verify", readbacks, 0);- // #1267 operational: one representative window, timed, before the- // walk. The semantic leg is preferred because its `term`+`exists`- // shape is exactly the one a server without the #1260 exists arm- // brute-scans; a generation with no semantic field probes a record- // window instead. One digest is enough — the brute path's cost is the- // index size, not the window width.- let probe = semantic_wanted- .iter()- .find_map(|((index, field), values)| {- let exists = serde_json::json!({"exists": {"field": field}});- values.first().map(|value| (index, value, vec![exists]))- })- .or_else(|| {- record_wanted- .iter()- .find_map(|(index, values)| values.first().map(|value| (index, value, vec![])))- });- if let Some((index, value, extra)) = probe {- let _probing = self.pr.file(index, 0);- Self::probe_server_generation(- self.pr,- readbacks,- Self::SERVER_PROBE_SLOW,- || {- let started = std::time::Instant::now();- self.windowed_value_counts(- index,- "ax_file",- std::slice::from_ref(value),- &extra,- )?;- Ok(started.elapsed())- },- || self.server_xerj_version(),- );+ // #1267 operational: a generation with no semantic field still times+ // one record window, so an old server is named before the walk. The+ // semantic leg's probe is the exists preflight below — a per-digest+ // `term` + `exists` sample is the query this change exists to stop+ // issuing when the field matches nothing. `watch_file` names the+ // index and does not credit an item: the probe is not one of+ // `readbacks`.+ if semantic_wanted.is_empty() {+ if let Some((index, value)) = record_wanted+ .iter()+ .find_map(|(index, values)| values.first().map(|value| (index.as_str(), value)))+ {+ let _probing = self.pr.watch_file(index, 0);+ Self::probe_server_generation(+ self.pr,+ readbacks,+ Self::SERVER_PROBE_SLOW,+ || {+ let started = std::time::Instant::now();+ self.windowed_value_counts(+ index,+ "ax_file",+ std::slice::from_ref(value),+ &[],+ false,+ )?;+ Ok(started.elapsed())+ },+ || self.server_xerj_version(),+ );+ }}let mut record_counts: HashMap<String, HashMap<String, u64>> = HashMap::new();for (index, values) in &record_wanted {- let _reading = self.pr.file(index, 0);- let counts = self.windowed_value_counts(index, "ax_file", values, &[])?;+ let _reading = self.pr.watch_file(index, 0);+ let counts = self.windowed_value_counts(index, "ax_file", values, &[], true)?;record_counts.insert(index.clone(), counts);}let mut semantic_counts: HashMap<(String, String), HashMap<String, u64>> = HashMap::new();+ let mut first_semantic = true;for ((index, field), values) in &semantic_wanted {- let _reading = self.pr.file(index, 0);- let exists = serde_json::json!({"exists": {"field": field}});- let counts = self.windowed_value_counts(index, "ax_file", values, &[exists])?;- semantic_counts.insert((index.clone(), field.clone()), counts);+ let _reading = self.pr.watch_file(index, 0);+ let probe = first_semantic;+ first_semantic = false;+ let total = self.semantic_exists_total(index, field, probe, windows(values.len()))?;+ credit_verify_items(self.pr, 1);+ let plan = plan_exists_probes(values.len(), Self::VERIFY_WINDOW, total);+ if plan.skip_windows > 0 {+ self.pr.note(&format!(+ "autoindex: semantic field {field} on {index} matches no documents; skipped \+ {} verify window(s) (#1267)",+ plan.skip_windows+ ));+ credit_verify_items(self.pr, plan.skip_windows);+ semantic_counts.insert(+ (index.clone(), field.clone()),+ values.iter().cloned().map(|value| (value, 0)).collect(),+ );+ } else {+ let exists = serde_json::json!({"exists": {"field": field}});+ let counts =+ self.windowed_value_counts(index, "ax_file", values, &[exists], true)?;+ semantic_counts.insert((index.clone(), field.clone()), counts);+ }}// Scoped to this generation's `run_id`, the same way the run-summary// read-back is (`lib.rs`). `file_key` is derived from CONTENT alone@@ -1796,7 +2163,7 @@ impl SyncOperationBackend for EsSyncBackend<'_> {let catalog_counts = if catalog_wanted.is_empty() {HashMap::new()} else {- let _reading = self.pr.file(crate::catalog::CATALOG_INDEX, 0);+ let _reading = self.pr.watch_file(crate::catalog::CATALOG_INDEX, 0);self.windowed_catalog_counts(&catalog_wanted, &snapshot.tx_id)?};@@ -5960,7 +6327,8 @@ mod tests {let dir = tempfile::tempdir().unwrap();let pr = crate::progress::Progress::silent();let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);- let result = backend.windowed_value_counts(INDEX, "ax_file", &["digest".into()], &[]);+ let result =+ backend.windowed_value_counts(INDEX, "ax_file", &["digest".into()], &[], false);let _ = tx.send(result.map_err(|error| format!("{error:#}")));});@@ -6030,12 +6398,147 @@ mod tests {"ax_file",&["digest-a".into(), "digest-b".into()],&[],+ false,).unwrap();assert_eq!(counts.get("digest-a"), Some(&3));assert_eq!(counts.get("digest-b"), Some(&0));server.join().unwrap();}++ #[test]+ fn a_zero_exists_total_skips_the_per_digest_windows() {+ // #1267 measured 2,619 term+exists probes, each a brute scan with zero+ // hits. After the windowed verify those are three windows of 1,024.+ // A field that matches nothing issues none of them.+ let skipped = plan_exists_probes(2619, EsSyncBackend::VERIFY_WINDOW, 0);+ assert_eq!(skipped.issue_windows, 0);+ assert_eq!(skipped.skip_windows, 3);+ let issued = plan_exists_probes(2619, EsSyncBackend::VERIFY_WINDOW, 1);+ assert_eq!(issued.issue_windows, 3);+ assert_eq!(issued.skip_windows, 0);+ assert_eq!(+ plan_exists_probes(0, EsSyncBackend::VERIFY_WINDOW, 0),+ ExistsProbePlan {+ issue_windows: 0,+ skip_windows: 0+ }+ );+ assert_eq!(plan_exists_probes(5, 0, 0).issue_windows, 0);+ }++ #[test]+ fn skipped_exists_windows_advance_the_verify_phase() {+ let (pr, buffer) = crate::progress::Progress::capture(+ crate::progress::Surface::Plain,+ std::time::Duration::from_millis(40),+ );+ let ticker = pr.spawn_ticker();+ let plan = plan_exists_probes(2619, EsSyncBackend::VERIFY_WINDOW, 0);+ pr.phase("finalize-verify", plan.skip_windows, 0);+ credit_verify_items(&pr, plan.skip_windows);++ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);+ loop {+ let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();+ if out.contains("phase=finalize-verify") && out.contains("items=3/3") {+ break;+ }+ assert!(+ std::time::Instant::now() < deadline,+ "the ticker never showed the skipped windows advance to items=3/3: {out}"+ );+ std::thread::yield_now();+ }+ drop(ticker);+ }++ #[test]+ fn an_unchanged_catalog_source_is_not_rewritten() {+ let same = serde_json::json!({"run_id": "g1", "path": "a"});+ let expected = BTreeMap::from([+ ("file:same".to_owned(), same.clone()),+ (+ "file:changed".to_owned(),+ serde_json::json!({"run_id": "g2", "path": "b"}),+ ),+ (+ "file:missing".to_owned(),+ serde_json::json!({"run_id": "g2", "path": "c"}),+ ),+ ]);+ let live = BTreeMap::from([+ ("file:same".to_owned(), same),+ (+ "file:changed".to_owned(),+ serde_json::json!({"run_id": "g1", "path": "b"}),+ ),+ ]);+ let write = catalog_docs_to_write(&expected, &live);+ assert!(+ !write.contains_key("file:same"),+ "a resume that already stored this projection must not bulk it again"+ );+ assert_eq!(write["file:changed"]["run_id"], "g2");+ assert!(write.contains_key("file:missing"));+ assert_eq!(write.len(), 2);+ }++ #[test]+ fn the_prior_run_remainder_is_the_kept_intersection() {+ let prior = BTreeSet::from([+ "kept".to_owned(),+ "rewritten".to_owned(),+ "not-kept".to_owned(),+ ]);+ let kept = BTreeSet::from(["kept".to_owned(), "born-kept".to_owned()]);+ assert_eq!(prior_run_expected_remainder(&prior, &kept), 1);+ }++ #[test]+ fn catalog_run_ids_round_trip_and_a_missing_file_is_an_upgrade() {+ let dir = tempfile::tempdir().unwrap();+ let snap = dir.path().join("sync-snapshots").join("tx-1267");+ std::fs::create_dir_all(&snap).unwrap();+ assert!(+ read_catalog_run_ids(dir.path(), "tx-1267")+ .unwrap()+ .is_none(),+ "a generation published before this file existed walks once"+ );+ let ids = ["b".to_owned(), "a".to_owned()];+ write_catalog_run_ids(dir.path(), "tx-1267", [&ids[0], &ids[1]]).unwrap();+ assert_eq!(+ read_catalog_run_ids(dir.path(), "tx-1267")+ .unwrap()+ .unwrap(),+ BTreeSet::from(["a".to_owned(), "b".to_owned()])+ );+ let bad = "a\nb".to_owned();+ let err = write_catalog_run_ids(dir.path(), "tx-1267", [&bad]).unwrap_err();+ assert!(+ err.to_string().contains("newline"),+ "a newline id must not split into two lines: {err}"+ );+ assert!(read_catalog_run_ids(dir.path(), "../tx").is_err());+ }++ #[test]+ fn mget_sources_keep_found_documents_and_reject_a_duplicate_id() {+ let body = serde_json::json!({"docs": [+ {"_id": "a", "found": true, "_source": {"k": 1}},+ {"_id": "b", "found": false}+ ]});+ let got = mget_sources(&body).unwrap();+ assert_eq!(got.len(), 1);+ assert_eq!(got["a"]["k"], 1);+ let dup = serde_json::json!({"docs": [+ {"_id": "a", "found": true, "_source": {}},+ {"_id": "a", "found": true, "_source": {}}+ ]});+ let err = mget_sources(&dup).unwrap_err().to_string();+ assert!(err.contains("duplicate ID a"), "{err}");+ }}/// Issue #482 regression guard.