Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): stop finalize-verify when its target index is deleted (#2225)

xerj-org383c732383f5
diff --git a/docs/recipes/zero-config-indexing.md b/docs/recipes/zero-config-indexing.md
index 025aaa1ee..dadc6ca17 100644
--- a/docs/recipes/zero-config-indexing.md
+++ b/docs/recipes/zero-config-indexing.md
@@ -419,8 +419,12 @@ Every long step is a phase of its own, in this order: `walk`, `hash`, `scan`,
every file, counted in source bytes), `index` (send the sealed bulk bytes, the
in-flight file named in `waiting_on`), `finalize-catalog`, `finalize-refresh`
(one refresh per dataset), `finalize-verify` (read-backs batched into
-per-index digest windows, one search per window). A
-resumed run starts at `replay`, and its `index` phase counts only the
+per-index digest windows, one search per window). A mapping scan that wants
+to delete an index — a JSON fixture the tabular sniffer turned into its own
+table — waits for the `xerj-done` line first. An index removed while
+`finalize-verify` is reading it ends the run with
+`index <name> was deleted during verify`, and the generation does not commit.
+A resumed run starts at `replay`, and its `index` phase counts only the
operations still to apply, so it starts at 0% of what remains rather than
crediting this run with an earlier one's writes.
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 7c4d2a94a..7db4f8af4 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -2158,6 +2158,90 @@ impl Es {
Ok(v)
}
+ /// How often [`Es::search_for_verify`] re-checks that its target still
+ /// exists while the window search is in flight (#2222). A fast window
+ /// returns before the first check, so the common path adds no request.
+ const VERIFY_INDEX_POLL: Duration = Duration::from_millis(500);
+
+ /// Bound on one existence probe. A probe that does not answer is "still
+ /// there": only a real 404 ends the wait, so a busy node cannot be read
+ /// as a deleted index.
+ const VERIFY_INDEX_PROBE_TIMEOUT: Duration = Duration::from_secs(2);
+
+ /// The search [`crate::sync_executor`] finalize-verify issues per window.
+ ///
+ /// `search` blocks the caller for the whole request. The engine resolves
+ /// the index when the request arrives, so a `DELETE` that lands while
+ /// that search is running does not turn it into a 404 — the handle is
+ /// already taken, and the client sits in `waiting_on` that index until
+ /// the transport budget (300 s, six attempts on a 5xx) runs out. The
+ /// generation never commits (#2222). This call keeps the search in flight
+ /// and, beside it, asks `HEAD /{index}`. The moment that answers 404 —
+ /// or the search itself does — it returns
+ /// `index {name} was deleted during verify` and the caller stops waiting.
+ /// A live index is unchanged: the search result comes back as `search`
+ /// would have returned it.
+ pub(crate) fn search_for_verify(&self, index: &str, body: &Value) -> Result<Value> {
+ let (tx, rx) = std::sync::mpsc::channel();
+ let es = self.clone();
+ let index_owned = index.to_owned();
+ let body = body.clone();
+ std::thread::Builder::new()
+ .name("xerj-verify-search".to_owned())
+ .spawn(move || {
+ let _ = tx.send(es.search(&index_owned, &body));
+ })
+ .context("spawn finalize-verify search")?;
+ loop {
+ match rx.recv_timeout(Self::VERIFY_INDEX_POLL) {
+ Ok(Ok(value)) => return Ok(value),
+ Ok(Err(error)) => {
+ if Self::verify_search_says_index_missing(&error)
+ || self.verify_index_present(index) == Some(false)
+ {
+ anyhow::bail!("index {index} was deleted during verify");
+ }
+ return Err(error);
+ }
+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
+ if self.verify_index_present(index) == Some(false) {
+ anyhow::bail!("index {index} was deleted during verify");
+ }
+ }
+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
+ anyhow::bail!("finalize-verify search for {index} ended without a result");
+ }
+ }
+ }
+ }
+
+ /// `Some(true)` the index is there, `Some(false)` the node answered 404,
+ /// `None` when this probe learned nothing (timeout, transport error, or
+ /// any other status). `None` must not be read as gone.
+ fn verify_index_present(&self, index: &str) -> Option<bool> {
+ match self
+ .req(reqwest::Method::HEAD, &format!("/{index}"))
+ .timeout(Self::VERIFY_INDEX_PROBE_TIMEOUT)
+ .send()
+ {
+ Ok(resp) => match resp.status().as_u16() {
+ 404 => Some(false),
+ 200..=299 => Some(true),
+ _ => None,
+ },
+ Err(_) => None,
+ }
+ }
+
+ /// Whether `search`'s error is the node saying the index is not there.
+ ///
+ /// `search` renders a non-2xx as `search /{index} HTTP {status}: {body}`.
+ /// 404 on `_search` is `index_not_found`; an empty index is HTTP 200.
+ fn verify_search_says_index_missing(error: &anyhow::Error) -> bool {
+ let text = format!("{error:#}");
+ text.contains("HTTP 404") || text.to_ascii_lowercase().contains("index_not_found")
+ }
+
/// `search`, but a missing index is an answer rather than a failure.
///
/// A probe whose whole job is to decide "does the server still hold what
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index 8ba48d02b..8f27abc0d 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -791,7 +791,11 @@ impl<'a> EsSyncBackend<'a> {
let mut filter = Vec::with_capacity(1 + extra_filters.len());
filter.push(serde_json::json!({"terms": {field: window}}));
filter.extend(extra_filters.iter().cloned());
- let response = self.es.search(
+ // #2222: `search_for_verify`, not `search`. A DELETE of this
+ // index while the window is in flight used to leave the caller
+ // parked on it (`waiting_on` the deleted name) until the
+ // transport budget ran out, and the generation never committed.
+ let response = self.es.search_for_verify(
index,
&serde_json::json!({
"size": 0,
@@ -5823,6 +5827,215 @@ mod tests {
"a full page with no sort key is a loud failure: {err}"
);
}
+
+ /// #2222: deleting a finalize-verify target while its window search is in
+ /// flight must end the wait. The search connection is held open on purpose
+ /// — that is the in-flight request whose handle the node already resolved,
+ /// so it does not become a 404 — and `HEAD` starts answering 404 once the
+ /// index is "deleted". Verify has to return the deleted-index error while
+ /// that search is still unanswered.
+ #[test]
+ fn finalize_verify_stops_when_its_target_index_is_deleted() {
+ use std::io::{Read, Write};
+ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+ use std::sync::mpsc;
+ use std::time::Duration;
+
+ const INDEX: &str = "xc-elastic-charts-cpu-profile-tree-mock";
+
+ fn read_http(stream: &mut std::net::TcpStream) -> String {
+ stream
+ .set_read_timeout(Some(Duration::from_secs(2)))
+ .unwrap();
+ let mut buf = Vec::new();
+ let mut tmp = [0u8; 2048];
+ loop {
+ match stream.read(&mut tmp) {
+ Ok(0) => break,
+ Ok(n) => {
+ buf.extend_from_slice(&tmp[..n]);
+ if buf.windows(4).any(|w| w == b"\r\n\r\n") {
+ break;
+ }
+ }
+ Err(_) => break,
+ }
+ }
+ String::from_utf8_lossy(&buf).into_owned()
+ }
+
+ fn write_status(stream: &mut std::net::TcpStream, status: &str) {
+ write!(
+ stream,
+ "HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+ )
+ .unwrap();
+ }
+
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ // Accepted sockets inherit non-blocking from the listener. Put each
+ // one back into blocking mode inside its handler. The accept loop
+ // itself stays non-blocking so a parked `_search` — handled on its
+ // own thread — cannot stop a later `HEAD` from being answered.
+ listener.set_nonblocking(true).unwrap();
+ let deleted = std::sync::Arc::new(AtomicBool::new(false));
+ let release = std::sync::Arc::new(AtomicBool::new(false));
+ let heads = std::sync::Arc::new(AtomicUsize::new(0));
+ let deleted_srv = std::sync::Arc::clone(&deleted);
+ let release_srv = std::sync::Arc::clone(&release);
+ let heads_srv = std::sync::Arc::clone(&heads);
+ let server = std::thread::spawn(move || {
+ // Hold the search well past the test's own deadline so a client
+ // that waits for the response fails the test instead of passing
+ // by accident. `release` lets the passing path close the socket.
+ let hold_until = std::time::Instant::now() + Duration::from_secs(45);
+ let started = std::time::Instant::now();
+ let mut handlers = Vec::new();
+ while started.elapsed() < Duration::from_secs(60) {
+ if release_srv.load(Ordering::SeqCst) {
+ break;
+ }
+ match listener.accept() {
+ Ok((stream, _)) => {
+ let deleted_one = std::sync::Arc::clone(&deleted_srv);
+ let release_one = std::sync::Arc::clone(&release_srv);
+ let heads_one = std::sync::Arc::clone(&heads_srv);
+ handlers.push(std::thread::spawn(move || {
+ let mut stream = stream;
+ stream.set_nonblocking(false).unwrap();
+ let request = read_http(&mut stream);
+ if request.starts_with("HEAD ") {
+ heads_one.fetch_add(1, Ordering::SeqCst);
+ let status = if deleted_one.load(Ordering::SeqCst) {
+ "404 Not Found"
+ } else {
+ "200 OK"
+ };
+ write_status(&mut stream, status);
+ } else if request.contains("_search") {
+ while !release_one.load(Ordering::SeqCst)
+ && std::time::Instant::now() < hold_until
+ {
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ write_status(&mut stream, "200 OK");
+ }
+ }));
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ Err(_) => break,
+ }
+ }
+ for handler in handlers {
+ let _ = handler.join();
+ }
+ });
+
+ let deleted_watch = std::sync::Arc::clone(&deleted);
+ let heads_watch = std::sync::Arc::clone(&heads);
+ std::thread::spawn(move || {
+ let deadline = std::time::Instant::now() + Duration::from_secs(10);
+ while heads_watch.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ deleted_watch.store(true, Ordering::SeqCst);
+ });
+
+ let url = format!("http://{address}");
+ let (tx, rx) = mpsc::channel();
+ // Release the parked search even if an assertion panics, so the
+ // server thread does not sit out the rest of its hold.
+ struct ReleaseOnDrop(std::sync::Arc<AtomicBool>);
+ impl Drop for ReleaseOnDrop {
+ fn drop(&mut self) {
+ self.0.store(true, Ordering::SeqCst);
+ }
+ }
+ let _release_on_drop = ReleaseOnDrop(std::sync::Arc::clone(&release));
+ std::thread::spawn(move || {
+ let es = crate::esclient::Es::new(&url, None).unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ let pr = crate::progress::Progress::silent();
+ let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);
+ let result = backend.windowed_value_counts(INDEX, "ax_file", &["digest".into()], &[]);
+ let _ = tx.send(result.map_err(|error| format!("{error:#}")));
+ });
+
+ let message = rx
+ .recv_timeout(Duration::from_secs(20))
+ .expect("finalize-verify hung on an index that was deleted during verify")
+ .expect_err("a deleted verify target must fail that item");
+ assert!(
+ !release.load(Ordering::SeqCst),
+ "verify returned only after the hung search was released"
+ );
+ assert!(message.contains("was deleted during verify"), "{message}");
+ assert!(message.contains(INDEX), "{message}");
+
+ release.store(true, Ordering::SeqCst);
+ // Unblock `accept` if it is parked, then let the held search finish.
+ let _ = std::net::TcpStream::connect(address);
+ server.join().unwrap();
+ }
+
+ /// The same window, when the index stays and the search answers, still
+ /// returns the bucket counts. The deletion check must not fire on a live
+ /// target.
+ #[test]
+ fn finalize_verify_window_counts_a_live_index() {
+ use std::io::{Read, Write};
+
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = std::thread::spawn(move || {
+ let (mut stream, _) = listener.accept().unwrap();
+ stream
+ .set_read_timeout(Some(std::time::Duration::from_secs(2)))
+ .unwrap();
+ let mut buf = Vec::new();
+ let mut tmp = [0u8; 2048];
+ loop {
+ match stream.read(&mut tmp) {
+ Ok(0) => break,
+ Ok(n) => {
+ buf.extend_from_slice(&tmp[..n]);
+ if buf.windows(4).any(|w| w == b"\r\n\r\n") {
+ break;
+ }
+ }
+ Err(_) => break,
+ }
+ }
+ let body =
+ br#"{"aggregations":{"values":{"buckets":[{"key":"digest-a","doc_count":3}]}}}"#;
+ write!(
+ stream,
+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+ body.len()
+ )
+ .unwrap();
+ stream.write_all(body).unwrap();
+ });
+
+ let es = crate::esclient::Es::new(&format!("http://{address}"), None).unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ let pr = crate::progress::Progress::silent();
+ let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);
+ let counts = backend
+ .windowed_value_counts(
+ "xc-live",
+ "ax_file",
+ &["digest-a".into(), "digest-b".into()],
+ &[],
+ )
+ .unwrap();
+ assert_eq!(counts.get("digest-a"), Some(&3));
+ assert_eq!(counts.get("digest-b"), Some(&0));
+ server.join().unwrap();
+ }
}
/// Issue #482 regression guard.
diff --git a/engine/crates/xerj-rerank/tests/egress_inventory.rs b/engine/crates/xerj-rerank/tests/egress_inventory.rs
index 28e278c96..c508f3b7e 100644
--- a/engine/crates/xerj-rerank/tests/egress_inventory.rs
+++ b/engine/crates/xerj-rerank/tests/egress_inventory.rs
@@ -139,6 +139,14 @@ const KNOWN: &[(&str, Role)] = &[
// accept loop on drop, label_http_tests.rs is a #[cfg(test)] module.
("xerj-autoindex/src/label.rs", Role::TestOnly),
("xerj-autoindex/src/label_http_tests.rs", Role::TestOnly),
+ // #2222's finalize-verify deletion test: production verify talks to the
+ // node through the shared Es client (esclient.rs, already classified as
+ // `xerj autoindex`). The only `TcpStream::connect` is inside `#[cfg(test)]`,
+ // a connect to that test's own `127.0.0.1` listener so a parked `accept`
+ // wakes after the index is "deleted". Nothing in this file is a node or
+ // client path, so the published list, DATA_EGRESS and the air-gapped
+ // recipe stay as they are.
+ ("xerj-autoindex/src/sync_executor.rs", Role::TestOnly),
// #1055's xerj_map module: the tool itself reuses the proxy's shared
// client (xerj-mcp/src/lib.rs, already classified); only its unit tests
// construct one, to point at an unroutable address.
diff --git a/tools/xerj-code/SKILL.md b/tools/xerj-code/SKILL.md
index ed02a635f..fff7f1044 100644
--- a/tools/xerj-code/SKILL.md
+++ b/tools/xerj-code/SKILL.md
@@ -234,6 +234,11 @@ Three rules that keep the choice honest:
answers during a rebuild.
- **During a rebuild both builds exist**, so the node briefly holds the corpus
twice. Budget disk for it on a large corpus.
+- **Wait for `xerj-done` before deleting an index a mapping scan flagged.**
+ A JSON fixture the tabular sniffer turned into its own index is removed with
+ `DELETE /{exact-name}` after `finalize-verify` has finished. Deleting one
+ while that phase is still reading it ends the run with
+ `index <name> was deleted during verify`, and the generation does not commit.
- Always verify: `curl -s "$URL/xc-<corpus>*/_count"` must be > 0. The command
does this itself and says so; do it again if you are scripting around it.