Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): report the real docs.deleted in _cat and the stats surfaces (#2140)

Tong-bit-art321e2370d252
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index e8c3ca799..cea502eb0 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -956,12 +956,18 @@ async fn cat_indices_inner(
Some(idx) => content_doc_count(idx).await,
None => info.doc_count,
};
+ // #2140: the deleted column was a hard-coded 0 even when updates had
+ // superseded documents or deletes had tombstoned them. Report the real
+ // pre-merge count (physical minus live), the same definition ES gives
+ // `docs.deleted`.
+ let deleted = idx.as_ref().map(|idx| idx.deleted_doc_count()).unwrap_or(0);
// Emit the stable REAL settings uuid (the same value
// GET /{index}/_settings returns), not a fresh random one per call.
rows.push((
info.name.clone(),
stable_index_uuid(&info.name),
doc_count,
+ deleted,
size,
human_bytes(size),
));
@@ -1006,7 +1012,7 @@ async fn cat_indices_inner(
],
rows: Vec::new(),
};
- for (name, uuid, doc_count, size, hsize) in &rows {
+ for (name, uuid, doc_count, deleted, size, hsize) in &rows {
table.rows.push(vec![
crate::cat::CatCell::plain("green"),
crate::cat::CatCell::plain("open"),
@@ -1015,7 +1021,7 @@ async fn cat_indices_inner(
crate::cat::CatCell::plain("1"),
crate::cat::CatCell::plain("0"),
crate::cat::CatCell::plain(doc_count.to_string()),
- crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::plain(deleted.to_string()),
crate::cat::CatCell::size(*size, hsize.clone()),
crate::cat::CatCell::size(*size, hsize.clone()),
]);
@@ -1320,6 +1326,67 @@ mod cat_format_json_tests {
assert_eq!(status, StatusCode::OK);
assert_eq!(body["hits"]["total"]["value"], 1);
}
+
+ /// A state whose merge thresholds are far above the test's segment count,
+ /// so "three flushes leave three segments" is a fact rather than a race
+ /// with a background merge.
+ fn test_state_without_merges() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ config.merge.min_merge_count = 100;
+ config.merge.max_merge_count = 100;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ /// #2140: `docs.deleted` is the pre-merge physical-minus-live count, not a
+ /// hard-coded 0. Three flushes of the same `_id` leave two superseded
+ /// versions in segments; `_cat/indices`, `_stats` and `_cluster/stats` must
+ /// all say 2, and a bare `_count` must still say 1.
+ #[tokio::test]
+ async fn deleted_doc_count_is_reported_by_cat_and_stats() {
+ let state = test_state_without_merges();
+ let app = crate::router::build_es_compat_router(state);
+ put_index(&app, "del").await;
+ for body in ["v1", "v2", "v3"] {
+ put_doc(&app, "del", "1", json!({"body": body})).await;
+ refresh(&app, "del").await;
+ }
+
+ let (status, count) = get(&app, "/del/_count").await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(count["count"], 1, "one live _id");
+
+ let (status, body) = get(
+ &app,
+ "/_cat/indices?format=json&h=index,docs.count,docs.deleted",
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ let row = body
+ .as_array()
+ .expect("json array")
+ .iter()
+ .find(|row| row["index"] == "del")
+ .expect("del row");
+ assert_eq!(row["docs.count"], "1");
+ assert_eq!(row["docs.deleted"], "2", "two superseded versions: {body}");
+
+ let (status, stats) = get(&app, "/del/_stats").await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(
+ stats["indices"]["del"]["primaries"]["docs"]["deleted"], 2,
+ "{stats}"
+ );
+ assert_eq!(stats["_all"]["primaries"]["docs"]["deleted"], 2, "{stats}");
+
+ let (status, cluster) = get(&app, "/_cluster/stats").await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(cluster["indices"]["docs"]["deleted"], 2, "{cluster}");
+ }
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -24076,6 +24143,7 @@ pub async fn index_stats(
if targets.len() > 1 || index == "_all" || index.contains('*') {
// Aggregate across multiple indices.
let mut all_doc_count = 0u64;
+ let mut all_deleted_doc_count = 0u64;
let mut all_store_bytes = 0u64;
// Aggregated so `GET /_all/_stats` carries the same
// `primaries.mappings.schema_persist_failures` the per-index form does.
@@ -24092,6 +24160,12 @@ pub async fn index_stats(
};
let doc_count = stats.doc_count;
all_doc_count += doc_count;
+ let deleted_doc_count = state
+ .engine
+ .get_index(name)
+ .map(|idx| idx.deleted_doc_count())
+ .unwrap_or(0);
+ all_deleted_doc_count += deleted_doc_count;
// Real on-disk footprint: recursive byte sum of the index's
// data dir (WAL + segments + everything else it persists).
let store_size_bytes = state
@@ -24125,7 +24199,7 @@ pub async fn index_stats(
.unwrap_or(0);
all_schema_persist_failures += schema_persist_failures;
let primaries = json!({
- "docs": { "count": doc_count, "deleted": 0 },
+ "docs": { "count": doc_count, "deleted": deleted_doc_count },
"store": { "size_in_bytes": store_size_bytes },
"dense_vector": dv,
"hnsw": hnsw,
@@ -24191,7 +24265,7 @@ pub async fn index_stats(
all_dv_value_count += dv.get("value_count").and_then(Value::as_u64).unwrap_or(0);
}
let all_primaries = json!({
- "docs": { "count": all_doc_count, "deleted": 0 },
+ "docs": { "count": all_doc_count, "deleted": all_deleted_doc_count },
"store": { "size_in_bytes": all_store_bytes },
"mappings": { "schema_persist_failures": all_schema_persist_failures },
"dense_vector": {
@@ -24220,6 +24294,14 @@ pub async fn index_stats(
};
let doc_count = stats.doc_count;
+ // #2140: `docs.deleted` is the pre-merge physical-minus-live count, not a
+ // hard-coded 0. It also has to agree with `_cat/indices` and the
+ // cluster/node rollups below.
+ let deleted_doc_count = state
+ .engine
+ .get_index(&index)
+ .map(|idx| idx.deleted_doc_count())
+ .unwrap_or(0);
// Real on-disk footprint: recursive byte sum of the index's data dir
// (WAL + segments + everything else it persists).
let store_size_bytes = state
@@ -24290,7 +24372,7 @@ pub async fn index_stats(
.map(|idx| idx.schema_persist_failures())
.unwrap_or(0);
let primaries = json!({
- "docs": { "count": doc_count, "deleted": 0 },
+ "docs": { "count": doc_count, "deleted": deleted_doc_count },
"store": { "size_in_bytes": store_size_bytes },
"dense_vector": dense_vector_stats,
"hnsw": hnsw_stats,
@@ -29701,7 +29783,7 @@ mod glob_match_simple_tests {
// ─────────────────────────────────────────────────────────────────────────────
pub async fn nodes_stats(State(state): State<AppState>) -> impl IntoResponse {
- let (_idx_count, total_docs, store_bytes) = real_index_totals(&state).await;
+ let (_idx_count, total_docs, total_deleted, store_bytes) = real_index_totals(&state).await;
// Live cumulative search + indexing counters, summed across all indices.
let (query_total, query_time_ms, index_total, index_time_ms) =
real_node_search_index_totals(&state).await;
@@ -29800,7 +29882,7 @@ pub async fn nodes_stats(State(state): State<AppState>) -> impl IntoResponse {
"bulk": { "threads": cpu_threads, "queue": 0, "active": 0, "rejected": 0 },
},
"indices": {
- "docs": { "count": total_docs, "deleted": 0 },
+ "docs": { "count": total_docs, "deleted": total_deleted },
"store": { "size_in_bytes": store_bytes },
// Real cumulative counters summed across all indices
// (RC4 W4 item 1): previously hardcoded 0, so node-stats
@@ -30041,7 +30123,7 @@ fn read_rss_bytes() -> Option<u64> {
pub async fn cluster_stats(State(state): State<AppState>) -> impl IntoResponse {
let health = state.engine.health().await;
- let (idx_count, total_docs, store_bytes) = real_index_totals(&state).await;
+ let (idx_count, total_docs, total_deleted, store_bytes) = real_index_totals(&state).await;
let rss_bytes = read_rss_bytes().unwrap_or(0);
let (mem_total, mem_avail) = read_meminfo().unwrap_or((rss_bytes * 4, rss_bytes * 3));
let mem_used = mem_total.saturating_sub(mem_avail);
@@ -30062,7 +30144,7 @@ pub async fn cluster_stats(State(state): State<AppState>) -> impl IntoResponse {
"indices": {
"count": idx_count,
"shards": { "total": idx_count, "primaries": idx_count, "replication": 0.0 },
- "docs": { "count": total_docs, "deleted": 0 },
+ "docs": { "count": total_docs, "deleted": total_deleted },
"store": { "size_in_bytes": store_bytes },
"fielddata": { "memory_size_in_bytes": 0, "evictions": 0 },
"query_cache": { "memory_size_in_bytes": 0, "total_count": 0, "hit_count": 0, "miss_count": 0 },
@@ -36455,7 +36537,7 @@ pub async fn nodes_info(
// Same resolution `es_info` (GET /) uses.
let compat_version = resolve_compat_version(&state, &headers);
let reported_version = compat_version.number.clone();
- let (_idx_count, total_docs, store_bytes) = real_index_totals(&state).await;
+ let (_idx_count, total_docs, total_deleted, store_bytes) = real_index_totals(&state).await;
let rss_bytes = read_rss_bytes().unwrap_or(0);
// Real host memory total from /proc/meminfo (falls back to an RSS-derived
// estimate if unreadable).
@@ -36565,7 +36647,7 @@ pub async fn nodes_info(
"plugins": [],
"modules": [],
"indices": {
- "docs": { "count": total_docs, "deleted": 0 },
+ "docs": { "count": total_docs, "deleted": total_deleted },
"store": { "size_in_bytes": store_bytes },
},
}),
@@ -47210,21 +47292,24 @@ fn closed_index_error(index: &str) -> axum::response::Response {
// ── Batch C helpers ──
-/// Real `(index_count, total_docs, store_size_in_bytes)` sampled from the
-/// live engine: doc counts come from `list_indices()`, store size is the
-/// recursive on-disk byte sum of each index's `data_dir`. Read-only; safe
-/// to call on a live engine.
-async fn real_index_totals(state: &AppState) -> (usize, u64, u64) {
+/// Real `(index_count, total_docs, total_deleted_docs, store_size_in_bytes)`
+/// sampled from the live engine: doc counts come from `list_indices()`, store
+/// size is the recursive on-disk byte sum of each index's `data_dir`, and the
+/// deleted count is the pre-merge physical-minus-live number the per-index
+/// `_stats`/`_cat` surfaces report. Read-only; safe to call on a live engine.
+async fn real_index_totals(state: &AppState) -> (usize, u64, u64, u64) {
let indices = state.engine.list_indices().await;
let mut total_docs = 0u64;
+ let mut total_deleted = 0u64;
let mut store_bytes = 0u64;
for info in &indices {
total_docs += info.doc_count;
if let Ok(idx) = state.engine.get_index(&info.name) {
+ total_deleted += idx.deleted_doc_count();
store_bytes += dir_size_bytes(idx.data_dir());
}
}
- (indices.len(), total_docs, store_bytes)
+ (indices.len(), total_docs, total_deleted, store_bytes)
}
/// Sum the live per-index search + indexing counters across every index,
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 5b1353f4d..0878bca94 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -26140,6 +26140,28 @@ impl Index {
self.store.version_map.live_count() as u64
}
+ /// Documents physically present but not live: superseded versions left by
+ /// updates and documents killed by deletes, until a merge folds them away.
+ /// This is the pre-merge meaning ES gives `docs.deleted`, derived from
+ /// segment metadata + the memtable + the version map, so a listing does not
+ /// decode every segment's tombstone section.
+ ///
+ /// It is NOT the same number as `Σ segment_rows().deleted_doc_count`:
+ /// xerj writes tombstones only for explicit deletes, while an update
+ /// supersedes the old version through the version map without tombstoning
+ /// it. Both kinds are absent from search, so both belong here.
+ #[inline]
+ pub fn deleted_doc_count(&self) -> u64 {
+ let snap = self.store.snapshot();
+ let physical: u64 = snap
+ .segments
+ .iter()
+ .map(|segment| segment.doc_count)
+ .sum::<u64>()
+ + self.memtable.doc_count() as u64;
+ physical.saturating_sub(self.live_doc_count())
+ }
+
/// Real per-segment rows for `_cat/segments` (#1202): one entry per LIVE
/// segment in the durable snapshot — its id, its docs, its deleted docs,
/// its on-disk bytes. The predecessor endpoint fabricated a single `_0`