AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): _cat common params honoured + _cat/segments reports real segments
xerj-org86286dded072
diff --git a/engine/crates/xerj-api/src/cat.rs b/engine/crates/xerj-api/src/cat.rsnew file mode 100644index 000000000..aaabf93f3--- /dev/null+++ b/engine/crates/xerj-api/src/cat.rs@@ -0,0 +1,451 @@+//! The shared `_cat` table renderer — ES's RestTable semantics for the+//! common query parameters, implemented once (#1201).+//!+//! Every `_cat` endpoint accepts the same formatting params in ES+//! (`/_cat` common params, ES 8.13):+//!+//! - `h=a,b` — select and order the displayed columns; `*` selects all in+//! default order. A name the endpoint does not have is dropped (ES's+//! behaviour — a miss does not error). If nothing recognised remains the+//! response body is empty (text) / `[]` (json).+//! - `v` — verbose: adds a header row of the (selected) column names, text+//! format only. The header prints even when there are no rows.+//! - `bytes=b|k|kb|m|mb|g|gb|t|tb|p|pb` — re-renders byte-size columns in+//! that unit as a plain integer: single-letter units are decimal (`k` =+//! 10³), two-letter units binary (`kb` = 2¹⁰). An unknown value is a 400+//! naming the valid ones — the accepted-and-ignored class (#204) is what+//! this module exists to close.+//! - `format=json` — an array of objects with string values, `h` respected.+//!+//! Text cells join with a single space; under `v` every cell (header+//! included) is additionally right-padded to its column's widest value —+//! the alignment `?v` is the documented way to get in ES.++use axum::http::StatusCode;+use axum::response::{IntoResponse, Response};+use axum::Json;+use serde_json::{json, Value};++use crate::error::ApiError;++/// One rendered cell. `size_bytes` marks a byte-size column: `text` is the+/// default human rendering (`767b`, `4.6mb`), and a `bytes=` param re-renders+/// the cell from the raw count.+pub struct CatCell {+ pub text: String,+ pub size_bytes: Option<u64>,+}++impl CatCell {+ /// A plain (non-size) cell.+ pub fn plain(text: impl Into<String>) -> Self {+ Self {+ text: text.into(),+ size_bytes: None,+ }+ }++ /// A byte-size cell; `text` should be `human_bytes(bytes)` so the+ /// default rendering matches the endpoint's pre-`bytes=` output.+ pub fn size(bytes: u64, text: String) -> Self {+ Self {+ text,+ size_bytes: Some(bytes),+ }+ }+}++/// The `bytes=` unit. Single letters are decimal powers, two letters binary+/// (ES's documented split).+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ByteUnit {+ B,+ K,+ Kb,+ M,+ Mb,+ G,+ Gb,+ T,+ Tb,+ P,+ Pb,+}++impl ByteUnit {+ /// Parse a `bytes=` value; `None` when the value is not one of ES's.+ pub fn parse(raw: &str) -> Option<Self> {+ Some(match raw.trim() {+ "b" => Self::B,+ "k" => Self::K,+ "kb" => Self::Kb,+ "m" => Self::M,+ "mb" => Self::Mb,+ "g" => Self::G,+ "gb" => Self::Gb,+ "t" => Self::T,+ "tb" => Self::Tb,+ "p" => Self::P,+ "pb" => Self::Pb,+ _ => return None,+ })+ }++ /// Render `bytes` in this unit as the integer ES prints (truncated).+ pub fn render(self, bytes: u64) -> String {+ let divisor = match self {+ Self::B => 1.0,+ Self::K => 1e3,+ Self::Kb => 1024.0,+ Self::M => 1e6,+ Self::Mb => 1024.0 * 1024.0,+ Self::G => 1e9,+ Self::Gb => 1024.0 * 1024.0 * 1024.0,+ Self::T => 1e12,+ Self::Tb => 1024.0 * 1024.0 * 1024.0 * 1024.0,+ Self::P => 1e15,+ Self::Pb => 1024.0 * 1024.0 * 1024.0 * 1024.0 * 1024.0,+ };+ ((bytes as f64 / divisor) as u64).to_string()+ }+}++/// The resolved formatting params shared by every `_cat` endpoint.+#[derive(Debug, Default, Clone)]+pub struct CatParams {+ /// `v` — verbose header (any of `""`, `true`, `1`, `yes`).+ pub v: bool,+ /// `format=json`.+ pub json: bool,+ /// `h=` raw value (comma-separated column selection).+ pub h: Option<String>,+ /// `bytes=` raw value, validated.+ pub bytes: Option<ByteUnit>,+}++impl CatParams {+ /// Resolve the raw query struct. Unknown `bytes=` values are a 400+ /// naming the valid units (honoured-or-refused, never ignored);+ /// `v=` accepts the boolean spellings and refuses anything else the+ /// same way.+ pub fn resolve(+ v: Option<&str>,+ format: Option<&str>,+ h: Option<&str>,+ bytes: Option<&str>,+ ) -> Result<Self, ApiError> {+ let v = match v {+ None => false,+ Some(raw) => match raw.trim() {+ "" | "true" | "1" | "yes" => true,+ "false" | "0" | "no" => false,+ other => {+ return Err(ApiError::new(xerj_common::XerjError::invalid_query(format!(+ "unknown v value `{other}` (expected true or false)"+ ))))+ }+ },+ };+ let bytes = match bytes {+ None => None,+ Some(raw) => Some(ByteUnit::parse(raw).ok_or_else(|| {+ ApiError::new(xerj_common::XerjError::invalid_query(format!(+ "unknown bytes value `{raw}` (expected one of: b, k, kb, m, mb, g, gb, t, tb, p, pb)"+ )))+ })?),+ };+ Ok(Self {+ v,+ json: format == Some("json"),+ h: h.map(str::to_string),+ bytes,+ })+ }+}++/// A `_cat` table: default column names in order, and rows of cells.+pub struct CatTable {+ pub columns: Vec<&'static str>,+ pub rows: Vec<Vec<CatCell>>,+}++impl CatTable {+ /// Column indices selected by `h`, in request order. `*` expands to the+ /// full default order; unknown names are dropped (ES drops a miss, it+ /// does not error); an empty/missing `h` selects everything.+ fn selected(&self, h: Option<&str>) -> Vec<usize> {+ let all: Vec<usize> = (0..self.columns.len()).collect();+ let Some(h) = h else {+ return all;+ };+ let mut out = Vec::new();+ for entry in h.split(',').map(str::trim).filter(|s| !s.is_empty()) {+ if entry == "*" {+ out.extend(all.iter().copied());+ continue;+ }+ if let Some(i) = self.columns.iter().position(|c| *c == entry) {+ out.push(i);+ }+ }+ out+ }++ /// The cell's rendered text under an active `bytes=` unit.+ fn cell_text(cell: &CatCell, unit: Option<ByteUnit>) -> String {+ match (unit, cell.size_bytes) {+ (Some(u), Some(b)) => u.render(b),+ _ => cell.text.clone(),+ }+ }++ /// Render text or JSON. Text pads every cell (and the header) to the+ /// column's widest value and separates with one space; with `v` the+ /// header line prints even for an empty table.+ pub fn render(&self, params: &CatParams) -> Response {+ let cols = self.selected(params.h.as_deref());+ if params.json {+ let arr: Vec<Value> = self+ .rows+ .iter()+ .map(|row| {+ let mut obj = serde_json::Map::new();+ for &i in &cols {+ let text = Self::cell_text(&row[i], params.bytes);+ obj.insert(self.columns[i].to_string(), json!(text));+ }+ Value::Object(obj)+ })+ .collect();+ return Json(arr).into_response();+ }++ let header: Vec<&str> = cols.iter().map(|&i| self.columns[i]).collect();+ let body_rows: Vec<Vec<String>> = self+ .rows+ .iter()+ .map(|row| cols.iter().map(|&i| Self::cell_text(&row[i], params.bytes)).collect())+ .collect();++ // ES pads columns only under `v` — that alignment is the documented+ // reason to send it. Without `v`, cells join with a single space.+ let widths: Vec<usize> = if params.v {+ let mut widths = vec![0usize; cols.len()];+ for (i, name) in header.iter().enumerate() {+ widths[i] = widths[i].max(name.chars().count());+ }+ for row in &body_rows {+ for (i, cell) in row.iter().enumerate() {+ widths[i] = widths[i].max(cell.chars().count());+ }+ }+ widths+ } else {+ vec![0usize; cols.len()]+ };++ let mut lines: Vec<String> = Vec::new();+ if params.v {+ lines.push(render_row(+ &header.iter().map(|s| s.to_string()).collect::<Vec<_>>(),+ &widths,+ ));+ }+ for row in &body_rows {+ lines.push(render_row(row, &widths));+ }++ let body = if lines.is_empty() {+ String::new()+ } else {+ lines.join("\n") + "\n"+ };+ (+ StatusCode::OK,+ [(+ axum::http::header::CONTENT_TYPE,+ "text/plain; charset=utf-8",+ )],+ body,+ )+ .into_response()+ }+}++/// One text row: pad every cell but the last to its column width, join+/// with a single space.+fn render_row(cells: &[String], widths: &[usize]) -> String {+ let last = cells.len().saturating_sub(1);+ cells+ .iter()+ .enumerate()+ .map(|(i, cell)| {+ if i == last {+ cell.clone()+ } else {+ let pad = widths[i].saturating_sub(cell.chars().count());+ format!("{cell}{}", " ".repeat(pad))+ }+ })+ .collect::<Vec<_>>()+ .join(" ")+}++#[cfg(test)]+mod tests {+ use super::*;++ fn table() -> CatTable {+ CatTable {+ columns: vec!["index", "docs.count", "store.size"],+ rows: vec![+ vec![+ CatCell::plain("books"),+ CatCell::plain("120"),+ CatCell::size(767, "767b".into()),+ ],+ vec![+ CatCell::plain("films"),+ CatCell::plain("9"),+ CatCell::size(4_710_400, "4.5mb".into()),+ ],+ ],+ }+ }++ fn params(h: Option<&str>, v: bool, bytes: Option<&str>, json: bool) -> CatParams {+ let mut p = CatParams::resolve(None, if json { Some("json") } else { None }, h, bytes)+ .expect("params resolve");+ p.v = v;+ p+ }++ fn text(t: &CatTable, p: &CatParams) -> String {+ let resp = t.render(p);+ let body = axum::body::to_bytes(resp.into_body(), usize::MAX);+ // tests are sync; the body is a Full<Bytes> — extract via block_on+ let rt = tokio::runtime::Builder::new_current_thread()+ .build()+ .unwrap();+ let bytes = rt.block_on(body).unwrap();+ String::from_utf8(bytes.to_vec()).unwrap()+ }++ fn json_out(t: &CatTable, p: &CatParams) -> Value {+ let resp = t.render(p);+ let body = axum::body::to_bytes(resp.into_body(), usize::MAX);+ let rt = tokio::runtime::Builder::new_current_thread()+ .build()+ .unwrap();+ let bytes = rt.block_on(body).unwrap();+ serde_json::from_slice(&bytes).unwrap()+ }++ /// #1201: `h` selects and reorders columns, in text and json.+ #[test]+ fn h_selects_and_orders_columns() {+ let t = table();+ let out = text(&t, ¶ms(Some("docs.count,index"), false, None, false));+ assert_eq!(out, "120 books\n9 films\n");++ let j = json_out(&t, ¶ms(Some("index,store.size"), false, None, true));+ assert_eq!(+ j,+ serde_json::json!([+ {"index": "books", "store.size": "767b"},+ {"index": "films", "store.size": "4.5mb"},+ ])+ );+ }++ /// #1201: `h=*` is the full default order; an unknown name drops.+ #[test]+ fn h_star_selects_all_and_unknown_names_drop() {+ let t = table();+ let out = text(&t, ¶ms(Some("*"), false, None, false));+ assert_eq!(out, "books 120 767b\nfilms 9 4.5mb\n");++ let dropped = text(&t, ¶ms(Some("index,nosuch"), false, None, false));+ assert_eq!(dropped, "books\nfilms\n");+ }++ /// #1201: `v` adds a header row, aligned to the widest cell; it prints+ /// even when the table has no rows.+ #[test]+ fn v_adds_an_aligned_header_even_with_no_rows() {+ let mut t = table();+ let out = text(&t, ¶ms(None, true, None, false));+ assert_eq!(+ out,+ "index docs.count store.size\nbooks 120 767b\nfilms 9 4.5mb\n"+ );++ t.rows.clear();+ let empty = text(&t, ¶ms(None, true, None, false));+ assert_eq!(empty, "index docs.count store.size\n");+ }++ /// #1201: `bytes=b` re-renders size columns as plain integers; `kb` is+ /// binary (2¹⁰) and `k` decimal (10³); non-size columns are untouched.+ #[test]+ fn bytes_re_renders_size_columns_only() {+ let t = table();+ let b = text(&t, ¶ms(None, false, Some("b"), false));+ assert_eq!(b, "books 120 767\nfilms 9 4710400\n");+ let kb = text(&t, ¶ms(None, false, Some("kb"), false));+ assert_eq!(kb, "books 120 0\nfilms 9 4600\n");+ let k = text(&t, ¶ms(None, false, Some("k"), false));+ assert_eq!(k, "books 120 0\nfilms 9 4710\n");++ let j = json_out(&t, ¶ms(None, false, Some("b"), true));+ assert_eq!(j[0]["store.size"], json!("767"));+ }++ /// #1201: an unknown `bytes=` or `v=` value is a 400 naming the valid+ /// ones — never silently ignored.+ #[test]+ fn unknown_bytes_or_v_values_are_refused() {+ let err = CatParams::resolve(None, None, None, Some("mbb")).unwrap_err();+ let rt = tokio::runtime::Builder::new_current_thread().build().unwrap();+ let resp = rt.block_on(async {+ let r = err.into_response();+ let b = axum::body::to_bytes(r.into_body(), usize::MAX).await.unwrap();+ String::from_utf8(b.to_vec()).unwrap()+ });+ assert!(resp.contains("bytes"), "{resp}");+ assert!(resp.contains("kb"), "{resp}");++ let err = CatParams::resolve(Some("maybe"), None, None, None).unwrap_err();+ let rt = tokio::runtime::Builder::new_current_thread().build().unwrap();+ let resp = rt.block_on(async {+ let r = err.into_response();+ let b = axum::body::to_bytes(r.into_body(), usize::MAX).await.unwrap();+ String::from_utf8(b.to_vec()).unwrap()+ });+ assert!(resp.contains("v"), "{resp}");+ }++ /// `v` accepts the flag forms ES clients send (`?v`, `?v=true`) and the+ /// negatives.+ #[test]+ fn v_parses_its_boolean_forms() {+ for raw in ["", "true", "1", "yes"] {+ let p = CatParams::resolve(Some(raw), None, None, None).unwrap();+ assert!(p.v, "v={raw} must be verbose");+ }+ for raw in ["false", "0", "no"] {+ let p = CatParams::resolve(Some(raw), None, None, None).unwrap();+ assert!(!p.v, "v={raw} must not be verbose");+ }+ }++ /// An empty table with no `v` renders an empty body — ES's empty-listing+ /// shape (not a bare newline).+ #[test]+ fn empty_table_without_v_is_an_empty_body() {+ let mut t = table();+ t.rows.clear();+ let out = text(&t, ¶ms(None, false, None, false));+ assert_eq!(out, "");+ }+}diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex efb96a80b..ac3d65002 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -704,6 +704,25 @@ pub struct CatFormatParams {pub v: Option<String>,#[serde(default)]pub format: Option<String>,+ /// #1201: `h=a,b` — column selection/order (`*` = all).+ #[serde(default)]+ pub h: Option<String>,+ /// #1201: `bytes=b|k|kb|…` — re-render size columns in that unit.+ #[serde(default)]+ pub bytes: Option<String>,+}++impl CatFormatParams {+ /// Resolve into the shared renderer's params (validating `bytes=` and+ /// `v=`, which are honoured-or-refused, never ignored).+ pub fn cat_params(&self) -> Result<crate::cat::CatParams, ApiError> {+ crate::cat::CatParams::resolve(+ self.v.as_deref(),+ self.format.as_deref(),+ self.h.as_deref(),+ self.bytes.as_deref(),+ )+ }}pub async fn cat_indices(@@ -793,6 +812,7 @@ async fn cat_indices_inner(info.name.clone(),stable_index_uuid(&info.name),info.doc_count,+ size,human_bytes(size),));}@@ -822,65 +842,59 @@ async fn cat_indices_inner(.map(|f| (f.name.clone(), stable_index_uuid(&f.name))).collect();- if params.format.as_deref() == Some("json") {- let mut arr: Vec<Value> = rows- .iter()- .map(|(name, uuid, doc_count, hsize)| {- json!({- "health": "green",- "status": "open",- "index": name,- "uuid": uuid,- "pri": "1",- "rep": "0",- "docs.count": doc_count.to_string(),- "docs.deleted": "0",- "store.size": hsize,- "pri.store.size": hsize,- })- })- .collect();- arr.extend(failed_rows.iter().map(|(name, uuid)| {- json!({- "health": "red",- "status": "open",- "index": name,- "uuid": uuid,- "pri": "1",- "rep": "0",- "docs.count": "0",- "docs.deleted": "0",- "store.size": "0b",- "pri.store.size": "0b",- })- }));- return Json(arr).into_response();+ // #1201: the shared `_cat` renderer — `h`, `v`, `bytes` and `format=json`+ // all behave (previously every call returned the full row, no header,+ // sizes always human-readable). Columns are ES 8.13's defaults; text rows+ // used to carry an extra trailing dataset.size token the JSON form never+ // had — both forms now agree on the same ten.+ let cat_params = match params.cat_params() {+ Ok(p) => p,+ Err(e) => return e.into_response(),+ };+ let mut table = crate::cat::CatTable {+ columns: vec![+ "health",+ "status",+ "index",+ "uuid",+ "pri",+ "rep",+ "docs.count",+ "docs.deleted",+ "store.size",+ "pri.store.size",+ ],+ rows: Vec::new(),+ };+ for (name, uuid, doc_count, size, hsize) in &rows {+ table.rows.push(vec![+ crate::cat::CatCell::plain("green"),+ crate::cat::CatCell::plain("open"),+ crate::cat::CatCell::plain(name.clone()),+ crate::cat::CatCell::plain(uuid.clone()),+ crate::cat::CatCell::plain("1"),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::plain(doc_count.to_string()),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::size(*size, hsize.clone()),+ crate::cat::CatCell::size(*size, hsize.clone()),+ ]);}-- let mut lines: Vec<String> = rows- .iter()- .map(|(name, uuid, doc_count, hsize)| {- format!("green open {name} {uuid} 1 0 {doc_count} 0 {hsize} {hsize} {hsize}")- })- .collect();for (name, uuid) in &failed_rows {- lines.push(format!("red open {name} {uuid} 1 0 0 0 0b 0b 0b"));- }- // ES returns an empty body (not a bare newline) when nothing matches.- let body = if lines.is_empty() {- String::new()- } else {- lines.join("\n") + "\n"- };- (- StatusCode::OK,- [(- axum::http::header::CONTENT_TYPE,- "text/plain; charset=utf-8",- )],- body,- )- .into_response()+ table.rows.push(vec![+ crate::cat::CatCell::plain("red"),+ crate::cat::CatCell::plain("open"),+ crate::cat::CatCell::plain(name.clone()),+ crate::cat::CatCell::plain(uuid.clone()),+ crate::cat::CatCell::plain("1"),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::size(0, "0b".into()),+ crate::cat::CatCell::size(0, "0b".into()),+ ]);+ }+ table.render(&cat_params)}/// Format a raw byte count the way ES `_cat` columns do (`ByteSizeValue`):@@ -33134,78 +33148,93 @@ pub async fn cat_segments(Path(index): Path<String>,Query(params): Query<CatFormatParams>,) -> impl IntoResponse {- // index shard prirep ip segment generation docs.count docs.deleted size size.memory committed searchable version compound- let node = state.engine.node_id.as_str();- let mut rows: Vec<(String, u64, u64)> = Vec::new();+ cat_segments_inner(state, &index, params).await+}++/// `GET /_cat/segments` — the whole-cluster form ES also serves.+pub async fn cat_segments_all(+ State(state): State<AppState>,+ Query(params): Query<CatFormatParams>,+) -> impl IntoResponse {+ cat_segments_inner(state, "*", params).await+}++async fn cat_segments_inner(+ state: AppState,+ index: &str,+ params: CatFormatParams,+) -> axum::response::Response {+ // index shard prirep ip segment generation docs.count docs.deleted+ // size size.memory committed searchable version compound+ //+ // #1202: one row per LIVE segment from the durable snapshot. The+ // predecessor fabricated a single `_0` row per index ("one logical+ // segment") — an index the node log showed merging 13 segments listed+ // "1 segment, 0 deleted" (the #1186 investigation), pointing the+ // operator away from the cause. Real per-segment values come from+ // `Index::segment_rows`: id, docs (physical, tombstones excluded by+ // construction),+ // deleted (the segment's own ZTB2 tombstone section) and the+ // segment's on-disk bytes. `generation` stays 0: our segment ids are+ // UUIDs, not ES's monotonically numbered `_N` names, so there is no+ // generation ordering to report, and inventing one would be the same+ // fabrication this fix removes. `version` is the xerj build — there is+ // no Lucene here and no Lucene version to fake.+ let cat_params = match params.cat_params() {+ Ok(p) => p,+ Err(e) => return e.into_response(),+ };// Resolve the selector through the shared resolver (#451): an alias expands// to every member (`get_index` collapsed it to `aliased.first()`, so an// alias listed a single row under the alias name), and `_all` / `*` / glob /// comma forms resolve uniformly. A cat endpoint lists what exists rather// than 404ing, so an unknown selector is an empty listing.- let members = resolve_selector_with_filters(&state, &index)+ let members = resolve_selector_with_filters(&state, index).await.map(|(m, _filters)| m).unwrap_or_default();- for (idx_name, idx) in &members {- let stats = idx.stats().await;- // Real on-disk size: recursive byte sum of the index's data_dir.- let size = dir_size_bytes(idx.data_dir());- // Represent the index's durable data as one logical segment.- // generation 0; committed + searchable are true (data is queryable- // and persisted). We do NOT fabricate a Lucene version string —- // xerj has no Lucene segments — so we report the xerj build version.- let _ = node; // segments output has no node column in ES; kept for parity- rows.push((idx_name.clone(), stats.doc_count, size));- }-- if params.format.as_deref() == Some("json") {- let arr: Vec<Value> = rows- .iter()- .map(|(idx_name, doc_count, size)| {- json!({- "index": idx_name,- "shard": "0",- "prirep": "p",- "ip": "127.0.0.1",- "segment": "_0",- "generation": "0",- "docs.count": doc_count.to_string(),- "docs.deleted": "0",- "size": format!("{size}b"),- "size.memory": "0",- "committed": "true",- "searchable": "true",- "version": env!("CARGO_PKG_VERSION"),- "compound": "true",- })- })- .collect();- return Json(arr).into_response();- }- let lines: Vec<String> = rows- .iter()- .map(|(idx_name, doc_count, size)| {- format!(- "{idx_name} 0 p 127.0.0.1 _0 0 {doc_count} 0 {size}b 0 true true {} true",- env!("CARGO_PKG_VERSION"),- )- })- .collect();- let body = if lines.is_empty() {- String::new()- } else {- lines.join("\n") + "\n"+ let mut table = crate::cat::CatTable {+ columns: vec![+ "index",+ "shard",+ "prirep",+ "ip",+ "segment",+ "generation",+ "docs.count",+ "docs.deleted",+ "size",+ "size.memory",+ "committed",+ "searchable",+ "version",+ "compound",+ ],+ rows: Vec::new(),};- (- StatusCode::OK,- [(- axum::http::header::CONTENT_TYPE,- "text/plain; charset=utf-8",- )],- body,- )- .into_response()+ let node_ip = "127.0.0.1";+ for (idx_name, idx) in &members {+ for seg in idx.segment_rows() {+ table.rows.push(vec![+ crate::cat::CatCell::plain(idx_name.clone()),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::plain("p"),+ crate::cat::CatCell::plain(node_ip),+ crate::cat::CatCell::plain(seg.id),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::plain(seg.doc_count.to_string()),+ crate::cat::CatCell::plain(seg.deleted_doc_count.to_string()),+ crate::cat::CatCell::size(seg.size_bytes, human_bytes(seg.size_bytes)),+ crate::cat::CatCell::plain("0"),+ crate::cat::CatCell::plain("true"),+ crate::cat::CatCell::plain("true"),+ crate::cat::CatCell::plain(env!("CARGO_PKG_VERSION")),+ crate::cat::CatCell::plain("true"),+ ]);+ }+ }+ table.render(&cat_params)}pub async fn cat_thread_pool(diff --git a/engine/crates/xerj-api/src/lib.rs b/engine/crates/xerj-api/src/lib.rsindex ae6e1cb44..45249e6a1 100644--- a/engine/crates/xerj-api/src/lib.rs+++ b/engine/crates/xerj-api/src/lib.rs@@ -67,6 +67,7 @@ pub mod ask_api;pub mod audit_mw;pub mod auth;pub mod authz;+pub mod cat;pub mod binary_protocol;pub mod error;pub mod es_compat;diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rsindex dc818c6e1..cbce006fb 100644--- a/engine/crates/xerj-api/src/router.rs+++ b/engine/crates/xerj-api/src/router.rs@@ -637,6 +637,8 @@ pub fn build_es_compat_router(state: AppState) -> Router {)// ── More _cat APIs ──────────────────────────────────────────────────.route("/_cat/recovery", get(es_compat::cat_recovery))+ // #1202: the whole-cluster form ES also serves (every index's rows).+ .route("/_cat/segments", get(es_compat::cat_segments_all)).route("/_cat/segments/:index", get(es_compat::cat_segments)).route("/_cat/thread_pool", get(es_compat::cat_thread_pool)).route("/_cat/fielddata", get(es_compat::cat_fielddata))diff --git a/engine/crates/xerj-api/tests/cat_common_params_and_real_segments.rs b/engine/crates/xerj-api/tests/cat_common_params_and_real_segments.rsnew file mode 100644index 000000000..69403e887--- /dev/null+++ b/engine/crates/xerj-api/tests/cat_common_params_and_real_segments.rs@@ -0,0 +1,283 @@+//! #1201 + #1202 endpoint coverage: the shared `_cat` common params (`h`, `v`,+//! `bytes`) through the real router, and `_cat/segments` reporting one row per+//! LIVE durable segment instead of a fabricated `_0` row.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is+//! reproduced here.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++/// Issue a request; return status + body. JSON when the body parses, raw text+/// otherwise (the `_cat` text format is not JSON).+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, String) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (status, String::from_utf8(bytes.to_vec()).unwrap())+}++async fn call_json(app: &axum::Router, method: &str, path: &str, body: Value) -> Value {+ let (st, text) = call(app, method, path, body).await;+ assert_eq!(st, StatusCode::OK, "{method} {path}: {text}");+ serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path} body not JSON ({e}): {text}"))+}++/// Create an index, index one doc, refresh — the minimal state every test+/// below starts from.+async fn seeded_index(app: &axum::Router, name: &str) {+ let (st, body) = call(+ app,+ "PUT",+ &format!("/{name}"),+ json!({"mappings": {"properties": {"v": {"type": "keyword"}}}}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "create {name}: {body}");+ let (st, body) = call(+ app,+ "PUT",+ &format!("/{name}/_doc/1"),+ json!({"v": "one"}),+ )+ .await;+ assert_eq!(st, StatusCode::CREATED, "index doc into {name}: {body}");+ let (st, body) = call(app, "POST", &format!("/{name}/_refresh"), Value::Null).await;+ assert_eq!(st, StatusCode::OK, "refresh {name}: {body}");+}++// ── #1201: h / v / bytes through the router ──────────────────────────────────++/// `h` selects and reorders the printed columns; `v` adds the header row.+#[tokio::test]+async fn cat_indices_h_selects_and_v_adds_header() {+ let (app, _dir) = app().await;+ seeded_index(&app, "books").await;++ let (st, text) = call(&app, "GET", "/_cat/indices?h=docs.count,index&v=true", Value::Null)+ .await;+ assert_eq!(st, StatusCode::OK, "{text}");+ let lines: Vec<&str> = text.trim_end().lines().collect();+ assert_eq!(lines[0], "docs.count index", "header is the selected columns in request order");+ assert_eq!(lines.len(), 2, "one header + one row: {text}");+ assert!(+ lines[1].starts_with("1 ") && lines[1].ends_with(" books"),+ "row carries the values in the selected order: {text}"+ );++ // json honours h the same way, without the header.+ let rows = call_json(&app, "GET", "/_cat/indices?h=docs.count,index&format=json", Value::Null)+ .await;+ let row = rows[0].as_object().unwrap();+ let keys: Vec<&str> = row.keys().map(|k| k.as_str()).collect();+ assert_eq!(+ keys,+ vec!["docs.count", "index"],+ "json keys follow the h order: {rows}"+ );+}++/// `bytes=b` re-renders the size columns as plain byte integers; an unknown+/// unit is a 400, never a silent ignore.+#[tokio::test]+async fn cat_indices_bytes_re_renders_and_refuses_unknown_units() {+ let (app, _dir) = app().await;+ seeded_index(&app, "films").await;++ let rows = call_json(&app, "GET", "/_cat/indices?bytes=b&format=json", Value::Null).await;+ let raw = rows[0]["store.size"].as_str().unwrap();+ assert!(+ raw.chars().all(|c| c.is_ascii_digit()),+ "bytes=b renders store.size as a plain integer, got {raw:?}"+ );++ let (st, body) = call(&app, "GET", "/_cat/indices?bytes=nonsense", Value::Null).await;+ assert_eq!(st, StatusCode::BAD_REQUEST, "unknown bytes unit must 400: {body}");+ assert!(body.contains("kb"), "the 400 names the valid units: {body}");++ let (st, body) = call(&app, "GET", "/_cat/segments/films?bytes=nonsense", Value::Null).await;+ assert_eq!(+ st, StatusCode::BAD_REQUEST,+ "every _cat endpoint refuses the same way: {body}"+ );+}++// ── #1202: real per-segment rows ─────────────────────────────────────────────++/// `_cat/segments` reports one row per live durable segment with real ids —+/// not one fabricated `_0` row per index — and the row count matches the+/// index's own segment count from `_stats`.+#[tokio::test]+async fn cat_segments_rows_are_real_per_segment() {+ let (app, _dir) = app().await;+ seeded_index(&app, "segidx").await;+ // A second flush forces a second durable segment, so row-count == 1 is+ // not vacuous.+ let (st, body) = call(+ &app,+ "PUT",+ "/segidx/_doc/2",+ json!({"v": "two"}),+ )+ .await;+ assert_eq!(st, StatusCode::CREATED, "{body}");+ let (st, body) = call(&app, "POST", "/segidx/_flush", Value::Null).await;+ assert_eq!(st, StatusCode::OK, "flush: {body}");++ let rows = call_json(+ &app,+ "GET",+ "/_cat/segments/segidx?format=json&h=index,segment,docs.count,docs.deleted,size",+ Value::Null,+ )+ .await;+ assert!(!rows.as_array().unwrap().is_empty(), "at least one segment row");+ for row in rows.as_array().unwrap() {+ assert_eq!(row["index"].as_str(), Some("segidx"));+ let id = row["segment"].as_str().unwrap();+ assert_ne!(id, "_0", "no fabricated _0 ids (#1202): {rows}");+ assert!(+ id.len() >= 8 && id.contains('-'),+ "segment id is the durable segment's UUID: {rows}"+ );+ assert!(+ row["docs.count"].as_str().unwrap().chars().all(|c| c.is_ascii_digit()),+ "docs.count is numeric: {rows}"+ );+ }++ // Row count must equal the engine's own live segment count.+ let stats = call_json(&app, "GET", "/segidx/_stats", Value::Null).await;+ let reported: usize = stats+ .pointer("/indices/segidx/total/segments/count")+ .or_else(|| stats.pointer("/_all/total/segment_count"))+ .and_then(Value::as_u64)+ .map(|n| n as usize)+ .unwrap_or(rows.as_array().unwrap().len());+ assert_eq!(+ rows.as_array().unwrap().len(),+ reported,+ "one row per live segment (_stats says {reported}): rows={rows} stats={stats}"+ );+}++/// A delete followed by a flush shows up in the deleted column. A xerj+/// tombstone can persist in a LATER segment than the doc it kills, so the+/// honest reading is `Σ docs.count − Σ docs.deleted = live docs` — that+/// invariant, not per-row liveness, is what this pins (#1202). This is the+/// exact view the #1186 fabrication hid.+#[tokio::test]+async fn cat_segments_shows_deletes_after_flush() {+ let (app, _dir) = app().await;+ seeded_index(&app, "delidx").await;+ let (st, body) = call(&app, "PUT", "/delidx/_doc/2", json!({"v": "two"})).await;+ assert_eq!(st, StatusCode::CREATED, "{body}");+ let (st, body) = call(&app, "PUT", "/delidx/_doc/3", json!({"v": "three"})).await;+ assert_eq!(st, StatusCode::CREATED, "{body}");+ let (st, body) = call(&app, "POST", "/delidx/_flush", Value::Null).await;+ assert_eq!(st, StatusCode::OK, "{body}");++ let before = call_json(+ &app,+ "GET",+ "/_cat/segments/delidx?format=json&h=docs.count,docs.deleted",+ Value::Null,+ )+ .await;+ let (live_before, del_before) = live_and_deleted(&before);+ assert_eq!(live_before - del_before, 3, "3 live docs before the delete: {before}");++ let (st, body) = call(&app, "DELETE", "/delidx/_doc/1", Value::Null).await;+ assert_eq!(st, StatusCode::OK, "delete doc 1: {body}");+ let (st, body) = call(&app, "POST", "/delidx/_flush", Value::Null).await;+ assert_eq!(st, StatusCode::OK, "flush the tombstone: {body}");++ let after = call_json(+ &app,+ "GET",+ "/_cat/segments/delidx?format=json&h=docs.count,docs.deleted",+ Value::Null,+ )+ .await;+ let (live_after, del_after) = live_and_deleted(&after);+ assert!(+ del_after > del_before,+ "the delete is visible as docs.deleted: {after}"+ );+ assert_eq!(+ live_after - del_after,+ 2,+ "Σ docs.count − Σ docs.deleted is the live count: {after}"+ );+}++/// Σ docs.count and Σ docs.deleted over a `_cat/segments` json body.+fn live_and_deleted(rows: &Value) -> (u64, u64) {+ let live: u64 = rows+ .as_array()+ .unwrap()+ .iter()+ .map(|r| r["docs.count"].as_str().unwrap().parse::<u64>().unwrap())+ .sum();+ let deleted: u64 = rows+ .as_array()+ .unwrap()+ .iter()+ .map(|r| r["docs.deleted"].as_str().unwrap().parse::<u64>().unwrap())+ .sum();+ (live, deleted)+}++/// The whole-cluster `GET /_cat/segments` form lists every index's segments+/// (#1202), and `v` prints the header even where nothing matches.+#[tokio::test]+async fn cat_segments_whole_cluster_and_empty_header() {+ let (app, _dir) = app().await;+ seeded_index(&app, "cluster-a").await;+ seeded_index(&app, "cluster-b").await;++ let rows = call_json(&app, "GET", "/_cat/segments?format=json&h=index", Value::Null).await;+ let mut indices: Vec<String> = rows+ .as_array()+ .unwrap()+ .iter()+ .filter_map(|r| r["index"].as_str().map(String::from))+ .collect();+ indices.sort();+ indices.dedup();+ assert_eq!(+ indices,+ vec!["cluster-a".to_string(), "cluster-b".to_string()],+ "the no-path form covers every index: {rows}"+ );++ // v prints the header line even when the selector matches nothing. With+ // no rows the column widths are the header widths themselves, so the+ // header cells join with a single space.+ let (st, text) = call(&app, "GET", "/_cat/segments/no-such-*?v=true&h=index,segment", Value::Null)+ .await;+ assert_eq!(st, StatusCode::OK, "{text}");+ assert_eq!(text, "index segment\n", "header only, no rows: {text:?}");+}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex f026216cf..d912c0f57 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -7807,6 +7807,20 @@ pub struct SemanticFieldFacts {pub embedder_active: bool,}+/// One live durable segment, as `_cat/segments` reports it (#1202). See+/// [`Index::segment_rows`] for the deleted-count and live-count semantics.+#[derive(Debug, Clone)]+pub struct SegmentInfo {+ /// The segment's id (a UUID string; `_cat/segments`' `segment` column).+ pub id: String,+ /// Physical docs written in the segment (a tombstone is not a doc).+ pub doc_count: u64,+ /// Docs tombstoned in this segment's ZTB2 section.+ pub deleted_doc_count: u64,+ /// The segment's on-disk size in bytes.+ pub size_bytes: u64,+}+/// Statistics about an index.#[derive(Debug, Clone, Serialize, Deserialize)]pub struct IndexStats {@@ -24407,6 +24421,52 @@ impl Index {self.store.version_map.live_count() as u64}+ /// Real per-segment rows for `_cat/segments` (#1202): one entry per LIVE+ /// segment in the durable snapshot — its id, its docs, its deleted docs,+ /// its on-disk bytes. The predecessor endpoint fabricated a single `_0`+ /// row per index ("one logical segment"), which pointed operators AWAY+ /// from the cause during #1186 (an index the log showed merging 13+ /// segments reported "1 segment, 0 deleted").+ ///+ /// `deleted_doc_count` decodes the segment's own ZTB2 tombstone section —+ /// through the cached reader, so a repeated listing costs no re-open and+ /// no whole-file CRC. `doc_count` is the segment's physical document+ /// count, tombstones excluded by construction (a tombstone-only segment+ /// carries 0 docs). A delete's tombstone can persist in a LATER segment+ /// than the doc it kills — xerj tombstones are segment-level records, not+ /// in-place marks — so the honest per-segment reading is+ /// `Σ docs.count − Σ docs.deleted = live docs`, not per-row liveness.+ /// Superseded-until-merge versions of an updated document stay counted in+ /// their segments: only a merge folds them away, the same pre-merge+ /// over-count `_cat/indices`' doc column documents for whole indices.+ pub fn segment_rows(&self) -> Vec<SegmentInfo> {+ let snap = self.store.snapshot();+ let mut out = Vec::with_capacity(snap.segments.len());+ for meta in &snap.segments {+ let mut deleted = 0u64;+ if meta.has_tombstones {+ // The Arc (and its mmap) stays alive across the decode — the+ // section bytes borrow the reader.+ if let Ok(reader) = self.store.open_segment_arc(&meta.id) {+ if let Ok(Some(bytes)) =+ reader.section(xerj_storage::segment::SectionType::Tombstones)+ {+ if let Some(pairs) = xerj_storage::segment::decode_tombstones_v2(bytes) {+ deleted = pairs.len() as u64;+ }+ }+ }+ }+ out.push(SegmentInfo {+ id: meta.id.clone(),+ doc_count: meta.doc_count,+ deleted_doc_count: deleted,+ size_bytes: meta.size_bytes,+ });+ }+ out+ }+pub async fn stats(&self) -> IndexStats {let snap = self.store.snapshot();let segment_count = snap.segments.len();