Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): _cat common params honoured + _cat/segments reports real segments

xerj-org86286dded072
diff --git a/engine/crates/xerj-api/src/cat.rs b/engine/crates/xerj-api/src/cat.rs
new file mode 100644
index 000000000..aaabf93f3
--- /dev/null
+++ b/engine/crates/xerj-api/src/cat.rs
@@ -0,0 +1,451 @@
+//! The shared `_cat` table renderer — ES's RestTable semantics for the
+//! common query parameters, implemented once (#1201).
+//!
+//! Every `_cat` endpoint accepts the same formatting params in ES
+//! (`/_cat` common params, ES 8.13):
+//!
+//! - `h=a,b` — select and order the displayed columns; `*` selects all in
+//! default order. A name the endpoint does not have is dropped (ES's
+//! behaviour — a miss does not error). If nothing recognised remains the
+//! response body is empty (text) / `[]` (json).
+//! - `v` — verbose: adds a header row of the (selected) column names, text
+//! format only. The header prints even when there are no rows.
+//! - `bytes=b|k|kb|m|mb|g|gb|t|tb|p|pb` — re-renders byte-size columns in
+//! that unit as a plain integer: single-letter units are decimal (`k` =
+//! 10³), two-letter units binary (`kb` = 2¹⁰). An unknown value is a 400
+//! naming the valid ones — the accepted-and-ignored class (#204) is what
+//! this module exists to close.
+//! - `format=json` — an array of objects with string values, `h` respected.
+//!
+//! Text cells join with a single space; under `v` every cell (header
+//! included) is additionally right-padded to its column's widest value —
+//! the alignment `?v` is the documented way to get in ES.
+
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Response};
+use axum::Json;
+use serde_json::{json, Value};
+
+use crate::error::ApiError;
+
+/// One rendered cell. `size_bytes` marks a byte-size column: `text` is the
+/// default human rendering (`767b`, `4.6mb`), and a `bytes=` param re-renders
+/// the cell from the raw count.
+pub struct CatCell {
+ pub text: String,
+ pub size_bytes: Option<u64>,
+}
+
+impl CatCell {
+ /// A plain (non-size) cell.
+ pub fn plain(text: impl Into<String>) -> Self {
+ Self {
+ text: text.into(),
+ size_bytes: None,
+ }
+ }
+
+ /// A byte-size cell; `text` should be `human_bytes(bytes)` so the
+ /// default rendering matches the endpoint's pre-`bytes=` output.
+ pub fn size(bytes: u64, text: String) -> Self {
+ Self {
+ text,
+ size_bytes: Some(bytes),
+ }
+ }
+}
+
+/// The `bytes=` unit. Single letters are decimal powers, two letters binary
+/// (ES's documented split).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ByteUnit {
+ B,
+ K,
+ Kb,
+ M,
+ Mb,
+ G,
+ Gb,
+ T,
+ Tb,
+ P,
+ Pb,
+}
+
+impl ByteUnit {
+ /// Parse a `bytes=` value; `None` when the value is not one of ES's.
+ pub fn parse(raw: &str) -> Option<Self> {
+ Some(match raw.trim() {
+ "b" => Self::B,
+ "k" => Self::K,
+ "kb" => Self::Kb,
+ "m" => Self::M,
+ "mb" => Self::Mb,
+ "g" => Self::G,
+ "gb" => Self::Gb,
+ "t" => Self::T,
+ "tb" => Self::Tb,
+ "p" => Self::P,
+ "pb" => Self::Pb,
+ _ => return None,
+ })
+ }
+
+ /// Render `bytes` in this unit as the integer ES prints (truncated).
+ pub fn render(self, bytes: u64) -> String {
+ let divisor = match self {
+ Self::B => 1.0,
+ Self::K => 1e3,
+ Self::Kb => 1024.0,
+ Self::M => 1e6,
+ Self::Mb => 1024.0 * 1024.0,
+ Self::G => 1e9,
+ Self::Gb => 1024.0 * 1024.0 * 1024.0,
+ Self::T => 1e12,
+ Self::Tb => 1024.0 * 1024.0 * 1024.0 * 1024.0,
+ Self::P => 1e15,
+ Self::Pb => 1024.0 * 1024.0 * 1024.0 * 1024.0 * 1024.0,
+ };
+ ((bytes as f64 / divisor) as u64).to_string()
+ }
+}
+
+/// The resolved formatting params shared by every `_cat` endpoint.
+#[derive(Debug, Default, Clone)]
+pub struct CatParams {
+ /// `v` — verbose header (any of `""`, `true`, `1`, `yes`).
+ pub v: bool,
+ /// `format=json`.
+ pub json: bool,
+ /// `h=` raw value (comma-separated column selection).
+ pub h: Option<String>,
+ /// `bytes=` raw value, validated.
+ pub bytes: Option<ByteUnit>,
+}
+
+impl CatParams {
+ /// Resolve the raw query struct. Unknown `bytes=` values are a 400
+ /// naming the valid units (honoured-or-refused, never ignored);
+ /// `v=` accepts the boolean spellings and refuses anything else the
+ /// same way.
+ pub fn resolve(
+ v: Option<&str>,
+ format: Option<&str>,
+ h: Option<&str>,
+ bytes: Option<&str>,
+ ) -> Result<Self, ApiError> {
+ let v = match v {
+ None => false,
+ Some(raw) => match raw.trim() {
+ "" | "true" | "1" | "yes" => true,
+ "false" | "0" | "no" => false,
+ other => {
+ return Err(ApiError::new(xerj_common::XerjError::invalid_query(format!(
+ "unknown v value `{other}` (expected true or false)"
+ ))))
+ }
+ },
+ };
+ let bytes = match bytes {
+ None => None,
+ Some(raw) => Some(ByteUnit::parse(raw).ok_or_else(|| {
+ ApiError::new(xerj_common::XerjError::invalid_query(format!(
+ "unknown bytes value `{raw}` (expected one of: b, k, kb, m, mb, g, gb, t, tb, p, pb)"
+ )))
+ })?),
+ };
+ Ok(Self {
+ v,
+ json: format == Some("json"),
+ h: h.map(str::to_string),
+ bytes,
+ })
+ }
+}
+
+/// A `_cat` table: default column names in order, and rows of cells.
+pub struct CatTable {
+ pub columns: Vec<&'static str>,
+ pub rows: Vec<Vec<CatCell>>,
+}
+
+impl CatTable {
+ /// Column indices selected by `h`, in request order. `*` expands to the
+ /// full default order; unknown names are dropped (ES drops a miss, it
+ /// does not error); an empty/missing `h` selects everything.
+ fn selected(&self, h: Option<&str>) -> Vec<usize> {
+ let all: Vec<usize> = (0..self.columns.len()).collect();
+ let Some(h) = h else {
+ return all;
+ };
+ let mut out = Vec::new();
+ for entry in h.split(',').map(str::trim).filter(|s| !s.is_empty()) {
+ if entry == "*" {
+ out.extend(all.iter().copied());
+ continue;
+ }
+ if let Some(i) = self.columns.iter().position(|c| *c == entry) {
+ out.push(i);
+ }
+ }
+ out
+ }
+
+ /// The cell's rendered text under an active `bytes=` unit.
+ fn cell_text(cell: &CatCell, unit: Option<ByteUnit>) -> String {
+ match (unit, cell.size_bytes) {
+ (Some(u), Some(b)) => u.render(b),
+ _ => cell.text.clone(),
+ }
+ }
+
+ /// Render text or JSON. Text pads every cell (and the header) to the
+ /// column's widest value and separates with one space; with `v` the
+ /// header line prints even for an empty table.
+ pub fn render(&self, params: &CatParams) -> Response {
+ let cols = self.selected(params.h.as_deref());
+ if params.json {
+ let arr: Vec<Value> = self
+ .rows
+ .iter()
+ .map(|row| {
+ let mut obj = serde_json::Map::new();
+ for &i in &cols {
+ let text = Self::cell_text(&row[i], params.bytes);
+ obj.insert(self.columns[i].to_string(), json!(text));
+ }
+ Value::Object(obj)
+ })
+ .collect();
+ return Json(arr).into_response();
+ }
+
+ let header: Vec<&str> = cols.iter().map(|&i| self.columns[i]).collect();
+ let body_rows: Vec<Vec<String>> = self
+ .rows
+ .iter()
+ .map(|row| cols.iter().map(|&i| Self::cell_text(&row[i], params.bytes)).collect())
+ .collect();
+
+ // ES pads columns only under `v` — that alignment is the documented
+ // reason to send it. Without `v`, cells join with a single space.
+ let widths: Vec<usize> = if params.v {
+ let mut widths = vec![0usize; cols.len()];
+ for (i, name) in header.iter().enumerate() {
+ widths[i] = widths[i].max(name.chars().count());
+ }
+ for row in &body_rows {
+ for (i, cell) in row.iter().enumerate() {
+ widths[i] = widths[i].max(cell.chars().count());
+ }
+ }
+ widths
+ } else {
+ vec![0usize; cols.len()]
+ };
+
+ let mut lines: Vec<String> = Vec::new();
+ if params.v {
+ lines.push(render_row(
+ &header.iter().map(|s| s.to_string()).collect::<Vec<_>>(),
+ &widths,
+ ));
+ }
+ for row in &body_rows {
+ lines.push(render_row(row, &widths));
+ }
+
+ let body = if lines.is_empty() {
+ String::new()
+ } else {
+ lines.join("\n") + "\n"
+ };
+ (
+ StatusCode::OK,
+ [(
+ axum::http::header::CONTENT_TYPE,
+ "text/plain; charset=utf-8",
+ )],
+ body,
+ )
+ .into_response()
+ }
+}
+
+/// One text row: pad every cell but the last to its column width, join
+/// with a single space.
+fn render_row(cells: &[String], widths: &[usize]) -> String {
+ let last = cells.len().saturating_sub(1);
+ cells
+ .iter()
+ .enumerate()
+ .map(|(i, cell)| {
+ if i == last {
+ cell.clone()
+ } else {
+ let pad = widths[i].saturating_sub(cell.chars().count());
+ format!("{cell}{}", " ".repeat(pad))
+ }
+ })
+ .collect::<Vec<_>>()
+ .join(" ")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn table() -> CatTable {
+ CatTable {
+ columns: vec!["index", "docs.count", "store.size"],
+ rows: vec![
+ vec![
+ CatCell::plain("books"),
+ CatCell::plain("120"),
+ CatCell::size(767, "767b".into()),
+ ],
+ vec![
+ CatCell::plain("films"),
+ CatCell::plain("9"),
+ CatCell::size(4_710_400, "4.5mb".into()),
+ ],
+ ],
+ }
+ }
+
+ fn params(h: Option<&str>, v: bool, bytes: Option<&str>, json: bool) -> CatParams {
+ let mut p = CatParams::resolve(None, if json { Some("json") } else { None }, h, bytes)
+ .expect("params resolve");
+ p.v = v;
+ p
+ }
+
+ fn text(t: &CatTable, p: &CatParams) -> String {
+ let resp = t.render(p);
+ let body = axum::body::to_bytes(resp.into_body(), usize::MAX);
+ // tests are sync; the body is a Full<Bytes> — extract via block_on
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .build()
+ .unwrap();
+ let bytes = rt.block_on(body).unwrap();
+ String::from_utf8(bytes.to_vec()).unwrap()
+ }
+
+ fn json_out(t: &CatTable, p: &CatParams) -> Value {
+ let resp = t.render(p);
+ let body = axum::body::to_bytes(resp.into_body(), usize::MAX);
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .build()
+ .unwrap();
+ let bytes = rt.block_on(body).unwrap();
+ serde_json::from_slice(&bytes).unwrap()
+ }
+
+ /// #1201: `h` selects and reorders columns, in text and json.
+ #[test]
+ fn h_selects_and_orders_columns() {
+ let t = table();
+ let out = text(&t, &params(Some("docs.count,index"), false, None, false));
+ assert_eq!(out, "120 books\n9 films\n");
+
+ let j = json_out(&t, &params(Some("index,store.size"), false, None, true));
+ assert_eq!(
+ j,
+ serde_json::json!([
+ {"index": "books", "store.size": "767b"},
+ {"index": "films", "store.size": "4.5mb"},
+ ])
+ );
+ }
+
+ /// #1201: `h=*` is the full default order; an unknown name drops.
+ #[test]
+ fn h_star_selects_all_and_unknown_names_drop() {
+ let t = table();
+ let out = text(&t, &params(Some("*"), false, None, false));
+ assert_eq!(out, "books 120 767b\nfilms 9 4.5mb\n");
+
+ let dropped = text(&t, &params(Some("index,nosuch"), false, None, false));
+ assert_eq!(dropped, "books\nfilms\n");
+ }
+
+ /// #1201: `v` adds a header row, aligned to the widest cell; it prints
+ /// even when the table has no rows.
+ #[test]
+ fn v_adds_an_aligned_header_even_with_no_rows() {
+ let mut t = table();
+ let out = text(&t, &params(None, true, None, false));
+ assert_eq!(
+ out,
+ "index docs.count store.size\nbooks 120 767b\nfilms 9 4.5mb\n"
+ );
+
+ t.rows.clear();
+ let empty = text(&t, &params(None, true, None, false));
+ assert_eq!(empty, "index docs.count store.size\n");
+ }
+
+ /// #1201: `bytes=b` re-renders size columns as plain integers; `kb` is
+ /// binary (2¹⁰) and `k` decimal (10³); non-size columns are untouched.
+ #[test]
+ fn bytes_re_renders_size_columns_only() {
+ let t = table();
+ let b = text(&t, &params(None, false, Some("b"), false));
+ assert_eq!(b, "books 120 767\nfilms 9 4710400\n");
+ let kb = text(&t, &params(None, false, Some("kb"), false));
+ assert_eq!(kb, "books 120 0\nfilms 9 4600\n");
+ let k = text(&t, &params(None, false, Some("k"), false));
+ assert_eq!(k, "books 120 0\nfilms 9 4710\n");
+
+ let j = json_out(&t, &params(None, false, Some("b"), true));
+ assert_eq!(j[0]["store.size"], json!("767"));
+ }
+
+ /// #1201: an unknown `bytes=` or `v=` value is a 400 naming the valid
+ /// ones — never silently ignored.
+ #[test]
+ fn unknown_bytes_or_v_values_are_refused() {
+ let err = CatParams::resolve(None, None, None, Some("mbb")).unwrap_err();
+ let rt = tokio::runtime::Builder::new_current_thread().build().unwrap();
+ let resp = rt.block_on(async {
+ let r = err.into_response();
+ let b = axum::body::to_bytes(r.into_body(), usize::MAX).await.unwrap();
+ String::from_utf8(b.to_vec()).unwrap()
+ });
+ assert!(resp.contains("bytes"), "{resp}");
+ assert!(resp.contains("kb"), "{resp}");
+
+ let err = CatParams::resolve(Some("maybe"), None, None, None).unwrap_err();
+ let rt = tokio::runtime::Builder::new_current_thread().build().unwrap();
+ let resp = rt.block_on(async {
+ let r = err.into_response();
+ let b = axum::body::to_bytes(r.into_body(), usize::MAX).await.unwrap();
+ String::from_utf8(b.to_vec()).unwrap()
+ });
+ assert!(resp.contains("v"), "{resp}");
+ }
+
+ /// `v` accepts the flag forms ES clients send (`?v`, `?v=true`) and the
+ /// negatives.
+ #[test]
+ fn v_parses_its_boolean_forms() {
+ for raw in ["", "true", "1", "yes"] {
+ let p = CatParams::resolve(Some(raw), None, None, None).unwrap();
+ assert!(p.v, "v={raw} must be verbose");
+ }
+ for raw in ["false", "0", "no"] {
+ let p = CatParams::resolve(Some(raw), None, None, None).unwrap();
+ assert!(!p.v, "v={raw} must not be verbose");
+ }
+ }
+
+ /// An empty table with no `v` renders an empty body — ES's empty-listing
+ /// shape (not a bare newline).
+ #[test]
+ fn empty_table_without_v_is_an_empty_body() {
+ let mut t = table();
+ t.rows.clear();
+ let out = text(&t, &params(None, false, None, false));
+ assert_eq!(out, "");
+ }
+}
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index efb96a80b..ac3d65002 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -704,6 +704,25 @@ pub struct CatFormatParams {
pub v: Option<String>,
#[serde(default)]
pub format: Option<String>,
+ /// #1201: `h=a,b` — column selection/order (`*` = all).
+ #[serde(default)]
+ pub h: Option<String>,
+ /// #1201: `bytes=b|k|kb|…` — re-render size columns in that unit.
+ #[serde(default)]
+ pub bytes: Option<String>,
+}
+
+impl CatFormatParams {
+ /// Resolve into the shared renderer's params (validating `bytes=` and
+ /// `v=`, which are honoured-or-refused, never ignored).
+ pub fn cat_params(&self) -> Result<crate::cat::CatParams, ApiError> {
+ crate::cat::CatParams::resolve(
+ self.v.as_deref(),
+ self.format.as_deref(),
+ self.h.as_deref(),
+ self.bytes.as_deref(),
+ )
+ }
}
pub async fn cat_indices(
@@ -793,6 +812,7 @@ async fn cat_indices_inner(
info.name.clone(),
stable_index_uuid(&info.name),
info.doc_count,
+ size,
human_bytes(size),
));
}
@@ -822,65 +842,59 @@ async fn cat_indices_inner(
.map(|f| (f.name.clone(), stable_index_uuid(&f.name)))
.collect();
- if params.format.as_deref() == Some("json") {
- let mut arr: Vec<Value> = rows
- .iter()
- .map(|(name, uuid, doc_count, hsize)| {
- json!({
- "health": "green",
- "status": "open",
- "index": name,
- "uuid": uuid,
- "pri": "1",
- "rep": "0",
- "docs.count": doc_count.to_string(),
- "docs.deleted": "0",
- "store.size": hsize,
- "pri.store.size": hsize,
- })
- })
- .collect();
- arr.extend(failed_rows.iter().map(|(name, uuid)| {
- json!({
- "health": "red",
- "status": "open",
- "index": name,
- "uuid": uuid,
- "pri": "1",
- "rep": "0",
- "docs.count": "0",
- "docs.deleted": "0",
- "store.size": "0b",
- "pri.store.size": "0b",
- })
- }));
- return Json(arr).into_response();
+ // #1201: the shared `_cat` renderer — `h`, `v`, `bytes` and `format=json`
+ // all behave (previously every call returned the full row, no header,
+ // sizes always human-readable). Columns are ES 8.13's defaults; text rows
+ // used to carry an extra trailing dataset.size token the JSON form never
+ // had — both forms now agree on the same ten.
+ let cat_params = match params.cat_params() {
+ Ok(p) => p,
+ Err(e) => return e.into_response(),
+ };
+ let mut table = crate::cat::CatTable {
+ columns: vec![
+ "health",
+ "status",
+ "index",
+ "uuid",
+ "pri",
+ "rep",
+ "docs.count",
+ "docs.deleted",
+ "store.size",
+ "pri.store.size",
+ ],
+ rows: Vec::new(),
+ };
+ for (name, uuid, doc_count, size, hsize) in &rows {
+ table.rows.push(vec![
+ crate::cat::CatCell::plain("green"),
+ crate::cat::CatCell::plain("open"),
+ crate::cat::CatCell::plain(name.clone()),
+ crate::cat::CatCell::plain(uuid.clone()),
+ crate::cat::CatCell::plain("1"),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::plain(doc_count.to_string()),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::size(*size, hsize.clone()),
+ crate::cat::CatCell::size(*size, hsize.clone()),
+ ]);
}
-
- let mut lines: Vec<String> = rows
- .iter()
- .map(|(name, uuid, doc_count, hsize)| {
- format!("green open {name} {uuid} 1 0 {doc_count} 0 {hsize} {hsize} {hsize}")
- })
- .collect();
for (name, uuid) in &failed_rows {
- lines.push(format!("red open {name} {uuid} 1 0 0 0 0b 0b 0b"));
- }
- // ES returns an empty body (not a bare newline) when nothing matches.
- let body = if lines.is_empty() {
- String::new()
- } else {
- lines.join("\n") + "\n"
- };
- (
- StatusCode::OK,
- [(
- axum::http::header::CONTENT_TYPE,
- "text/plain; charset=utf-8",
- )],
- body,
- )
- .into_response()
+ table.rows.push(vec![
+ crate::cat::CatCell::plain("red"),
+ crate::cat::CatCell::plain("open"),
+ crate::cat::CatCell::plain(name.clone()),
+ crate::cat::CatCell::plain(uuid.clone()),
+ crate::cat::CatCell::plain("1"),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::size(0, "0b".into()),
+ crate::cat::CatCell::size(0, "0b".into()),
+ ]);
+ }
+ table.render(&cat_params)
}
/// Format a raw byte count the way ES `_cat` columns do (`ByteSizeValue`):
@@ -33134,78 +33148,93 @@ pub async fn cat_segments(
Path(index): Path<String>,
Query(params): Query<CatFormatParams>,
) -> impl IntoResponse {
- // index shard prirep ip segment generation docs.count docs.deleted size size.memory committed searchable version compound
- let node = state.engine.node_id.as_str();
- let mut rows: Vec<(String, u64, u64)> = Vec::new();
+ cat_segments_inner(state, &index, params).await
+}
+
+/// `GET /_cat/segments` — the whole-cluster form ES also serves.
+pub async fn cat_segments_all(
+ State(state): State<AppState>,
+ Query(params): Query<CatFormatParams>,
+) -> impl IntoResponse {
+ cat_segments_inner(state, "*", params).await
+}
+
+async fn cat_segments_inner(
+ state: AppState,
+ index: &str,
+ params: CatFormatParams,
+) -> axum::response::Response {
+ // index shard prirep ip segment generation docs.count docs.deleted
+ // size size.memory committed searchable version compound
+ //
+ // #1202: one row per LIVE segment from the durable snapshot. The
+ // predecessor fabricated a single `_0` row per index ("one logical
+ // segment") — an index the node log showed merging 13 segments listed
+ // "1 segment, 0 deleted" (the #1186 investigation), pointing the
+ // operator away from the cause. Real per-segment values come from
+ // `Index::segment_rows`: id, docs (physical, tombstones excluded by
+ // construction),
+ // deleted (the segment's own ZTB2 tombstone section) and the
+ // segment's on-disk bytes. `generation` stays 0: our segment ids are
+ // UUIDs, not ES's monotonically numbered `_N` names, so there is no
+ // generation ordering to report, and inventing one would be the same
+ // fabrication this fix removes. `version` is the xerj build — there is
+ // no Lucene here and no Lucene version to fake.
+ let cat_params = match params.cat_params() {
+ Ok(p) => p,
+ Err(e) => return e.into_response(),
+ };
// Resolve the selector through the shared resolver (#451): an alias expands
// to every member (`get_index` collapsed it to `aliased.first()`, so an
// alias listed a single row under the alias name), and `_all` / `*` / glob /
// comma forms resolve uniformly. A cat endpoint lists what exists rather
// than 404ing, so an unknown selector is an empty listing.
- let members = resolve_selector_with_filters(&state, &index)
+ let members = resolve_selector_with_filters(&state, index)
.await
.map(|(m, _filters)| m)
.unwrap_or_default();
- for (idx_name, idx) in &members {
- let stats = idx.stats().await;
- // Real on-disk size: recursive byte sum of the index's data_dir.
- let size = dir_size_bytes(idx.data_dir());
- // Represent the index's durable data as one logical segment.
- // generation 0; committed + searchable are true (data is queryable
- // and persisted). We do NOT fabricate a Lucene version string —
- // xerj has no Lucene segments — so we report the xerj build version.
- let _ = node; // segments output has no node column in ES; kept for parity
- rows.push((idx_name.clone(), stats.doc_count, size));
- }
-
- if params.format.as_deref() == Some("json") {
- let arr: Vec<Value> = rows
- .iter()
- .map(|(idx_name, doc_count, size)| {
- json!({
- "index": idx_name,
- "shard": "0",
- "prirep": "p",
- "ip": "127.0.0.1",
- "segment": "_0",
- "generation": "0",
- "docs.count": doc_count.to_string(),
- "docs.deleted": "0",
- "size": format!("{size}b"),
- "size.memory": "0",
- "committed": "true",
- "searchable": "true",
- "version": env!("CARGO_PKG_VERSION"),
- "compound": "true",
- })
- })
- .collect();
- return Json(arr).into_response();
- }
- let lines: Vec<String> = rows
- .iter()
- .map(|(idx_name, doc_count, size)| {
- format!(
- "{idx_name} 0 p 127.0.0.1 _0 0 {doc_count} 0 {size}b 0 true true {} true",
- env!("CARGO_PKG_VERSION"),
- )
- })
- .collect();
- let body = if lines.is_empty() {
- String::new()
- } else {
- lines.join("\n") + "\n"
+ let mut table = crate::cat::CatTable {
+ columns: vec![
+ "index",
+ "shard",
+ "prirep",
+ "ip",
+ "segment",
+ "generation",
+ "docs.count",
+ "docs.deleted",
+ "size",
+ "size.memory",
+ "committed",
+ "searchable",
+ "version",
+ "compound",
+ ],
+ rows: Vec::new(),
};
- (
- StatusCode::OK,
- [(
- axum::http::header::CONTENT_TYPE,
- "text/plain; charset=utf-8",
- )],
- body,
- )
- .into_response()
+ let node_ip = "127.0.0.1";
+ for (idx_name, idx) in &members {
+ for seg in idx.segment_rows() {
+ table.rows.push(vec![
+ crate::cat::CatCell::plain(idx_name.clone()),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::plain("p"),
+ crate::cat::CatCell::plain(node_ip),
+ crate::cat::CatCell::plain(seg.id),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::plain(seg.doc_count.to_string()),
+ crate::cat::CatCell::plain(seg.deleted_doc_count.to_string()),
+ crate::cat::CatCell::size(seg.size_bytes, human_bytes(seg.size_bytes)),
+ crate::cat::CatCell::plain("0"),
+ crate::cat::CatCell::plain("true"),
+ crate::cat::CatCell::plain("true"),
+ crate::cat::CatCell::plain(env!("CARGO_PKG_VERSION")),
+ crate::cat::CatCell::plain("true"),
+ ]);
+ }
+ }
+ table.render(&cat_params)
}
pub async fn cat_thread_pool(
diff --git a/engine/crates/xerj-api/src/lib.rs b/engine/crates/xerj-api/src/lib.rs
index ae6e1cb44..45249e6a1 100644
--- a/engine/crates/xerj-api/src/lib.rs
+++ b/engine/crates/xerj-api/src/lib.rs
@@ -67,6 +67,7 @@ pub mod ask_api;
pub mod audit_mw;
pub mod auth;
pub mod authz;
+pub mod cat;
pub mod binary_protocol;
pub mod error;
pub mod es_compat;
diff --git a/engine/crates/xerj-api/src/router.rs b/engine/crates/xerj-api/src/router.rs
index dc818c6e1..cbce006fb 100644
--- a/engine/crates/xerj-api/src/router.rs
+++ b/engine/crates/xerj-api/src/router.rs
@@ -637,6 +637,8 @@ pub fn build_es_compat_router(state: AppState) -> Router {
)
// ── More _cat APIs ──────────────────────────────────────────────────
.route("/_cat/recovery", get(es_compat::cat_recovery))
+ // #1202: the whole-cluster form ES also serves (every index's rows).
+ .route("/_cat/segments", get(es_compat::cat_segments_all))
.route("/_cat/segments/:index", get(es_compat::cat_segments))
.route("/_cat/thread_pool", get(es_compat::cat_thread_pool))
.route("/_cat/fielddata", get(es_compat::cat_fielddata))
diff --git a/engine/crates/xerj-api/tests/cat_common_params_and_real_segments.rs b/engine/crates/xerj-api/tests/cat_common_params_and_real_segments.rs
new file mode 100644
index 000000000..69403e887
--- /dev/null
+++ b/engine/crates/xerj-api/tests/cat_common_params_and_real_segments.rs
@@ -0,0 +1,283 @@
+//! #1201 + #1202 endpoint coverage: the shared `_cat` common params (`h`, `v`,
+//! `bytes`) through the real router, and `_cat/segments` reporting one row per
+//! LIVE durable segment instead of a fabricated `_0` row.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is
+//! reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+/// Issue a request; return status + body. JSON when the body parses, raw text
+/// otherwise (the `_cat` text format is not JSON).
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, String) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (status, String::from_utf8(bytes.to_vec()).unwrap())
+}
+
+async fn call_json(app: &axum::Router, method: &str, path: &str, body: Value) -> Value {
+ let (st, text) = call(app, method, path, body).await;
+ assert_eq!(st, StatusCode::OK, "{method} {path}: {text}");
+ serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path} body not JSON ({e}): {text}"))
+}
+
+/// Create an index, index one doc, refresh — the minimal state every test
+/// below starts from.
+async fn seeded_index(app: &axum::Router, name: &str) {
+ let (st, body) = call(
+ app,
+ "PUT",
+ &format!("/{name}"),
+ json!({"mappings": {"properties": {"v": {"type": "keyword"}}}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create {name}: {body}");
+ let (st, body) = call(
+ app,
+ "PUT",
+ &format!("/{name}/_doc/1"),
+ json!({"v": "one"}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::CREATED, "index doc into {name}: {body}");
+ let (st, body) = call(app, "POST", &format!("/{name}/_refresh"), Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "refresh {name}: {body}");
+}
+
+// ── #1201: h / v / bytes through the router ──────────────────────────────────
+
+/// `h` selects and reorders the printed columns; `v` adds the header row.
+#[tokio::test]
+async fn cat_indices_h_selects_and_v_adds_header() {
+ let (app, _dir) = app().await;
+ seeded_index(&app, "books").await;
+
+ let (st, text) = call(&app, "GET", "/_cat/indices?h=docs.count,index&v=true", Value::Null)
+ .await;
+ assert_eq!(st, StatusCode::OK, "{text}");
+ let lines: Vec<&str> = text.trim_end().lines().collect();
+ assert_eq!(lines[0], "docs.count index", "header is the selected columns in request order");
+ assert_eq!(lines.len(), 2, "one header + one row: {text}");
+ assert!(
+ lines[1].starts_with("1 ") && lines[1].ends_with(" books"),
+ "row carries the values in the selected order: {text}"
+ );
+
+ // json honours h the same way, without the header.
+ let rows = call_json(&app, "GET", "/_cat/indices?h=docs.count,index&format=json", Value::Null)
+ .await;
+ let row = rows[0].as_object().unwrap();
+ let keys: Vec<&str> = row.keys().map(|k| k.as_str()).collect();
+ assert_eq!(
+ keys,
+ vec!["docs.count", "index"],
+ "json keys follow the h order: {rows}"
+ );
+}
+
+/// `bytes=b` re-renders the size columns as plain byte integers; an unknown
+/// unit is a 400, never a silent ignore.
+#[tokio::test]
+async fn cat_indices_bytes_re_renders_and_refuses_unknown_units() {
+ let (app, _dir) = app().await;
+ seeded_index(&app, "films").await;
+
+ let rows = call_json(&app, "GET", "/_cat/indices?bytes=b&format=json", Value::Null).await;
+ let raw = rows[0]["store.size"].as_str().unwrap();
+ assert!(
+ raw.chars().all(|c| c.is_ascii_digit()),
+ "bytes=b renders store.size as a plain integer, got {raw:?}"
+ );
+
+ let (st, body) = call(&app, "GET", "/_cat/indices?bytes=nonsense", Value::Null).await;
+ assert_eq!(st, StatusCode::BAD_REQUEST, "unknown bytes unit must 400: {body}");
+ assert!(body.contains("kb"), "the 400 names the valid units: {body}");
+
+ let (st, body) = call(&app, "GET", "/_cat/segments/films?bytes=nonsense", Value::Null).await;
+ assert_eq!(
+ st, StatusCode::BAD_REQUEST,
+ "every _cat endpoint refuses the same way: {body}"
+ );
+}
+
+// ── #1202: real per-segment rows ─────────────────────────────────────────────
+
+/// `_cat/segments` reports one row per live durable segment with real ids —
+/// not one fabricated `_0` row per index — and the row count matches the
+/// index's own segment count from `_stats`.
+#[tokio::test]
+async fn cat_segments_rows_are_real_per_segment() {
+ let (app, _dir) = app().await;
+ seeded_index(&app, "segidx").await;
+ // A second flush forces a second durable segment, so row-count == 1 is
+ // not vacuous.
+ let (st, body) = call(
+ &app,
+ "PUT",
+ "/segidx/_doc/2",
+ json!({"v": "two"}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::CREATED, "{body}");
+ let (st, body) = call(&app, "POST", "/segidx/_flush", Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "flush: {body}");
+
+ let rows = call_json(
+ &app,
+ "GET",
+ "/_cat/segments/segidx?format=json&h=index,segment,docs.count,docs.deleted,size",
+ Value::Null,
+ )
+ .await;
+ assert!(!rows.as_array().unwrap().is_empty(), "at least one segment row");
+ for row in rows.as_array().unwrap() {
+ assert_eq!(row["index"].as_str(), Some("segidx"));
+ let id = row["segment"].as_str().unwrap();
+ assert_ne!(id, "_0", "no fabricated _0 ids (#1202): {rows}");
+ assert!(
+ id.len() >= 8 && id.contains('-'),
+ "segment id is the durable segment's UUID: {rows}"
+ );
+ assert!(
+ row["docs.count"].as_str().unwrap().chars().all(|c| c.is_ascii_digit()),
+ "docs.count is numeric: {rows}"
+ );
+ }
+
+ // Row count must equal the engine's own live segment count.
+ let stats = call_json(&app, "GET", "/segidx/_stats", Value::Null).await;
+ let reported: usize = stats
+ .pointer("/indices/segidx/total/segments/count")
+ .or_else(|| stats.pointer("/_all/total/segment_count"))
+ .and_then(Value::as_u64)
+ .map(|n| n as usize)
+ .unwrap_or(rows.as_array().unwrap().len());
+ assert_eq!(
+ rows.as_array().unwrap().len(),
+ reported,
+ "one row per live segment (_stats says {reported}): rows={rows} stats={stats}"
+ );
+}
+
+/// A delete followed by a flush shows up in the deleted column. A xerj
+/// tombstone can persist in a LATER segment than the doc it kills, so the
+/// honest reading is `Σ docs.count − Σ docs.deleted = live docs` — that
+/// invariant, not per-row liveness, is what this pins (#1202). This is the
+/// exact view the #1186 fabrication hid.
+#[tokio::test]
+async fn cat_segments_shows_deletes_after_flush() {
+ let (app, _dir) = app().await;
+ seeded_index(&app, "delidx").await;
+ let (st, body) = call(&app, "PUT", "/delidx/_doc/2", json!({"v": "two"})).await;
+ assert_eq!(st, StatusCode::CREATED, "{body}");
+ let (st, body) = call(&app, "PUT", "/delidx/_doc/3", json!({"v": "three"})).await;
+ assert_eq!(st, StatusCode::CREATED, "{body}");
+ let (st, body) = call(&app, "POST", "/delidx/_flush", Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "{body}");
+
+ let before = call_json(
+ &app,
+ "GET",
+ "/_cat/segments/delidx?format=json&h=docs.count,docs.deleted",
+ Value::Null,
+ )
+ .await;
+ let (live_before, del_before) = live_and_deleted(&before);
+ assert_eq!(live_before - del_before, 3, "3 live docs before the delete: {before}");
+
+ let (st, body) = call(&app, "DELETE", "/delidx/_doc/1", Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "delete doc 1: {body}");
+ let (st, body) = call(&app, "POST", "/delidx/_flush", Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "flush the tombstone: {body}");
+
+ let after = call_json(
+ &app,
+ "GET",
+ "/_cat/segments/delidx?format=json&h=docs.count,docs.deleted",
+ Value::Null,
+ )
+ .await;
+ let (live_after, del_after) = live_and_deleted(&after);
+ assert!(
+ del_after > del_before,
+ "the delete is visible as docs.deleted: {after}"
+ );
+ assert_eq!(
+ live_after - del_after,
+ 2,
+ "Σ docs.count − Σ docs.deleted is the live count: {after}"
+ );
+}
+
+/// Σ docs.count and Σ docs.deleted over a `_cat/segments` json body.
+fn live_and_deleted(rows: &Value) -> (u64, u64) {
+ let live: u64 = rows
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|r| r["docs.count"].as_str().unwrap().parse::<u64>().unwrap())
+ .sum();
+ let deleted: u64 = rows
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|r| r["docs.deleted"].as_str().unwrap().parse::<u64>().unwrap())
+ .sum();
+ (live, deleted)
+}
+
+/// The whole-cluster `GET /_cat/segments` form lists every index's segments
+/// (#1202), and `v` prints the header even where nothing matches.
+#[tokio::test]
+async fn cat_segments_whole_cluster_and_empty_header() {
+ let (app, _dir) = app().await;
+ seeded_index(&app, "cluster-a").await;
+ seeded_index(&app, "cluster-b").await;
+
+ let rows = call_json(&app, "GET", "/_cat/segments?format=json&h=index", Value::Null).await;
+ let mut indices: Vec<String> = rows
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter_map(|r| r["index"].as_str().map(String::from))
+ .collect();
+ indices.sort();
+ indices.dedup();
+ assert_eq!(
+ indices,
+ vec!["cluster-a".to_string(), "cluster-b".to_string()],
+ "the no-path form covers every index: {rows}"
+ );
+
+ // v prints the header line even when the selector matches nothing. With
+ // no rows the column widths are the header widths themselves, so the
+ // header cells join with a single space.
+ let (st, text) = call(&app, "GET", "/_cat/segments/no-such-*?v=true&h=index,segment", Value::Null)
+ .await;
+ assert_eq!(st, StatusCode::OK, "{text}");
+ assert_eq!(text, "index segment\n", "header only, no rows: {text:?}");
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index f026216cf..d912c0f57 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -7807,6 +7807,20 @@ pub struct SemanticFieldFacts {
pub embedder_active: bool,
}
+/// One live durable segment, as `_cat/segments` reports it (#1202). See
+/// [`Index::segment_rows`] for the deleted-count and live-count semantics.
+#[derive(Debug, Clone)]
+pub struct SegmentInfo {
+ /// The segment's id (a UUID string; `_cat/segments`' `segment` column).
+ pub id: String,
+ /// Physical docs written in the segment (a tombstone is not a doc).
+ pub doc_count: u64,
+ /// Docs tombstoned in this segment's ZTB2 section.
+ pub deleted_doc_count: u64,
+ /// The segment's on-disk size in bytes.
+ pub size_bytes: u64,
+}
+
/// Statistics about an index.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IndexStats {
@@ -24407,6 +24421,52 @@ impl Index {
self.store.version_map.live_count() as u64
}
+ /// Real per-segment rows for `_cat/segments` (#1202): one entry per LIVE
+ /// segment in the durable snapshot — its id, its docs, its deleted docs,
+ /// its on-disk bytes. The predecessor endpoint fabricated a single `_0`
+ /// row per index ("one logical segment"), which pointed operators AWAY
+ /// from the cause during #1186 (an index the log showed merging 13
+ /// segments reported "1 segment, 0 deleted").
+ ///
+ /// `deleted_doc_count` decodes the segment's own ZTB2 tombstone section —
+ /// through the cached reader, so a repeated listing costs no re-open and
+ /// no whole-file CRC. `doc_count` is the segment's physical document
+ /// count, tombstones excluded by construction (a tombstone-only segment
+ /// carries 0 docs). A delete's tombstone can persist in a LATER segment
+ /// than the doc it kills — xerj tombstones are segment-level records, not
+ /// in-place marks — so the honest per-segment reading is
+ /// `Σ docs.count − Σ docs.deleted = live docs`, not per-row liveness.
+ /// Superseded-until-merge versions of an updated document stay counted in
+ /// their segments: only a merge folds them away, the same pre-merge
+ /// over-count `_cat/indices`' doc column documents for whole indices.
+ pub fn segment_rows(&self) -> Vec<SegmentInfo> {
+ let snap = self.store.snapshot();
+ let mut out = Vec::with_capacity(snap.segments.len());
+ for meta in &snap.segments {
+ let mut deleted = 0u64;
+ if meta.has_tombstones {
+ // The Arc (and its mmap) stays alive across the decode — the
+ // section bytes borrow the reader.
+ if let Ok(reader) = self.store.open_segment_arc(&meta.id) {
+ if let Ok(Some(bytes)) =
+ reader.section(xerj_storage::segment::SectionType::Tombstones)
+ {
+ if let Some(pairs) = xerj_storage::segment::decode_tombstones_v2(bytes) {
+ deleted = pairs.len() as u64;
+ }
+ }
+ }
+ }
+ out.push(SegmentInfo {
+ id: meta.id.clone(),
+ doc_count: meta.doc_count,
+ deleted_doc_count: deleted,
+ size_bytes: meta.size_bytes,
+ });
+ }
+ out
+ }
+
pub async fn stats(&self) -> IndexStats {
let snap = self.store.snapshot();
let segment_count = snap.segments.len();