AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(memory): paginate _memory list + document its auth model (item 12)
xerj-orga89cfbd316ab
diff --git a/engine/crates/xerj-api/src/memory_api.rs b/engine/crates/xerj-api/src/memory_api.rsindex b7f7a9537..d2a943208 100644--- a/engine/crates/xerj-api/src/memory_api.rs+++ b/engine/crates/xerj-api/src/memory_api.rs@@ -10,11 +10,34 @@//! ```text//! POST /_memory/{namespace} store — {text, metadata?, vector?, id?, dedup?, dedup_threshold?} → {id, created} | {id, created:false, deduplicated:true}//! POST /_memory/{namespace}/_recall recall — {query?, vector?, semantic?, k?, filter?, recency_weight?} → {hits:[…]}-//! GET /_memory/{namespace} list — {count, entries:[…]} (bounded, recent first)+//! GET /_memory/{namespace}?from&size list — {count, entries:[…], next} (paged, recent first)//! DELETE /_memory/{namespace}/{id} forget — delete one entry//! DELETE /_memory/{namespace} drop — drop the whole namespace//! ```//!+//! ## Pagination (item 12)+//!+//! `GET /_memory/{namespace}` pages the namespace with `from` (offset, default+//! 0; `after` is accepted as an alias) and `size` (page size, default 100,+//! capped at [`MAX_LIST_SIZE`]). Before this the endpoint silently truncated at+//! 100 entries with no way to see the rest — an agent introspecting a large+//! namespace got a partial, unmarked view. The response now carries `count`+//! (the true total) and a `next` cursor (`{from, size}` for the next page, or+//! `null` on the last page). Deep offsets are bounded by the backing index's+//! `max_result_window`; for exhaustive retrieval prefer `_recall`.+//!+//! ## Authorization model (item 12)+//!+//! ⚠️ The `/_memory/*` endpoints are guarded ONLY by the process-wide API-key+//! auth middleware (the admin key or any key minted via+//! `POST /_security/api_key`). There is **no per-namespace authorization**: any+//! credential that can reach the node can read, write, and drop **every**+//! namespace. Namespaces isolate data *by name* (a recall in `agent-a` never+//! returns `agent-b`'s memories), NOT by credential — do not treat a namespace+//! as a security boundary between mutually-distrusting tenants. Per-namespace+//! access control depends on the deferred RBAC enforcement (see+//! `xerj_engine::rbac`).+//!//! Recall has three modes, tried in order: an explicit query `vector` (kNN over//! a caller-supplied embedding) → `semantic: true` (the server embeds `query`//! with the same embedder used at store time and recalls by vector similarity)@@ -47,9 +70,15 @@ use crate::state::AppState;/// through `/_memory/*`.const MEMORY_PREFIX: &str = ".xerj-memory-";-/// Maximum number of entries returned by the introspection (`GET`) endpoint.+/// Default page size for the introspection (`GET`) endpoint when `size` is+/// omitted. Callers page further with `from`/`size` (see [`ListParams`]).const LIST_LIMIT: usize = 100;+/// Hard cap on a single `list` page `size`, so one call can't ask the backing+/// index for an unbounded result set (memory-safety). Deep pagination uses+/// `from` across multiple calls.+const MAX_LIST_SIZE: usize = 1000;+/// Default number of memories returned by `_recall` when `k` is omitted.const DEFAULT_K: usize = 10;@@ -625,11 +654,43 @@ fn blend_recency(search_response: &Value, w: f32, k: usize) -> Vec<Value> {// List / introspect// ─────────────────────────────────────────────────────────────────────────────-/// `GET /_memory/{namespace}` — count + most-recent entries (bounded).-pub async fn list(State(state): State<AppState>, Path(namespace): Path<String>) -> Response {+/// Query params for the paged `list` endpoint (item 12).+///+/// Deliberately lenient on unknown params (no `deny_unknown_fields`): the+/// previous handler took no query at all and ignored everything, so a client+/// appending a benign ES-ism like `?pretty` / `?human` / `?format=json` must+/// keep working. Only `from` / `after` / `size` are interpreted.+#[derive(Debug, Default, Deserialize)]+pub struct ListParams {+ /// Zero-based offset of the first entry to return. Default 0.+ #[serde(default)]+ pub from: Option<usize>,+ /// Alias for `from` (an `after`-style cursor). `from` wins if both given.+ #[serde(default)]+ pub after: Option<usize>,+ /// Page size. Default [`LIST_LIMIT`] (100), capped at [`MAX_LIST_SIZE`].+ #[serde(default)]+ pub size: Option<usize>,+}++/// `GET /_memory/{namespace}` — count + a page of most-recent entries.+///+/// Pages with `from`/`size` (item 12); see the module-level docs for the+/// pagination and (no-per-namespace) authorization model.+pub async fn list(+ State(state): State<AppState>,+ Path(namespace): Path<String>,+ axum::extract::Query(params): axum::extract::Query<ListParams>,+) -> Response {if let Err(reason) = validate_namespace(&namespace) {return error_response(StatusCode::BAD_REQUEST, reason);}+ // `from`/`after` are aliases (offset cursor); `size` is clamped to+ // [1, MAX_LIST_SIZE] so a caller can neither page from a negative offset+ // nor pull an unbounded set in one call.+ let from = params.from.or(params.after).unwrap_or(0);+ let size = params.size.unwrap_or(LIST_LIMIT).clamp(1, MAX_LIST_SIZE);+let index = backing_index(&namespace);if !index_exists(&state, &index) {return Json(json!({@@ -637,13 +698,17 @@ pub async fn list(State(state): State<AppState>, Path(namespace): Path<String>)"exists": false,"count": 0,"entries": [],+ "from": from,+ "size": size,+ "next": Value::Null,})).into_response();}let search_body = EsSearchBody {query: Some(json!({ "match_all": {} })),- size: LIST_LIMIT,+ from,+ size,// Recent-first. `unmapped_type` guards a namespace whose entries were// all created before `stored_at` existed (defensive; we always store it).sort: Some(json!([{ "stored_at": { "order": "desc", "unmapped_type": "long" } }])),@@ -670,11 +735,20 @@ pub async fn list(State(state): State<AppState>, Path(namespace): Path<String>).and_then(Value::as_u64).unwrap_or(0);let entries = extract_hits(&body);+ // `next` cursor: present only when more entries remain past this page.+ let next = if (from as u64) + (entries.len() as u64) < count {+ json!({ "from": from + entries.len(), "size": size })+ } else {+ Value::Null+ };Json(json!({"namespace": namespace,"exists": true,"count": count,"entries": entries,+ "from": from,+ "size": size,+ "next": next,})).into_response()}@@ -958,11 +1032,124 @@ mod tests {}async fn count_of(state: &AppState, ns: &str) -> u64 {- let resp = list(State(state.clone()), Path(ns.to_string())).await;+ let resp = list(+ State(state.clone()),+ Path(ns.to_string()),+ axum::extract::Query(ListParams::default()),+ )+ .await;let (_, body) = drain_json(resp).await;body["count"].as_u64().unwrap()}+ async fn list_page(state: &AppState, ns: &str, params: ListParams) -> Value {+ let resp = list(+ State(state.clone()),+ Path(ns.to_string()),+ axum::extract::Query(params),+ )+ .await;+ let (s, body) = drain_json(resp).await;+ assert_eq!(s, StatusCode::OK, "list must 200");+ body+ }++ /// Item 12: `list` pages a namespace larger than the default page with+ /// `from`/`size` and exposes a `next` cursor — the old endpoint silently+ /// capped at 100 with no way to reach the rest.+ #[tokio::test]+ async fn list_paginates_beyond_default_limit() {+ let state = test_state();+ // 150 entries — more than the default page (LIST_LIMIT = 100).+ for i in 0..150 {+ let (s, _) = store_mem(+ &state,+ "big",+ json!({"text": format!("memory number {i}"), "id": format!("m{i:03}")}),+ )+ .await;+ assert_eq!(s, StatusCode::CREATED);+ }++ // Default page: true count is 150, exactly 100 entries returned, and a+ // `next` cursor points at offset 100.+ let p1 = list_page(&state, "big", ListParams::default()).await;+ assert_eq!(+ p1["count"].as_u64().unwrap(),+ 150,+ "count is the true total"+ );+ assert_eq!(+ p1["entries"].as_array().unwrap().len(),+ 100,+ "default page returns LIST_LIMIT entries"+ );+ assert_eq!(+ p1["next"]["from"].as_u64().unwrap(),+ 100,+ "next cursor at 100"+ );+ assert_eq!(p1["next"]["size"].as_u64().unwrap(), 100);++ // Second page via the cursor: the remaining 50, and `next` is null.+ let p2 = list_page(+ &state,+ "big",+ ListParams {+ from: Some(100),+ after: None,+ size: Some(100),+ },+ )+ .await;+ assert_eq!(+ p2["entries"].as_array().unwrap().len(),+ 50,+ "from=100 returns the final 50 entries"+ );+ assert!(p2["next"].is_null(), "last page has no next cursor");++ // `size` is clamped to MAX_LIST_SIZE (asking for more still succeeds).+ let big = list_page(+ &state,+ "big",+ ListParams {+ from: Some(0),+ after: None,+ size: Some(10_000),+ },+ )+ .await;+ assert_eq!(+ big["size"].as_u64().unwrap(),+ MAX_LIST_SIZE as u64,+ "size clamped"+ );+ assert_eq!(+ big["entries"].as_array().unwrap().len(),+ 150,+ "a large (clamped) page still returns all 150"+ );+ assert!(big["next"].is_null());++ // `after` is accepted as an alias for `from`.+ let aliased = list_page(+ &state,+ "big",+ ListParams {+ from: None,+ after: Some(100),+ size: Some(100),+ },+ )+ .await;+ assert_eq!(+ aliased["entries"].as_array().unwrap().len(),+ 50,+ "after=100 aliases from=100"+ );+ }+/// Opt-in semantic dedup: storing the same text twice with `dedup:true`/// keeps the namespace at one entry (2nd store is deduplicated), a distinct/// text still stores, and omitting `dedup` keeps the backward-compatible