AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): memtable circuit breaker ignores cgroup memory limit (#163)
Vinz2168dccf73e0507d
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rsindex bc7d828cd..a7466843e 100644--- a/engine/crates/xerj-common/src/config.rs+++ b/engine/crates/xerj-common/src/config.rs@@ -982,8 +982,10 @@ pub struct LimitsConfig {/// past the cap.pub max_buckets: usize,/// Process-wide ceiling on the total bytes buffered across ALL indices'- /// memtables (default: `0` = auto-derive to 25% of system RAM, floored- /// at 2048 MiB). This is the parent circuit breaker for the ingest path:+ /// memtables (default: `0` = auto-derive to 25% of the effective+ /// cgroup/system memory limit, floored at 2048 MiB, capped at 50% of+ /// that same limit). This is the parent circuit breaker for the ingest+ /// path:/// per-index back-pressure only bounds one index at `~3×flush_size_mb`,/// so `N` indices could buffer `N × 1.5 GiB` with no global ceiling —/// the structural cause of the 112 GiB OOM. When the summed memtable@@ -1029,7 +1031,7 @@ impl Default for LimitsConfig {max_result_window: 10_000,max_mget_docs: 10_000,max_buckets: 65_536,- max_total_memtable_mb: 0, // 0 = auto-derive (25% RAM, floor 2 GiB)+ max_total_memtable_mb: 0, // 0 = auto-derive (25% effective limit, floor 2 GiB, cap 50%)max_segment_hydration_cache_mb: 0, // 0 = 20% effective memory, no floormemory_watermark_percent: 95,disk_flood_stage_percent: 95,diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsindex 2e1f70254..f003cf165 100644--- a/engine/crates/xerj-engine/src/governor.rs+++ b/engine/crates/xerj-engine/src/governor.rs@@ -379,6 +379,21 @@ pub struct GovernorSnapshot {pub max_concurrent_bulks: usize,}+/// Auto-derive the memtable byte ceiling from the effective (cgroup-aware)+/// memory limit: 25% of the limit, floored at 2 GiB, but never past 50% of+/// the same limit. The 50% cap keeps the floor itself from exceeding the+/// container under a small enough effective limit.+///+/// `effective_limit` must already be the cgroup-aware value (see+/// `effective_memory_limit_bytes`) — this function has no way to detect a+/// host-RAM value passed in by mistake, so getting that argument right is+/// the caller's responsibility.+fn auto_memtable_budget(effective_limit: u64) -> u64 {+ (effective_limit / 4)+ .max(2 * 1024 * 1024 * 1024)+ .min(effective_limit / 2)+}+#[derive(Debug, Clone, Copy, Eq, PartialEq)]pub enum SegmentHydrationBudgetSource {Auto,@@ -480,16 +495,22 @@ pub fn global() -> Option<Arc<ResourceGovernor>> {fn build(config: &Config) -> ResourceGovernor {let limits = &config.limits;- // ── memtable budget: 0 = auto-derive to 25% RAM, floored at 2 GiB ──+ // ── RSS watermark against the effective memory limit ──+ let memory_limit_bytes = effective_memory_limit_bytes();++ // ── memtable budget: 0 = auto-derive from the effective (cgroup-aware)+ // memory limit — see `auto_memtable_budget`. Must be derived from the+ // same effective limit as every sibling budget in this function, not+ // raw host RAM: `/proc/meminfo` is not namespace-virtualized, so a+ // host-RAM-based ceiling silently ignores any cgroup / `systemd-run -p+ // MemoryMax=` cap smaller than the host's physical RAM. The 2 GiB floor+ // needs its own ceiling for the same reason — it can exceed the whole+ // effective limit under a small enough cap on its own.let memtable_budget_bytes = if limits.max_total_memtable_mb != 0 {limits.max_total_memtable_mb.saturating_mul(1024 * 1024)} else {- let sys = system_total_bytes();- (sys / 4).max(2 * 1024 * 1024 * 1024)+ auto_memtable_budget(memory_limit_bytes)};-- // ── RSS watermark against the effective memory limit ──- let memory_limit_bytes = effective_memory_limit_bytes();let resolved_segment_hydration = resolve_segment_hydration_budget(memory_limit_bytes,limits.max_segment_hydration_cache_mb,@@ -630,6 +651,12 @@ fn effective_memory_limit(sys: u64, cgroup: Option<u64>) -> u64 {/// (`memory.max`, walking up to a parent when a leaf reads `max`) and falls/// back to cgroup v1 (`memory.limit_in_bytes`). Returns `None` when no/// finite limit applies.+///+/// A limit file that is missing, unreadable, malformed, `0`, or the v1/v2+/// "no limit" sentinel (`max` / `9223372036854771712`) contributes nothing+/// rather than failing: the caller then falls back to host RAM. Nothing on+/// this path panics, so an unexpected cgroupfs layout degrades to the+/// pre-cgroup behaviour instead of taking the process down at startup.#[cfg(target_os = "linux")]fn fold_cgroup_memory_limit(current: Option<u64>, raw: &str) -> Option<u64> {const UNLIMITED_V1: u64 = 9_223_372_036_854_771_712;@@ -645,43 +672,97 @@ fn fold_cgroup_memory_limit(current: Option<u64>, raw: &str) -> Option<u64> {}}+/// Walk from `rel` up to the root, folding the tightest finite limit found+/// in `<root><rel>/<file>` at each level. cgroup memory limits are+/// hierarchical in BOTH v1 and v2 — a finite child value does not override a+/// smaller finite ancestor — so the effective limit is the minimum over the+/// whole chain. Levels whose file is missing or unreadable are skipped, so a+/// partially visible hierarchy still yields whatever it does expose.+///+/// `root` is a parameter rather than a constant so the hierarchy walk is+/// testable against a temp-dir fixture without a real cgroupfs mount.#[cfg(target_os = "linux")]-fn cgroup_memory_limit_bytes() -> Option<u64> {- // cgroup v2: /proc/self/cgroup has a single "0::<path>" line.- if let Ok(cg) = std::fs::read_to_string("/proc/self/cgroup") {- for line in cg.lines() {- if let Some(path) = line.strip_prefix("0::") {- // Walk from leaf to root and take the tightest finite limit.- // cgroup-v2 limits are hierarchical: a finite child value- // does not override a smaller finite ancestor.- let mut rel = path.trim().to_string();- let mut tightest = None;- loop {- let full = format!("/sys/fs/cgroup{rel}/memory.max");- if let Ok(s) = std::fs::read_to_string(&full) {- tightest = fold_cgroup_memory_limit(tightest, &s);- }- if rel.is_empty() || rel == "/" {- break;- }- match rel.rfind('/') {- Some(0) => rel = String::new(), // step to root next iter- Some(i) => rel.truncate(i),- None => break,- }- }- if tightest.is_some() {- return tightest;- }+fn tightest_limit_up_hierarchy(root: &str, rel: &str, file: &str) -> Option<u64> {+ let mut rel = rel.trim().to_string();+ let mut tightest = None;+ loop {+ let full = format!("{root}{rel}/{file}");+ if let Ok(s) = std::fs::read_to_string(&full) {+ tightest = fold_cgroup_memory_limit(tightest, &s);+ }+ if rel.is_empty() || rel == "/" {+ break;+ }+ match rel.rfind('/') {+ Some(0) => rel = String::new(), // step to root next iter+ Some(i) => rel.truncate(i),+ None => break,+ }+ }+ tightest+}++/// The cgroup-relative path of the v1 `memory` controller from one+/// `/proc/self/cgroup` line (`<hierarchy-id>:<controllers>:<path>`).+/// Returns `None` for v2 lines (`0::<path>`, empty controller list) and for+/// controller sets that do not include `memory`.+#[cfg(target_os = "linux")]+fn cgroup_v1_memory_path(line: &str) -> Option<&str> {+ let mut parts = line.splitn(3, ':');+ let _hierarchy_id = parts.next()?;+ let controllers = parts.next()?;+ let path = parts.next()?;+ controllers+ .split(',')+ .any(|c| c == "memory")+ .then_some(path.trim())+}++/// Resolve the memory limit from the contents of `/proc/self/cgroup`,+/// against the v2 unified mount at `v2_root` and the v1 memory-controller+/// mount at `v1_root`. Split out from [`cgroup_memory_limit_bytes`] so the+/// whole resolution — v2 line, v1 line, and the mount-root fallback — is+/// testable against a fixture instead of the host's real cgroupfs.+#[cfg(target_os = "linux")]+fn cgroup_memory_limit_from(self_cgroup: &str, v2_root: &str, v1_root: &str) -> Option<u64> {+ // cgroup v2: /proc/self/cgroup has a single "0::<path>" line, and the+ // limits live at <v2_root><path>/memory.max.+ for line in self_cgroup.lines() {+ if let Some(path) = line.strip_prefix("0::") {+ let tightest = tightest_limit_up_hierarchy(v2_root, path.trim(), "memory.max");+ if tightest.is_some() {+ return tightest;}}}- // cgroup v1 fallback.- if let Ok(s) = std::fs::read_to_string("/sys/fs/cgroup/memory/memory.limit_in_bytes") {- return fold_cgroup_memory_limit(None, &s);+ // cgroup v1: the memory controller is mounted separately and+ // /proc/self/cgroup carries one "<id>:memory:<path>" line per hierarchy.+ // Walk that path from leaf to root exactly as for v2. Under Docker the+ // cgroupfs is bind-mounted at the container's own directory, so the leaf+ // path from /proc/self/cgroup does not resolve and only the mount root+ // reads — which is the container's real limit. Under a bare+ // `systemd-run -p MemoryMax=` on a v1/hybrid host it is the other way+ // round: the process sits in a leaf slice and the mount root reads+ // unlimited. Folding the whole chain is correct in both cases.+ for line in self_cgroup.lines() {+ if let Some(path) = cgroup_v1_memory_path(line) {+ let tightest = tightest_limit_up_hierarchy(v1_root, path, "memory.limit_in_bytes");+ if tightest.is_some() {+ return tightest;+ }+ }}- None++ // Last resort: /proc/self/cgroup was unreadable or named no memory+ // controller, so read the v1 mount root directly.+ tightest_limit_up_hierarchy(v1_root, "", "memory.limit_in_bytes")+}++#[cfg(target_os = "linux")]+fn cgroup_memory_limit_bytes() -> Option<u64> {+ let self_cgroup = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default();+ cgroup_memory_limit_from(&self_cgroup, "/sys/fs/cgroup", "/sys/fs/cgroup/memory")}#[cfg(not(target_os = "linux"))]@@ -803,6 +884,35 @@ mod tests {assert!(zero.warning.is_some());}+ /// Regression test for the bug fixed in this PR: `auto_memtable_budget`+ /// must be derived from the effective (cgroup-aware) limit passed in,+ /// not from raw host RAM read separately inside the function. Before+ /// the fix this formula lived inline in `build()` and called+ /// `system_total_bytes()` directly, so it had no coverage at all —+ /// this table pins the exact values a reviewer reproduced live on a+ /// real cgroup-v2 host (1 GiB cap → 512 MiB, not the pre-fix ~30 GiB).+ #[test]+ fn auto_memtable_budget_is_cgroup_proportional_and_bounded() {+ const GIB: u64 = 1024 * 1024 * 1024;+ let cases = [+ (GIB, 512 * 1024 * 1024),+ (4 * GIB, 2 * GIB),+ (8 * GIB, 2 * GIB),+ (64 * GIB, 16 * GIB),+ ];+ for (effective_limit, expected) in cases {+ let budget = auto_memtable_budget(effective_limit);+ assert_eq!(+ budget, expected,+ "auto_memtable_budget({effective_limit}) = {budget}, expected {expected}"+ );+ assert!(+ budget <= effective_limit / 2,+ "auto_memtable_budget({effective_limit}) = {budget} exceeds 50% of the limit"+ );+ }+ }+#[test]fn effective_limit_is_positive() {assert!(effective_memory_limit_bytes() > 0);@@ -833,6 +943,169 @@ mod tests {assert_eq!(effective_memory_limit(8 * GIB, Some(4 * GIB)), 4 * GIB);}+ /// Regression test for the cgroup-v1 half of the limit lookup, and for+ /// the v2 leaf that reads the literal string `max`.+ ///+ /// Before this, only the v2 `0::<path>` line of `/proc/self/cgroup` was+ /// parsed and the v1 side blindly read the mount root+ /// `/sys/fs/cgroup/memory/memory.limit_in_bytes`. That root read is the+ /// container's own limit under Docker's v1 bind-mount, but on a v1 or+ /// hybrid host under `systemd-run -p MemoryMax=` the process sits in a+ /// leaf slice while the root reads unlimited — so no limit was found,+ /// the effective limit fell back to host RAM, and every cgroup-aware+ /// budget in `build()` (including the memtable circuit breaker) silently+ /// ignored the cap.+ ///+ /// The walk is driven against a temp-dir fixture rather than a real+ /// cgroupfs so it runs identically on any host, v1, v2 or neither.+ #[cfg(target_os = "linux")]+ #[test]+ fn cgroup_hierarchy_walk_handles_v1_v2_max_and_missing_files() {+ const GIB: u64 = 1024 * 1024 * 1024;+ // The v1 "no limit" sentinel; v2 spells the same thing "max".+ const UNLIMITED_V1: &str = "9223372036854771712";++ let tmp = tempfile::tempdir().unwrap();+ let root = tmp.path().to_str().unwrap().to_string();+ let write = |rel: &str, file: &str, body: &str| {+ let dir = format!("{root}{rel}");+ std::fs::create_dir_all(&dir).unwrap();+ std::fs::write(format!("{dir}/{file}"), body).unwrap();+ };++ // ── cgroup v2: leaf reads the literal "max", a finite ancestor caps ──+ // This is the shape `systemd-run --user --scope -p MemoryMax=1G`+ // produces: the limit lands on the slice, not on the leaf scope.+ write("", "memory.max", "max");+ write("/user.slice", "memory.max", "1073741824");+ write("/user.slice/app.scope", "memory.max", "max");+ assert_eq!(+ tightest_limit_up_hierarchy(&root, "/user.slice/app.scope", "memory.max"),+ Some(GIB),+ "a v2 leaf reading the literal \"max\" must inherit its ancestor's finite limit"+ );++ // A tighter leaf still wins over a looser ancestor, and vice versa.+ write("/user.slice/tight.scope", "memory.max", "536870912");+ assert_eq!(+ tightest_limit_up_hierarchy(&root, "/user.slice/tight.scope", "memory.max"),+ Some(GIB / 2)+ );++ // ── cgroup v1: same walk, different file name and mount root ──+ write("/memory", "memory.limit_in_bytes", UNLIMITED_V1);+ write(+ "/memory/system.slice/xerj.service",+ "memory.limit_in_bytes",+ "2147483648",+ );+ let v1_root = format!("{root}/memory");+ assert_eq!(+ tightest_limit_up_hierarchy(+ &v1_root,+ "/system.slice/xerj.service",+ "memory.limit_in_bytes"+ ),+ Some(2 * GIB),+ "a v1 leaf slice limit must be found, not just the mount root"+ );++ // ── unlimited everywhere, and missing files, both mean "no limit" ──+ // Not 0, and not a panic: `effective_memory_limit` then keeps host RAM.+ write("/unlimited", "memory.max", "max");+ assert_eq!(+ tightest_limit_up_hierarchy(&root, "/unlimited", "memory.max"),+ None+ );+ assert_eq!(+ tightest_limit_up_hierarchy(&root, "/user.slice/no-such.scope", "memory.max"),+ Some(GIB),+ "a missing leaf is skipped, not fatal — ancestors still apply"+ );+ // Nothing readable anywhere on the chain: no limit, no panic, not 0.+ assert_eq!(+ tightest_limit_up_hierarchy(&root, "/does/not/exist", "memory.max"),+ None+ );+ assert_eq!(+ tightest_limit_up_hierarchy("/no/such/cgroupfs", "/a/b", "memory.max"),+ None+ );+ let sys = 4 * GIB;+ assert_eq!(effective_memory_limit(sys, None), sys);++ // ── /proc/self/cgroup line parsing: v1 memory lines vs everything else ──+ assert_eq!(+ cgroup_v1_memory_path("5:memory:/system.slice/xerj.service"),+ Some("/system.slice/xerj.service")+ );+ // Co-mounted controllers are comma-separated.+ assert_eq!(+ cgroup_v1_memory_path("4:cpu,cpuacct,memory:/foo"),+ Some("/foo")+ );+ assert_eq!(cgroup_v1_memory_path("3:cpuset:/foo"), None);+ // A v2 line has an empty controller list and must not match here.+ assert_eq!(cgroup_v1_memory_path("0::/user.slice/app.scope"), None);+ // Malformed lines are ignored rather than panicking.+ assert_eq!(cgroup_v1_memory_path(""), None);+ assert_eq!(cgroup_v1_memory_path("garbage"), None);++ // ── full resolution, as `cgroup_memory_limit_bytes` runs it ──+ // A pure-v2 host: the v2 line resolves and the v1 mount is absent.+ assert_eq!(+ cgroup_memory_limit_from("0::/user.slice/app.scope\n", &root, &v1_root),+ Some(GIB),+ "v2: leaf reads \"max\", the finite ancestor must win"+ );+ // A pure-v1 host under `systemd-run -p MemoryMax=`: there is no v2+ // line to resolve, and the limit is on the leaf slice while the v1+ // mount root reads unlimited. This is the case that was a no-op+ // before: the v1 "<id>:memory:<path>" line was never parsed at all.+ assert_eq!(+ cgroup_memory_limit_from(+ "7:memory:/system.slice/xerj.service\n5:cpuset:/\n",+ "/no/such/cgroupfs",+ &v1_root,+ ),+ Some(2 * GIB),+ "v1: the leaf slice limit must be found, not just the mount root"+ );+ // A hybrid host: a v2 line exists but carries no memory limit, so+ // resolution must fall through to the v1 memory controller.+ assert_eq!(+ cgroup_memory_limit_from(+ "0::/unlimited\n7:memory:/system.slice/xerj.service\n",+ &root,+ &v1_root,+ ),+ Some(2 * GIB),+ "hybrid: an unlimited v2 line must not shadow the v1 limit"+ );+ // Docker v1: the leaf path does not exist under the bind-mounted+ // cgroupfs, so the mount root is the container's real limit.+ write("/docker", "memory.limit_in_bytes", "3221225472");+ assert_eq!(+ cgroup_memory_limit_from(+ "7:memory:/docker/deadbeef\n",+ "/no/such/cgroupfs",+ &format!("{root}/docker"),+ ),+ Some(3 * GIB),+ "v1 bind-mount: an unresolvable leaf must fall back to the mount root"+ );+ // Unreadable /proc/self/cgroup, and no cgroupfs at all: no limit,+ // no panic, no 0 — the caller keeps host RAM.+ assert_eq!(+ cgroup_memory_limit_from("", "/no/such/cgroupfs", "/no/such/cgroupfs/memory"),+ None+ );++ // The real lookup must never panic or report 0 on this host,+ // whatever its cgroup layout is.+ assert!(cgroup_memory_limit_bytes().is_none_or(|limit| limit > 0));+ }+#[test]fn disk_used_pct_bounded() {let p = disk_used_pct("/");