Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(resource): native Windows system RAM, process RSS, and disk probes (#1766)

Roedy Rustam6ca619daf18b
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 0d7441a84..42d84c48d 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -32804,41 +32804,11 @@ pub async fn cat_allocation(
.into_response()
}
-/// Real `(total_bytes, avail_bytes)` for the filesystem backing `path`, via
-/// `statvfs(2)`. Returns `None` if the syscall fails (caller falls back).
-///
-/// The `as u64` casts are PORTABILITY, not noise: `statvfs` field widths
-/// differ per libc target — `f_blocks`/`f_bavail` are u64 on Linux but u32
-/// on macOS, and `f_frsize`/`f_bsize` are c_ulong. Clippy on Linux flags
-/// them as same-type casts; removing them breaks the Apple release builds.
-#[cfg(unix)]
-#[allow(clippy::unnecessary_cast)]
+/// Real `(total_bytes, avail_bytes)` for the filesystem backing `path`,
+/// delegated to `xerj_engine::governor::disk_stats` (`statvfs` on Unix,
+/// `GetDiskFreeSpaceExW` on Windows). Returns `None` if the probe fails.
fn read_disk_stats(path: &str) -> Option<(u64, u64)> {
- let c = std::ffi::CString::new(path).ok()?;
- // SAFETY: `statvfs` fully initialises the struct it writes to; we only
- // read scalar fields afterwards and never alias the buffer.
- let mut st: libc::statvfs = unsafe { std::mem::zeroed() };
- if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 {
- return None;
- }
- let bsize = if st.f_frsize > 0 {
- st.f_frsize as u64
- } else {
- st.f_bsize as u64
- };
- let total = (st.f_blocks as u64).saturating_mul(bsize);
- let avail = (st.f_bavail as u64).saturating_mul(bsize);
- if total == 0 {
- return None;
- }
- Some((total, avail))
-}
-
-/// Non-unix: no statvfs. Return `None`; the caller already falls back to
-/// a conservative default estimate.
-#[cfg(not(unix))]
-fn read_disk_stats(_path: &str) -> Option<(u64, u64)> {
- None
+ xerj_engine::governor::disk_stats(path)
}
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-common/src/hub.rs b/engine/crates/xerj-common/src/hub.rs
index d3f99f5d5..c4b2e9792 100644
--- a/engine/crates/xerj-common/src/hub.rs
+++ b/engine/crates/xerj-common/src/hub.rs
@@ -429,7 +429,10 @@ mod tests {
// sed over this file. If a reformat ever wrapped the declaration, they
// would read "" and go quietly dormant instead of publishing.
let line = format!("\npub const HUB_PUBKEY_HEX: &str = \"{HUB_PUBKEY_HEX}\";\n");
- assert!(include_str!("hub.rs").contains(&line), "{line}");
+ assert!(
+ include_str!("hub.rs").replace("\r\n", "\n").contains(&line),
+ "{line}"
+ );
}
#[test]
diff --git a/engine/crates/xerj-common/src/resource.rs b/engine/crates/xerj-common/src/resource.rs
index 9f4a52ae9..2461f6917 100644
--- a/engine/crates/xerj-common/src/resource.rs
+++ b/engine/crates/xerj-common/src/resource.rs
@@ -419,7 +419,92 @@ fn probe_total_memory() -> Option<u64> {
sysctl_u64(c"hw.memsize")
}
-#[cfg(not(any(target_os = "linux", target_os = "macos")))]
+#[cfg(target_os = "windows")]
+#[allow(clippy::upper_case_acronyms)]
+mod win_mem {
+ #[repr(C)]
+ #[allow(non_snake_case)]
+ pub struct MEMORYSTATUSEX {
+ pub dwLength: u32,
+ pub dwMemoryLoad: u32,
+ pub ullTotalPhys: u64,
+ pub ullAvailPhys: u64,
+ pub ullTotalPageFile: u64,
+ pub ullAvailPageFile: u64,
+ pub ullTotalVirtual: u64,
+ pub ullAvailVirtual: u64,
+ pub ullAvailExtendedVirtual: u64,
+ }
+
+ #[repr(C)]
+ #[allow(non_snake_case)]
+ pub struct PROCESS_MEMORY_COUNTERS {
+ pub cb: u32,
+ pub PageFaultCount: u32,
+ pub PeakWorkingSetSize: usize,
+ pub WorkingSetSize: usize,
+ pub QuotaPeakPagedPoolUsage: usize,
+ pub QuotaPagedPoolUsage: usize,
+ pub QuotaPeakNonPagedPoolUsage: usize,
+ pub QuotaNonPagedPoolUsage: usize,
+ pub PagefileUsage: usize,
+ pub PeakPagefileUsage: usize,
+ }
+
+ #[link(name = "kernel32")]
+ extern "system" {
+ pub fn GlobalMemoryStatusEx(lpBuffer: *mut MEMORYSTATUSEX) -> i32;
+ pub fn GetCurrentProcess() -> *mut std::ffi::c_void;
+ pub fn K32GetProcessMemoryInfo(
+ hProcess: *mut std::ffi::c_void,
+ ppsmemCounters: *mut PROCESS_MEMORY_COUNTERS,
+ cb: u32,
+ ) -> i32;
+ }
+
+ pub fn status() -> Option<MEMORYSTATUSEX> {
+ let mut s = MEMORYSTATUSEX {
+ dwLength: std::mem::size_of::<MEMORYSTATUSEX>() as u32,
+ dwMemoryLoad: 0,
+ ullTotalPhys: 0,
+ ullAvailPhys: 0,
+ ullTotalPageFile: 0,
+ ullAvailPageFile: 0,
+ ullTotalVirtual: 0,
+ ullAvailVirtual: 0,
+ ullAvailExtendedVirtual: 0,
+ };
+ // SAFETY: GlobalMemoryStatusEx writes dwLength bytes into s.
+ let ok = unsafe { GlobalMemoryStatusEx(&mut s) };
+ (ok != 0 && s.ullTotalPhys > 0).then_some(s)
+ }
+
+ pub fn rss() -> Option<u64> {
+ let mut counters = PROCESS_MEMORY_COUNTERS {
+ cb: std::mem::size_of::<PROCESS_MEMORY_COUNTERS>() as u32,
+ PageFaultCount: 0,
+ PeakWorkingSetSize: 0,
+ WorkingSetSize: 0,
+ QuotaPeakPagedPoolUsage: 0,
+ QuotaPagedPoolUsage: 0,
+ QuotaPeakNonPagedPoolUsage: 0,
+ QuotaNonPagedPoolUsage: 0,
+ PagefileUsage: 0,
+ PeakPagefileUsage: 0,
+ };
+ // SAFETY: K32GetProcessMemoryInfo writes cb bytes to counters for the current process.
+ let ok =
+ unsafe { K32GetProcessMemoryInfo(GetCurrentProcess(), &mut counters, counters.cb) };
+ (ok != 0 && counters.WorkingSetSize > 0).then_some(counters.WorkingSetSize as u64)
+ }
+}
+
+#[cfg(target_os = "windows")]
+fn probe_total_memory() -> Option<u64> {
+ win_mem::status().map(|s| s.ullTotalPhys)
+}
+
+#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
fn probe_total_memory() -> Option<u64> {
None
}
@@ -432,7 +517,12 @@ fn probe_available_memory() -> Option<u64> {
)
}
-#[cfg(not(target_os = "linux"))]
+#[cfg(target_os = "windows")]
+fn probe_available_memory() -> Option<u64> {
+ win_mem::status().map(|s| s.ullAvailPhys)
+}
+
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
fn probe_available_memory() -> Option<u64> {
None
}
@@ -481,7 +571,12 @@ fn probe_rss() -> Option<u64> {
Some(unsafe { info.assume_init() }.pti_resident_size)
}
-#[cfg(not(any(target_os = "linux", target_os = "macos")))]
+#[cfg(target_os = "windows")]
+fn probe_rss() -> Option<u64> {
+ win_mem::rss()
+}
+
+#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
fn probe_rss() -> Option<u64> {
None
}
@@ -733,13 +828,13 @@ mod tests {
// Windows PDF-throttling bug this API shape exists to prevent.
assert_eq!(memory_safe_zone_bytes(), None);
}
- #[cfg(any(target_os = "linux", target_os = "macos"))]
+ #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
{
let rss = current_rss_bytes().expect("RSS must be probeable here");
assert!(rss > 0, "a running process has non-zero RSS");
assert_eq!(
available_memory_bytes().is_some(),
- cfg!(target_os = "linux")
+ cfg!(any(target_os = "linux", target_os = "windows"))
);
}
}
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index d0b485905..4ee515534 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -1170,7 +1170,8 @@ fn cgroup_memory_limit_bytes() -> Option<u64> {
}
/// `(total_bytes, avail_bytes)` for the filesystem backing `path`, via
-/// `statvfs(2)`. Returns `None` on syscall failure.
+/// `statvfs(2)` on Unix or `GetDiskFreeSpaceExW` on Windows.
+/// Returns `None` on syscall failure.
#[cfg(unix)]
#[allow(clippy::unnecessary_cast)]
pub fn disk_stats(path: &str) -> Option<(u64, u64)> {
@@ -1193,7 +1194,62 @@ pub fn disk_stats(path: &str) -> Option<(u64, u64)> {
Some((total, avail))
}
-#[cfg(not(unix))]
+#[cfg(target_os = "windows")]
+pub fn disk_stats(path: &str) -> Option<(u64, u64)> {
+ use std::os::windows::ffi::OsStrExt as _;
+
+ let p = if path.is_empty() {
+ std::path::Path::new(".")
+ } else {
+ std::path::Path::new(path)
+ };
+ let abs = std::path::absolute(p).ok()?;
+ let candidate = if abs.exists() {
+ abs
+ } else {
+ abs.ancestors().find(|a| a.exists())?.to_path_buf()
+ };
+
+ let mut os_str = candidate.into_os_string();
+ let s = os_str.to_string_lossy();
+ if s.ends_with(':') {
+ os_str.push("\\");
+ }
+
+ let mut wide: Vec<u16> = os_str.encode_wide().collect();
+ wide.push(0);
+
+ let mut free_bytes_available: u64 = 0;
+ let mut total_bytes: u64 = 0;
+ let mut total_free_bytes: u64 = 0;
+
+ #[link(name = "kernel32")]
+ extern "system" {
+ fn GetDiskFreeSpaceExW(
+ lpDirectoryName: *const u16,
+ lpFreeBytesAvailableToCaller: *mut u64,
+ lpTotalNumberOfBytes: *mut u64,
+ lpTotalNumberOfFreeBytes: *mut u64,
+ ) -> i32;
+ }
+
+ let ok = unsafe {
+ GetDiskFreeSpaceExW(
+ wide.as_ptr(),
+ &mut free_bytes_available,
+ &mut total_bytes,
+ &mut total_free_bytes,
+ )
+ };
+
+ if ok != 0 && total_bytes > 0 {
+ Some((total_bytes, free_bytes_available))
+ } else {
+ None
+ }
+}
+
+#[cfg(not(any(unix, target_os = "windows")))]
pub fn disk_stats(_path: &str) -> Option<(u64, u64)> {
None
}
@@ -1789,6 +1845,18 @@ mod tests {
assert!(p <= 100);
}
+ #[test]
+ fn this_machine_disk_stats_are_sane() {
+ if cfg!(any(unix, target_os = "windows")) {
+ let (total, avail) =
+ disk_stats(".").expect("disk_stats must succeed on the current directory");
+ assert!(total > 0, "total disk space must be positive");
+ assert!(avail <= total, "available disk space cannot exceed total");
+ let pct = disk_used_pct(".");
+ assert!(pct <= 100, "disk used percentage must be <= 100");
+ }
+ }
+
#[test]
fn human_bytes_scales() {
assert_eq!(human_bytes(512), "512b");