AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
//! Bounded, local profiling for controlled benchmark runs.
//!
//! Compiled only with `--features debug-profiling`. There is deliberately no
//! HTTP endpoint. The output directory must exist and captures are requested
//! before process start:
//!
//! ```text
//! XERJ_DEBUG_PROFILE_DIR=/private/run \
//! XERJ_DEBUG_CPU_SECONDS=30 XERJ_DEBUG_HEAP_SECONDS=30 xerj
//! ```
use std::fs::{File, OpenOptions};
use std::io::{self, Write};
use std::os::unix::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Condvar, Mutex};
use std::time::Duration;
use xerj_common::thread_spawn::SpawnedThread;
use anyhow::{anyhow, Context, Result};
use pprof::protos::Message;
const MAX_CAPTURE_SECONDS: u64 = 300;
const MAX_CPU_HZ: i32 = 1_000;
#[derive(Debug, Clone, PartialEq, Eq)]
struct Config {
output_dir: PathBuf,
cpu_seconds: Option<u64>,
heap_seconds: Option<u64>,
cpu_hz: i32,
delay_seconds: u64,
}
impl Config {
fn from_env() -> Result<Option<Self>> {
let Some(output) = std::env::var_os("XERJ_DEBUG_PROFILE_DIR") else {
return Ok(None);
};
Self::parse(
PathBuf::from(output),
std::env::var("XERJ_DEBUG_CPU_SECONDS").ok().as_deref(),
std::env::var("XERJ_DEBUG_HEAP_SECONDS").ok().as_deref(),
std::env::var("XERJ_DEBUG_CPU_HZ").ok().as_deref(),
std::env::var("XERJ_DEBUG_PROFILE_DELAY_SECONDS")
.ok()
.as_deref(),
)
.map(Some)
}
fn parse(
output_dir: PathBuf,
cpu_seconds: Option<&str>,
heap_seconds: Option<&str>,
cpu_hz: Option<&str>,
delay_seconds: Option<&str>,
) -> Result<Self> {
let output_dir = output_dir.canonicalize().with_context(|| {
format!("profile directory does not exist: {}", output_dir.display())
})?;
if !output_dir.is_dir() {
return Err(anyhow!(
"profile output is not a directory: {}",
output_dir.display()
));
}
let cpu_seconds = parse_duration("XERJ_DEBUG_CPU_SECONDS", cpu_seconds)?;
let heap_seconds = parse_duration("XERJ_DEBUG_HEAP_SECONDS", heap_seconds)?;
if cpu_seconds.is_none() && heap_seconds.is_none() {
return Err(anyhow!(
"set XERJ_DEBUG_CPU_SECONDS and/or XERJ_DEBUG_HEAP_SECONDS"
));
}
let cpu_hz = match cpu_hz {
Some(value) => value
.parse::<i32>()
.with_context(|| format!("XERJ_DEBUG_CPU_HZ is not an integer: {value}"))?,
None => 100,
};
if !(1..=MAX_CPU_HZ).contains(&cpu_hz) {
return Err(anyhow!(
"XERJ_DEBUG_CPU_HZ must be in 1..={MAX_CPU_HZ}, got {cpu_hz}"
));
}
let delay_seconds = match delay_seconds {
Some(value) => value.parse::<u64>().with_context(|| {
format!("XERJ_DEBUG_PROFILE_DELAY_SECONDS is not an integer: {value}")
})?,
None => 0,
};
if delay_seconds > MAX_CAPTURE_SECONDS {
return Err(anyhow!(
"XERJ_DEBUG_PROFILE_DELAY_SECONDS must be in 0..={MAX_CAPTURE_SECONDS}, got {delay_seconds}"
));
}
Ok(Self {
output_dir,
cpu_seconds,
heap_seconds,
cpu_hz,
delay_seconds,
})
}
}
fn parse_duration(name: &str, value: Option<&str>) -> Result<Option<u64>> {
let Some(value) = value else {
return Ok(None);
};
let seconds = value
.parse::<u64>()
.with_context(|| format!("{name} is not an integer: {value}"))?;
if !(1..=MAX_CAPTURE_SECONDS).contains(&seconds) {
return Err(anyhow!(
"{name} must be in 1..={MAX_CAPTURE_SECONDS}, got {seconds}"
));
}
Ok(Some(seconds))
}
pub struct ProfileRuntime {
workers: Vec<SpawnedThread>,
stop: Arc<(Mutex<bool>, Condvar)>,
lock_path: Option<PathBuf>,
_lock: File,
}
impl Drop for ProfileRuntime {
fn drop(&mut self) {
*self
.stop
.0
.lock()
.unwrap_or_else(|error| error.into_inner()) = true;
self.stop.1.notify_all();
for worker in self.workers.drain(..) {
let _ = worker.join();
}
if let Some(path) = self.lock_path.take() {
let _ = std::fs::remove_file(path);
}
}
}
pub fn spawn() -> Result<Option<ProfileRuntime>> {
let Some(config) = Config::from_env()? else {
return Ok(None);
};
spawn_configured(config).map(Some)
}
/// #1122: `std::thread::Builder::spawn` panics inside the child when the
/// alternate signal stack cannot be mapped, and the release profile is
/// `panic = "abort"`. A profile worker that cannot be created fails this
/// startup step with an error instead of aborting the process.
fn spawn_profile_thread<F>(name: &'static str, job: F) -> Result<SpawnedThread>
where
F: FnOnce() + Send + 'static,
{
xerj_common::thread_spawn::try_spawn_unguarded(name, job).map_err(|failure| {
let error = failure.error;
drop(failure.job);
anyhow!("spawn {name} worker: {error}")
})
}
fn spawn_configured(config: Config) -> Result<ProfileRuntime> {
let lock_path = config.output_dir.join(".xerj-debug-profile.lock");
let lock = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&lock_path)
.with_context(|| {
format!(
"profile directory is already in use or contains stale lock: {}",
lock_path.display()
)
})?;
let stop = Arc::new((Mutex::new(false), Condvar::new()));
let mut runtime = ProfileRuntime {
workers: Vec::new(),
stop,
lock_path: Some(lock_path),
_lock: lock,
};
if let Some(seconds) = config.cpu_seconds {
let directory = config.output_dir.clone();
let stop = Arc::clone(&runtime.stop);
let frequency = config.cpu_hz;
let delay = config.delay_seconds;
runtime
.workers
.push(spawn_profile_thread("xerj-debug-cpu-profile", move || {
if wait_for_duration_or_stop(&stop, Duration::from_secs(delay)) {
return;
}
if let Err(error) = capture_cpu(&directory, seconds, frequency, &stop) {
write_profile_error(&directory, "cpu", &error);
tracing::error!(%error, "CPU profile capture failed");
}
})?);
}
if let Some(seconds) = config.heap_seconds {
let directory = config.output_dir.clone();
let stop = Arc::clone(&runtime.stop);
let delay = config.delay_seconds;
runtime.workers.push(spawn_profile_thread(
"xerj-debug-heap-profile",
move || {
if wait_for_duration_or_stop(&stop, Duration::from_secs(delay)) {
return;
}
if let Err(error) = capture_heap(&directory, seconds, &stop) {
write_profile_error(&directory, "heap", &error);
tracing::error!(%error, "heap profile capture failed");
}
},
)?);
}
Ok(runtime)
}
fn capture_cpu(
output_dir: &Path,
seconds: u64,
frequency: i32,
stop: &(Mutex<bool>, Condvar),
) -> Result<()> {
let guard = pprof::ProfilerGuardBuilder::default()
.frequency(frequency)
.blocklist(&["libc", "libgcc", "pthread", "vdso"])
.build()
.context("start SIGPROF CPU sampler")?;
wait_for_duration_or_stop(stop, Duration::from_secs(seconds));
let report = guard.report().build().context("build CPU profile")?;
let profile = report.pprof().context("convert CPU profile to pprof")?;
let mut bytes = Vec::new();
profile.encode(&mut bytes).context("encode CPU profile")?;
write_new_atomic(output_dir, "cpu.pb", &bytes)
}
fn capture_heap(output_dir: &Path, seconds: u64, stop: &(Mutex<bool>, Condvar)) -> Result<()> {
let control = Arc::clone(
jemalloc_pprof::PROF_CTL
.as_ref()
.ok_or_else(|| anyhow!("jemalloc was not compiled/configured with profiling"))?,
);
let mut ctl = control.blocking_lock();
if ctl.activated() {
return Err(anyhow!("jemalloc profiling is already active"));
}
ctl.activate().context("activate jemalloc profiling")?;
drop(ctl);
wait_for_duration_or_stop(stop, Duration::from_secs(seconds));
let mut ctl = control.blocking_lock();
let dump = ctl.dump_pprof().context("dump heap pprof");
let deactivate = ctl.deactivate().context("deactivate jemalloc profiling");
let bytes = dump?;
deactivate?;
write_new_atomic(output_dir, "heap.pb.gz", &bytes)
}
fn wait_for_duration_or_stop(stop: &(Mutex<bool>, Condvar), duration: Duration) -> bool {
let guard = stop.0.lock().unwrap_or_else(|error| error.into_inner());
let (guard, _) = stop
.1
.wait_timeout_while(guard, duration, |stopped| !*stopped)
.unwrap_or_else(|error| error.into_inner());
*guard
}
fn write_new_atomic(output_dir: &Path, name: &str, contents: &[u8]) -> Result<()> {
let final_path = output_dir.join(name);
let temporary = output_dir.join(format!(".{name}.{}.tmp", std::process::id()));
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&temporary)
.with_context(|| format!("create profile temporary file: {}", temporary.display()))?;
let result = (|| -> io::Result<()> {
file.write_all(contents)?;
file.sync_all()?;
// hard_link fails if the final name exists; rename would overwrite it.
std::fs::hard_link(&temporary, &final_path)?;
std::fs::remove_file(&temporary)
})();
if result.is_err() {
let _ = std::fs::remove_file(&temporary);
}
result.with_context(|| format!("publish profile: {}", final_path.display()))
}
fn write_profile_error(output_dir: &Path, kind: &str, error: &anyhow::Error) {
let payload = serde_json::json!({
"schema": "xerj.debug_profile.error.v1",
"profiler": kind,
"error": format!("{error:#}"),
});
if let Ok(bytes) = serde_json::to_vec_pretty(&payload) {
let _ = write_new_atomic(output_dir, &format!("{kind}-error.json"), &bytes);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn configuration_requires_a_bounded_capture() {
let dir = tempfile::tempdir().unwrap();
assert!(Config::parse(dir.path().into(), None, None, None, None).is_err());
for invalid in ["0", "301", "garbage"] {
assert!(Config::parse(dir.path().into(), Some(invalid), None, None, None).is_err());
}
}
#[test]
fn configuration_defaults_to_low_rate_cpu_sampling() {
let dir = tempfile::tempdir().unwrap();
let config = Config::parse(dir.path().into(), Some("2"), None, None, Some("3")).unwrap();
assert_eq!(config.cpu_seconds, Some(2));
assert_eq!(config.cpu_hz, 100);
assert_eq!(config.delay_seconds, 3);
}
#[test]
fn atomic_publication_never_overwrites_evidence() {
let dir = tempfile::tempdir().unwrap();
write_new_atomic(dir.path(), "profile.pb", b"first").unwrap();
assert!(write_new_atomic(dir.path(), "profile.pb", b"second").is_err());
assert_eq!(
std::fs::read(dir.path().join("profile.pb")).unwrap(),
b"first"
);
use std::os::unix::fs::PermissionsExt;
assert_eq!(
std::fs::metadata(dir.path().join("profile.pb"))
.unwrap()
.permissions()
.mode()
& 0o777,
0o600
);
}
#[test]
fn capture_wait_is_interruptible() {
let stop = Arc::new((Mutex::new(false), Condvar::new()));
let thread_stop = Arc::clone(&stop);
let started = std::time::Instant::now();
let worker = std::thread::spawn(move || {
wait_for_duration_or_stop(&thread_stop, Duration::from_secs(300))
});
*stop.0.lock().unwrap() = true;
stop.1.notify_all();
worker.join().unwrap();
assert!(started.elapsed() < Duration::from_secs(1));
}
#[test]
fn heap_capture_is_a_pprof_gzip() {
use std::io::Read;
let dir = tempfile::tempdir().unwrap();
let stop = (Mutex::new(false), Condvar::new());
let allocation = std::thread::spawn(|| vec![0x5a_u8; 8 * 1024 * 1024]);
capture_heap(dir.path(), 1, &stop).unwrap();
let bytes = std::fs::read(dir.path().join("heap.pb.gz")).unwrap();
assert_eq!(bytes.get(..2), Some(&[0x1f, 0x8b][..]));
let mut protobuf = Vec::new();
flate2::read::GzDecoder::new(bytes.as_slice())
.read_to_end(&mut protobuf)
.unwrap();
pprof::protos::Profile::decode(protobuf.as_slice()).unwrap();
drop(allocation.join().unwrap());
}
#[test]
fn cpu_capture_is_a_nonempty_pprof_protobuf() {
let dir = tempfile::tempdir().unwrap();
let stop = (Mutex::new(false), Condvar::new());
let work = std::thread::spawn(|| {
let deadline = std::time::Instant::now() + Duration::from_secs(1);
let mut value = 1_u64;
while std::time::Instant::now() < deadline {
value = std::hint::black_box(value.wrapping_mul(6364136223846793005));
}
});
capture_cpu(dir.path(), 1, 100, &stop).unwrap();
let bytes = std::fs::read(dir.path().join("cpu.pb")).unwrap();
pprof::protos::Profile::decode(bytes.as_slice()).unwrap();
work.join().unwrap();
}
}