AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix: macOS indexing stalls under a memory cap, and shutdown hangs (#1677)
Gil Portnoyed98fefcf2db
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex cea4bc70c..bf46a2cb6 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -7755,6 +7755,7 @@ dependencies = ["serde_json","tempfile","tikv-jemalloc-ctl",+ "tikv-jemalloc-sys","tikv-jemallocator","tokio","tokio-stream",diff --git a/engine/crates/xerj-common/src/fsio.rs b/engine/crates/xerj-common/src/fsio.rsindex 5a03f721a..d408786b7 100644--- a/engine/crates/xerj-common/src/fsio.rs+++ b/engine/crates/xerj-common/src/fsio.rs@@ -9,7 +9,8 @@//! acked-data loss.//!//! Every side-car / manifest write on the publish chain must go through-//! [`write_file_durable`] or [`replace_file_durable`]. Unix makes the rename+//! [`write_file_durable`] or [`replace_file_durable`], or, for many files in+//! one directory, [`stage_file`] plus one [`publish_staged`]. Unix makes the rename//! durable with parent-directory fsync; Windows uses a same-directory Win32//! write-through replacement because it has no equivalent directory-fsync//! contract.@@ -27,7 +28,151 @@ use std::os::windows::ffi::OsStrExt as _;#[cfg(not(windows))]pub fn fsync_dir(dir: &Path) -> std::io::Result<()> {let d = std::fs::File::open(dir)?;- d.sync_all()+ sync_file_full(&d)+}++/// How many full flushes ([`sync_file_full`]) this process has issued.+static FULL_FLUSHES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);++/// Number of full flushes this process has issued through this module.+///+/// A full flush is `File::sync_all`, which on Apple targets is+/// `fcntl(F_FULLFSYNC)`: it drains the whole device write cache and costs+/// milliseconds each. Exposed so tests can bound how many a publish costs.+#[doc(hidden)]+pub fn full_flush_count() -> u64 {+ FULL_FLUSHES.load(std::sync::atomic::Ordering::Relaxed)+}++/// `file.sync_all()`, counted (see [`full_flush_count`]).+pub fn sync_file_full(file: &std::fs::File) -> std::io::Result<()> {+ FULL_FLUSHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ file.sync_all()+}++/// Sync a staged file's bytes ahead of a later device flush.+///+/// On Apple targets this is plain `fsync(2)`: it hands the data to the+/// device but leaves it in the device's write cache, which the one+/// `F_FULLFSYNC` in [`publish_staged`] then drains for every staged file at+/// once. Everywhere else it is the same full `sync_all` a durable write+/// always used, because there `fsync` is already the per-file barrier and+/// there is no cheaper device-wide one.+fn sync_file_for_batch(file: &std::fs::File) -> std::io::Result<()> {+ #[cfg(target_vendor = "apple")]+ {+ use std::os::fd::AsRawFd as _;+ // SAFETY: `fsync` on a descriptor this `File` owns for the call.+ if unsafe { libc::fsync(file.as_raw_fd()) } == 0 {+ Ok(())+ } else {+ Err(std::io::Error::last_os_error())+ }+ }+ #[cfg(not(target_vendor = "apple"))]+ {+ sync_file_full(file)+ }+}++/// A file written to a same-directory temp name and synced, waiting for+/// [`publish_staged`] to move it into place. Dropping it unpublished removes+/// the temp file.+#[derive(Debug)]+pub struct StagedFile {+ tmp: std::path::PathBuf,+ dest: std::path::PathBuf,+ published: bool,+}++impl Drop for StagedFile {+ fn drop(&mut self) {+ if !self.published {+ let _ = std::fs::remove_file(&self.tmp);+ }+ }+}++fn temp_sibling(path: &Path) -> std::path::PathBuf {+ let nonce = format!("{:x}-{:?}", std::process::id(), std::thread::current().id());+ let file_name = path+ .file_name()+ .map(|n| n.to_string_lossy().into_owned())+ .unwrap_or_else(|| "file".to_string());+ path.with_file_name(format!("{file_name}.tmp.{nonce}"))+}++/// Write `bytes` to a temp file beside `path` and sync it, for a later+/// [`publish_staged`]. Nothing is visible at `path` until then.+pub fn stage_file(path: &Path, bytes: &[u8]) -> std::io::Result<StagedFile> {+ let tmp = temp_sibling(path);+ require_same_directory(&tmp, path)?;+ let staged = StagedFile {+ tmp,+ dest: path.to_path_buf(),+ published: false,+ };+ let mut f = std::fs::File::create(&staged.tmp)?;+ f.write_all(bytes)?;+ sync_file_for_batch(&f)?;+ Ok(staged)+}++/// Stage a temp file the caller wrote itself (streamed output). `file` is the+/// open handle to `tmp`, fully written and flushed by the caller; this syncs+/// it like [`stage_file`] does.+pub fn stage_written_file(+ tmp: &Path,+ dest: &Path,+ file: &std::fs::File,+) -> std::io::Result<StagedFile> {+ let staged = StagedFile {+ tmp: tmp.to_path_buf(),+ dest: dest.to_path_buf(),+ published: false,+ };+ require_same_directory(&staged.tmp, &staged.dest)?;+ sync_file_for_batch(file)?;+ Ok(staged)+}++/// Move every staged file in `dir` into place, durably, with the same+/// guarantee as one [`write_file_durable`] per file:+///+/// 1. every staged file's bytes reach stable storage before any rename (on+/// Apple targets, one `F_FULLFSYNC`, which drains the device cache for+/// every earlier `fsync`; elsewhere each file was already fully synced+/// when it was staged);+/// 2. each file replaces its target (rename on Unix, write-through+/// `MoveFileExW` on Windows);+/// 3. one parent-directory fsync makes every rename durable (Unix).+///+/// A crash at any point leaves each target either absent, old or complete,+/// never torn, exactly as per-file durable writes did. What changes is the+/// cost: two device flushes for the batch instead of two per file, which on+/// Apple targets is the difference between ~250 cache drains per segment and+/// two. If a rename fails, the files already moved stay moved and the rest are+/// removed, which is also what a failure part-way through a loop of+/// [`write_file_durable`] calls left behind.+pub fn publish_staged(dir: &Path, files: Vec<StagedFile>) -> std::io::Result<()> {+ if files.is_empty() {+ return Ok(());+ }+ if let Some(stray) = files.iter().find(|f| f.dest.parent() != Some(dir)) {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidInput,+ format!("staged file {:?} is not in {:?}", stray.dest, dir),+ ));+ }+ #[cfg(target_vendor = "apple")]+ sync_file_full(&std::fs::File::open(dir)?)?;+ for mut file in files {+ replace_file_for_durable_write(&file.tmp, &file.dest)?;+ file.published = true;+ }+ #[cfg(not(windows))]+ fsync_dir(dir)?;+ Ok(())}/// Windows has no supported directory-flush primitive to call here:@@ -214,7 +359,7 @@ pub fn write_file_durable_with_hook({let mut f = std::fs::File::create(&tmp)?;f.write_all(bytes)?;- f.sync_all()?;+ sync_file_full(&f)?;}if let Err(error) = hook(DurableWriteStage::BeforeRename) {let _ = std::fs::remove_file(&tmp);@@ -290,6 +435,49 @@ mod tests {assert!(!source.exists());}+ #[test]+ fn staged_batch_publishes_every_file_and_leaves_no_temp() {+ let dir = tempfile::tempdir().unwrap();+ let a = dir.path().join("a.post");+ let b = dir.path().join("b.meta");+ std::fs::write(&b, b"old").unwrap();+ let staged = vec![+ stage_file(&a, b"alpha").unwrap(),+ stage_file(&b, b"beta").unwrap(),+ ];+ // Nothing is visible until the publish.+ assert!(!a.exists());+ assert_eq!(std::fs::read(&b).unwrap(), b"old");+ publish_staged(dir.path(), staged).unwrap();+ assert_eq!(std::fs::read(&a).unwrap(), b"alpha");+ assert_eq!(std::fs::read(&b).unwrap(), b"beta");+ assert!(std::fs::read_dir(dir.path()).unwrap().all(|e| !e+ .unwrap()+ .file_name()+ .to_string_lossy()+ .contains(".tmp.")));+ }++ #[test]+ fn an_unpublished_stage_removes_its_temp_file() {+ let dir = tempfile::tempdir().unwrap();+ let a = dir.path().join("a.post");+ drop(stage_file(&a, b"alpha").unwrap());+ assert!(!a.exists());+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);+ }++ #[test]+ fn publish_staged_rejects_a_file_outside_the_directory() {+ let dir = tempfile::tempdir().unwrap();+ let other = tempfile::tempdir().unwrap();+ let staged = vec![stage_file(&other.path().join("x"), b"x").unwrap()];+ let error = publish_staged(dir.path(), staged).unwrap_err();+ assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);+ assert!(!other.path().join("x").exists());+ assert_eq!(std::fs::read_dir(other.path()).unwrap().count(), 0);+ }+#[test]fn durable_replace_rejects_cross_directory_move() {let source_dir = tempfile::tempdir().unwrap();diff --git a/engine/crates/xerj-common/src/resource.rs b/engine/crates/xerj-common/src/resource.rsindex 2461f6917..0dea044c0 100644--- a/engine/crates/xerj-common/src/resource.rs+++ b/engine/crates/xerj-common/src/resource.rs@@ -402,8 +402,9 @@ pub fn available_memory_bytes() -> Option<u64> {probe_available_memory()}-/// Resident set size of this process, or `None` where this platform has no-/// probe. `None` must disable an RSS-driven decision, never read as zero.+/// Resident set size of this process (on macOS its physical footprint, see+/// the macOS `probe_rss`), or `None` where this platform has no probe.+/// `None` must disable an RSS-driven decision, never read as zero.pub fn current_rss_bytes() -> Option<u64> {probe_rss()}@@ -549,26 +550,35 @@ fn probe_rss() -> Option<u64> {Some(pages.saturating_mul(page))}+/// macOS: the process's physical footprint (`ri_phys_footprint`), not its+/// resident size.+///+/// Resident size leaves out the pages the compressor has taken and the pages+/// swapped out, which are still this process's memory, and it counts clean+/// file-backed pages (mmapped segments) the kernel can drop at will. Under a+/// 1 GiB cap a node read 0.8 GB resident while vmmap showed a 2.2 GB+/// footprint, 1.3 GB of it compressed, so the breaker kept admitting writes;+/// and its "released" edges followed the compressor, not a drain. The+/// footprint is what `/usr/bin/time -l` reports as "peak memory footprint"+/// and what the kernel's memory-pressure kills act on.#[cfg(target_os = "macos")]fn probe_rss() -> Option<u64> {- let mut info = std::mem::MaybeUninit::<libc::proc_taskinfo>::zeroed();- let size = std::mem::size_of::<libc::proc_taskinfo>() as libc::c_int;- // SAFETY: `proc_pidinfo` writes at most `size` bytes into `info` and- // reports how many it wrote; anything short is treated as a failure.- let written = unsafe {- libc::proc_pidinfo(+ let mut info = std::mem::MaybeUninit::<libc::rusage_info_v2>::zeroed();+ // SAFETY: with flavor RUSAGE_INFO_V2, `proc_pid_rusage` writes one+ // `rusage_info_v2` into the buffer it is given and returns 0, or returns+ // -1 and writes nothing usable.+ let rc = unsafe {+ libc::proc_pid_rusage(std::process::id() as libc::c_int,- libc::PROC_PIDTASKINFO,- 0,- info.as_mut_ptr() as *mut libc::c_void,- size,+ libc::RUSAGE_INFO_V2,+ info.as_mut_ptr() as *mut libc::rusage_info_t,)};- if written != size {+ if rc != 0 {return None;}- // SAFETY: proc_pidinfo filled the whole struct (checked above).- Some(unsafe { info.assume_init() }.pti_resident_size)+ // SAFETY: proc_pid_rusage returned 0, so it filled the struct.+ Some(unsafe { info.assume_init() }.ri_phys_footprint)}#[cfg(target_os = "windows")]diff --git a/engine/crates/xerj-common/tests/macos_memory_probe.rs b/engine/crates/xerj-common/tests/macos_memory_probe.rsnew file mode 100644index 000000000..cf4e02562--- /dev/null+++ b/engine/crates/xerj-common/tests/macos_memory_probe.rs@@ -0,0 +1,100 @@+//! On macOS the memory breaker must see the process's physical footprint, not+//! its resident set.+//!+//! The two diverge in both directions, and both hurt the breaker:+//!+//! * Pages the compressor has taken (or the pager swapped) leave the resident+//! set but are still this process's memory. Under a 1 GiB cap a node read+//! 0.8 GB resident while it held 2.2 GB, so the breaker let it keep growing.+//! * Clean file-backed pages (mmapped segment files) count as resident but+//! the kernel can drop them at any time, so they are not memory the process+//! can be asked to give back.+//!+//! The first case cannot be forced from a test. The second can, and it tells+//! the two probes apart: map a file, read every page, and the resident set+//! grows by the file's size while the footprint does not move.+//!+//! One test binary of its own, so no other test allocates while it measures.++#![cfg(target_os = "macos")]++use std::io::Write as _;+use std::os::fd::AsRawFd as _;++use xerj_common::resource::current_rss_bytes;++const MIB: u64 = 1024 * 1024;+const SIZE: usize = 128 * 1024 * 1024;++fn probe() -> u64 {+ current_rss_bytes().expect("the macOS memory probe must work")+}++#[test]+fn the_memory_probe_counts_owned_memory_not_mapped_file_pages() {+ // Mapped, read, clean file pages.+ let dir = tempfile::tempdir().expect("tempdir");+ let path = dir.path().join("mapped.bin");+ {+ let mut f = std::fs::File::create(&path).expect("create");+ let chunk = vec![0x5au8; 1024 * 1024];+ for _ in 0..(SIZE / chunk.len()) {+ f.write_all(&chunk).expect("write");+ }+ f.sync_all().expect("sync");+ }+ let file = std::fs::File::open(&path).expect("open");+ let before_map = probe();+ // SAFETY: read-only shared mapping of a file we own, unmapped below.+ let addr = unsafe {+ libc::mmap(+ std::ptr::null_mut(),+ SIZE,+ libc::PROT_READ,+ libc::MAP_SHARED,+ file.as_raw_fd(),+ 0,+ )+ };+ assert_ne!(addr, libc::MAP_FAILED, "mmap failed");+ let bytes = addr as *const u8;+ let mut sum = 0u64;+ for off in (0..SIZE).step_by(4096) {+ // SAFETY: `off < SIZE`, inside the mapping.+ sum += u64::from(unsafe { std::ptr::read_volatile(bytes.add(off)) });+ }+ std::hint::black_box(sum);++ // The premise: the pages really are resident, or this test proves nothing.+ let page = 16 * 1024usize;+ let mut vec = vec![0 as libc::c_char; SIZE / page];+ // SAFETY: `vec` holds one byte per page of the mapping.+ let rc = unsafe { libc::mincore(addr, SIZE, vec.as_mut_ptr()) };+ assert_eq!(rc, 0, "mincore failed");+ let resident = vec.iter().filter(|&&b| b & 1 != 0).count();+ assert!(+ resident * 10 >= vec.len() * 9,+ "only {resident} of {} mapped pages are resident",+ vec.len()+ );++ let mapped_growth = probe().saturating_sub(before_map);+ // SAFETY: the mapping created above, with its length.+ unsafe { libc::munmap(addr, SIZE) };+ assert!(+ mapped_growth < 32 * MIB,+ "probe grew {} MiB for 128 MiB of clean mapped file pages",+ mapped_growth / MIB+ );++ // Positive control: memory the process owns does move the probe.+ let before_heap = probe();+ let heap: Vec<u8> = (0..SIZE).map(|i| (i % 251) as u8).collect();+ std::hint::black_box(&heap);+ let heap_growth = probe().saturating_sub(before_heap);+ assert!(+ heap_growth >= 96 * MIB,+ "probe grew only {} MiB for 128 MiB of touched heap",+ heap_growth / MIB+ );+}diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsindex 4ee515534..64758c243 100644--- a/engine/crates/xerj-engine/src/governor.rs+++ b/engine/crates/xerj-engine/src/governor.rs@@ -1004,7 +1004,7 @@ fn build(config: &Config) -> ResourceGovernor {/// Current resident set size of this process, in bytes, via/// [`xerj_common::resource::current_rss_bytes`] (`/proc/self/statm` on Linux,-/// `proc_pidinfo(PROC_PIDTASKINFO)` on macOS). Returns 0 where the platform+/// the physical footprint from `proc_pid_rusage` on macOS). Returns 0 where the platform/// exposes no probe — the RSS watermark then never trips, which is safe but/// means no admission control, so [`build`] says so out loud at startup.///diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rsindex 70f08355f..e13939033 100644--- a/engine/crates/xerj-fts/src/index.rs+++ b/engine/crates/xerj-fts/src/index.rs@@ -1420,6 +1420,11 @@ impl FtsIndexWriter {.map(|(name, _stats, fd)| (name.as_str(), fd.config.analyzer.as_str())).filter(|(_name, analyzer)| *analyzer != "standard").collect();+ // Every side-car of this segment is staged (temp file written and+ // synced) and published together at the end: see `publish_staged`+ // for why that is as durable as one durable write per file, and for+ // the cost it removes (two device flushes per file on Apple targets).+ let mut staged: Vec<xerj_common::fsio::StagedFile> = Vec::new();if !recorded.is_empty() {#[derive(serde::Serialize)]struct Marker<'a> {@@ -1428,17 +1433,19 @@ impl FtsIndexWriter {let bytes = serde_json::to_vec(&Marker { fields: recorded }).with_context(|| "encoding FTS analyzer marker")?;let path = segment_analyzer_marker_path(&segment_dir, &segment_id);- xerj_common::fsio::write_file_durable(&path, &bytes)- .with_context(|| format!("writing FTS analyzer marker {:?}", path))?;+ staged.push(+ xerj_common::fsio::stage_file(&path, &bytes)+ .with_context(|| format!("writing FTS analyzer marker {:?}", path))?,+ );}// Parallel field writes. `write_field_static` is a pure function// of its inputs and touches only files named after the field, so// there's no cross-thread contention.- let results: Vec<Result<(String, FieldStats)>> = fields+ let results: Vec<Result<(String, FieldStats, Vec<xerj_common::fsio::StagedFile>)>> = fields.into_par_iter().map(|(field_name, stats, field_data)| {- Self::write_field_static(+ let files = Self::write_field_static(&segment_dir,&segment_id,&field_name,@@ -1446,16 +1453,20 @@ impl FtsIndexWriter {zstd_level,).with_context(|| format!("writing field '{}'", field_name))?;- Ok((field_name, stats))+ Ok((field_name, stats, files))}).collect();- // Surface any error; otherwise build the stats map.+ // Surface any error (dropping `staged` removes every temp file);+ // otherwise build the stats map and publish the side-cars.let mut all_stats = HashMap::new();for r in results {- let (name, stats) = r?;+ let (name, stats, files) = r?;+ staged.extend(files);all_stats.insert(name, stats);}+ xerj_common::fsio::publish_staged(&segment_dir, staged)+ .with_context(|| format!("publishing FTS side-cars in {:?}", segment_dir))?;Ok(all_stats)}@@ -1466,7 +1477,8 @@ impl FtsIndexWriter {field_name: &str,field_data: FieldData,zstd_level: i32,- ) -> Result<()> {+ ) -> Result<Vec<xerj_common::fsio::StagedFile>> {+ let mut staged = Vec::with_capacity(4);// NOTE: `PathBuf::with_extension` replaces the final `.ext` in the path,// so using `segment_dir.join("segment_id.field_name")` followed by// `with_extension("fst")` would strip `.field_name` and collapse every@@ -1546,16 +1558,19 @@ impl FtsIndexWriter {out.extend_from_slice(&compressed);out};- // RC4 W1 #10 — every FTS side-car write below goes through durable- // temporary-file sync plus platform-appropriate publication: rename- // and parent-directory fsync on Unix, write-through replacement on- // Windows. These files are part of the segment publish chain: the WAL- // entries they cover are pruned ~1 s after the flush, so side-cars- // sitting only in the volatile page cache meant a power loss could- // leave a registered segment with missing/torn FTS data (silently- // wrong query results or unreadable fields) with no WAL to recover from.- xerj_common::fsio::write_file_durable(&post_path, &post_bytes_wrapped)- .with_context(|| format!("writing postings to {:?}", post_path))?;+ // RC4 W1 #10 — every FTS side-car below is staged (synced temp file)+ // and published durably by `finish` through `fsio::publish_staged`:+ // rename and parent-directory fsync on Unix, write-through+ // replacement on Windows. These files are part of the segment publish+ // chain: the WAL entries they cover are pruned ~1 s after the flush,+ // so side-cars sitting only in the volatile page cache meant a power+ // loss could leave a registered segment with missing/torn FTS data+ // (silently wrong query results or unreadable fields) with no WAL to+ // recover from.+ staged.push(+ xerj_common::fsio::stage_file(&post_path, &post_bytes_wrapped)+ .with_context(|| format!("writing postings to {:?}", post_path))?,+ );// 3. Build and write FST (term → meta byte offset).//@@ -1591,13 +1606,11 @@ impl FtsIndexWriter {}let mut fst_out = fst_builder.into_inner().with_context(|| "finishing FST")?;fst_out.flush().with_context(|| "flushing FST")?;- fst_out- .get_ref()- .sync_all()- .with_context(|| "fsyncing FST")?;+ staged.push(+ xerj_common::fsio::stage_written_file(&fst_tmp, &fst_path, fst_out.get_ref())+ .with_context(|| "fsyncing FST")?,+ );drop(fst_out);- xerj_common::fsio::replace_file_durable(&fst_tmp, &fst_path)- .with_context(|| format!("publishing FST {:?}", fst_path))?;// 4. Write meta in the ZFM5 format — columnar varint streams// (df, ttf-delta, gaps, lengths) under the Zstd envelope;@@ -1612,8 +1625,10 @@ impl FtsIndexWriter {&term_postings,zstd_level,)?;- xerj_common::fsio::write_file_durable(&meta_path, &meta_bytes)- .with_context(|| format!("writing meta to {:?}", meta_path))?;+ staged.push(+ xerj_common::fsio::stage_file(&meta_path, &meta_bytes)+ .with_context(|| format!("writing meta to {:?}", meta_path))?,+ );// 5. Write norms file — V4 M4.7 compact format.//@@ -1658,10 +1673,12 @@ impl FtsIndexWriter {norms_bytes.extend_from_slice(&(dense_len as u32).to_le_bytes());norms_bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes());norms_bytes.extend_from_slice(payload);- xerj_common::fsio::write_file_durable(&norms_path, &norms_bytes)- .with_context(|| format!("writing norms to {:?}", norms_path))?;+ staged.push(+ xerj_common::fsio::stage_file(&norms_path, &norms_bytes)+ .with_context(|| format!("writing norms to {:?}", norms_path))?,+ );- Ok(())+ Ok(staged)}}diff --git a/engine/crates/xerj-fts/tests/finish_full_flushes.rs b/engine/crates/xerj-fts/tests/finish_full_flushes.rsnew file mode 100644index 000000000..9841dabb7--- /dev/null+++ b/engine/crates/xerj-fts/tests/finish_full_flushes.rs@@ -0,0 +1,90 @@+//! A segment's FTS side-cars must not cost a full device flush per file.+//!+//! `File::sync_all` is `fcntl(F_FULLFSYNC)` on Apple targets, which drains the+//! whole device write cache and costs milliseconds per call. Writing each of a+//! segment's side-cars as its own durable write (sync the temp file, rename,+//! sync the directory) paid two of those per file: about 250 for a 31-field+//! segment, and on a Mac those flushes were most of the indexing time.+//!+//! One device flush after the temp files are written, and one after they are+//! renamed into place (Apple targets; elsewhere `fsync` stays per file, and+//! only the per-file directory sync goes), give the segment the same guarantee at the moment+//! `finish` returns: every side-car's bytes are on stable storage before its+//! name points at them, and every name is on stable storage before the caller+//! publishes the segment.+//!+//! This file holds one test on purpose: the flush counter is process-wide, and+//! a test binary of its own is the only way no other test writes alongside it.++use std::collections::HashMap;+use std::sync::Arc;++use xerj_fts::analyzer::AnalyzerRegistry;+use xerj_fts::index::{FieldValues, FtsIndexReader, FtsIndexWriter};++const FIELDS: usize = 31;+const DOCS: u32 = 50;++fn field(i: usize) -> String {+ format!("field_{i:02}")+}++#[test]+fn finishing_a_wide_segment_costs_a_bounded_number_of_full_flushes() {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut writer =+ FtsIndexWriter::new(dir.path(), "seg_1", Arc::new(AnalyzerRegistry::default()));+ for doc in 0..DOCS {+ let fields: HashMap<String, FieldValues> = (0..FIELDS)+ .map(|i| {+ (+ field(i),+ FieldValues::One(format!("alpha beta value{} doc{doc}", i % 5)),+ )+ })+ .collect();+ writer.add_document(doc, &fields);+ }+ writer+ .publish_encoded_filename_layout()+ .expect("layout marker");++ let before = xerj_common::fsio::full_flush_count();+ let stats = writer.finish().expect("finish");+ let flushes = xerj_common::fsio::full_flush_count() - before;+ assert_eq!(stats.len(), FIELDS);++ // The old per-file shape issued 2 per side-car (temp file, then the+ // directory after its rename): 8 per field, 248 here.+ //+ // Apple targets: one device flush before the renames and one after.+ // Elsewhere `fsync` is the per-file barrier and stays per file, so the+ // saving is the per-file directory sync: one per side-car plus one.+ let side_cars = (FIELDS * 4) as u64;+ let bound = if cfg!(target_vendor = "apple") {+ 2+ } else {+ side_cars + 1+ };+ assert!(+ flushes <= bound,+ "finish issued {flushes} full flushes for {FIELDS} fields; expected at most {bound}"+ );++ // Same segment, still readable: every field opens and answers.+ let names: Vec<String> = (0..FIELDS).map(field).collect();+ let refs: Vec<&str> = names.iter().map(String::as_str).collect();+ let reader = FtsIndexReader::open(dir.path(), "seg_1", &refs).expect("open");+ for name in &names {+ assert!(reader.field_stats(name).is_some(), "{name} has no stats");+ assert_eq!(reader.term_doc_freq(name, "alpha"), Some(DOCS), "{name}");+ }++ // No temp file survives a successful finish.+ let leftovers: Vec<String> = std::fs::read_dir(dir.path())+ .expect("read_dir")+ .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned())+ .filter(|n| n.contains(".tmp"))+ .collect();+ assert!(leftovers.is_empty(), "temp files left: {leftovers:?}");+}diff --git a/engine/crates/xerj-server/Cargo.toml b/engine/crates/xerj-server/Cargo.tomlindex eb9c2c5c3..de5ecaa1b 100644--- a/engine/crates/xerj-server/Cargo.toml+++ b/engine/crates/xerj-server/Cargo.toml@@ -95,6 +95,11 @@ debug-profiling = [[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator = "0.6"tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }+# `mallctl` itself, for the void `arena.<i>.purge` control that the ctl+# crate's typed read/write helpers cannot call (see+# `purge_all_jemalloc_arenas`). Already in the graph under+# tikv-jemallocator and tikv-jemalloc-ctl, so this adds an edge, not a crate.+tikv-jemalloc-sys = "0.6"libc = "0.2"[target.'cfg(all(target_os = "linux", not(target_env = "msvc")))'.dependencies]diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 3ab1a46ae..296b95c2c 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -2338,7 +2338,33 @@ fn main() -> Result<()> {.filter(|&n| n > 0).unwrap_or_else(|| (cores * 8).clamp(64, 512));let rt = build_runtime(worker_threads)?;- rt.block_on(async_main())+ run_to_exit(rt, async_main())+}++/// How long exit waits for blocking jobs still running after `async_main`+/// returns ("Goodbye" has been logged and the final flush is done).+const RUNTIME_SHUTDOWN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);++/// Drive the server future to completion, then tear the runtime down without+/// waiting more than [`RUNTIME_SHUTDOWN_TIMEOUT`] for blocking jobs.+///+/// Dropping a `Runtime` joins its blocking pool with no time limit. After+/// SIGTERM the one job left there is usually a background merge, detached by+/// `spawn_blocking` and parked in the merge pool, which can spend minutes+/// compressing a large column at zstd 19 on a low-priority thread. The+/// process then outlived "Goodbye" by however long that merge had left+/// (90 s+ measured on macOS), and a supervisor's stop timeout killed it.+///+/// Walking away from that merge is safe: a merge publishes atomically through+/// `apply_merge`, and an unfinished batch leaves its input segments untouched+/// (the LOSS FIREWALL notes in `xerj-engine`'s `index.rs`). The final+/// `flush_all_force` has completed before `async_main` returns, and leaving a+/// job unfinished here is no different from the SIGKILL or power loss the+/// engine is already built to recover from.+fn run_to_exit<F: std::future::Future>(rt: tokio::runtime::Runtime, fut: F) -> F::Output {+ let out = rt.block_on(fut);+ rt.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT);+ out}/// Explicit worker-thread stack size for the production Tokio runtime.@@ -2976,9 +3002,87 @@ mod decide_flag_tests {}}+/// Return every jemalloc arena's dirty pages to the OS (`arena.4096.purge`,+/// 4096 being `MALLCTL_ARENAS_ALL`).+///+/// `arena.<i>.purge` is a void control: jemalloc rejects it with EPERM unless+/// all four of `oldp`, `oldlenp`, `newp` and `newlen` are null/0+/// (`NEITHER_READ_NOR_WRITE` in jemalloc's `ctl.c`). The earlier+/// `tikv_jemalloc_ctl::raw::write(.., 0u64)` passed a value, so every purge+/// failed and the error was discarded. On Linux `background_thread:true`+/// hid it; on macOS there is no background thread, and the drain freed+/// nothing back to the OS.+#[cfg(not(target_env = "msvc"))]+fn purge_all_jemalloc_arenas() -> Result<(), i32> {+ // SAFETY: the name is a NUL-terminated literal naming a real ctl node,+ // and a void control reads and writes nothing, so null pointers and a+ // zero length are exactly what it requires.+ let rc = unsafe {+ tikv_jemalloc_sys::mallctl(+ c"arena.4096.purge".as_ptr(),+ std::ptr::null_mut(),+ std::ptr::null_mut(),+ std::ptr::null_mut(),+ 0,+ )+ };+ if rc == 0 {+ Ok(())+ } else {+ Err(rc)+ }+}++#[cfg(all(test, not(target_env = "msvc")))]+mod allocator_purge_tests {+ use super::purge_all_jemalloc_arenas;++ /// The memory drain's purge must actually run. `arena.<i>.purge` is a+ /// void control: jemalloc answers EPERM to any call that passes a new+ /// value (`ctl.c`, `NEITHER_READ_NOR_WRITE`), so a "write 0" purge is a+ /// silent no-op and the drain returns nothing to the OS.+ #[test]+ fn arena_purge_succeeds() {+ // Dirty some pages first so the purge has work to do.+ let blocks: Vec<Vec<u8>> = (0..64).map(|i| vec![i as u8; 256 * 1024]).collect();+ drop(std::hint::black_box(blocks));+ assert_eq!(+ purge_all_jemalloc_arenas(),+ Ok(()),+ "arena.4096.purge must return 0"+ );+ }+}+#[cfg(test)]mod runtime_tests {- use super::build_runtime;+ use super::{build_runtime, run_to_exit};++ /// SIGTERM → "Goodbye" → exit must not wait on a blocking job that is+ /// still running (in production: a background merge compressing a large+ /// column on a low-priority thread). Dropping a Runtime joins its blocking+ /// pool with no timeout, so the process outlived "Goodbye" by as long as+ /// the job had left.+ #[test]+ fn exit_does_not_wait_for_a_running_blocking_job() {+ let rt = build_runtime(2).expect("prod runtime builds");+ let started = std::time::Instant::now();+ run_to_exit(rt, async {+ let (tx, rx) = tokio::sync::oneshot::channel::<()>();+ // Detached, like the merge task: nothing awaits it.+ drop(tokio::task::spawn_blocking(move || {+ let _ = tx.send(());+ std::thread::sleep(std::time::Duration::from_secs(30));+ }));+ // Make sure the job is really running before the server returns.+ rx.await.expect("blocking job started");+ });+ let waited = started.elapsed();+ assert!(+ waited < std::time::Duration::from_secs(10),+ "exit waited {waited:?} for a detached blocking job"+ );+ }#[test]fn runtime_builds_and_boots() {@@ -3800,11 +3904,14 @@ async fn async_main() -> Result<()> {// MALLCTL_ARENAS_ALL. Both are best-effort: a failing mallctl// must never propagate out of the drain path.let _ = tikv_jemalloc_ctl::epoch::advance();- // SAFETY: `arena.<i>.purge` with i = MALLCTL_ARENAS_ALL (4096)- // is a plain "purge every arena" action mallctl; the value is an- // unused unsigned scalar. The name is NUL-terminated and names a- // real jemalloc ctl node, so the write cannot read out of bounds.- let _ = unsafe { tikv_jemalloc_ctl::raw::write(b"arena.4096.purge\0", 0u64) };+ // Best-effort as before, but no longer silent: a purge that+ // fails is a drain that returns nothing to the OS. Warn once.+ if let Err(rc) = purge_all_jemalloc_arenas() {+ static WARNED: std::sync::Once = std::sync::Once::new();+ WARNED.call_once(|| {+ warn!("jemalloc arena.4096.purge failed (mallctl returned {rc}); memory drains will not return pages to the OS");+ });+ }}));state.engine.spawn_resource_sampler();}