Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix: macOS indexing stalls under a memory cap, and shutdown hangs (#1677)

Gil Portnoyed98fefcf2db
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index cea4bc70c..bf46a2cb6 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7755,6 +7755,7 @@ dependencies = [
"serde_json",
"tempfile",
"tikv-jemalloc-ctl",
+ "tikv-jemalloc-sys",
"tikv-jemallocator",
"tokio",
"tokio-stream",
diff --git a/engine/crates/xerj-common/src/fsio.rs b/engine/crates/xerj-common/src/fsio.rs
index 5a03f721a..d408786b7 100644
--- a/engine/crates/xerj-common/src/fsio.rs
+++ b/engine/crates/xerj-common/src/fsio.rs
@@ -9,7 +9,8 @@
//! acked-data loss.
//!
//! Every side-car / manifest write on the publish chain must go through
-//! [`write_file_durable`] or [`replace_file_durable`]. Unix makes the rename
+//! [`write_file_durable`] or [`replace_file_durable`], or, for many files in
+//! one directory, [`stage_file`] plus one [`publish_staged`]. Unix makes the rename
//! durable with parent-directory fsync; Windows uses a same-directory Win32
//! write-through replacement because it has no equivalent directory-fsync
//! contract.
@@ -27,7 +28,151 @@ use std::os::windows::ffi::OsStrExt as _;
#[cfg(not(windows))]
pub fn fsync_dir(dir: &Path) -> std::io::Result<()> {
let d = std::fs::File::open(dir)?;
- d.sync_all()
+ sync_file_full(&d)
+}
+
+/// How many full flushes ([`sync_file_full`]) this process has issued.
+static FULL_FLUSHES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+
+/// Number of full flushes this process has issued through this module.
+///
+/// A full flush is `File::sync_all`, which on Apple targets is
+/// `fcntl(F_FULLFSYNC)`: it drains the whole device write cache and costs
+/// milliseconds each. Exposed so tests can bound how many a publish costs.
+#[doc(hidden)]
+pub fn full_flush_count() -> u64 {
+ FULL_FLUSHES.load(std::sync::atomic::Ordering::Relaxed)
+}
+
+/// `file.sync_all()`, counted (see [`full_flush_count`]).
+pub fn sync_file_full(file: &std::fs::File) -> std::io::Result<()> {
+ FULL_FLUSHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ file.sync_all()
+}
+
+/// Sync a staged file's bytes ahead of a later device flush.
+///
+/// On Apple targets this is plain `fsync(2)`: it hands the data to the
+/// device but leaves it in the device's write cache, which the one
+/// `F_FULLFSYNC` in [`publish_staged`] then drains for every staged file at
+/// once. Everywhere else it is the same full `sync_all` a durable write
+/// always used, because there `fsync` is already the per-file barrier and
+/// there is no cheaper device-wide one.
+fn sync_file_for_batch(file: &std::fs::File) -> std::io::Result<()> {
+ #[cfg(target_vendor = "apple")]
+ {
+ use std::os::fd::AsRawFd as _;
+ // SAFETY: `fsync` on a descriptor this `File` owns for the call.
+ if unsafe { libc::fsync(file.as_raw_fd()) } == 0 {
+ Ok(())
+ } else {
+ Err(std::io::Error::last_os_error())
+ }
+ }
+ #[cfg(not(target_vendor = "apple"))]
+ {
+ sync_file_full(file)
+ }
+}
+
+/// A file written to a same-directory temp name and synced, waiting for
+/// [`publish_staged`] to move it into place. Dropping it unpublished removes
+/// the temp file.
+#[derive(Debug)]
+pub struct StagedFile {
+ tmp: std::path::PathBuf,
+ dest: std::path::PathBuf,
+ published: bool,
+}
+
+impl Drop for StagedFile {
+ fn drop(&mut self) {
+ if !self.published {
+ let _ = std::fs::remove_file(&self.tmp);
+ }
+ }
+}
+
+fn temp_sibling(path: &Path) -> std::path::PathBuf {
+ let nonce = format!("{:x}-{:?}", std::process::id(), std::thread::current().id());
+ let file_name = path
+ .file_name()
+ .map(|n| n.to_string_lossy().into_owned())
+ .unwrap_or_else(|| "file".to_string());
+ path.with_file_name(format!("{file_name}.tmp.{nonce}"))
+}
+
+/// Write `bytes` to a temp file beside `path` and sync it, for a later
+/// [`publish_staged`]. Nothing is visible at `path` until then.
+pub fn stage_file(path: &Path, bytes: &[u8]) -> std::io::Result<StagedFile> {
+ let tmp = temp_sibling(path);
+ require_same_directory(&tmp, path)?;
+ let staged = StagedFile {
+ tmp,
+ dest: path.to_path_buf(),
+ published: false,
+ };
+ let mut f = std::fs::File::create(&staged.tmp)?;
+ f.write_all(bytes)?;
+ sync_file_for_batch(&f)?;
+ Ok(staged)
+}
+
+/// Stage a temp file the caller wrote itself (streamed output). `file` is the
+/// open handle to `tmp`, fully written and flushed by the caller; this syncs
+/// it like [`stage_file`] does.
+pub fn stage_written_file(
+ tmp: &Path,
+ dest: &Path,
+ file: &std::fs::File,
+) -> std::io::Result<StagedFile> {
+ let staged = StagedFile {
+ tmp: tmp.to_path_buf(),
+ dest: dest.to_path_buf(),
+ published: false,
+ };
+ require_same_directory(&staged.tmp, &staged.dest)?;
+ sync_file_for_batch(file)?;
+ Ok(staged)
+}
+
+/// Move every staged file in `dir` into place, durably, with the same
+/// guarantee as one [`write_file_durable`] per file:
+///
+/// 1. every staged file's bytes reach stable storage before any rename (on
+/// Apple targets, one `F_FULLFSYNC`, which drains the device cache for
+/// every earlier `fsync`; elsewhere each file was already fully synced
+/// when it was staged);
+/// 2. each file replaces its target (rename on Unix, write-through
+/// `MoveFileExW` on Windows);
+/// 3. one parent-directory fsync makes every rename durable (Unix).
+///
+/// A crash at any point leaves each target either absent, old or complete,
+/// never torn, exactly as per-file durable writes did. What changes is the
+/// cost: two device flushes for the batch instead of two per file, which on
+/// Apple targets is the difference between ~250 cache drains per segment and
+/// two. If a rename fails, the files already moved stay moved and the rest are
+/// removed, which is also what a failure part-way through a loop of
+/// [`write_file_durable`] calls left behind.
+pub fn publish_staged(dir: &Path, files: Vec<StagedFile>) -> std::io::Result<()> {
+ if files.is_empty() {
+ return Ok(());
+ }
+ if let Some(stray) = files.iter().find(|f| f.dest.parent() != Some(dir)) {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidInput,
+ format!("staged file {:?} is not in {:?}", stray.dest, dir),
+ ));
+ }
+ #[cfg(target_vendor = "apple")]
+ sync_file_full(&std::fs::File::open(dir)?)?;
+ for mut file in files {
+ replace_file_for_durable_write(&file.tmp, &file.dest)?;
+ file.published = true;
+ }
+ #[cfg(not(windows))]
+ fsync_dir(dir)?;
+ Ok(())
}
/// Windows has no supported directory-flush primitive to call here:
@@ -214,7 +359,7 @@ pub fn write_file_durable_with_hook(
{
let mut f = std::fs::File::create(&tmp)?;
f.write_all(bytes)?;
- f.sync_all()?;
+ sync_file_full(&f)?;
}
if let Err(error) = hook(DurableWriteStage::BeforeRename) {
let _ = std::fs::remove_file(&tmp);
@@ -290,6 +435,49 @@ mod tests {
assert!(!source.exists());
}
+ #[test]
+ fn staged_batch_publishes_every_file_and_leaves_no_temp() {
+ let dir = tempfile::tempdir().unwrap();
+ let a = dir.path().join("a.post");
+ let b = dir.path().join("b.meta");
+ std::fs::write(&b, b"old").unwrap();
+ let staged = vec![
+ stage_file(&a, b"alpha").unwrap(),
+ stage_file(&b, b"beta").unwrap(),
+ ];
+ // Nothing is visible until the publish.
+ assert!(!a.exists());
+ assert_eq!(std::fs::read(&b).unwrap(), b"old");
+ publish_staged(dir.path(), staged).unwrap();
+ assert_eq!(std::fs::read(&a).unwrap(), b"alpha");
+ assert_eq!(std::fs::read(&b).unwrap(), b"beta");
+ assert!(std::fs::read_dir(dir.path()).unwrap().all(|e| !e
+ .unwrap()
+ .file_name()
+ .to_string_lossy()
+ .contains(".tmp.")));
+ }
+
+ #[test]
+ fn an_unpublished_stage_removes_its_temp_file() {
+ let dir = tempfile::tempdir().unwrap();
+ let a = dir.path().join("a.post");
+ drop(stage_file(&a, b"alpha").unwrap());
+ assert!(!a.exists());
+ assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
+ }
+
+ #[test]
+ fn publish_staged_rejects_a_file_outside_the_directory() {
+ let dir = tempfile::tempdir().unwrap();
+ let other = tempfile::tempdir().unwrap();
+ let staged = vec![stage_file(&other.path().join("x"), b"x").unwrap()];
+ let error = publish_staged(dir.path(), staged).unwrap_err();
+ assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
+ assert!(!other.path().join("x").exists());
+ assert_eq!(std::fs::read_dir(other.path()).unwrap().count(), 0);
+ }
+
#[test]
fn durable_replace_rejects_cross_directory_move() {
let source_dir = tempfile::tempdir().unwrap();
diff --git a/engine/crates/xerj-common/src/resource.rs b/engine/crates/xerj-common/src/resource.rs
index 2461f6917..0dea044c0 100644
--- a/engine/crates/xerj-common/src/resource.rs
+++ b/engine/crates/xerj-common/src/resource.rs
@@ -402,8 +402,9 @@ pub fn available_memory_bytes() -> Option<u64> {
probe_available_memory()
}
-/// Resident set size of this process, or `None` where this platform has no
-/// probe. `None` must disable an RSS-driven decision, never read as zero.
+/// Resident set size of this process (on macOS its physical footprint, see
+/// the macOS `probe_rss`), or `None` where this platform has no probe.
+/// `None` must disable an RSS-driven decision, never read as zero.
pub fn current_rss_bytes() -> Option<u64> {
probe_rss()
}
@@ -549,26 +550,35 @@ fn probe_rss() -> Option<u64> {
Some(pages.saturating_mul(page))
}
+/// macOS: the process's physical footprint (`ri_phys_footprint`), not its
+/// resident size.
+///
+/// Resident size leaves out the pages the compressor has taken and the pages
+/// swapped out, which are still this process's memory, and it counts clean
+/// file-backed pages (mmapped segments) the kernel can drop at will. Under a
+/// 1 GiB cap a node read 0.8 GB resident while vmmap showed a 2.2 GB
+/// footprint, 1.3 GB of it compressed, so the breaker kept admitting writes;
+/// and its "released" edges followed the compressor, not a drain. The
+/// footprint is what `/usr/bin/time -l` reports as "peak memory footprint"
+/// and what the kernel's memory-pressure kills act on.
#[cfg(target_os = "macos")]
fn probe_rss() -> Option<u64> {
- let mut info = std::mem::MaybeUninit::<libc::proc_taskinfo>::zeroed();
- let size = std::mem::size_of::<libc::proc_taskinfo>() as libc::c_int;
- // SAFETY: `proc_pidinfo` writes at most `size` bytes into `info` and
- // reports how many it wrote; anything short is treated as a failure.
- let written = unsafe {
- libc::proc_pidinfo(
+ let mut info = std::mem::MaybeUninit::<libc::rusage_info_v2>::zeroed();
+ // SAFETY: with flavor RUSAGE_INFO_V2, `proc_pid_rusage` writes one
+ // `rusage_info_v2` into the buffer it is given and returns 0, or returns
+ // -1 and writes nothing usable.
+ let rc = unsafe {
+ libc::proc_pid_rusage(
std::process::id() as libc::c_int,
- libc::PROC_PIDTASKINFO,
- 0,
- info.as_mut_ptr() as *mut libc::c_void,
- size,
+ libc::RUSAGE_INFO_V2,
+ info.as_mut_ptr() as *mut libc::rusage_info_t,
)
};
- if written != size {
+ if rc != 0 {
return None;
}
- // SAFETY: proc_pidinfo filled the whole struct (checked above).
- Some(unsafe { info.assume_init() }.pti_resident_size)
+ // SAFETY: proc_pid_rusage returned 0, so it filled the struct.
+ Some(unsafe { info.assume_init() }.ri_phys_footprint)
}
#[cfg(target_os = "windows")]
diff --git a/engine/crates/xerj-common/tests/macos_memory_probe.rs b/engine/crates/xerj-common/tests/macos_memory_probe.rs
new file mode 100644
index 000000000..cf4e02562
--- /dev/null
+++ b/engine/crates/xerj-common/tests/macos_memory_probe.rs
@@ -0,0 +1,100 @@
+//! On macOS the memory breaker must see the process's physical footprint, not
+//! its resident set.
+//!
+//! The two diverge in both directions, and both hurt the breaker:
+//!
+//! * Pages the compressor has taken (or the pager swapped) leave the resident
+//! set but are still this process's memory. Under a 1 GiB cap a node read
+//! 0.8 GB resident while it held 2.2 GB, so the breaker let it keep growing.
+//! * Clean file-backed pages (mmapped segment files) count as resident but
+//! the kernel can drop them at any time, so they are not memory the process
+//! can be asked to give back.
+//!
+//! The first case cannot be forced from a test. The second can, and it tells
+//! the two probes apart: map a file, read every page, and the resident set
+//! grows by the file's size while the footprint does not move.
+//!
+//! One test binary of its own, so no other test allocates while it measures.
+
+#![cfg(target_os = "macos")]
+
+use std::io::Write as _;
+use std::os::fd::AsRawFd as _;
+
+use xerj_common::resource::current_rss_bytes;
+
+const MIB: u64 = 1024 * 1024;
+const SIZE: usize = 128 * 1024 * 1024;
+
+fn probe() -> u64 {
+ current_rss_bytes().expect("the macOS memory probe must work")
+}
+
+#[test]
+fn the_memory_probe_counts_owned_memory_not_mapped_file_pages() {
+ // Mapped, read, clean file pages.
+ let dir = tempfile::tempdir().expect("tempdir");
+ let path = dir.path().join("mapped.bin");
+ {
+ let mut f = std::fs::File::create(&path).expect("create");
+ let chunk = vec![0x5au8; 1024 * 1024];
+ for _ in 0..(SIZE / chunk.len()) {
+ f.write_all(&chunk).expect("write");
+ }
+ f.sync_all().expect("sync");
+ }
+ let file = std::fs::File::open(&path).expect("open");
+ let before_map = probe();
+ // SAFETY: read-only shared mapping of a file we own, unmapped below.
+ let addr = unsafe {
+ libc::mmap(
+ std::ptr::null_mut(),
+ SIZE,
+ libc::PROT_READ,
+ libc::MAP_SHARED,
+ file.as_raw_fd(),
+ 0,
+ )
+ };
+ assert_ne!(addr, libc::MAP_FAILED, "mmap failed");
+ let bytes = addr as *const u8;
+ let mut sum = 0u64;
+ for off in (0..SIZE).step_by(4096) {
+ // SAFETY: `off < SIZE`, inside the mapping.
+ sum += u64::from(unsafe { std::ptr::read_volatile(bytes.add(off)) });
+ }
+ std::hint::black_box(sum);
+
+ // The premise: the pages really are resident, or this test proves nothing.
+ let page = 16 * 1024usize;
+ let mut vec = vec![0 as libc::c_char; SIZE / page];
+ // SAFETY: `vec` holds one byte per page of the mapping.
+ let rc = unsafe { libc::mincore(addr, SIZE, vec.as_mut_ptr()) };
+ assert_eq!(rc, 0, "mincore failed");
+ let resident = vec.iter().filter(|&&b| b & 1 != 0).count();
+ assert!(
+ resident * 10 >= vec.len() * 9,
+ "only {resident} of {} mapped pages are resident",
+ vec.len()
+ );
+
+ let mapped_growth = probe().saturating_sub(before_map);
+ // SAFETY: the mapping created above, with its length.
+ unsafe { libc::munmap(addr, SIZE) };
+ assert!(
+ mapped_growth < 32 * MIB,
+ "probe grew {} MiB for 128 MiB of clean mapped file pages",
+ mapped_growth / MIB
+ );
+
+ // Positive control: memory the process owns does move the probe.
+ let before_heap = probe();
+ let heap: Vec<u8> = (0..SIZE).map(|i| (i % 251) as u8).collect();
+ std::hint::black_box(&heap);
+ let heap_growth = probe().saturating_sub(before_heap);
+ assert!(
+ heap_growth >= 96 * MIB,
+ "probe grew only {} MiB for 128 MiB of touched heap",
+ heap_growth / MIB
+ );
+}
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index 4ee515534..64758c243 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -1004,7 +1004,7 @@ fn build(config: &Config) -> ResourceGovernor {
/// Current resident set size of this process, in bytes, via
/// [`xerj_common::resource::current_rss_bytes`] (`/proc/self/statm` on Linux,
-/// `proc_pidinfo(PROC_PIDTASKINFO)` on macOS). Returns 0 where the platform
+/// the physical footprint from `proc_pid_rusage` on macOS). Returns 0 where the platform
/// exposes no probe — the RSS watermark then never trips, which is safe but
/// means no admission control, so [`build`] says so out loud at startup.
///
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 70f08355f..e13939033 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -1420,6 +1420,11 @@ impl FtsIndexWriter {
.map(|(name, _stats, fd)| (name.as_str(), fd.config.analyzer.as_str()))
.filter(|(_name, analyzer)| *analyzer != "standard")
.collect();
+ // Every side-car of this segment is staged (temp file written and
+ // synced) and published together at the end: see `publish_staged`
+ // for why that is as durable as one durable write per file, and for
+ // the cost it removes (two device flushes per file on Apple targets).
+ let mut staged: Vec<xerj_common::fsio::StagedFile> = Vec::new();
if !recorded.is_empty() {
#[derive(serde::Serialize)]
struct Marker<'a> {
@@ -1428,17 +1433,19 @@ impl FtsIndexWriter {
let bytes = serde_json::to_vec(&Marker { fields: recorded })
.with_context(|| "encoding FTS analyzer marker")?;
let path = segment_analyzer_marker_path(&segment_dir, &segment_id);
- xerj_common::fsio::write_file_durable(&path, &bytes)
- .with_context(|| format!("writing FTS analyzer marker {:?}", path))?;
+ staged.push(
+ xerj_common::fsio::stage_file(&path, &bytes)
+ .with_context(|| format!("writing FTS analyzer marker {:?}", path))?,
+ );
}
// Parallel field writes. `write_field_static` is a pure function
// of its inputs and touches only files named after the field, so
// there's no cross-thread contention.
- let results: Vec<Result<(String, FieldStats)>> = fields
+ let results: Vec<Result<(String, FieldStats, Vec<xerj_common::fsio::StagedFile>)>> = fields
.into_par_iter()
.map(|(field_name, stats, field_data)| {
- Self::write_field_static(
+ let files = Self::write_field_static(
&segment_dir,
&segment_id,
&field_name,
@@ -1446,16 +1453,20 @@ impl FtsIndexWriter {
zstd_level,
)
.with_context(|| format!("writing field '{}'", field_name))?;
- Ok((field_name, stats))
+ Ok((field_name, stats, files))
})
.collect();
- // Surface any error; otherwise build the stats map.
+ // Surface any error (dropping `staged` removes every temp file);
+ // otherwise build the stats map and publish the side-cars.
let mut all_stats = HashMap::new();
for r in results {
- let (name, stats) = r?;
+ let (name, stats, files) = r?;
+ staged.extend(files);
all_stats.insert(name, stats);
}
+ xerj_common::fsio::publish_staged(&segment_dir, staged)
+ .with_context(|| format!("publishing FTS side-cars in {:?}", segment_dir))?;
Ok(all_stats)
}
@@ -1466,7 +1477,8 @@ impl FtsIndexWriter {
field_name: &str,
field_data: FieldData,
zstd_level: i32,
- ) -> Result<()> {
+ ) -> Result<Vec<xerj_common::fsio::StagedFile>> {
+ let mut staged = Vec::with_capacity(4);
// NOTE: `PathBuf::with_extension` replaces the final `.ext` in the path,
// so using `segment_dir.join("segment_id.field_name")` followed by
// `with_extension("fst")` would strip `.field_name` and collapse every
@@ -1546,16 +1558,19 @@ impl FtsIndexWriter {
out.extend_from_slice(&compressed);
out
};
- // RC4 W1 #10 — every FTS side-car write below goes through durable
- // temporary-file sync plus platform-appropriate publication: rename
- // and parent-directory fsync on Unix, write-through replacement on
- // Windows. These files are part of the segment publish chain: the WAL
- // entries they cover are pruned ~1 s after the flush, so side-cars
- // sitting only in the volatile page cache meant a power loss could
- // leave a registered segment with missing/torn FTS data (silently
- // wrong query results or unreadable fields) with no WAL to recover from.
- xerj_common::fsio::write_file_durable(&post_path, &post_bytes_wrapped)
- .with_context(|| format!("writing postings to {:?}", post_path))?;
+ // RC4 W1 #10 — every FTS side-car below is staged (synced temp file)
+ // and published durably by `finish` through `fsio::publish_staged`:
+ // rename and parent-directory fsync on Unix, write-through
+ // replacement on Windows. These files are part of the segment publish
+ // chain: the WAL entries they cover are pruned ~1 s after the flush,
+ // so side-cars sitting only in the volatile page cache meant a power
+ // loss could leave a registered segment with missing/torn FTS data
+ // (silently wrong query results or unreadable fields) with no WAL to
+ // recover from.
+ staged.push(
+ xerj_common::fsio::stage_file(&post_path, &post_bytes_wrapped)
+ .with_context(|| format!("writing postings to {:?}", post_path))?,
+ );
// 3. Build and write FST (term → meta byte offset).
//
@@ -1591,13 +1606,11 @@ impl FtsIndexWriter {
}
let mut fst_out = fst_builder.into_inner().with_context(|| "finishing FST")?;
fst_out.flush().with_context(|| "flushing FST")?;
- fst_out
- .get_ref()
- .sync_all()
- .with_context(|| "fsyncing FST")?;
+ staged.push(
+ xerj_common::fsio::stage_written_file(&fst_tmp, &fst_path, fst_out.get_ref())
+ .with_context(|| "fsyncing FST")?,
+ );
drop(fst_out);
- xerj_common::fsio::replace_file_durable(&fst_tmp, &fst_path)
- .with_context(|| format!("publishing FST {:?}", fst_path))?;
// 4. Write meta in the ZFM5 format — columnar varint streams
// (df, ttf-delta, gaps, lengths) under the Zstd envelope;
@@ -1612,8 +1625,10 @@ impl FtsIndexWriter {
&term_postings,
zstd_level,
)?;
- xerj_common::fsio::write_file_durable(&meta_path, &meta_bytes)
- .with_context(|| format!("writing meta to {:?}", meta_path))?;
+ staged.push(
+ xerj_common::fsio::stage_file(&meta_path, &meta_bytes)
+ .with_context(|| format!("writing meta to {:?}", meta_path))?,
+ );
// 5. Write norms file — V4 M4.7 compact format.
//
@@ -1658,10 +1673,12 @@ impl FtsIndexWriter {
norms_bytes.extend_from_slice(&(dense_len as u32).to_le_bytes());
norms_bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes());
norms_bytes.extend_from_slice(payload);
- xerj_common::fsio::write_file_durable(&norms_path, &norms_bytes)
- .with_context(|| format!("writing norms to {:?}", norms_path))?;
+ staged.push(
+ xerj_common::fsio::stage_file(&norms_path, &norms_bytes)
+ .with_context(|| format!("writing norms to {:?}", norms_path))?,
+ );
- Ok(())
+ Ok(staged)
}
}
diff --git a/engine/crates/xerj-fts/tests/finish_full_flushes.rs b/engine/crates/xerj-fts/tests/finish_full_flushes.rs
new file mode 100644
index 000000000..9841dabb7
--- /dev/null
+++ b/engine/crates/xerj-fts/tests/finish_full_flushes.rs
@@ -0,0 +1,90 @@
+//! A segment's FTS side-cars must not cost a full device flush per file.
+//!
+//! `File::sync_all` is `fcntl(F_FULLFSYNC)` on Apple targets, which drains the
+//! whole device write cache and costs milliseconds per call. Writing each of a
+//! segment's side-cars as its own durable write (sync the temp file, rename,
+//! sync the directory) paid two of those per file: about 250 for a 31-field
+//! segment, and on a Mac those flushes were most of the indexing time.
+//!
+//! One device flush after the temp files are written, and one after they are
+//! renamed into place (Apple targets; elsewhere `fsync` stays per file, and
+//! only the per-file directory sync goes), give the segment the same guarantee at the moment
+//! `finish` returns: every side-car's bytes are on stable storage before its
+//! name points at them, and every name is on stable storage before the caller
+//! publishes the segment.
+//!
+//! This file holds one test on purpose: the flush counter is process-wide, and
+//! a test binary of its own is the only way no other test writes alongside it.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use xerj_fts::analyzer::AnalyzerRegistry;
+use xerj_fts::index::{FieldValues, FtsIndexReader, FtsIndexWriter};
+
+const FIELDS: usize = 31;
+const DOCS: u32 = 50;
+
+fn field(i: usize) -> String {
+ format!("field_{i:02}")
+}
+
+#[test]
+fn finishing_a_wide_segment_costs_a_bounded_number_of_full_flushes() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut writer =
+ FtsIndexWriter::new(dir.path(), "seg_1", Arc::new(AnalyzerRegistry::default()));
+ for doc in 0..DOCS {
+ let fields: HashMap<String, FieldValues> = (0..FIELDS)
+ .map(|i| {
+ (
+ field(i),
+ FieldValues::One(format!("alpha beta value{} doc{doc}", i % 5)),
+ )
+ })
+ .collect();
+ writer.add_document(doc, &fields);
+ }
+ writer
+ .publish_encoded_filename_layout()
+ .expect("layout marker");
+
+ let before = xerj_common::fsio::full_flush_count();
+ let stats = writer.finish().expect("finish");
+ let flushes = xerj_common::fsio::full_flush_count() - before;
+ assert_eq!(stats.len(), FIELDS);
+
+ // The old per-file shape issued 2 per side-car (temp file, then the
+ // directory after its rename): 8 per field, 248 here.
+ //
+ // Apple targets: one device flush before the renames and one after.
+ // Elsewhere `fsync` is the per-file barrier and stays per file, so the
+ // saving is the per-file directory sync: one per side-car plus one.
+ let side_cars = (FIELDS * 4) as u64;
+ let bound = if cfg!(target_vendor = "apple") {
+ 2
+ } else {
+ side_cars + 1
+ };
+ assert!(
+ flushes <= bound,
+ "finish issued {flushes} full flushes for {FIELDS} fields; expected at most {bound}"
+ );
+
+ // Same segment, still readable: every field opens and answers.
+ let names: Vec<String> = (0..FIELDS).map(field).collect();
+ let refs: Vec<&str> = names.iter().map(String::as_str).collect();
+ let reader = FtsIndexReader::open(dir.path(), "seg_1", &refs).expect("open");
+ for name in &names {
+ assert!(reader.field_stats(name).is_some(), "{name} has no stats");
+ assert_eq!(reader.term_doc_freq(name, "alpha"), Some(DOCS), "{name}");
+ }
+
+ // No temp file survives a successful finish.
+ let leftovers: Vec<String> = std::fs::read_dir(dir.path())
+ .expect("read_dir")
+ .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned())
+ .filter(|n| n.contains(".tmp"))
+ .collect();
+ assert!(leftovers.is_empty(), "temp files left: {leftovers:?}");
+}
diff --git a/engine/crates/xerj-server/Cargo.toml b/engine/crates/xerj-server/Cargo.toml
index eb9c2c5c3..de5ecaa1b 100644
--- a/engine/crates/xerj-server/Cargo.toml
+++ b/engine/crates/xerj-server/Cargo.toml
@@ -95,6 +95,11 @@ debug-profiling = [
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = "0.6"
tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
+# `mallctl` itself, for the void `arena.<i>.purge` control that the ctl
+# crate's typed read/write helpers cannot call (see
+# `purge_all_jemalloc_arenas`). Already in the graph under
+# tikv-jemallocator and tikv-jemalloc-ctl, so this adds an edge, not a crate.
+tikv-jemalloc-sys = "0.6"
libc = "0.2"
[target.'cfg(all(target_os = "linux", not(target_env = "msvc")))'.dependencies]
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 3ab1a46ae..296b95c2c 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -2338,7 +2338,33 @@ fn main() -> Result<()> {
.filter(|&n| n > 0)
.unwrap_or_else(|| (cores * 8).clamp(64, 512));
let rt = build_runtime(worker_threads)?;
- rt.block_on(async_main())
+ run_to_exit(rt, async_main())
+}
+
+/// How long exit waits for blocking jobs still running after `async_main`
+/// returns ("Goodbye" has been logged and the final flush is done).
+const RUNTIME_SHUTDOWN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
+
+/// Drive the server future to completion, then tear the runtime down without
+/// waiting more than [`RUNTIME_SHUTDOWN_TIMEOUT`] for blocking jobs.
+///
+/// Dropping a `Runtime` joins its blocking pool with no time limit. After
+/// SIGTERM the one job left there is usually a background merge, detached by
+/// `spawn_blocking` and parked in the merge pool, which can spend minutes
+/// compressing a large column at zstd 19 on a low-priority thread. The
+/// process then outlived "Goodbye" by however long that merge had left
+/// (90 s+ measured on macOS), and a supervisor's stop timeout killed it.
+///
+/// Walking away from that merge is safe: a merge publishes atomically through
+/// `apply_merge`, and an unfinished batch leaves its input segments untouched
+/// (the LOSS FIREWALL notes in `xerj-engine`'s `index.rs`). The final
+/// `flush_all_force` has completed before `async_main` returns, and leaving a
+/// job unfinished here is no different from the SIGKILL or power loss the
+/// engine is already built to recover from.
+fn run_to_exit<F: std::future::Future>(rt: tokio::runtime::Runtime, fut: F) -> F::Output {
+ let out = rt.block_on(fut);
+ rt.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT);
+ out
}
/// Explicit worker-thread stack size for the production Tokio runtime.
@@ -2976,9 +3002,87 @@ mod decide_flag_tests {
}
}
+/// Return every jemalloc arena's dirty pages to the OS (`arena.4096.purge`,
+/// 4096 being `MALLCTL_ARENAS_ALL`).
+///
+/// `arena.<i>.purge` is a void control: jemalloc rejects it with EPERM unless
+/// all four of `oldp`, `oldlenp`, `newp` and `newlen` are null/0
+/// (`NEITHER_READ_NOR_WRITE` in jemalloc's `ctl.c`). The earlier
+/// `tikv_jemalloc_ctl::raw::write(.., 0u64)` passed a value, so every purge
+/// failed and the error was discarded. On Linux `background_thread:true`
+/// hid it; on macOS there is no background thread, and the drain freed
+/// nothing back to the OS.
+#[cfg(not(target_env = "msvc"))]
+fn purge_all_jemalloc_arenas() -> Result<(), i32> {
+ // SAFETY: the name is a NUL-terminated literal naming a real ctl node,
+ // and a void control reads and writes nothing, so null pointers and a
+ // zero length are exactly what it requires.
+ let rc = unsafe {
+ tikv_jemalloc_sys::mallctl(
+ c"arena.4096.purge".as_ptr(),
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ 0,
+ )
+ };
+ if rc == 0 {
+ Ok(())
+ } else {
+ Err(rc)
+ }
+}
+
+#[cfg(all(test, not(target_env = "msvc")))]
+mod allocator_purge_tests {
+ use super::purge_all_jemalloc_arenas;
+
+ /// The memory drain's purge must actually run. `arena.<i>.purge` is a
+ /// void control: jemalloc answers EPERM to any call that passes a new
+ /// value (`ctl.c`, `NEITHER_READ_NOR_WRITE`), so a "write 0" purge is a
+ /// silent no-op and the drain returns nothing to the OS.
+ #[test]
+ fn arena_purge_succeeds() {
+ // Dirty some pages first so the purge has work to do.
+ let blocks: Vec<Vec<u8>> = (0..64).map(|i| vec![i as u8; 256 * 1024]).collect();
+ drop(std::hint::black_box(blocks));
+ assert_eq!(
+ purge_all_jemalloc_arenas(),
+ Ok(()),
+ "arena.4096.purge must return 0"
+ );
+ }
+}
+
#[cfg(test)]
mod runtime_tests {
- use super::build_runtime;
+ use super::{build_runtime, run_to_exit};
+
+ /// SIGTERM → "Goodbye" → exit must not wait on a blocking job that is
+ /// still running (in production: a background merge compressing a large
+ /// column on a low-priority thread). Dropping a Runtime joins its blocking
+ /// pool with no timeout, so the process outlived "Goodbye" by as long as
+ /// the job had left.
+ #[test]
+ fn exit_does_not_wait_for_a_running_blocking_job() {
+ let rt = build_runtime(2).expect("prod runtime builds");
+ let started = std::time::Instant::now();
+ run_to_exit(rt, async {
+ let (tx, rx) = tokio::sync::oneshot::channel::<()>();
+ // Detached, like the merge task: nothing awaits it.
+ drop(tokio::task::spawn_blocking(move || {
+ let _ = tx.send(());
+ std::thread::sleep(std::time::Duration::from_secs(30));
+ }));
+ // Make sure the job is really running before the server returns.
+ rx.await.expect("blocking job started");
+ });
+ let waited = started.elapsed();
+ assert!(
+ waited < std::time::Duration::from_secs(10),
+ "exit waited {waited:?} for a detached blocking job"
+ );
+ }
#[test]
fn runtime_builds_and_boots() {
@@ -3800,11 +3904,14 @@ async fn async_main() -> Result<()> {
// MALLCTL_ARENAS_ALL. Both are best-effort: a failing mallctl
// must never propagate out of the drain path.
let _ = tikv_jemalloc_ctl::epoch::advance();
- // SAFETY: `arena.<i>.purge` with i = MALLCTL_ARENAS_ALL (4096)
- // is a plain "purge every arena" action mallctl; the value is an
- // unused unsigned scalar. The name is NUL-terminated and names a
- // real jemalloc ctl node, so the write cannot read out of bounds.
- let _ = unsafe { tikv_jemalloc_ctl::raw::write(b"arena.4096.purge\0", 0u64) };
+ // Best-effort as before, but no longer silent: a purge that
+ // fails is a drain that returns nothing to the OS. Warn once.
+ if let Err(rc) = purge_all_jemalloc_arenas() {
+ static WARNED: std::sync::Once = std::sync::Once::new();
+ WARNED.call_once(|| {
+ warn!("jemalloc arena.4096.purge failed (mallctl returned {rc}); memory drains will not return pages to the OS");
+ });
+ }
}));
state.engine.spawn_resource_sampler();
}