Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(engine): an idle index costs 0.14 MB, not 0.64 MB (#873) (#900)

xerj-org72948aa2aa55
diff --git a/engine/crates/xerj-common/src/resource.rs b/engine/crates/xerj-common/src/resource.rs
index b1ab124cc..9f4a52ae9 100644
--- a/engine/crates/xerj-common/src/resource.rs
+++ b/engine/crates/xerj-common/src/resource.rs
@@ -213,6 +213,52 @@ pub const fn threads_for_cores(workload: Workload, cores: usize) -> usize {
}
}
+// ─────────────────────────────────────────────────────────────────────────────
+// Per-index concurrent-map sharding
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Ceiling on the shard count of a **per-index** concurrent map (#873).
+///
+/// `dashmap`'s own default is `(available_parallelism * 4).next_power_of_two()`,
+/// which is the right answer for a map there is ONE of per process and the
+/// wrong answer for a map there is one of per index: the shard array is
+/// `Box<[CachePadded<RwLock<HashMap>>]>`, 128 bytes per shard on x86_64,
+/// written at construction and therefore resident forever. Every open index
+/// carries ~23 of these maps, so on a 32-core host the default cost 128 × 128 B
+/// × 23 ≈ 368 KiB per index before a single document existed — measured as the
+/// largest single term in the idle-RSS floor of #873.
+///
+/// 16 is a ceiling, not a target: a machine whose dashmap default is already
+/// smaller keeps that smaller value. Striping wider than this buys nothing here
+/// because the contention these maps see is bounded by the concurrency on ONE
+/// index, and it is read-dominated — hydration inserts are rare, and readers
+/// share the shard lock.
+pub const MAX_PER_INDEX_MAP_SHARDS: usize = 16;
+
+/// Shard count for a concurrent map that exists once per index.
+///
+/// Never larger than the `dashmap` default this replaced, so no machine ends up
+/// with *more* striping than before; capped at [`MAX_PER_INDEX_MAP_SHARDS`] so
+/// a 64-core host does not pay 256-way striping on every cache of every open
+/// index. Must stay a power of two — `DashMap::with_shard_amount` panics
+/// otherwise.
+pub fn per_index_map_shards() -> usize {
+ static SHARDS: OnceLock<usize> = OnceLock::new();
+ *SHARDS.get_or_init(|| per_index_map_shards_for(cores()))
+}
+
+/// Pure rule behind [`per_index_map_shards`] — the tested half.
+pub const fn per_index_map_shards_for(cores: usize) -> usize {
+ let cores = if cores == 0 { 1 } else { cores };
+ // `dashmap::default_shard_amount()`, then capped.
+ let dashmap_default = (cores * 4).next_power_of_two();
+ if dashmap_default < MAX_PER_INDEX_MAP_SHARDS {
+ dashmap_default
+ } else {
+ MAX_PER_INDEX_MAP_SHARDS
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Thread priority
// ─────────────────────────────────────────────────────────────────────────────
@@ -543,6 +589,41 @@ mod tests {
assert!(note.is_some(), "an undetectable machine must say so");
}
+ /// #873 - a per-index concurrent map must not stripe with the machine.
+ ///
+ /// The rule is asserted at explicit core counts rather than at the host's,
+ /// because a 2-core CI runner would otherwise agree with the unfixed
+ /// behaviour and report a green that means nothing.
+ #[test]
+ fn a_per_index_map_is_capped_and_never_wider_than_the_dashmap_default() {
+ // The value this replaced, restated so the comparison is explicit.
+ let dashmap_default = |cores: usize| (cores * 4).next_power_of_two();
+ for cores in [1usize, 2, 3, 4, 8, 12, 16, 32, 64, 128] {
+ let n = per_index_map_shards_for(cores);
+ assert!(n.is_power_of_two(), "cores={cores} n={n} must be pow2");
+ assert!(n >= 1, "cores={cores}");
+ assert!(
+ n <= MAX_PER_INDEX_MAP_SHARDS,
+ "cores={cores} n={n} exceeds the cap"
+ );
+ assert!(
+ n <= dashmap_default(cores),
+ "cores={cores}: must never stripe WIDER than dashmap would have"
+ );
+ }
+ // Below the cap the machine still decides; above it, it does not.
+ assert_eq!(per_index_map_shards_for(1), 4);
+ assert_eq!(per_index_map_shards_for(2), 8);
+ assert_eq!(per_index_map_shards_for(4), 16);
+ assert_eq!(per_index_map_shards_for(8), 16);
+ assert_eq!(per_index_map_shards_for(64), 16);
+ // A machine that reports zero cores must not produce a zero shard
+ // count: `DashMap::with_shard_amount(0)` panics.
+ assert_eq!(per_index_map_shards_for(0), 4);
+ // The cached host answer obeys the same rule.
+ assert_eq!(per_index_map_shards(), per_index_map_shards_for(cores()));
+ }
+
#[test]
fn latency_work_gets_every_core_and_maintenance_gets_a_reserve() {
for cores in [1usize, 2, 4, 8, 12, 16, 32, 64] {
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index 4dc9e3f38..df749b25b 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -79,6 +79,11 @@ stackprobe = []
[dev-dependencies]
xerj-storage = { path = "../xerj-storage", features = ["test-hooks"] }
+# `raw-api` exposes `DashMap::shards()`, which is the only way to assert what
+# #873 fixed: that a per-index cache's shard array is capped rather than sized
+# from the host core count. Dev-only: with resolver = "2" it never reaches the
+# release binary's graph.
+dashmap = { workspace = true, features = ["raw-api"] }
tempfile = { workspace = true }
# `test-util` is what lets a test pause tokio's clock (`start_paused = true`)
# and step a background timer deterministically instead of sleeping through it.
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index a17dfbf42..3efebec56 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -3515,6 +3515,72 @@ mod merge_publication_transaction_tests {
hand_driven_index_with_config(config(dir), name, schema)
}
+ /// #873: an index that is not being written to holds NO WAL file
+ /// descriptors and no WAL write buffers, whatever its ingest-shard count.
+ ///
+ /// The two states that matter are the ones a quiet node is actually in:
+ /// freshly opened, and freshly flushed. Both are covered here, either side
+ /// of a write that proves the descriptors do appear when they are needed.
+ /// Pre-fix every shard opened a file and a 64 KiB buffer at
+ /// `IndexStore::open` and kept both for the process lifetime — 8 of them
+ /// on a 16-core laptop, 16 here, multiplied by every open index, which is
+ /// what made a few hundred idle autoindex datasets expensive.
+ #[tokio::test]
+ async fn an_index_that_is_not_being_written_to_holds_no_wal_descriptors() {
+ let dir = TempDir::new().unwrap();
+ let mut cfg = config(&dir);
+ // Pin well above 1: the default is a fraction of the host core count,
+ // so on a 1- or 2-core runner this test would assert nothing.
+ cfg.engine.ingest_shards = 8;
+ let (_engine, index) = hand_driven_index_with_config(cfg, "idle-wal", Schema::empty());
+ assert_eq!(
+ index.store.wal_shard_count(),
+ 8,
+ "fixture must actually open 8 shards"
+ );
+ assert_eq!(
+ index.store.materialized_wal_shards(),
+ 0,
+ "a freshly opened index must hold no WAL descriptors"
+ );
+
+ // Enough documents to spread across shards (routing is
+ // `xxh3(doc_id) & (shards - 1)`), so this is not a one-shard result.
+ for i in 0..64 {
+ index
+ .index_document(Some(format!("d{i}")), serde_json::json!({"m": i}))
+ .await
+ .unwrap();
+ }
+ assert!(
+ index.store.materialized_wal_shards() > 1,
+ "writes must materialize the shards they route to (got {})",
+ index.store.materialized_wal_shards()
+ );
+
+ // The flush path (`force_wal_maintenance` → `force_rotate` per shard)
+ // freezes every written generation and leaves an empty one active.
+ index.flush().await.unwrap();
+ assert_eq!(
+ index.store.materialized_wal_shards(),
+ 0,
+ "a flushed index must hand every WAL descriptor and buffer back"
+ );
+
+ // And the data is still there, through the memtable drain and the
+ // released descriptors.
+ let hits = index
+ .search(
+ &xerj_query::parse_request(&serde_json::json!({
+ "query": {"match_all": {}}, "size": 0
+ }))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(hits.total.value, 64);
+ }
+
/// #873: a non-empty memtable BELOW the size thresholds flushes once its
/// contents have sat unchanged for `flush_idle_secs`. Idleness is
/// crossed via the test rewind hook, never a sleep. Also proves the two
@@ -7994,35 +8060,35 @@ impl Index {
embedding_config: config.embedding.clone(),
max_fields_per_index: config.limits.max_fields_per_index,
embedder: Arc::new(RwLock::new(effective_embedder)),
- dv_cache: Arc::new(dashmap::DashMap::new()),
- sort_shadow_cache: Arc::new(dashmap::DashMap::new()),
- sort_shadow_fields: Arc::new(dashmap::DashMap::new()),
+ dv_cache: Arc::new(per_index_map()),
+ sort_shadow_cache: Arc::new(per_index_map()),
+ sort_shadow_fields: Arc::new(per_index_map()),
millis_date_fields: Arc::new(parking_lot::RwLock::new(Arc::new(
millis_date_fields_snapshot,
))),
- stored_slices_build_locks: Arc::new(dashmap::DashMap::new()),
- range_prefilter_cache: Arc::new(dashmap::DashMap::new()),
- id_pos_cache: Arc::new(dashmap::DashMap::new()),
- row_seq_cache: Arc::new(dashmap::DashMap::new()),
- stored_value_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_build_locks: Arc::new(per_index_map()),
+ range_prefilter_cache: Arc::new(per_index_map()),
+ id_pos_cache: Arc::new(per_index_map()),
+ row_seq_cache: Arc::new(per_index_map()),
+ stored_value_cache: Arc::new(per_index_map()),
segment_cache_lifecycle: Arc::new(parking_lot::RwLock::new(())),
segment_hydration_budget,
- stored_value_loads: Arc::new(dashmap::DashMap::new()),
+ stored_value_loads: Arc::new(per_index_map()),
stored_value_load_semaphore: Arc::new(Semaphore::new(4)),
- stored_slices_cache: Arc::new(dashmap::DashMap::new()),
- decoded_stored_cache: Arc::new(dashmap::DashMap::new()),
- fts_reader_cache: Arc::new(dashmap::DashMap::new()),
- shortcut_count_cache: Arc::new(dashmap::DashMap::new()),
- ghost_positions_cache: Arc::new(dashmap::DashMap::new()),
- regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
- fast_date_cache: Arc::new(dashmap::DashMap::new()),
- fast_date_sorted_cache: Arc::new(dashmap::DashMap::new()),
- query_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_cache: Arc::new(per_index_map()),
+ decoded_stored_cache: Arc::new(per_index_map()),
+ fts_reader_cache: Arc::new(per_index_map()),
+ shortcut_count_cache: Arc::new(per_index_map()),
+ ghost_positions_cache: Arc::new(per_index_map()),
+ regexp_expand_cache: Arc::new(per_index_map()),
+ fast_date_cache: Arc::new(per_index_map()),
+ fast_date_sorted_cache: Arc::new(per_index_map()),
+ query_cache: Arc::new(per_index_map()),
dataset_version: Arc::new(AtomicU64::new(0)),
- query_inflight: Arc::new(dashmap::DashMap::new()),
+ query_inflight: Arc::new(per_index_map()),
metric_singleflight_coalesced: Arc::new(AtomicU64::new(0)),
flush_signal: Arc::new(SyncFlushCoord::new()),
- external_versions: Arc::new(dashmap::DashMap::new()),
+ external_versions: Arc::new(per_index_map()),
merge_task: Arc::new(parking_lot::Mutex::new(None)),
merge_check_armed: std::sync::atomic::AtomicBool::new(false),
merge_events: AtomicU64::new(0),
@@ -8381,35 +8447,35 @@ impl Index {
embedding_config: config.embedding.clone(),
max_fields_per_index: config.limits.max_fields_per_index,
embedder: Arc::new(RwLock::new(effective_embedder)),
- dv_cache: Arc::new(dashmap::DashMap::new()),
- sort_shadow_cache: Arc::new(dashmap::DashMap::new()),
- sort_shadow_fields: Arc::new(dashmap::DashMap::new()),
+ dv_cache: Arc::new(per_index_map()),
+ sort_shadow_cache: Arc::new(per_index_map()),
+ sort_shadow_fields: Arc::new(per_index_map()),
millis_date_fields: Arc::new(parking_lot::RwLock::new(Arc::new(
millis_date_fields_snapshot,
))),
- stored_slices_build_locks: Arc::new(dashmap::DashMap::new()),
- range_prefilter_cache: Arc::new(dashmap::DashMap::new()),
- id_pos_cache: Arc::new(dashmap::DashMap::new()),
- row_seq_cache: Arc::new(dashmap::DashMap::new()),
- stored_value_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_build_locks: Arc::new(per_index_map()),
+ range_prefilter_cache: Arc::new(per_index_map()),
+ id_pos_cache: Arc::new(per_index_map()),
+ row_seq_cache: Arc::new(per_index_map()),
+ stored_value_cache: Arc::new(per_index_map()),
segment_cache_lifecycle: Arc::new(parking_lot::RwLock::new(())),
segment_hydration_budget,
- stored_value_loads: Arc::new(dashmap::DashMap::new()),
+ stored_value_loads: Arc::new(per_index_map()),
stored_value_load_semaphore: Arc::new(Semaphore::new(4)),
- stored_slices_cache: Arc::new(dashmap::DashMap::new()),
- decoded_stored_cache: Arc::new(dashmap::DashMap::new()),
- fts_reader_cache: Arc::new(dashmap::DashMap::new()),
- shortcut_count_cache: Arc::new(dashmap::DashMap::new()),
- ghost_positions_cache: Arc::new(dashmap::DashMap::new()),
- regexp_expand_cache: Arc::new(dashmap::DashMap::new()),
- fast_date_cache: Arc::new(dashmap::DashMap::new()),
- fast_date_sorted_cache: Arc::new(dashmap::DashMap::new()),
- query_cache: Arc::new(dashmap::DashMap::new()),
+ stored_slices_cache: Arc::new(per_index_map()),
+ decoded_stored_cache: Arc::new(per_index_map()),
+ fts_reader_cache: Arc::new(per_index_map()),
+ shortcut_count_cache: Arc::new(per_index_map()),
+ ghost_positions_cache: Arc::new(per_index_map()),
+ regexp_expand_cache: Arc::new(per_index_map()),
+ fast_date_cache: Arc::new(per_index_map()),
+ fast_date_sorted_cache: Arc::new(per_index_map()),
+ query_cache: Arc::new(per_index_map()),
dataset_version: Arc::new(AtomicU64::new(0)),
- query_inflight: Arc::new(dashmap::DashMap::new()),
+ query_inflight: Arc::new(per_index_map()),
metric_singleflight_coalesced: Arc::new(AtomicU64::new(0)),
flush_signal: Arc::new(SyncFlushCoord::new()),
- external_versions: Arc::new(dashmap::DashMap::new()),
+ external_versions: Arc::new(per_index_map()),
merge_task: Arc::new(parking_lot::Mutex::new(None)),
merge_check_armed: std::sync::atomic::AtomicBool::new(false),
merge_events: AtomicU64::new(0),
@@ -32823,6 +32889,82 @@ fn wal_shards_override_from_settings(settings: &Value) -> Option<usize> {
/// prevent.
const MAX_PER_INDEX_WAL_SHARDS: u64 = 256;
+/// A concurrent map that exists **once per index** (#873).
+///
+/// `DashMap::new()` sizes its shard array from the host core count —
+/// `(cores * 4).next_power_of_two()`, 128 shards on a 32-core box — and each
+/// shard is a `CachePadded<RwLock<HashMap>>`: 128 bytes on x86_64, written at
+/// construction, resident for the life of the index whether or not the map
+/// ever holds an entry. An `Index` builds 20 of these, so the default cost
+/// ~328 KiB of resident memory per open index on a 32-core host *before* its
+/// first document — measured (300 tiny indices, restarted, idle) as the
+/// dominant term of the #873 idle floor: 641 KB/index at 32 cores against
+/// 270 KB/index with the same build pinned to 4 cores.
+///
+/// [`per_index_map_shards`] caps that at 16 shards, and never raises it above
+/// what dashmap would have chosen. Per-index concurrency is bounded by the
+/// traffic on one index and these maps are read-dominated (hydration inserts
+/// are rare; readers share the shard lock), so the striping that is lost was
+/// not doing work.
+///
+/// [`per_index_map_shards`]: xerj_common::resource::per_index_map_shards
+fn per_index_map<K: Eq + std::hash::Hash, V>() -> dashmap::DashMap<K, V> {
+ dashmap::DashMap::with_shard_amount(xerj_common::resource::per_index_map_shards())
+}
+
+#[cfg(test)]
+mod per_index_map_tests {
+ use xerj_common::resource::{per_index_map_shards, MAX_PER_INDEX_MAP_SHARDS};
+
+ /// #873 - the maps an `Index` builds must be capped, not striped by the
+ /// host core count.
+ ///
+ /// `dashmap::DashMap::new()` allocates
+ /// `(cores * 4).next_power_of_two()` shards, each a
+ /// `CachePadded<RwLock<HashMap>>` written at construction: 128 shards x
+ /// 128 B = 16 KiB per map on a 32-core host, and an `Index` builds 20 of
+ /// them. Measured end to end (300 tiny indices, restart, idle) that was
+ /// the largest term of the #873 floor.
+ ///
+ /// The strict-inequality arm is guarded on the host actually being wide
+ /// enough for the cap to bite, so this cannot report a false green on a
+ /// 2-core runner - the class of blind spot that has produced green CI over
+ /// real breakage in this repo before.
+ #[test]
+ fn a_per_index_cache_is_capped_not_striped_by_host_cores() {
+ let capped: dashmap::DashMap<String, u32> = super::per_index_map();
+ let dashmap_default: dashmap::DashMap<String, u32> = dashmap::DashMap::new();
+
+ assert_eq!(
+ capped.shards().len(),
+ per_index_map_shards(),
+ "the Index constructor helper must use the capped shard count"
+ );
+ assert!(
+ capped.shards().len() <= MAX_PER_INDEX_MAP_SHARDS,
+ "a per-index map must never exceed the cap"
+ );
+ assert!(
+ capped.shards().len() <= dashmap_default.shards().len(),
+ "and must never stripe wider than the default it replaced"
+ );
+ if dashmap_default.shards().len() > MAX_PER_INDEX_MAP_SHARDS {
+ assert!(
+ capped.shards().len() < dashmap_default.shards().len(),
+ "on a host wide enough for the cap to bite it must actually bite: \
+ capped={} default={}",
+ capped.shards().len(),
+ dashmap_default.shards().len()
+ );
+ }
+
+ // Still a working map.
+ capped.insert("k".to_string(), 7);
+ assert_eq!(*capped.get("k").unwrap(), 7);
+ assert_eq!(capped.len(), 1);
+ }
+}
+
fn store_config_from(config: &Config, wal_shards_override: Option<usize>) -> IndexStoreConfig {
let sync_mode = match config.storage.wal_sync {
xerj_common::config::WalSync::Sync => SyncMode::Strict,
diff --git a/engine/crates/xerj-engine/src/write_publication.rs b/engine/crates/xerj-engine/src/write_publication.rs
index ea6770cb2..0ebd2e903 100644
--- a/engine/crates/xerj-engine/src/write_publication.rs
+++ b/engine/crates/xerj-engine/src/write_publication.rs
@@ -10,13 +10,25 @@ use dashmap::mapref::entry::Entry;
use dashmap::DashMap;
use tokio::sync::{Mutex, OwnedMutexGuard};
-#[derive(Default)]
pub(crate) struct WritePublicationCoordinator {
locks: DashMap<String, Weak<Mutex<()>>>,
#[cfg(test)]
before_mutex_await: parking_lot::RwLock<Option<Arc<dyn Fn(&str) + Send + Sync + 'static>>>,
}
+impl Default for WritePublicationCoordinator {
+ fn default() -> Self {
+ Self {
+ // #873 — one coordinator per index; dashmap's core-count-derived
+ // default shard array is resident from construction, so it is
+ // capped like every other per-index map.
+ locks: DashMap::with_shard_amount(xerj_common::resource::per_index_map_shards()),
+ #[cfg(test)]
+ before_mutex_await: parking_lot::RwLock::new(None),
+ }
+ }
+}
+
pub(crate) struct WritePublicationGuard {
coordinator: Arc<WritePublicationCoordinator>,
key: String,
diff --git a/engine/crates/xerj-fts/src/analyzer.rs b/engine/crates/xerj-fts/src/analyzer.rs
index 9b16262f8..0cc6fca43 100644
--- a/engine/crates/xerj-fts/src/analyzer.rs
+++ b/engine/crates/xerj-fts/src/analyzer.rs
@@ -1049,14 +1049,46 @@ pub struct AnalyzerRegistry {
analyzers: std::collections::HashMap<String, Arc<AnalyzerPipeline>>,
}
+/// The built-in analyzer pipelines, constructed once for the whole process
+/// (#873).
+///
+/// Every open index owns an `AnalyzerRegistry`, and building the built-ins per
+/// registry meant every index got its own copy of the English stop-word set,
+/// the Snowball stemmers, the e-commerce synonym table and the rest —
+/// **92.7 KB of heap per registry, measured** (500 registries, RSS delta,
+/// x86-64 release), duplicated across every index on the node for data that is
+/// immutable and identical in all of them. That was the second-largest term in
+/// the #873 idle-RSS floor after the per-index concurrent maps.
+///
+/// The pipelines are pure immutable trait objects behind `Arc`s (no interior
+/// mutability anywhere in this module), and `register` replaces a *map entry*
+/// rather than mutating a pipeline, so sharing them across registries is
+/// invisible: a registry that overrides `standard` from its index settings
+/// still gets its own entry, and no other index sees it.
+fn builtin_analyzers() -> &'static HashMap<String, Arc<AnalyzerPipeline>> {
+ static BUILTINS: std::sync::OnceLock<HashMap<String, Arc<AnalyzerPipeline>>> =
+ std::sync::OnceLock::new();
+ BUILTINS.get_or_init(|| {
+ let mut registry = AnalyzerRegistry {
+ analyzers: HashMap::new(),
+ };
+ registry.register_defaults();
+ registry.analyzers
+ })
+}
+
impl AnalyzerRegistry {
/// Creates a registry pre-populated with all built-in analyzers.
+ ///
+ /// The built-ins themselves are process-wide (see [`builtin_analyzers`]);
+ /// what this allocates per registry is the name → `Arc` table, so a
+ /// registry that adds nothing costs a few hundred bytes rather than the
+ /// ~92 KB of stop-word sets, stemmers and synonym tables it used to
+ /// rebuild (#873).
pub fn with_defaults() -> Self {
- let mut registry = Self {
- analyzers: std::collections::HashMap::new(),
- };
- registry.register_defaults();
- registry
+ Self {
+ analyzers: builtin_analyzers().clone(),
+ }
}
fn register_defaults(&mut self) {
@@ -2526,6 +2558,63 @@ const ENGLISH_STOP_WORDS: &[&str] = &[
mod tests {
use super::*;
+ /// #873 - two registries must SHARE their built-in pipelines rather than
+ /// each build its own copy of the stop-word sets, stemmers and synonym
+ /// tables. One registry is built per open index, and rebuilding the
+ /// built-ins per registry measured 92.7 KB of heap each.
+ ///
+ /// Pointer identity is the property, not a byte count: it is what makes
+ /// the cost O(1) in the number of indices instead of O(n). The negative
+ /// arm matters just as much - an index that overrides a built-in from its
+ /// own settings must get its own pipeline and must not disturb anyone
+ /// else's.
+ #[test]
+ fn built_in_analyzers_are_shared_between_registries_not_rebuilt() {
+ let a = AnalyzerRegistry::with_defaults();
+ let b = AnalyzerRegistry::with_defaults();
+ assert!(
+ !a.analyzers.is_empty(),
+ "fixture: the registry must have built-ins to share"
+ );
+ for (name, pipeline) in &a.analyzers {
+ let other = b
+ .get_analyzer(name)
+ .unwrap_or_else(|| panic!("{name} missing from the second registry"));
+ assert!(
+ Arc::ptr_eq(pipeline, &other),
+ "built-in analyzer `{name}` was rebuilt instead of shared"
+ );
+ }
+
+ // Overriding one is local to the registry that did it.
+ let mut c = AnalyzerRegistry::with_defaults();
+ c.register(
+ "standard",
+ AnalyzerPipeline::new(vec![], Arc::new(WhitespaceTokenizer), vec![]),
+ );
+ assert!(
+ !Arc::ptr_eq(
+ &c.get_analyzer("standard").unwrap(),
+ &a.get_analyzer("standard").unwrap()
+ ),
+ "an override must replace this registry's entry"
+ );
+ assert!(
+ Arc::ptr_eq(
+ &a.get_analyzer("standard").unwrap(),
+ &b.get_analyzer("standard").unwrap()
+ ),
+ "and must not disturb any other registry"
+ );
+ // The shared built-in still analyzes exactly as it did.
+ assert_eq!(
+ a.get_analyzer("standard")
+ .unwrap()
+ .analyze_to_terms("Hello, World!"),
+ vec!["hello".to_string(), "world".to_string()]
+ );
+ }
+
#[test]
fn standard_tokenizer_splits_unicode_words() {
let tok = StandardTokenizer;
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 6bc86d312..60f1d7f81 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -729,7 +729,11 @@ impl IndexStore {
seq_counter,
memtable_shards,
memtable_bytes: AtomicU64::new(0),
- seg_reader_cache: dashmap::DashMap::new(),
+ // #873 — per-index map: capped shard array, see
+ // `xerj_common::resource::per_index_map_shards`.
+ seg_reader_cache: dashmap::DashMap::with_shard_amount(
+ xerj_common::resource::per_index_map_shards(),
+ ),
last_wal_maintenance_ms: AtomicU64::new(0),
read_leases: std::sync::atomic::AtomicUsize::new(0),
retired_segments: Mutex::new(Vec::new()),
@@ -2886,6 +2890,25 @@ impl IndexStore {
.collect()
}
+ /// Number of WAL shards this store opened — the routing modulus for both
+ /// the WAL and the storage memtable.
+ pub fn wal_shard_count(&self) -> usize {
+ self.wal_shards.len()
+ }
+
+ /// How many of this store's WAL shards currently hold an open file
+ /// descriptor and a `WAL_BUF_CAP` userspace write buffer (#873).
+ ///
+ /// Instrumentation for the property the laziness exists to provide, so it
+ /// can be asserted rather than argued: a store that has not been written
+ /// to since its last flush must report `0`, whatever its shard count.
+ pub fn materialized_wal_shards(&self) -> usize {
+ self.wal_shards
+ .iter()
+ .filter(|s| s.lock().unwrap().is_materialized())
+ .count()
+ }
+
/// Unconditionally run verified WAL maintenance across all shards.
/// Bypasses the `WAL_MAINTENANCE_INTERVAL_MS` gate that
/// `finalize_flush_with_publisher` uses on the hot flush path.
@@ -4327,15 +4350,30 @@ impl IndexStore {
mod tests {
use super::*;
- /// Structural bound: 16 empty indices × 8 WAL shards each must retain
- /// exactly `128 × 64 KiB` of userspace buffer capacity — not the old
- /// `128 × 8 MiB` reservation. Asserts `buffer_capacity()` (deterministic)
- /// rather than process RSS.
+ /// Structural bound: idle WAL shards retain NO userspace buffer at all,
+ /// and a written store retains one buffer per shard it actually wrote to
+ /// (#873).
+ ///
+ /// The previous version of this test pinned the bound at
+ /// `16 indices × 8 shards × 64 KiB` — 8 MiB of buffers held by 16 indices
+ /// holding nothing, which is the cost the #873 field reports were paying
+ /// (`(cpus/2).next_power_of_two()` shards per index × every open index).
+ /// Buffers are now materialized by the first append to a shard and handed
+ /// back by the flush-time `force_rotate`, so the idle total is zero and
+ /// the old ceiling survives only as the *materialized* bound. Asserts
+ /// `buffer_capacity()` (deterministic) rather than process RSS.
#[test]
fn empty_multi_index_wal_buffers_have_a_bounded_total_capacity() {
let root = tempfile::tempdir().unwrap();
let mut stores = Vec::new();
- let mut total_capacity = 0usize;
+
+ let total_capacity = |stores: &Vec<Arc<IndexStore>>| -> usize {
+ stores
+ .iter()
+ .flat_map(|s| s.wal_shards.iter())
+ .map(|writer| writer.lock().unwrap().buffer_capacity())
+ .sum()
+ };
for index in 0..16 {
let store = IndexStore::open(
@@ -4349,19 +4387,46 @@ mod tests {
)
.unwrap();
assert_eq!(store.wal_shards.len(), 8);
- total_capacity += store
- .wal_shards
- .iter()
- .map(|writer| writer.lock().unwrap().buffer_capacity())
- .sum::<usize>();
+ assert_eq!(
+ store.materialized_wal_shards(),
+ 0,
+ "an index that has not been written to must hold no WAL \
+ descriptors"
+ );
stores.push(store);
}
assert_eq!(stores.len(), 16);
- assert_eq!(total_capacity, 16 * 8 * 64 * 1024);
+ assert_eq!(
+ total_capacity(&stores),
+ 0,
+ "128 idle WAL shards must retain no buffers at all"
+ );
+
+ // One batch per store. A batch routes to exactly ONE shard, so each
+ // store materializes exactly one of its eight.
+ for store in &stores {
+ let validated = IndexStore::validate_raw_batch(vec![(
+ "d1".to_owned(),
+ Arc::<[u8]>::from(br#"{"m":"tiny"}"#.as_slice()),
+ )])
+ .unwrap();
+ store.wal_append_batch_raw(&validated).unwrap();
+ assert_eq!(
+ store.materialized_wal_shards(),
+ 1,
+ "a write must materialize the shard it routed to, and only it"
+ );
+ }
+ assert_eq!(
+ total_capacity(&stores),
+ 16 * 64 * 1024,
+ "a written shard holds exactly one WAL_BUF_CAP buffer"
+ );
assert!(
- total_capacity <= 8 * 1024 * 1024,
- "128 idle WAL shards retained {total_capacity} bytes of buffers"
+ total_capacity(&stores) <= 8 * 1024 * 1024,
+ "128 WAL shards retained {} bytes of buffers",
+ total_capacity(&stores)
);
}
diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rs
index 90e0e81ee..9a166b129 100644
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@ -244,7 +244,11 @@ impl VersionMap {
/// Create an empty version map.
pub fn new() -> Self {
Self {
- inner: DashMap::new(),
+ // #873 — one of these exists per open index, and dashmap's default
+ // shard array is sized from the host core count (128 shards ×
+ // 128 B on a 32-core box, resident from construction). Cap it: the
+ // concurrency this map sees is the write concurrency on ONE index.
+ inner: DashMap::with_shard_amount(xerj_common::resource::per_index_map_shards()),
live: AtomicI64::new(0),
ghost_events: std::sync::atomic::AtomicU64::new(0),
}
diff --git a/engine/crates/xerj-storage/src/wal.rs b/engine/crates/xerj-storage/src/wal.rs
index c4adc1636..81d35220f 100644
--- a/engine/crates/xerj-storage/src/wal.rs
+++ b/engine/crates/xerj-storage/src/wal.rs
@@ -320,7 +320,31 @@ impl Write for WalFile {
pub struct WalWriter {
dir: PathBuf,
generation: u64,
- writer: BufWriter<WalFile>,
+ /// The active generation's file handle and its `WAL_BUF_CAP` userspace
+ /// buffer — **materialized on demand** (#873).
+ ///
+ /// `None` means the active generation is EMPTY: the file exists on disk
+ /// (created by `open`/`rotate`, exactly as before) but holds nothing past
+ /// its 16-byte header, so no append, replay, prune, checkpoint or tap path
+ /// has anything to read from it. Holding an open descriptor and a 64 KiB
+ /// buffer for that state is pure resident overhead, and it is the *normal*
+ /// state of an idle index: every index opens with it, and every flush
+ /// returns to it (`wal_maintain_all_verified` → `force_rotate`). With one
+ /// shard per ingest shard — `(cpus/2).next_power_of_two()`, so 8 on a
+ /// 16-core laptop and 16 here — that overhead is multiplied by the shard
+ /// count and again by the number of open indices, which is what made a
+ /// few hundred idle autoindex datasets cost hundreds of MB (#873).
+ ///
+ /// [`ensure_materialized`](Self::ensure_materialized) reopens the file for
+ /// append at the first write; `rotate` drops it again. Nothing on disk
+ /// changes: the file set, the generation numbering and the header bytes are
+ /// byte-for-byte what they were before this became lazy.
+ writer: Option<BufWriter<WalFile>>,
+ /// Append position in the active generation. Always exactly
+ /// `WAL_HEADER_LEN` while `writer` is `None` — that equality *is* the
+ /// unmaterialized invariant, and it is what makes `checkpoint`,
+ /// `force_rotate` and `rotate_if_large` no-op on an unmaterialized shard
+ /// through the offset guards they already had.
current_offset: u64,
max_size_bytes: u64,
sync_mode: SyncMode,
@@ -368,11 +392,60 @@ pub struct WalWriter {
impl WalWriter {
/// Userspace `BufWriter` capacity for this writer (test/instrumentation).
///
- /// Always equals [`WAL_BUF_CAP`] after open, rotate, truncate-reseat, and
- /// fresh-generation recovery. Does not grow with record size.
+ /// Always equals [`WAL_BUF_CAP`] once materialized — after the first
+ /// append, and after rotate, truncate-reseat and fresh-generation
+ /// recovery. Does not grow with record size. `0` while the shard is
+ /// unmaterialized (#873): an empty active generation holds no buffer at
+ /// all, which is the property the size bound now rests on.
#[cfg(test)]
pub(crate) fn buffer_capacity(&self) -> usize {
- self.writer.capacity()
+ self.writer.as_ref().map_or(0, |w| w.capacity())
+ }
+
+ /// Whether this shard currently holds an open file descriptor and a
+ /// userspace buffer for its active generation (#873).
+ ///
+ /// Exposed so the resident-cost property is assertable rather than
+ /// argued: an index that has not been written to since its last flush
+ /// must report `false` on every shard.
+ pub fn is_materialized(&self) -> bool {
+ self.writer.is_some()
+ }
+
+ /// Open the active generation's file for append and allocate its buffer,
+ /// if that has not happened yet (#873). Idempotent.
+ ///
+ /// The file is expected to exist — `open` and `rotate` both create it —
+ /// so the normal path is one `open(2)` in append mode. It is created here
+ /// only as a defensive fallback (an operator deleting WAL files under a
+ /// live node), which is strictly better than the alternative of failing an
+ /// acknowledged write.
+ fn ensure_materialized(&mut self) -> Result<()> {
+ if self.writer.is_some() {
+ return Ok(());
+ }
+ let path = wal_path(&self.dir, self.generation);
+ let file = if path.exists() {
+ OpenOptions::new().append(true).open(&path)?
+ } else {
+ warn!(
+ generation = self.generation,
+ "WAL active generation file was missing at first append — recreating it"
+ );
+ create_wal_file(&path, self.generation)?
+ };
+ self.writer = Some(BufWriter::with_capacity(WAL_BUF_CAP, WalFile::new(file)));
+ Ok(())
+ }
+
+ /// The materialized writer. Every caller reaches this only after
+ /// [`ensure_materialized`](Self::ensure_materialized) on the same
+ /// `&mut self`, which is why it can be infallible.
+ #[inline]
+ fn writer(&mut self) -> &mut BufWriter<WalFile> {
+ self.writer
+ .as_mut()
+ .expect("WAL writer materialized before use")
}
/// Open (or create) the WAL in `dir`.
@@ -467,10 +540,22 @@ impl WalWriter {
(generation, f, WAL_HEADER_LEN)
};
+ // #873 — an active generation that holds nothing past its header is
+ // the state every index opens in and every flush returns to. Drop the
+ // descriptor and skip the 64 KiB buffer for it; the first append
+ // reopens the same file for append at the same offset. The file itself
+ // was created above exactly as before, so nothing on disk differs.
+ let writer = if current_offset == WAL_HEADER_LEN {
+ drop(file);
+ None
+ } else {
+ Some(BufWriter::with_capacity(WAL_BUF_CAP, WalFile::new(file)))
+ };
+
Ok(Self {
dir,
generation,
- writer: BufWriter::with_capacity(WAL_BUF_CAP, WalFile::new(file)),
+ writer,
current_offset,
max_size_bytes,
sync_mode,
@@ -608,11 +693,12 @@ impl WalWriter {
let crc = hasher.finalize();
let entry_len = payload.len() as u32;
- self.writer.write_u32::<LittleEndian>(entry_len)?;
- self.writer.write_u64::<LittleEndian>(seq_no)?;
- self.writer.write_u8(op_code)?;
- self.writer.write_all(payload)?;
- self.writer.write_u32::<LittleEndian>(crc)?;
+ let w = self.writer();
+ w.write_u32::<LittleEndian>(entry_len)?;
+ w.write_u64::<LittleEndian>(seq_no)?;
+ w.write_u8(op_code)?;
+ w.write_all(payload)?;
+ w.write_u32::<LittleEndian>(crc)?;
let written = WAL_FRAME_OVERHEAD as u64 + payload.len() as u64;
self.current_offset += written;
@@ -626,7 +712,7 @@ impl WalWriter {
// Skips fsync(2) — bytes survive process death (kernel keeps
// the page cache) but not power loss until the next sync().
// Mirrors what `wal_append_batch` does after its frame write.
- self.writer.flush()?;
+ self.writer().flush()?;
}
Ok(())
}
@@ -725,13 +811,13 @@ impl WalWriter {
}
let frame_start = self.current_offset;
let res: Result<()> = (|| {
- self.writer.write_all(frames)?;
+ self.writer().write_all(frames)?;
self.current_offset += total_written;
self.mark_dirty();
if self.sync_mode == SyncMode::Strict {
self.sync()?;
} else {
- self.writer.flush()?;
+ self.writer().flush()?;
}
Ok(())
})();
@@ -819,13 +905,13 @@ impl WalWriter {
// roll the WAL back to the batch boundary and NACK the batch.
let frame_start = self.current_offset;
let res: Result<()> = (|| {
- self.writer.write_all(&out)?;
+ self.writer().write_all(&out)?;
self.current_offset += written_total;
self.mark_dirty();
if self.sync_mode == SyncMode::Strict {
self.sync()?;
} else {
- self.writer.flush()?;
+ self.writer().flush()?;
}
Ok(())
})();
@@ -837,9 +923,18 @@ impl WalWriter {
}
/// Flush the write buffer and call `fsync`.
+ ///
+ /// A no-op on an unmaterialized shard (#873): its active generation holds
+ /// nothing past the header it was created with and already fsynced, and
+ /// there is no buffer to drain — so there is nothing an fsync could make
+ /// durable that is not durable already.
pub fn sync(&mut self) -> Result<()> {
- self.writer.flush()?;
- self.writer.get_ref().file().sync_all()?;
+ let Some(w) = self.writer.as_mut() else {
+ self.dirty = false;
+ return Ok(());
+ };
+ w.flush()?;
+ w.get_ref().file().sync_all()?;
self.dirty = false;
Ok(())
}
@@ -853,7 +948,13 @@ impl WalWriter {
/// instead of writing after a torn frame.
fn ensure_writable(&mut self) -> Result<()> {
if !self.poisoned {
- return Ok(());
+ // #873 — the one chokepoint every append path already goes
+ // through, which is why materialization is armed here rather than
+ // in each append method: a future append path cannot forget to
+ // open its own file. `rotate` deliberately re-seats the writer
+ // itself, so the rotate-on-size check that follows every call to
+ // this gate cannot leave the writer unmaterialized.
+ return self.ensure_materialized();
}
if self.try_reseat_fresh_generation() {
warn!(
@@ -884,7 +985,14 @@ impl WalWriter {
/// keep re-trying step 2 via `ensure_writable`).
fn recover_after_write_error(&mut self, frame_start: u64, cause: &StorageError) {
let reseat: io::Result<WalFile> = (|| {
- let dup = self.writer.get_ref().file().try_clone()?;
+ // Only reachable from a failed append, which materialized the
+ // writer before it wrote (#873) — but recovery is the last thing
+ // that should panic, so an unmaterialized shard falls through to
+ // the fresh-generation heal below instead.
+ let Some(w) = self.writer.as_ref() else {
+ return Err(io::Error::other("WAL writer not materialized"));
+ };
+ let dup = w.get_ref().file().try_clone()?;
// Defensive min(): under the drain-per-append invariant the
// file can only be AT or PAST the boundary; never extend it
// (set_len past EOF would zero-fill — manufactured garbage).
@@ -896,12 +1004,13 @@ impl WalWriter {
})();
match reseat {
Ok(fresh) => {
- let old = std::mem::replace(
- &mut self.writer,
- BufWriter::with_capacity(WAL_BUF_CAP, fresh),
- );
+ let old = self
+ .writer
+ .replace(BufWriter::with_capacity(WAL_BUF_CAP, fresh));
// Discard buffered torn bytes WITHOUT flushing them.
- let _ = old.into_parts();
+ if let Some(old) = old {
+ let _ = old.into_parts();
+ }
self.current_offset = frame_start;
self.poisoned = false;
warn!(
@@ -944,11 +1053,12 @@ impl WalWriter {
match create_wal_file(&wal_path(&self.dir, next), next) {
Ok(f) => {
self.generation = next;
- let old = std::mem::replace(
- &mut self.writer,
- BufWriter::with_capacity(WAL_BUF_CAP, WalFile::new(f)),
- );
- let _ = old.into_parts();
+ let old = self
+ .writer
+ .replace(BufWriter::with_capacity(WAL_BUF_CAP, WalFile::new(f)));
+ if let Some(old) = old {
+ let _ = old.into_parts();
+ }
self.current_offset = WAL_HEADER_LEN;
self.poisoned = false;
true
@@ -962,7 +1072,9 @@ impl WalWriter {
/// filling disk. `None` clears the fault.
#[cfg(test)]
pub(crate) fn inject_write_fault(&mut self, budget: Option<u64>) {
- self.writer.get_mut().fail_after = budget;
+ self.ensure_materialized()
+ .expect("materialize WAL for fault injection");
+ self.writer().get_mut().fail_after = budget;
}
/// True when bytes were appended since the last `fsync`. Consumed by
@@ -984,7 +1096,10 @@ impl WalWriter {
/// the committed `.seg` file, so acknowledged-and-flushed
/// documents are still durable.
pub fn soft_flush(&mut self) -> Result<()> {
- self.writer.flush()?;
+ // Unmaterialized ⇒ no buffer, nothing to drain (#873).
+ if let Some(w) = self.writer.as_mut() {
+ w.flush()?;
+ }
Ok(())
}
@@ -1150,6 +1265,14 @@ impl WalWriter {
Ok(removed)
}
+ /// Close the active generation and start the next one, seated and ready
+ /// to be appended to.
+ ///
+ /// This one deliberately leaves the writer MATERIALIZED (#873): its
+ /// size-driven callers are mid-append and write into the new generation
+ /// immediately. Releasing an empty generation is
+ /// [`force_rotate`](Self::force_rotate)'s job — that is the post-flush
+ /// path, where no write follows.
fn rotate(&mut self) -> Result<()> {
self.sync()?;
// Bump the generation only AFTER the new file exists: bumping
@@ -1160,7 +1283,7 @@ impl WalWriter {
let path = wal_path(&self.dir, next);
let file = create_wal_file(&path, next)?;
self.generation = next;
- self.writer = BufWriter::with_capacity(WAL_BUF_CAP, WalFile::new(file));
+ self.writer = Some(BufWriter::with_capacity(WAL_BUF_CAP, WalFile::new(file)));
self.current_offset = WAL_HEADER_LEN;
debug!(
generation = self.generation,
@@ -1206,8 +1329,31 @@ impl WalWriter {
if self.current_offset > WAL_HEADER_LEN {
self.rotate()?;
}
+ // #873 — this is the post-flush path (`wal_maintain_all_verified`
+ // calls it on every shard of every flushed index), so the generation
+ // it leaves active is empty by construction. Release its descriptor
+ // and its 64 KiB buffer: an index that stops being written to must
+ // stop paying for a write buffer, which — multiplied by the ingest
+ // shard count and by hundreds of small indices — is a large part of
+ // the resident floor a quiet node was carrying.
+ self.release_if_empty();
Ok(())
}
+
+ /// Drop the active generation's descriptor and userspace buffer when that
+ /// generation holds nothing past its header (#873).
+ ///
+ /// Safe to call at any quiescent point: `current_offset == WAL_HEADER_LEN`
+ /// means no frame has been appended to this generation, and every append
+ /// drains the buffer before it returns, so there is nothing buffered to
+ /// lose. The file stays on disk; [`ensure_materialized`](Self::ensure_materialized)
+ /// reopens it for append at the same offset.
+ fn release_if_empty(&mut self) {
+ if self.current_offset == WAL_HEADER_LEN {
+ self.writer = None;
+ self.dirty = false;
+ }
+ }
}
/// M5.8 — lock-free WAL frame builder.
@@ -2476,6 +2622,89 @@ mod tests {
.unwrap()
}
+ /// #873 — an EMPTY active generation holds no file descriptor and no
+ /// 64 KiB buffer, and nothing about the on-disk log changes.
+ ///
+ /// This is the state every index opens in and every flush returns to
+ /// (`wal_maintain_all_verified` → `force_rotate`), multiplied by the
+ /// ingest-shard count and again by the number of open indices — so the
+ /// property under test is "an index that is not being written to stops
+ /// paying for a write buffer", not a micro-optimisation of one shard.
+ #[test]
+ fn an_empty_wal_generation_holds_no_descriptor_and_no_buffer() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut w = make_writer(dir.path());
+
+ // Fresh shard: unmaterialized, but the generation-0 file was created
+ // on disk exactly as before — the laziness is in memory only.
+ assert!(
+ !w.is_materialized(),
+ "a fresh WAL shard must not hold a descriptor"
+ );
+ assert_eq!(w.buffer_capacity(), 0, "and must not allocate a buffer");
+ assert!(
+ wal_path(dir.path(), 0).exists(),
+ "the active generation file must still be created on disk"
+ );
+ assert_eq!(
+ fs::metadata(wal_path(dir.path(), 0)).unwrap().len(),
+ WAL_HEADER_LEN,
+ "an empty generation is exactly its header"
+ );
+
+ // The first append materializes it.
+ w.append(&doc("d1")).unwrap();
+ assert!(w.is_materialized(), "an append must materialize the shard");
+ assert_eq!(w.buffer_capacity(), WAL_BUF_CAP);
+
+ // The post-flush path hands the buffer back: `force_rotate` freezes
+ // the generation it wrote and leaves an empty one active.
+ w.force_rotate().unwrap();
+ assert!(
+ !w.is_materialized(),
+ "force_rotate must release the descriptor of the empty generation \
+ it leaves active"
+ );
+ assert_eq!(w.buffer_capacity(), 0);
+ assert_eq!(w.active_generation(), 1);
+
+ // A write after the release re-materializes and lands correctly.
+ w.append(&doc("d2")).unwrap();
+ assert!(w.is_materialized());
+ drop(w);
+
+ // Reopening a shard whose active generation is empty stays lazy, and
+ // every entry ever written is still replayable.
+ let mut reopened = make_writer(dir.path());
+ assert!(
+ reopened.is_materialized(),
+ "gen 1 has data, so it is opened"
+ );
+ reopened.force_rotate().unwrap();
+ drop(reopened);
+ let reopened = make_writer(dir.path());
+ assert!(
+ !reopened.is_materialized(),
+ "a cleanly rotated shard reopens without a descriptor or a buffer"
+ );
+ assert_eq!(reopened.buffer_capacity(), 0);
+ drop(reopened);
+
+ let ids: Vec<String> = WalReader::new(dir.path())
+ .replay()
+ .unwrap()
+ .map(|e| match e.unwrap().entry {
+ WalEntry::Index { doc_id, .. } => doc_id,
+ other => panic!("unexpected entry {other:?}"),
+ })
+ .collect();
+ assert_eq!(
+ ids,
+ vec!["d1".to_string(), "d2".to_string()],
+ "laziness must not cost a single acknowledged entry"
+ );
+ }
+
#[test]
fn round_trip_index_entry() {
let dir = tempfile::tempdir().unwrap();
@@ -2911,7 +3140,9 @@ mod tests {
Arc::clone(&seq_counter),
)
.unwrap();
- assert_eq!(writer.buffer_capacity(), WAL_BUF_CAP);
+ // #873 — a shard with an empty active generation holds no buffer at
+ // all; the bound below is what the first append allocates.
+ assert_eq!(writer.buffer_capacity(), 0);
let first = WalEntry::Index {
doc_id: "before-rotate".into(),
@@ -2958,7 +3189,7 @@ mod tests {
Arc::clone(&seq_counter),
)
.unwrap();
- assert_eq!(writer.buffer_capacity(), WAL_BUF_CAP);
+ assert_eq!(writer.buffer_capacity(), 0, "empty generation, no buffer");
// High-entropy printable so LZ4 cannot collapse oversized payloads
// back under the buffer capacity.
@@ -3002,7 +3233,9 @@ mod tests {
}
}
writer.force_rotate().unwrap();
- assert_eq!(writer.buffer_capacity(), WAL_BUF_CAP);
+ // #873 — force_rotate leaves an EMPTY generation active and hands the
+ // buffer back, which is what keeps a flushed index from holding one.
+ assert_eq!(writer.buffer_capacity(), 0);
// One more append after rotate to prove the new generation is live.
let final_id = "post-rotate".to_string();
@@ -3022,6 +3255,8 @@ mod tests {
Arc::clone(&seq_counter),
)
.unwrap();
+ // The post-rotate append above left data in the active generation, so
+ // the reopen materializes it — at the same bounded capacity (#873).
assert_eq!(reopened.buffer_capacity(), WAL_BUF_CAP);
drop(reopened);
@@ -3059,7 +3294,7 @@ mod tests {
handles.push(thread::spawn(move || {
let mut w =
WalWriter::open(&dir, 64 * 1024 * 1024, SyncMode::Batched, seq).unwrap();
- assert_eq!(w.buffer_capacity(), WAL_BUF_CAP);
+ assert_eq!(w.buffer_capacity(), 0, "fresh shard holds no buffer");
barrier.wait();
for i in 0..50 {
let pad = if i % 10 == 0 {