Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(fts): make unsafe field sidecars durable and unambiguous

buger6f3d43f4def3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 23cc2c8fc..446e1ad45 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
tolerance made the enforced floor `interval/2` shorter — 12.5 s at the
shipped defaults. Measured at `--progress-interval 10`: 11 of 11 same-phase
gaps under 15 s (min 10.0 s) before, none after (min 19.3 s).
+- **Text field names containing path separators can be flushed and reopened.**
+ FTS side-car filenames previously interpolated the raw field name, so a name
+ such as `styles_blocks_core/site-title` became a child path and prevented the
+ segment from being published. Unsafe or overlong field names now use one
+ deterministic bounded filename component. Existing portable names retain
+ their byte-identical paths. Each segment has an immutable filename-layout
+ discriminator: absence means the historical raw layout; an explicit marker
+ means the encoded layout. Readers never infer the layout from side-car file
+ existence, preventing partial files and raw names that resemble digests from
+ aliasing another field. Before the discriminator or first encoded side-car
+ is written, the index durably advances its data-directory marker to format 2;
+ older binaries refuse that index instead of silently missing the new layout.
+ A retry after a post-rename marker failure must re-establish durable marker
+ publication before writing the discriminator or encoded FTS paths. Unix
+ confirms the parent-directory fsync; Windows uses a same-directory Win32
+ write-through replacement rather than claiming its directory-sync no-op is
+ durable. Safe-only indices remain format 1.
## [1.0.0-rc.15] - 2026-08-10
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index eb9c46db3..3d22162d9 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -6658,6 +6658,7 @@ dependencies = [
"rust-stemmers",
"serde",
"serde_json",
+ "sha2 0.10.9",
"tempfile",
"thiserror 2.0.18",
"tracing",
diff --git a/engine/crates/xerj-common/src/fsio.rs b/engine/crates/xerj-common/src/fsio.rs
index 080064491..5a03f721a 100644
--- a/engine/crates/xerj-common/src/fsio.rs
+++ b/engine/crates/xerj-common/src/fsio.rs
@@ -9,13 +9,17 @@
//! acked-data loss.
//!
//! Every side-car / manifest write on the publish chain must go through
-//! [`write_file_durable`], and every `rename` that publishes a file must
-//! be followed by [`fsync_dir`] on the parent (rename durability is a
-//! property of the *directory*, not the file).
+//! [`write_file_durable`] or [`replace_file_durable`]. Unix makes the rename
+//! durable with parent-directory fsync; Windows uses a same-directory Win32
+//! write-through replacement because it has no equivalent directory-fsync
+//! contract.
use std::io::Write as _;
use std::path::Path;
+#[cfg(windows)]
+use std::os::windows::ffi::OsStrExt as _;
+
/// fsync a directory so previously-renamed/created entries in it survive
/// power loss. No-op errors are surfaced to the caller; on filesystems
/// where directories cannot be fsynced (rare), callers may choose to
@@ -26,12 +30,14 @@ pub fn fsync_dir(dir: &Path) -> std::io::Result<()> {
d.sync_all()
}
-/// Windows has no directory-flush primitive to call: `File::open` on a
+/// Windows has no supported directory-flush primitive to call here:
+/// `File::open` on a
/// directory fails with `ERROR_ACCESS_DENIED` (os error 5) because std
/// cannot pass `FILE_FLAG_BACKUP_SEMANTICS`, and `FlushFileBuffers` is
-/// not defined for directory handles. Durability here rests on the
-/// file-level `sync_all` the callers already perform plus NTFS metadata
-/// journalling of the rename.
+/// not defined for directory handles. This compatibility shim makes no
+/// durability claim. Code whose correctness requires a durable namespace
+/// replacement must use [`write_file_durable`] or [`replace_file_durable`],
+/// whose Windows implementation uses `MoveFileExW` with write-through.
///
/// This is not a cosmetic cfg: the Unix body returned `Err` for *every*
/// call on Windows, and `save_snapshot` propagates that error, so the
@@ -42,36 +48,185 @@ pub fn fsync_dir(_dir: &Path) -> std::io::Result<()> {
Ok(())
}
+#[cfg(not(windows))]
+fn replace_file_for_durable_write(source: &Path, destination: &Path) -> std::io::Result<()> {
+ std::fs::rename(source, destination)
+}
+
+fn require_same_directory(source: &Path, destination: &Path) -> std::io::Result<()> {
+ if source.parent() == destination.parent() {
+ Ok(())
+ } else {
+ Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidInput,
+ "durable file replacement requires source and destination in the same directory",
+ ))
+ }
+}
+
+/// Atomically replace one same-directory file and wait for the move to reach
+/// disk on Windows.
+///
+/// Windows cannot use the Unix rename + parent-directory-fsync sequence:
+/// `std::fs::File` cannot open a directory with `FILE_FLAG_BACKUP_SEMANTICS`,
+/// and `FlushFileBuffers` does not define a directory-handle contract. Win32's
+/// `MoveFileExW`, however, defines `MOVEFILE_WRITE_THROUGH` to wait until the
+/// move is on disk. Callers create the temporary file beside `destination`, so
+/// this remains a same-volume namespace replacement rather than the API's
+/// optional copy/delete fallback.
+#[cfg(windows)]
+fn replace_file_for_durable_write(source: &Path, destination: &Path) -> std::io::Result<()> {
+ const MOVEFILE_REPLACE_EXISTING: u32 = 0x0000_0001;
+ const MOVEFILE_WRITE_THROUGH: u32 = 0x0000_0008;
+
+ #[link(name = "kernel32")]
+ unsafe extern "system" {
+ fn MoveFileExW(
+ existing_file_name: *const u16,
+ new_file_name: *const u16,
+ flags: u32,
+ ) -> i32;
+ }
+
+ fn wide_path(path: &Path) -> std::io::Result<Vec<u16>> {
+ const BACKSLASH: u16 = b'\\' as u16;
+ const QUESTION: u16 = b'?' as u16;
+ const UPPER_U: u16 = b'U' as u16;
+ const UPPER_N: u16 = b'N' as u16;
+ const UPPER_C: u16 = b'C' as u16;
+
+ let absolute = std::path::absolute(path)?;
+ let original: Vec<u16> = absolute.as_os_str().encode_wide().collect();
+ if original.contains(&0) {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidInput,
+ "path contains an embedded NUL",
+ ));
+ }
+
+ // MoveFileExW does not get std's automatic long-path conversion.
+ // Supply an absolute verbatim path so the durable replacement keeps
+ // working beyond MAX_PATH. Preserve an existing verbatim prefix and
+ // translate ordinary UNC paths to `\\?\UNC\...`.
+ let mut wide = if original.starts_with(&[BACKSLASH, BACKSLASH, QUESTION, BACKSLASH]) {
+ original
+ } else if original.starts_with(&[BACKSLASH, BACKSLASH]) {
+ let mut verbatim = vec![
+ BACKSLASH, BACKSLASH, QUESTION, BACKSLASH, UPPER_U, UPPER_N, UPPER_C, BACKSLASH,
+ ];
+ verbatim.extend_from_slice(&original[2..]);
+ verbatim
+ } else {
+ let mut verbatim = vec![BACKSLASH, BACKSLASH, QUESTION, BACKSLASH];
+ verbatim.extend_from_slice(&original);
+ verbatim
+ };
+ wide.push(0);
+ Ok(wide)
+ }
+
+ let source = wide_path(source)?;
+ let destination = wide_path(destination)?;
+ // SAFETY: both buffers are NUL-terminated and live through the call.
+ // The flags request replacement plus synchronous persistence; no
+ // COPY_ALLOWED flag is present, and our temp file is same-directory.
+ let succeeded = unsafe {
+ MoveFileExW(
+ source.as_ptr(),
+ destination.as_ptr(),
+ MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
+ )
+ };
+ if succeeded == 0 {
+ Err(std::io::Error::last_os_error())
+ } else {
+ Ok(())
+ }
+}
+
+/// Replace `destination` with an already-written, already-synced sibling file
+/// and wait for the namespace replacement to become durable.
+///
+/// Both paths must have the same lexical parent. Unix uses rename followed by
+/// a parent-directory fsync. Windows uses a same-volume
+/// `MoveFileExW(MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)` call and
+/// does not claim that directory handles can be flushed.
+pub fn replace_file_durable(source: &Path, destination: &Path) -> std::io::Result<()> {
+ require_same_directory(source, destination)?;
+ replace_file_for_durable_write(source, destination)?;
+ #[cfg(not(windows))]
+ if let Some(parent) = destination.parent() {
+ fsync_dir(parent)?;
+ }
+ Ok(())
+}
+
/// Write `bytes` to `path` atomically **and durably**:
///
/// 1. write to a same-directory temp file,
/// 2. `fsync` the temp file (data + metadata),
-/// 3. `rename` over the target,
-/// 4. `fsync` the parent directory (makes the rename itself durable).
+/// 3. replace the target with the temp file,
+/// 4. wait for that replacement to become durable: parent-directory `fsync`
+/// on Unix, or `MoveFileExW(MOVEFILE_WRITE_THROUGH)` on Windows.
///
/// A crash or power loss at any point leaves either the old file or the
/// complete new file — never a torn one, and never a "file that
/// evaporates on power loss because only the page cache had it".
///
-/// The temp name embeds a uuid + thread id so concurrent writers to the
+/// The temp name embeds the process + thread ids so concurrent writers to the
/// same target never clobber each other's temp file (see the
/// `save_snapshot` race note in `xerj-storage`).
pub fn write_file_durable(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
+ write_file_durable_with_hook(path, bytes, |_| Ok(()))
+}
+
+/// Durable-write boundaries exposed for deterministic cross-crate fault tests.
+///
+/// Production callers should use [`write_file_durable`]. The hook-capable form
+/// exists so publication code can prove that every failure boundary prevents
+/// later, format-dependent writes without relying on filesystem permissions.
+#[doc(hidden)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DurableWriteStage {
+ BeforeTempWrite,
+ BeforeRename,
+ /// The target name has switched. On Unix the parent-directory fsync has
+ /// not run yet; on Windows the write-through replacement has completed,
+ /// but callers can still inject a conservative post-replace failure before
+ /// recording their process-local durability confirmation.
+ BeforeParentFsync,
+}
+
+/// [`write_file_durable`] with a deterministic observer/failure hook.
+#[doc(hidden)]
+pub fn write_file_durable_with_hook(
+ path: &Path,
+ bytes: &[u8],
+ mut hook: impl FnMut(DurableWriteStage) -> std::io::Result<()>,
+) -> std::io::Result<()> {
let nonce = format!("{:x}-{:?}", std::process::id(), std::thread::current().id());
let file_name = path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "file".to_string());
let tmp = path.with_file_name(format!("{file_name}.tmp.{nonce}"));
+ hook(DurableWriteStage::BeforeTempWrite)?;
{
let mut f = std::fs::File::create(&tmp)?;
f.write_all(bytes)?;
f.sync_all()?;
}
- if let Err(e) = std::fs::rename(&tmp, path) {
+ if let Err(error) = hook(DurableWriteStage::BeforeRename) {
+ let _ = std::fs::remove_file(&tmp);
+ return Err(error);
+ }
+ require_same_directory(&tmp, path)?;
+ if let Err(e) = replace_file_for_durable_write(&tmp, path) {
let _ = std::fs::remove_file(&tmp);
return Err(e);
}
+ hook(DurableWriteStage::BeforeParentFsync)?;
+ #[cfg(not(windows))]
if let Some(parent) = path.parent() {
fsync_dir(parent)?;
}
@@ -100,9 +255,80 @@ mod tests {
assert!(leftovers.is_empty(), "temp files left: {leftovers:?}");
}
+ #[cfg(not(windows))]
#[test]
fn fsync_dir_works() {
let dir = tempfile::tempdir().unwrap();
fsync_dir(dir.path()).unwrap();
}
+
+ #[cfg(windows)]
+ #[test]
+ fn fsync_dir_is_only_a_compatibility_shim_on_windows() {
+ let dir = tempfile::tempdir().unwrap();
+ fsync_dir(dir.path()).unwrap();
+ // This return value intentionally carries no durability proof. The
+ // durable replacement tests below exercise the write-through API that
+ // publication code must use on Windows.
+ }
+
+ #[test]
+ fn durable_replace_overwrites_synced_sibling() {
+ use std::io::Write as _;
+
+ let dir = tempfile::tempdir().unwrap();
+ let source = dir.path().join("replacement.tmp");
+ let destination = dir.path().join("marker.json");
+ std::fs::write(&destination, b"old").unwrap();
+ let mut staged = std::fs::File::create(&source).unwrap();
+ staged.write_all(b"new").unwrap();
+ staged.sync_all().unwrap();
+ drop(staged);
+
+ replace_file_durable(&source, &destination).unwrap();
+ assert_eq!(std::fs::read(&destination).unwrap(), b"new");
+ assert!(!source.exists());
+ }
+
+ #[test]
+ fn durable_replace_rejects_cross_directory_move() {
+ let source_dir = tempfile::tempdir().unwrap();
+ let destination_dir = tempfile::tempdir().unwrap();
+ let source = source_dir.path().join("replacement.tmp");
+ let destination = destination_dir.path().join("marker.json");
+ std::fs::write(&source, b"new").unwrap();
+ let error = replace_file_durable(&source, &destination).unwrap_err();
+ assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
+ assert!(source.exists());
+ assert!(!destination.exists());
+ }
+
+ #[test]
+ fn durable_write_hook_classifies_pre_and_post_rename_failures() {
+ for (stage, expected) in [
+ (DurableWriteStage::BeforeTempWrite, b"old".as_slice()),
+ (DurableWriteStage::BeforeRename, b"old".as_slice()),
+ (DurableWriteStage::BeforeParentFsync, b"new".as_slice()),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("marker.json");
+ std::fs::write(&path, b"old").unwrap();
+ let result = write_file_durable_with_hook(&path, b"new", |boundary| {
+ if boundary == stage {
+ Err(std::io::Error::other("injected durable-write failure"))
+ } else {
+ Ok(())
+ }
+ });
+ assert!(result.is_err());
+ assert_eq!(std::fs::read(&path).unwrap(), expected);
+ assert!(std::fs::read_dir(dir.path()).unwrap().all(|entry| {
+ !entry
+ .unwrap()
+ .file_name()
+ .to_string_lossy()
+ .contains(".tmp.")
+ }));
+ }
+ }
}
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index 028484792..6a6bbe93b 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -55,6 +55,7 @@ onnx-experimental = ["xerj-ai/onnx-experimental"]
stackprobe = []
[dev-dependencies]
+xerj-storage = { path = "../xerj-storage", features = ["test-hooks"] }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
criterion = { workspace = true }
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 277c2bed9..f6f973e2e 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -829,6 +829,13 @@ mod flush_publication_recovery_tests {
static FLUSH_FAULT_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+ fn data_dir_format_version(index: &Index) -> u64 {
+ let marker: serde_json::Value =
+ serde_json::from_slice(&std::fs::read(index.data_dir.join("xerj_meta.json")).unwrap())
+ .unwrap();
+ marker["format_version"].as_u64().unwrap()
+ }
+
fn text_schema() -> Schema {
let mut schema = Schema::empty();
schema
@@ -837,6 +844,14 @@ mod flush_publication_recovery_tests {
schema
}
+ fn slash_field_schema() -> Schema {
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new("bad/field", FieldType::Text))
+ .unwrap();
+ schema
+ }
+
fn vector_schema() -> Schema {
let mut schema = Schema::empty();
schema
@@ -1144,6 +1159,203 @@ mod flush_publication_recovery_tests {
assert_eq!(idx.search(&request).await.unwrap().hits[0].id, "survivor");
}
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn slash_field_flush_publishes_and_restarts_searchable() {
+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config.clone()).unwrap();
+ engine
+ .create_index("slash-field-publish", slash_field_schema())
+ .unwrap();
+ let idx = engine.get_index("slash-field-publish").unwrap();
+ idx.abort_background_tasks();
+ assert_eq!(data_dir_format_version(&idx), 1);
+ idx.index_document(Some("survivor".into()), json!({"bad/field": "slashneedle"}))
+ .await
+ .unwrap();
+
+ idx.flush().await.unwrap();
+ assert_eq!(data_dir_format_version(&idx), 2);
+ assert_eq!(idx.memtable.doc_count(), 0);
+ assert_eq!(idx.store.snapshot().segments.len(), 1);
+ assert!(idx.data_dir.join("snapshot.json").is_file());
+ let segment_entries: Vec<_> = std::fs::read_dir(idx.data_dir.join("segments"))
+ .unwrap()
+ .map(|entry| entry.unwrap())
+ .collect();
+ assert!(segment_entries
+ .iter()
+ .all(|entry| entry.file_type().unwrap().is_file()));
+ assert!(segment_entries.iter().any(|entry| entry
+ .file_name()
+ .to_string_lossy()
+ .ends_with(".fts-layout-v2")));
+
+ let request = xerj_query::parse_request(&json!({
+ "query": {"match": {"bad/field": "slashneedle"}},
+ "size": 10,
+ "track_total_hits": true
+ }))
+ .unwrap();
+ let result = idx.search(&request).await.unwrap();
+ assert_eq!(result.total.value, 1);
+ assert_eq!(result.hits[0].id, "survivor");
+ drop(idx);
+ drop(engine);
+
+ let restarted = crate::Engine::new(config).unwrap();
+ let idx = restarted.get_index("slash-field-publish").unwrap();
+ idx.abort_background_tasks();
+ assert_eq!(data_dir_format_version(&idx), 2);
+ assert_eq!(idx.store.snapshot().segments.len(), 1);
+ assert_eq!(idx.memtable.doc_count(), 0);
+ let result = idx.search(&request).await.unwrap();
+ assert_eq!(result.total.value, 1);
+ assert_eq!(result.hits[0].id, "survivor");
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn portable_field_flush_keeps_baseline_data_dir_format() {
+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config.clone()).unwrap();
+ engine
+ .create_index("portable-field-publish", text_schema())
+ .unwrap();
+ let idx = engine.get_index("portable-field-publish").unwrap();
+ idx.abort_background_tasks();
+ assert_eq!(data_dir_format_version(&idx), 1);
+ idx.index_document(Some("safe".into()), json!({"body": "portable"}))
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+ assert_eq!(data_dir_format_version(&idx), 1);
+ drop(idx);
+ drop(engine);
+
+ let restarted = crate::Engine::new(config).unwrap();
+ let idx = restarted.get_index("portable-field-publish").unwrap();
+ idx.abort_background_tasks();
+ assert_eq!(data_dir_format_version(&idx), 1);
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn marker_preflight_failure_writes_no_encoded_sidecars_and_retry_recovers() {
+ use xerj_storage::index_store::DataDirFormatWriteFailpoint;
+
+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
+ for (position, failpoint, expected_version) in [
+ (
+ "temp write",
+ DataDirFormatWriteFailpoint::BeforeTempWrite,
+ 1,
+ ),
+ ("rename", DataDirFormatWriteFailpoint::BeforeRename, 1),
+ (
+ "parent fsync",
+ DataDirFormatWriteFailpoint::BeforeParentFsync,
+ 2,
+ ),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ let index_name = format!("slash-field-marker-{}", position.replace(' ', "-"));
+ engine
+ .create_index(&index_name, slash_field_schema())
+ .unwrap();
+ let idx = engine.get_index(&index_name).unwrap();
+ idx.abort_background_tasks();
+ idx.index_document(Some("survivor".into()), json!({"bad/field": "slashneedle"}))
+ .await
+ .unwrap();
+ idx.store
+ .set_data_dir_format_write_failpoint_for_test(failpoint);
+
+ let error = idx.flush().await.unwrap_err().to_string();
+ assert!(
+ error.contains("marker persistence failure"),
+ "unexpected {position} error: {error}"
+ );
+ assert_eq!(
+ data_dir_format_version(&idx),
+ expected_version,
+ "{position}"
+ );
+ let entries: Vec<String> = std::fs::read_dir(idx.data_dir.join("segments"))
+ .unwrap()
+ .flatten()
+ .map(|entry| entry.file_name().to_string_lossy().into_owned())
+ .collect();
+ assert!(
+ entries
+ .iter()
+ .all(|name| !name.contains("__xerj_fts_field_sha256_")),
+ "{position} failure wrote an encoded side-car: {entries:?}"
+ );
+ assert!(
+ entries.iter().all(|name| !name.ends_with(".fts-layout-v2")),
+ "{position} failure published a layout discriminator: {entries:?}"
+ );
+ assert_eq!(
+ idx.store.fts_v2_marker_durability_confirmations_for_test(),
+ 0,
+ "{position} must not claim marker durability after failure"
+ );
+ assert_eq!(idx.memtable.doc_count(), 1);
+
+ // Split the retry at its preflight boundary: confirmation must
+ // cross one platform-appropriate durable-publication boundary
+ // (Unix parent fsync or Windows write-through replacement) while
+ // no discriminator, encoded temporary path, or encoded final path
+ // exists yet.
+ idx.store
+ .ensure_fts_encoded_field_component_format()
+ .unwrap();
+ assert_eq!(
+ idx.store.fts_v2_marker_durability_confirmations_for_test(),
+ 1,
+ "{position} retry must establish durability exactly once"
+ );
+ let after_confirmation: Vec<String> = std::fs::read_dir(idx.data_dir.join("segments"))
+ .unwrap()
+ .flatten()
+ .map(|entry| entry.file_name().to_string_lossy().into_owned())
+ .collect();
+ assert!(after_confirmation.iter().all(|name| {
+ !name.contains("__xerj_fts_field_sha256_") && !name.ends_with(".fts-layout-v2")
+ }));
+
+ idx.flush().await.unwrap();
+ assert_eq!(
+ idx.store.fts_v2_marker_durability_confirmations_for_test(),
+ 1,
+ "{position} publication must reuse the confirmed preflight"
+ );
+ assert_eq!(data_dir_format_version(&idx), 2);
+ assert_eq!(idx.memtable.doc_count(), 0);
+ assert!(std::fs::read_dir(idx.data_dir.join("segments"))
+ .unwrap()
+ .flatten()
+ .any(|entry| entry
+ .file_name()
+ .to_string_lossy()
+ .contains("__xerj_fts_field_sha256_")));
+ assert!(std::fs::read_dir(idx.data_dir.join("segments"))
+ .unwrap()
+ .flatten()
+ .any(|entry| entry
+ .file_name()
+ .to_string_lossy()
+ .ends_with(".fts-layout-v2")));
+ }
+ }
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn failed_post_sidecar_segment_has_no_recovery_marker() {
let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
@@ -1830,6 +2042,131 @@ mod merge_publication_transaction_tests {
config
}
+ fn dot_field_schema() -> Schema {
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new(".", FieldType::Text))
+ .unwrap();
+ schema
+ }
+
+ fn marker_version(index: &Index) -> u64 {
+ let marker: serde_json::Value =
+ serde_json::from_slice(&std::fs::read(index.data_dir.join("xerj_meta.json")).unwrap())
+ .unwrap();
+ marker["format_version"].as_u64().unwrap()
+ }
+
+ async fn v1_raw_dot_field_merge_fixture(
+ dir: &TempDir,
+ index_name: &str,
+ ) -> (Engine, Arc<Index>, String, PathBuf) {
+ use sha2::{Digest, Sha256};
+
+ let engine = Engine::new(config(dir)).unwrap();
+ engine.create_index(index_name, dot_field_schema()).unwrap();
+ let index = engine.get_index(index_name).unwrap();
+ index.abort_background_tasks();
+ for (id, value) in [("one", "alpha"), ("two", "beta")] {
+ index
+ .index_document(Some(id.into()), serde_json::json!({".": value}))
+ .await
+ .unwrap();
+ index.flush().await.unwrap();
+ }
+ assert_eq!(index.store.snapshot().segments.len(), 2);
+ assert_eq!(marker_version(&index), 2);
+
+ // Rewrite the two input families into the exact v1 raw layout and
+ // restore a v1 marker. This models segments published by the old
+ // writer without needing an old binary inside this unit test.
+ let component = format!("__xerj_fts_field_sha256_{:x}", Sha256::digest(b"."));
+ let segments_dir = index.data_dir.join("segments");
+ for meta in index.store.snapshot().segments.iter() {
+ for extension in ["fst", "post", "meta", "norms"] {
+ let current = segments_dir.join(format!("{}.{}.{}", meta.id, component, extension));
+ let legacy = segments_dir.join(format!("{}...{}", meta.id, extension));
+ std::fs::rename(current, legacy).unwrap();
+ }
+ std::fs::remove_file(segments_dir.join(format!("{}.fts-layout-v2", meta.id))).unwrap();
+ }
+ std::fs::write(
+ index.data_dir.join("xerj_meta.json"),
+ serde_json::to_vec(&serde_json::json!({
+ "format_version": 1,
+ "xerj_version": "v1-test-fixture"
+ }))
+ .unwrap(),
+ )
+ .unwrap();
+ index
+ .store
+ .clear_fts_v2_marker_durability_confirmation_for_test();
+ assert_eq!(marker_version(&index), 1);
+ (engine, index, component, segments_dir)
+ }
+
+ #[tokio::test]
+ async fn merging_v1_raw_unsafe_fields_advances_marker_before_encoded_output() {
+ let dir = TempDir::new().unwrap();
+ let (_engine, index, component, segments_dir) =
+ v1_raw_dot_field_merge_fixture(&dir, "v1-raw-fts-merge").await;
+
+ assert_eq!(index.run_merge_once().await.unwrap(), 1);
+ assert_eq!(marker_version(&index), 2);
+ let snapshot = index.store.snapshot();
+ assert_eq!(snapshot.segments.len(), 1);
+ let output = &snapshot.segments[0];
+ assert!(segments_dir
+ .join(format!("{}.fts-layout-v2", output.id))
+ .is_file());
+ for extension in ["fst", "post", "meta", "norms"] {
+ assert!(segments_dir
+ .join(format!("{}.{}.{}", output.id, component, extension))
+ .is_file());
+ }
+ }
+
+ #[tokio::test]
+ async fn merge_marker_failure_publishes_only_v1_compatible_no_fts_output() {
+ use xerj_storage::index_store::DataDirFormatWriteFailpoint;
+
+ let dir = TempDir::new().unwrap();
+ let (_engine, index, component, segments_dir) =
+ v1_raw_dot_field_merge_fixture(&dir, "v1-raw-fts-merge-failure").await;
+ index.store.set_data_dir_format_write_failpoint_for_test(
+ DataDirFormatWriteFailpoint::BeforeRename,
+ );
+
+ // Merge has historically allowed a correct stored-scan output when
+ // optional FTS side-car construction fails. Preserve that policy:
+ // marker failure skips finish(), so no v2 filename exists and the v1
+ // marker remains truthful.
+ assert_eq!(index.run_merge_once().await.unwrap(), 1);
+ assert_eq!(marker_version(&index), 1);
+ let snapshot = index.store.snapshot();
+ assert_eq!(snapshot.segments.len(), 1);
+ let output = &snapshot.segments[0];
+ assert!(["fst", "post", "meta", "norms"].iter().all(|extension| {
+ !segments_dir
+ .join(format!("{}.{}.{}", output.id, component, extension))
+ .exists()
+ }));
+ assert!(!segments_dir
+ .join(format!("{}.fts-layout-v2", output.id))
+ .exists());
+
+ let request = xerj_query::parse_request(&serde_json::json!({
+ "query": {"match": {".": "alpha"}},
+ "size": 10,
+ "track_total_hits": true
+ }))
+ .unwrap();
+ let result = index.search(&request).await.unwrap();
+ assert_eq!(result.total.value, 1);
+ assert_eq!(result.hits[0].id, "one");
+ }
+
async fn fixture(dir: &TempDir) -> (Engine, Arc<Index>) {
let engine = Engine::new(config(dir)).unwrap();
engine
@@ -8538,6 +8875,24 @@ impl Index {
// the whole batch already runs inside merge_pool.
crate::merge_pool().install(|| {
fts_writer.add_documents_parallel(&fts_input);
+ if fts_writer.uses_encoded_field_filename_components() {
+ if let Err(e) = store_for_task
+ .ensure_fts_encoded_field_component_format()
+ {
+ tracing::warn!(
+ "merge: encoded FTS filename format preflight failed: {e}"
+ );
+ return;
+ }
+ if let Err(e) =
+ fts_writer.publish_encoded_filename_layout()
+ {
+ tracing::warn!(
+ "merge: FTS filename-layout publication failed: {e}"
+ );
+ return;
+ }
+ }
if let Err(e) = fts_writer.finish() {
tracing::warn!("merge: FTS build failed: {e}");
}
@@ -23573,6 +23928,16 @@ async fn do_flush_shard(
fts_writer.add_documents_parallel(&drained_fts_for_build);
fts_add_us = t_add.elapsed().as_micros();
let t_fin = std::time::Instant::now();
+ if fts_writer.uses_encoded_field_filename_components() {
+ store_for_warm.ensure_fts_encoded_field_component_format()?;
+ fts_writer
+ .publish_encoded_filename_layout()
+ .map_err(|error| {
+ xerj_storage::StorageError::Io(std::io::Error::other(format!(
+ "FTS filename-layout publication failed: {error}"
+ )))
+ })?;
+ }
fts_writer.finish().map_err(|error| {
xerj_storage::StorageError::Io(std::io::Error::other(format!(
"FTS side-car build failed: {error}"
diff --git a/engine/crates/xerj-fts/Cargo.toml b/engine/crates/xerj-fts/Cargo.toml
index 7584cae3c..c24044c15 100644
--- a/engine/crates/xerj-fts/Cargo.toml
+++ b/engine/crates/xerj-fts/Cargo.toml
@@ -31,6 +31,7 @@ unicode-segmentation = "1"
unicode-normalization = "0.1"
rust-stemmers = "1.2"
regex = "1"
+sha2 = "0.10"
[dev-dependencies]
tempfile.workspace = true
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 2218b7f6e..696bdc374 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -2,16 +2,20 @@
//!
//! ## On-disk layout
//!
-//! For each indexed field `<field>` a segment produces two files:
+//! For each indexed field `<field-component>` a segment produces four files:
//!
//! ```text
-//! seg-<id>.<field>.fst — FST term dictionary
+//! seg-<id>.<field-component>.fst — FST term dictionary
//! value = byte offset into .post file
-//! seg-<id>.<field>.post — concatenated posting lists
-//! seg-<id>.<field>.meta — JSON: FieldStats + per-term TermPostings headers
-//! seg-<id>.<field>.norms — u16 per doc (field length, capped at 65535)
+//! seg-<id>.<field-component>.post — concatenated posting lists
+//! seg-<id>.<field-component>.meta — FieldStats + per-term metadata
+//! seg-<id>.<field-component>.norms — encoded per-doc field lengths
+//! seg-<id>.fts-layout-v2 — exact marker when components are encoded
//! ```
//!
+//! Short portable field names are used literally for on-disk compatibility;
+//! all other logical field names map to a bounded digest component.
+//!
//! The FST key is the term text (UTF-8 bytes, lexicographically sorted by construction).
//! The FST output value is the byte offset in the `.post` file.
//! `TermPostings` metadata (doc_freq, ttf) is stored in the `.meta` JSON.
@@ -21,12 +25,14 @@ use crate::{
bm25::FieldStats,
postings::{PostingsWriter, TermPostings},
};
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use fst::{Map, MapBuilder};
use memmap2::Mmap;
use serde::Deserialize;
+use sha2::{Digest, Sha256};
use std::{
+ borrow::Cow,
collections::HashMap,
fs::{self, File},
io::{BufWriter, Write},
@@ -34,6 +40,127 @@ use std::{
sync::Arc,
};
+/// Longest field name that can remain literal inside an FTS side-car filename.
+///
+/// The complete filename also carries a segment UUID, separators, an extension,
+/// and sometimes `.tmp`. Keeping the user-controlled component at 128 bytes
+/// leaves comfortable room below the common 255-byte component limit.
+const MAX_LITERAL_FIELD_COMPONENT_BYTES: usize = 128;
+
+/// Namespace for field names that cannot safely be used as portable filesystem
+/// components. Literal names in this namespace are encoded too, so a user field
+/// cannot alias another field's digest-derived component.
+const ENCODED_FIELD_COMPONENT_PREFIX: &str = "__xerj_fts_field_sha256_";
+
+/// Immutable per-segment discriminator for encoded field filename components.
+///
+/// This name has no FTS side-car extension, so it cannot equal any v1 file:
+/// every historical file is `<segment>.<raw-field>.(fst|post|meta|norms)`.
+/// It retains the segment prefix so publication manifests, rollback, orphan
+/// recovery, and retirement treat it as part of the same artifact family.
+const FTS_FILENAME_LAYOUT_V2_MARKER_SUFFIX: &str = "fts-layout-v2";
+const FTS_FILENAME_LAYOUT_V2_MARKER_BYTES: &[u8] = b"XERJ_FTS_FILENAME_LAYOUT_V2\n";
+
+fn segment_filename_layout_v2_marker_path(segment_dir: &Path, segment_id: &str) -> PathBuf {
+ segment_dir.join(format!(
+ "{segment_id}.{FTS_FILENAME_LAYOUT_V2_MARKER_SUFFIX}"
+ ))
+}
+
+/// Absence means v1/raw. Exact magic means v2/encoded. Any other visible
+/// marker is corruption. File-family existence is never layout evidence.
+fn segment_uses_encoded_filename_layout(segment_dir: &Path, segment_id: &str) -> Result<bool> {
+ let marker = segment_filename_layout_v2_marker_path(segment_dir, segment_id);
+ match fs::read(&marker) {
+ Ok(bytes) if bytes == FTS_FILENAME_LAYOUT_V2_MARKER_BYTES => Ok(true),
+ Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", marker),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
+ Err(error) => Err(error)
+ .with_context(|| format!("reading FTS filename-layout discriminator {:?}", marker)),
+ }
+}
+
+fn is_windows_reserved_device_name(field: &str) -> bool {
+ let stem = field.split('.').next().unwrap_or(field);
+ if ["CON", "PRN", "AUX", "NUL"]
+ .iter()
+ .any(|reserved| stem.eq_ignore_ascii_case(reserved))
+ {
+ return true;
+ }
+ let bytes = stem.as_bytes();
+ bytes.len() == 4
+ && (bytes[..3].eq_ignore_ascii_case(b"COM") || bytes[..3].eq_ignore_ascii_case(b"LPT"))
+ && matches!(bytes[3], b'1'..=b'9')
+}
+
+/// Return the bounded, portable component used for every side-car of `field`.
+///
+/// Existing short field names that are portable across supported filesystems
+/// remain byte-for-byte compatible with the historical on-disk layout.
+/// Everything else is represented by a SHA-256 digest. In particular, path
+/// separators, traversal-like names, platform-reserved punctuation, controls,
+/// and overlong names can never create a child path or exceed a normal
+/// filesystem's component limit. Unicode, `@`, and internal spaces are valid
+/// portable literals; trailing spaces and dots are not portable to Windows.
+/// Preserving literals also preserves v1's pre-existing limitation: distinct
+/// portable names can alias on a filesystem that case-folds or normalizes
+/// Unicode. Digest-derived names remain distinct by exact UTF-8 bytes modulo a
+/// SHA-256 collision; eliminating literal aliases would require a new all-name
+/// encoding or name table and is outside filename layout v2.
+fn field_file_component(field: &str) -> Cow<'_, str> {
+ let portable_literal = !field.is_empty()
+ && field.len() <= MAX_LITERAL_FIELD_COMPONENT_BYTES
+ && field != "."
+ && field != ".."
+ && !field.starts_with(ENCODED_FIELD_COMPONENT_PREFIX)
+ && !is_windows_reserved_device_name(field)
+ && !field.ends_with(' ')
+ && !field.ends_with('.')
+ && field.chars().all(|character| {
+ !character.is_control()
+ && !matches!(
+ character,
+ '/' | '\\' | '<' | '>' | ':' | '"' | '|' | '?' | '*'
+ )
+ });
+
+ if portable_literal {
+ return Cow::Borrowed(field);
+ }
+
+ let digest = Sha256::digest(field.as_bytes());
+ Cow::Owned(format!("{ENCODED_FIELD_COMPONENT_PREFIX}{digest:x}"))
+}
+
+/// Derive one FTS side-car path from controlled components only.
+fn field_sidecar_path(
+ segment_dir: &Path,
+ segment_id: &str,
+ field_name: &str,
+ extension: &str,
+) -> PathBuf {
+ let field_component = field_file_component(field_name);
+ segment_dir.join(format!("{segment_id}.{field_component}.{extension}"))
+}
+
+/// Return the exact pre-fix raw path only when it remains one child of the
+/// segment directory on this host.
+///
+/// This reader-only bridge keeps already-published Linux fields such as a raw
+/// backslash, a reserved digest-prefix literal, or a formerly accepted long
+/// component readable after upgrade. It never creates a path and it refuses
+/// slash/backslash-as-separator shapes through the parent equality check.
+fn legacy_field_sidecar_path(
+ segment_dir: &Path,
+ segment_id: &str,
+ field_name: &str,
+ extension: &str,
+) -> Option<PathBuf> {
+ let candidate = segment_dir.join(format!("{segment_id}.{field_name}.{extension}"));
+ (candidate.parent() == Some(segment_dir)).then_some(candidate)
+}
+
// ── FieldIndexConfig ──────────────────────────────────────────────────────────
/// Per-field indexing configuration.
@@ -415,6 +542,10 @@ fn decode_field_meta_binary(bytes: &[u8]) -> Result<FieldMeta> {
/// ```text
/// let mut writer = FtsIndexWriter::new(dir, segment_id, registry);
/// for doc in docs { writer.add_document(doc_id, &fields); }
+/// if writer.uses_encoded_field_filename_components() {
+/// // The engine advances its data-directory format before this step.
+/// writer.publish_encoded_filename_layout()?;
+/// }
/// writer.finish()?;
/// ```
pub struct FtsIndexWriter {
@@ -423,6 +554,7 @@ pub struct FtsIndexWriter {
registry: Arc<AnalyzerRegistry>,
/// Per-field: (config, postings_writer, field_stats, norms)
fields: HashMap<String, FieldData>,
+ encoded_filename_layout_published: bool,
}
struct FieldData {
@@ -445,6 +577,7 @@ impl FtsIndexWriter {
segment_id: segment_id.into(),
registry,
fields: HashMap::new(),
+ encoded_filename_layout_published: false,
}
}
@@ -467,6 +600,67 @@ impl FtsIndexWriter {
);
}
+ /// Whether finishing this writer can create a v2 digest-derived field
+ /// filename.
+ ///
+ /// Engine callers use this after configuration/document ingestion and
+ /// before [`Self::finish`] to durably advance the data-directory marker.
+ pub fn uses_encoded_field_filename_components(&self) -> bool {
+ self.fields
+ .keys()
+ .any(|field| matches!(field_file_component(field), Cow::Owned(_)))
+ }
+
+ /// Durably publish this segment's immutable v2 filename discriminator.
+ ///
+ /// The engine must durably advance the data-directory format first, call
+ /// this method second, and call [`Self::finish`] last. `finish` refuses to
+ /// emit encoded side-cars without this proof. An existing discriminator is
+ /// accepted only when it contains the exact v2 magic.
+ pub fn publish_encoded_filename_layout(&mut self) -> Result<()> {
+ if !self.uses_encoded_field_filename_components() {
+ return Ok(());
+ }
+ fs::create_dir_all(&self.segment_dir)
+ .with_context(|| format!("creating segment dir {:?}", self.segment_dir))?;
+ let marker = segment_filename_layout_v2_marker_path(&self.segment_dir, &self.segment_id);
+ match fs::read(&marker) {
+ Ok(bytes) if bytes == FTS_FILENAME_LAYOUT_V2_MARKER_BYTES => {
+ // A previous attempt can have made the rename visible and
+ // then failed its durability confirmation. A fresh writer has
+ // no process-local proof, so confirm the directory entry
+ // before authorizing encoded side-cars.
+ #[cfg(not(windows))]
+ xerj_common::fsio::fsync_dir(&self.segment_dir).with_context(|| {
+ format!("confirming FTS filename-layout discriminator {:?}", marker)
+ })?;
+ // Windows has no parent-directory fsync contract. Replacing
+ // the exact marker through MoveFileExW WRITE_THROUGH also
+ // repairs a visible discriminator produced by an older build
+ // whose Windows directory-sync shim was a no-op.
+ #[cfg(windows)]
+ xerj_common::fsio::write_file_durable(&marker, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)
+ .with_context(|| {
+ format!("confirming FTS filename-layout discriminator {:?}", marker)
+ })?;
+ }
+ Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", marker),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ xerj_common::fsio::write_file_durable(&marker, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)
+ .with_context(|| {
+ format!("publishing FTS filename-layout discriminator {:?}", marker)
+ })?;
+ }
+ Err(error) => {
+ return Err(error).with_context(|| {
+ format!("reading FTS filename-layout discriminator {:?}", marker)
+ });
+ }
+ }
+ self.encoded_filename_layout_published = true;
+ Ok(())
+ }
+
/// Index all text fields of one document.
///
/// `fields` is a map of field name → field text value.
@@ -643,6 +837,11 @@ impl FtsIndexWriter {
pub fn finish(self) -> Result<HashMap<String, FieldStats>> {
use rayon::prelude::*;
+ if self.uses_encoded_field_filename_components() && !self.encoded_filename_layout_published
+ {
+ bail!("encoded FTS field filenames require a durable per-segment layout discriminator");
+ }
+
fs::create_dir_all(&self.segment_dir)
.with_context(|| format!("creating segment dir {:?}", self.segment_dir))?;
@@ -692,9 +891,9 @@ impl FtsIndexWriter {
// NOTE: `PathBuf::with_extension` replaces the final `.ext` in the path,
// so using `segment_dir.join("segment_id.field_name")` followed by
// `with_extension("fst")` would strip `.field_name` and collapse every
- // field to the same file. Build filenames by hand to avoid that.
- let filename =
- |ext: &str| segment_dir.join(format!("{}.{}.{}", segment_id, field_name, ext));
+ // field to the same file. The shared helper also keeps untrusted field
+ // names from becoming paths.
+ let filename = |ext: &str| field_sidecar_path(segment_dir, segment_id, field_name, ext);
let fst_path = filename("fst");
let post_path = filename("post");
let meta_path = filename("meta");
@@ -768,13 +967,14 @@ impl FtsIndexWriter {
out.extend_from_slice(&compressed);
out
};
- // RC4 W1 #10 — every FTS side-car write below goes through the
- // durable tmp+fsync+rename+dir-fsync pattern. These files are
- // part of the segment publish chain: the WAL entries they cover
- // are pruned ~1 s after the flush, so side-cars sitting only in
- // the volatile page cache meant a power loss could leave a
- // registered segment with missing/torn FTS data (silently wrong
- // query results or unreadable fields) with no WAL to recover from.
+ // RC4 W1 #10 — every FTS side-car write below goes through durable
+ // temporary-file sync plus platform-appropriate publication: rename
+ // and parent-directory fsync on Unix, write-through replacement on
+ // Windows. These files are part of the segment publish chain: the WAL
+ // entries they cover are pruned ~1 s after the flush, so side-cars
+ // sitting only in the volatile page cache meant a power loss could
+ // leave a registered segment with missing/torn FTS data (silently
+ // wrong query results or unreadable fields) with no WAL to recover from.
xerj_common::fsio::write_file_durable(&post_path, &post_bytes_wrapped)
.with_context(|| format!("writing postings to {:?}", post_path))?;
@@ -790,9 +990,9 @@ impl FtsIndexWriter {
// holds df + ttf + postings_offset + length, so one FST hit →
// one bounded mmap read.
// FST streams into a same-directory temp file; fsync + rename +
- // dir-fsync publishes it durably (RC4 W1 #10 — see the postings
- // note above).
- let fst_tmp = segment_dir.join(format!("{}.{}.fst.tmp", segment_id, field_name));
+ // platform-specific durable replacement publishes it (RC4 W1 #10 —
+ // see the postings note above).
+ let fst_tmp = filename("fst.tmp");
let fst_file = BufWriter::new(
File::create(&fst_tmp).with_context(|| format!("creating FST file {:?}", fst_tmp))?,
);
@@ -817,9 +1017,8 @@ impl FtsIndexWriter {
.sync_all()
.with_context(|| "fsyncing FST")?;
drop(fst_out);
- fs::rename(&fst_tmp, &fst_path)
+ xerj_common::fsio::replace_file_durable(&fst_tmp, &fst_path)
.with_context(|| format!("publishing FST {:?}", fst_path))?;
- xerj_common::fsio::fsync_dir(segment_dir).with_context(|| "fsyncing segment dir (fst)")?;
// 4. Write meta in the ZFM4 format — Zstd-19 envelope around
// the per-term records section; ZFM3-compatible header so
@@ -1037,17 +1236,35 @@ impl FtsIndexReader {
) -> Result<Self> {
let segment_dir = segment_dir.as_ref().to_path_buf();
let segment_id = segment_id.into();
+ let encoded_layout = segment_uses_encoded_filename_layout(&segment_dir, &segment_id)?;
let mut fields = HashMap::new();
for &field_name in field_names {
- // Build filenames explicitly — see note in `write_field_static`
- // for why `with_extension()` would be wrong here.
- let filename =
- |ext: &str| segment_dir.join(format!("{}.{}.{}", segment_id, field_name, ext));
- let fst_path = filename("fst");
- let post_path = filename("post");
- let meta_path = filename("meta");
- let norms_path = filename("norms");
+ // The immutable segment discriminator is the sole authority.
+ // Existence is deliberately irrelevant: otherwise a v1 field
+ // whose literal name equals another field's digest component can
+ // alias, and partial or stray files can alter layout selection.
+ let filename = |ext: &str| -> Option<PathBuf> {
+ if encoded_layout {
+ Some(field_sidecar_path(
+ &segment_dir,
+ &segment_id,
+ field_name,
+ ext,
+ ))
+ } else {
+ legacy_field_sidecar_path(&segment_dir, &segment_id, field_name, ext)
+ }
+ };
+ let Some(fst_path) = filename("fst") else {
+ continue;
+ };
+ let post_path =
+ filename("post").expect("FTS family path containment is extension-invariant");
+ let meta_path =
+ filename("meta").expect("FTS family path containment is extension-invariant");
+ let norms_path =
+ filename("norms").expect("FTS family path containment is extension-invariant");
// Skip fields that haven't been indexed yet
if !fst_path.exists() {
@@ -1478,6 +1695,7 @@ impl FtsIndexReader {
#[cfg(test)]
mod tests {
use super::*;
+ use std::collections::HashSet;
use tempfile::TempDir;
fn make_registry() -> Arc<AnalyzerRegistry> {
@@ -1586,4 +1804,403 @@ mod tests {
let data = reader.postings_data("title", &tp);
assert!(data.is_some() && !data.unwrap().is_empty());
}
+
+ #[test]
+ fn safe_field_component_preserves_existing_filename_layout() {
+ let dir = TempDir::new().unwrap();
+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-safe", make_registry());
+ let fields = ["title.body-2", "@timestamp", "space field", "配置.名字"];
+ let document: HashMap<String, String> = fields
+ .into_iter()
+ .enumerate()
+ .map(|(index, field)| (field.to_owned(), format!("needle{index}")))
+ .collect();
+ writer.add_document(0, &document);
+ writer.finish().unwrap();
+
+ for field in fields {
+ assert_eq!(field_file_component(field), Cow::Borrowed(field));
+ for extension in ["fst", "post", "meta", "norms"] {
+ assert!(dir
+ .path()
+ .join(format!("seg-safe.{field}.{extension}"))
+ .is_file());
+ }
+ }
+ }
+
+ fn write_legacy_raw_field_fixture(segment_dir: &Path, segment_id: &str, legacy_field: &str) {
+ let source_field = "legacy_fixture_source";
+ let mut writer = FtsIndexWriter::new(segment_dir, segment_id, make_registry());
+ let document = [(source_field.to_owned(), "legacyneedle".to_owned())]
+ .into_iter()
+ .collect();
+ writer.add_document(0, &document);
+ writer.finish().unwrap();
+ for extension in ["fst", "post", "meta", "norms"] {
+ let source = field_sidecar_path(segment_dir, segment_id, source_field, extension);
+ let destination =
+ legacy_field_sidecar_path(segment_dir, segment_id, legacy_field, extension)
+ .expect("test fixture must be one component on this host");
+ fs::rename(source, destination).unwrap();
+ }
+ }
+
+ #[test]
+ fn new_reader_opens_existing_single_component_legacy_fields() {
+ let root = TempDir::new().unwrap();
+ let segment_dir = root.path().join("segments");
+ fs::create_dir_all(&segment_dir).unwrap();
+ let overlong = "a".repeat(MAX_LITERAL_FIELD_COMPONENT_BYTES + 1);
+ let platform_fields: &[&str] = if cfg!(windows) {
+ &[]
+ } else {
+ &[r"legacy\backslash", "legacy:colon*question?", "trailing "]
+ };
+ let fields: Vec<&str> = vec![
+ "@timestamp",
+ "space field",
+ "配置.名字",
+ "CON.txt",
+ ENCODED_FIELD_COMPONENT_PREFIX,
+ overlong.as_str(),
+ ]
+ .into_iter()
+ .chain(platform_fields.iter().copied())
+ .collect();
+
+ for (index, field) in fields.iter().enumerate() {
+ let segment_id = format!("legacy-{index}");
+ write_legacy_raw_field_fixture(&segment_dir, &segment_id, field);
+ let raw_fst = legacy_field_sidecar_path(&segment_dir, &segment_id, field, "fst")
+ .expect("legacy field must remain contained");
+ assert!(raw_fst.is_file());
+
+ let reader = FtsIndexReader::open(&segment_dir, &segment_id, &[*field]).unwrap();
+ assert!(reader.term_exists(field, "legacyneedle"));
+ }
+ }
+
+ fn collision_fields() -> (String, String) {
+ let unsafe_name = "CON".to_owned();
+ let legacy_alias = format!(
+ "{ENCODED_FIELD_COMPONENT_PREFIX}{:x}",
+ Sha256::digest(unsafe_name.as_bytes())
+ );
+ (unsafe_name, legacy_alias)
+ }
+
+ fn write_v1_collision_fixture(segment_dir: &Path, segment_id: &str) -> (String, String) {
+ let (unsafe_name, legacy_alias) = collision_fields();
+ let source_fields = ["source_unsafe", "source_alias"];
+ let mut writer = FtsIndexWriter::new(segment_dir, segment_id, make_registry());
+ let document = [
+ (source_fields[0].to_owned(), "unsafeneedle".to_owned()),
+ (source_fields[1].to_owned(), "aliasneedle".to_owned()),
+ ]
+ .into_iter()
+ .collect();
+ writer.add_document(0, &document);
+ writer.finish().unwrap();
+ for (source, destination) in source_fields
+ .into_iter()
+ .zip([unsafe_name.as_str(), legacy_alias.as_str()])
+ {
+ for extension in ["fst", "post", "meta", "norms"] {
+ fs::rename(
+ field_sidecar_path(segment_dir, segment_id, source, extension),
+ legacy_field_sidecar_path(segment_dir, segment_id, destination, extension)
+ .unwrap(),
+ )
+ .unwrap();
+ }
+ }
+ (unsafe_name, legacy_alias)
+ }
+
+ fn assert_collision_fields_read(
+ segment_dir: &Path,
+ segment_id: &str,
+ unsafe_name: &str,
+ legacy_alias: &str,
+ ) {
+ for stats_only in [false, true] {
+ let reader = if stats_only {
+ FtsIndexReader::open_stats_only(
+ segment_dir,
+ segment_id,
+ &[unsafe_name, legacy_alias],
+ )
+ } else {
+ FtsIndexReader::open(segment_dir, segment_id, &[unsafe_name, legacy_alias])
+ }
+ .unwrap();
+ assert!(reader.term_exists(unsafe_name, "unsafeneedle"));
+ assert!(!reader.term_exists(unsafe_name, "aliasneedle"));
+ assert!(reader.term_exists(legacy_alias, "aliasneedle"));
+ assert!(!reader.term_exists(legacy_alias, "unsafeneedle"));
+ }
+ }
+
+ #[test]
+ fn discriminator_prevents_same_segment_v1_digest_alias_collision_across_reader_reopens() {
+ let root = TempDir::new().unwrap();
+ let segment_dir = root.path().join("segments");
+ fs::create_dir_all(&segment_dir).unwrap();
+ let segment_id = "v1-collision";
+ let (unsafe_name, legacy_alias) = write_v1_collision_fixture(&segment_dir, segment_id);
+ assert!(!segment_filename_layout_v2_marker_path(&segment_dir, segment_id).exists());
+ assert_collision_fields_read(&segment_dir, segment_id, &unsafe_name, &legacy_alias);
+ // Reopening creates a fresh reader and must make the same selection.
+ assert_collision_fields_read(&segment_dir, segment_id, &unsafe_name, &legacy_alias);
+ }
+
+ #[test]
+ fn discriminator_keeps_same_segment_v2_digest_alias_fields_distinct_across_reader_reopens() {
+ let root = TempDir::new().unwrap();
+ let segment_dir = root.path().join("segments");
+ let segment_id = "v2-collision";
+ let (unsafe_name, legacy_alias) = collision_fields();
+ let mut writer = FtsIndexWriter::new(&segment_dir, segment_id, make_registry());
+ let document = [
+ (unsafe_name.clone(), "unsafeneedle".to_owned()),
+ (legacy_alias.clone(), "aliasneedle".to_owned()),
+ ]
+ .into_iter()
+ .collect();
+ writer.add_document(0, &document);
+ writer.publish_encoded_filename_layout().unwrap();
+ writer.finish().unwrap();
+ assert!(segment_filename_layout_v2_marker_path(&segment_dir, segment_id).is_file());
+ assert_ne!(
+ field_sidecar_path(&segment_dir, segment_id, &unsafe_name, "fst"),
+ field_sidecar_path(&segment_dir, segment_id, &legacy_alias, "fst")
+ );
+ assert_collision_fields_read(&segment_dir, segment_id, &unsafe_name, &legacy_alias);
+ assert_collision_fields_read(&segment_dir, segment_id, &unsafe_name, &legacy_alias);
+ }
+
+ #[test]
+ fn v1_discriminator_state_ignores_stray_and_partial_v2_families_and_fails_on_raw_corruption() {
+ let root = TempDir::new().unwrap();
+ let segment_dir = root.path().join("segments");
+ fs::create_dir_all(&segment_dir).unwrap();
+ let segment_id = "v1-stray-v2";
+ let field = ".";
+ write_legacy_raw_field_fixture(&segment_dir, segment_id, field);
+ let raw_fst = legacy_field_sidecar_path(&segment_dir, segment_id, field, "fst").unwrap();
+ let encoded_fst = field_sidecar_path(&segment_dir, segment_id, field, "fst");
+ let encoded_meta = field_sidecar_path(&segment_dir, segment_id, field, "meta");
+ fs::copy(&raw_fst, &encoded_fst).unwrap();
+ fs::write(&encoded_meta, b"corrupt-stray-v2-meta").unwrap();
+
+ let full = FtsIndexReader::open(&segment_dir, segment_id, &[field]).unwrap();
+ assert!(full.term_exists(field, "legacyneedle"));
+ let stats = FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field]).unwrap();
+ assert!(stats.term_exists(field, "legacyneedle"));
+
+ // The raw v1 family does not make an invalid visible discriminator
+ // ignorable. Layout selection is discriminator-only and corruption
+ // fails closed before either family is opened.
+ let marker = segment_filename_layout_v2_marker_path(&segment_dir, segment_id);
+ fs::write(&marker, b"corrupt-layout-marker-over-raw-v1-family").unwrap();
+ assert!(FtsIndexReader::open(&segment_dir, segment_id, &[field]).is_err());
+ assert!(FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field]).is_err());
+ fs::remove_file(&marker).unwrap();
+
+ let raw_post = legacy_field_sidecar_path(&segment_dir, segment_id, field, "post").unwrap();
+ let raw_post_bytes = fs::read(&raw_post).unwrap();
+ fs::remove_file(&raw_post).unwrap();
+ assert!(FtsIndexReader::open(&segment_dir, segment_id, &[field]).is_err());
+ assert!(
+ FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field])
+ .unwrap()
+ .term_exists(field, "legacyneedle")
+ );
+ fs::write(&raw_post, raw_post_bytes).unwrap();
+
+ let raw_meta = legacy_field_sidecar_path(&segment_dir, segment_id, field, "meta").unwrap();
+ fs::write(&raw_meta, b"corrupt-selected-v1-meta").unwrap();
+ assert!(FtsIndexReader::open(&segment_dir, segment_id, &[field]).is_err());
+ assert!(FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field]).is_err());
+ }
+
+ #[test]
+ fn v2_discriminator_state_ignores_raw_strays_and_fails_closed_on_partial_or_corrupt_state() {
+ let root = TempDir::new().unwrap();
+ let segment_dir = root.path().join("segments");
+ let segment_id = "v2-partial";
+ let field = ".";
+ let mut writer = FtsIndexWriter::new(&segment_dir, segment_id, make_registry());
+ writer.add_document(
+ 0,
+ &[(field.to_owned(), "encodedneedle".to_owned())]
+ .into_iter()
+ .collect(),
+ );
+ writer.publish_encoded_filename_layout().unwrap();
+ writer.finish().unwrap();
+
+ // A corrupt complete raw family is a stray in explicit v2 state.
+ for extension in ["fst", "post", "meta", "norms"] {
+ let raw =
+ legacy_field_sidecar_path(&segment_dir, segment_id, field, extension).unwrap();
+ fs::write(raw, b"corrupt-raw-stray").unwrap();
+ }
+ assert!(FtsIndexReader::open(&segment_dir, segment_id, &[field])
+ .unwrap()
+ .term_exists(field, "encodedneedle"));
+ assert!(
+ FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field])
+ .unwrap()
+ .term_exists(field, "encodedneedle")
+ );
+
+ // Full mode needs postings and norms; stats-only intentionally does not.
+ let encoded_post = field_sidecar_path(&segment_dir, segment_id, field, "post");
+ let encoded_post_bytes = fs::read(&encoded_post).unwrap();
+ fs::remove_file(&encoded_post).unwrap();
+ assert!(FtsIndexReader::open(&segment_dir, segment_id, &[field]).is_err());
+ assert!(
+ FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field])
+ .unwrap()
+ .term_exists(field, "encodedneedle")
+ );
+ fs::write(&encoded_post, encoded_post_bytes).unwrap();
+
+ let encoded_meta = field_sidecar_path(&segment_dir, segment_id, field, "meta");
+ let encoded_meta_bytes = fs::read(&encoded_meta).unwrap();
+ fs::write(&encoded_meta, b"corrupt-selected-v2-meta").unwrap();
+ assert!(FtsIndexReader::open(&segment_dir, segment_id, &[field]).is_err());
+ assert!(FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field]).is_err());
+ fs::write(&encoded_meta, encoded_meta_bytes).unwrap();
+
+ fs::write(
+ segment_filename_layout_v2_marker_path(&segment_dir, segment_id),
+ b"corrupt-layout-marker",
+ )
+ .unwrap();
+ assert!(FtsIndexReader::open(&segment_dir, segment_id, &[field]).is_err());
+ assert!(FtsIndexReader::open_stats_only(&segment_dir, segment_id, &[field]).is_err());
+ }
+
+ #[test]
+ fn writer_reports_when_filename_v2_preflight_is_required() {
+ let dir = TempDir::new().unwrap();
+ let mut writer = FtsIndexWriter::new(dir.path(), "seg", make_registry());
+ writer.configure_field("title", FieldIndexConfig::default());
+ assert!(!writer.uses_encoded_field_filename_components());
+ writer.configure_field("bad/field", FieldIndexConfig::default());
+ assert!(writer.uses_encoded_field_filename_components());
+ assert!(writer.finish().is_err());
+ }
+
+ #[test]
+ fn unsafe_field_components_are_bounded_portable_and_distinct() {
+ let overlong = "a".repeat(MAX_LITERAL_FIELD_COMPONENT_BYTES + 1);
+ let encoded_alias = format!(
+ "{ENCODED_FIELD_COMPONENT_PREFIX}{:x}",
+ Sha256::digest(b"bad/field")
+ );
+ let fields = vec![
+ "bad/field",
+ r"bad\field",
+ "../traversal",
+ ".",
+ "..",
+ "trailing ",
+ "line\nfield",
+ "nul\0field",
+ "bad:windows*name?",
+ "CON",
+ "con.txt",
+ "PRN",
+ "aux.log",
+ "NUL",
+ "COM1",
+ "com9.log",
+ "LPT1",
+ "lpt9.txt",
+ ENCODED_FIELD_COMPONENT_PREFIX,
+ encoded_alias.as_str(),
+ overlong.as_str(),
+ ];
+ let components: Vec<String> = fields
+ .iter()
+ .map(|field| field_file_component(field).into_owned())
+ .collect();
+
+ assert_eq!(
+ components.len(),
+ components.iter().collect::<HashSet<_>>().len()
+ );
+ for (field, component) in fields.iter().zip(&components) {
+ assert_eq!(component.as_str(), field_file_component(field).as_ref());
+ assert!(component.starts_with(ENCODED_FIELD_COMPONENT_PREFIX));
+ assert!(component.is_ascii());
+ assert!(component
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_'));
+ assert!(component.len() < MAX_LITERAL_FIELD_COMPONENT_BYTES);
+ assert!(!component.contains('/'));
+ assert!(!component.contains('\\'));
+ }
+ assert_ne!(
+ field_file_component("bad/field"),
+ field_file_component(&encoded_alias),
+ "an encoded component used as a literal field must enter the reserved namespace"
+ );
+ for field in ["COM0", "COM10", "LPT0", "CONSOLE"] {
+ assert_eq!(field_file_component(field), Cow::Borrowed(field));
+ }
+ }
+
+ #[test]
+ fn unsafe_field_names_round_trip_without_creating_child_paths() {
+ let root = TempDir::new().unwrap();
+ let segment_dir = root.path().join("segments");
+ let overlong = "long".repeat(80);
+ let fields = [
+ "bad/field",
+ r"bad\field",
+ "../../escape",
+ "nul\0field",
+ "bad:windows*name?",
+ overlong.as_str(),
+ ];
+ for field in ["bad/field", "../../escape"] {
+ assert!(legacy_field_sidecar_path(&segment_dir, "seg-unsafe", field, "fst").is_none());
+ }
+ let mut writer = FtsIndexWriter::new(&segment_dir, "seg-unsafe", make_registry());
+ let document: HashMap<String, String> = fields
+ .iter()
+ .enumerate()
+ .map(|(index, field)| ((*field).to_owned(), format!("needle{index}")))
+ .collect();
+ writer.add_document(0, &document);
+ writer.publish_encoded_filename_layout().unwrap();
+ writer.finish().unwrap();
+
+ let field_refs: Vec<&str> = fields.to_vec();
+ let reader = FtsIndexReader::open(&segment_dir, "seg-unsafe", &field_refs).unwrap();
+ for (index, field) in fields.iter().enumerate() {
+ assert!(reader.term_exists(field, &format!("needle{index}")));
+ for extension in ["fst", "post", "meta", "norms"] {
+ let path = field_sidecar_path(&segment_dir, "seg-unsafe", field, extension);
+ assert_eq!(path.parent(), Some(segment_dir.as_path()));
+ assert!(path.is_file(), "missing {}", path.display());
+ }
+ assert!(!field_sidecar_path(&segment_dir, "seg-unsafe", field, "fst.tmp").exists());
+ }
+
+ let entries: Vec<_> = fs::read_dir(&segment_dir)
+ .unwrap()
+ .map(|entry| entry.unwrap())
+ .collect();
+ assert_eq!(entries.len(), fields.len() * 4 + 1);
+ assert!(entries
+ .iter()
+ .all(|entry| entry.file_type().unwrap().is_file()));
+ assert!(!root.path().join("escape").exists());
+ }
}
diff --git a/engine/crates/xerj-storage/Cargo.toml b/engine/crates/xerj-storage/Cargo.toml
index 4acb526ec..eeae1e44c 100644
--- a/engine/crates/xerj-storage/Cargo.toml
+++ b/engine/crates/xerj-storage/Cargo.toml
@@ -6,6 +6,10 @@ license.workspace = true
authors.workspace = true
description = "Storage engine for xerj: WAL, segment format, mmap reads, pluggable backends"
+[features]
+default = []
+test-hooks = []
+
[dependencies]
xerj-common = { path = "../xerj-common" }
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index e9d46950d..f8c3c7408 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -24,7 +24,7 @@
use arc_swap::ArcSwap;
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use tracing::{debug, info, instrument, warn};
use uuid::Uuid;
@@ -37,13 +37,30 @@ use crate::{Result, SeqNo, StorageError};
// ── Data-directory format marker (RC4 W3 #10) ─────────────────────────────────
-/// On-disk format version of the *data directory* as a whole — distinct from
-/// the per-segment `.seg` `FORMAT_VERSION` in `segment.rs`. Bumped only on an
-/// incompatible change to the data-dir layout / manifest. A binary refuses to
-/// open a data dir whose recorded marker version is greater than this, because
-/// opening data written in a format it does not understand risks silent
-/// corruption or loss.
-const DATA_DIR_FORMAT_VERSION: u32 = 1;
+/// Highest on-disk data-directory format this binary can read.
+///
+/// Version 2 reserves digest-derived FTS field filename components. A v2
+/// binary can read v1 raw side-cars, but a v1 binary must refuse a directory
+/// after the first v2 side-car can be created.
+const DATA_DIR_FORMAT_VERSION: u32 = 2;
+
+/// Baseline written for fresh and pre-marker directories.
+///
+/// Merely opening a directory with a newer binary must not make rollback
+/// impossible. The marker advances to v2 only at the durable preflight for a
+/// writer that will create an encoded FTS field filename.
+const DATA_DIR_BASE_FORMAT_VERSION: u32 = 1;
+
+/// First layout version that permits digest-derived FTS field components.
+const DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION: u32 = 2;
+
+#[cfg(any(test, feature = "test-hooks"))]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DataDirFormatWriteFailpoint {
+ BeforeTempWrite = 1,
+ BeforeRename = 2,
+ BeforeParentFsync = 3,
+}
/// Name of the format-marker file written at the data-dir root.
const DATA_DIR_META_FILE: &str = "xerj_meta.json";
@@ -437,6 +454,22 @@ pub struct IndexStore {
/// Root directory for this index's data files.
pub data_dir: PathBuf,
config: IndexStoreConfig,
+ /// Serializes monotonic data-directory marker advancement. The durable
+ /// writer itself is atomic, but without this latch concurrent future
+ /// version transitions could let a lower version overwrite a higher one.
+ data_dir_format_lock: Mutex<()>,
+ /// Process-local proof that the v2 marker has crossed the platform's
+ /// durable-publication boundary: parent-directory fsync on Unix, or
+ /// same-directory Win32 write-through replacement on Windows. Merely
+ /// observing v2 is insufficient: Unix may expose a rename before its
+ /// parent fsync, and an older Windows build may have used the non-durable
+ /// directory-sync shim. The format lock serializes the false -> true
+ /// transition so concurrent flush/merge retries cannot bypass confirmation.
+ fts_v2_marker_durability_confirmed: AtomicBool,
+ #[cfg(any(test, feature = "test-hooks"))]
+ data_dir_format_write_failpoint: std::sync::atomic::AtomicU8,
+ #[cfg(any(test, feature = "test-hooks"))]
+ fts_v2_marker_durability_confirmations: AtomicU64,
/// Actual memtable shard count, derived at open time from
/// `IndexStoreConfig.num_wal_shards.max(1).next_power_of_two()`.
num_shards: usize,
@@ -641,6 +674,12 @@ impl IndexStore {
let store = Arc::new(Self {
data_dir: data_dir.clone(),
config,
+ data_dir_format_lock: Mutex::new(()),
+ fts_v2_marker_durability_confirmed: AtomicBool::new(false),
+ #[cfg(any(test, feature = "test-hooks"))]
+ data_dir_format_write_failpoint: std::sync::atomic::AtomicU8::new(0),
+ #[cfg(any(test, feature = "test-hooks"))]
+ fts_v2_marker_durability_confirmations: AtomicU64::new(0),
num_shards,
snapshot: ArcSwap::from_pointee(snapshot),
wal_shards,
@@ -1173,7 +1212,7 @@ impl IndexStore {
return false;
}
let rest = &name[prefix.len()..];
- if matches!(rest, "seg" | "sidx" | "ids" | "dv") {
+ if matches!(rest, "seg" | "sidx" | "ids" | "dv" | "fts-layout-v2") {
if !roles.insert(rest) {
return false;
}
@@ -1317,7 +1356,8 @@ impl IndexStore {
/// Unlink every on-disk file belonging to the given segment ids — the
/// primary `.seg` plus all side-cars (`.sidx`, `.ids`, `.dv`,
- /// `.<field>.post` / `.fst` / `.meta` / `.norms`).
+ /// `.<field>.post` / `.fst` / `.meta` / `.norms`) and the optional
+ /// `.fts-layout-v2` discriminator.
///
/// Disk-space fix (2026-07): called by the engine merge task right
/// after `apply_merge` commits, so merged-away input segments are
@@ -3027,6 +3067,12 @@ impl IndexStore {
/// (a newer xerj wrote this dir; we may not understand its layout).
/// - Marker present but UNPARSEABLE → REFUSE (corrupt marker).
fn check_data_dir_version(data_dir: &Path) -> Result<()> {
+ Self::check_data_dir_version_with_max(data_dir, DATA_DIR_FORMAT_VERSION)
+ }
+
+ /// Version-parameterized form used by the compatibility regression to
+ /// exercise exactly what a v1 binary does when it sees a v2 marker.
+ fn check_data_dir_version_with_max(data_dir: &Path, max_supported: u32) -> Result<()> {
let path = Self::data_dir_meta_path(data_dir);
let bytes = match std::fs::read(&path) {
Ok(b) => b,
@@ -3041,14 +3087,14 @@ impl IndexStore {
path.display()
))
})?;
- if meta.format_version > DATA_DIR_FORMAT_VERSION {
+ if meta.format_version > max_supported {
return Err(StorageError::IncompatibleDataDir(format!(
"data dir {} was written by a newer xerj (data-dir format \
version {}); this binary supports up to {}. Refusing to open \
— upgrade xerj to a build that understands format {}.",
data_dir.display(),
meta.format_version,
- DATA_DIR_FORMAT_VERSION,
+ max_supported,
meta.format_version
)));
}
@@ -3067,7 +3113,7 @@ impl IndexStore {
return Ok(());
}
let meta = DataDirMeta {
- format_version: DATA_DIR_FORMAT_VERSION,
+ format_version: DATA_DIR_BASE_FORMAT_VERSION,
xerj_version: env!("CARGO_PKG_VERSION").to_string(),
};
let bytes = serde_json::to_vec(&meta)?;
@@ -3077,6 +3123,158 @@ impl IndexStore {
Ok(())
}
+ /// Durably fence a writer before it creates digest-derived FTS side-cars.
+ ///
+ /// The marker reaches disk before the first v2 filename. A v1 process
+ /// therefore either sees a v1-only directory or refuses the complete v2
+ /// directory; it can never open v2 filenames as if they were v1. The
+ /// transition is intentionally monotonic: if later segment publication
+ /// fails, retaining v2 is the safe crash/retry outcome.
+ ///
+ /// Product-level cross-process exclusion is provided by Engine's
+ /// `<server-data-dir>/node.lock`. The mutex here covers concurrent flush
+ /// and merge callers inside that one supported writer process; opening
+ /// multiple raw `IndexStore`s on the same directory remains unsupported.
+ pub fn ensure_fts_encoded_field_component_format(&self) -> Result<()> {
+ let _format_guard = self.data_dir_format_lock.lock().unwrap();
+ if self
+ .fts_v2_marker_durability_confirmed
+ .load(Ordering::Acquire)
+ {
+ return Ok(());
+ }
+ let path = Self::data_dir_meta_path(&self.data_dir);
+ let mut document = match std::fs::read(&path) {
+ Ok(bytes) => Some(serde_json::from_slice::<serde_json::Value>(&bytes).map_err(
+ |error| {
+ StorageError::IncompatibleDataDir(format!(
+ "data-dir format marker {} is present but unparseable ({error}). Refusing to write encoded FTS side-cars.",
+ path.display()
+ ))
+ },
+ )?),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
+ Err(error) => return Err(error.into()),
+ };
+
+ if let Some(value) = document.as_ref() {
+ let meta: DataDirMeta = serde_json::from_value(value.clone()).map_err(|error| {
+ StorageError::IncompatibleDataDir(format!(
+ "data-dir format marker {} has an incompatible shape ({error}). Refusing to write encoded FTS side-cars.",
+ path.display()
+ ))
+ })?;
+ if meta.format_version > DATA_DIR_FORMAT_VERSION {
+ return Err(StorageError::IncompatibleDataDir(format!(
+ "data dir {} records unsupported format version {}; this binary supports up to {}",
+ self.data_dir.display(),
+ meta.format_version,
+ DATA_DIR_FORMAT_VERSION
+ )));
+ }
+ if meta.format_version >= DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION {
+ // Do not trust visibility as proof of durability. A previous
+ // attempt may have renamed v2 into place and then failed its
+ // durability confirmation. Re-establish that proof now, while
+ // still holding the transition lock, before permitting any
+ // encoded FTS filename to be created.
+ #[cfg(not(windows))]
+ if let Some(parent) = path.parent() {
+ xerj_common::fsio::fsync_dir(parent)?;
+ }
+ // Windows has no parent-directory fsync contract. Rewrite the
+ // exact visible marker through the Win32 write-through replace
+ // path instead. Besides confirming current writes, this safely
+ // upgrades a marker left visible by an older build whose
+ // Windows directory-sync shim was a no-op.
+ #[cfg(windows)]
+ xerj_common::fsio::write_file_durable(&path, &std::fs::read(&path)?)?;
+ self.note_fts_v2_marker_durability_confirmed();
+ return Ok(());
+ }
+ }
+
+ let document = document.get_or_insert_with(|| serde_json::json!({}));
+ let object = document.as_object_mut().ok_or_else(|| {
+ StorageError::IncompatibleDataDir(format!(
+ "data-dir format marker {} must be a JSON object",
+ path.display()
+ ))
+ })?;
+ object.insert(
+ "format_version".to_string(),
+ serde_json::Value::from(DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION),
+ );
+ object.insert(
+ "xerj_version".to_string(),
+ serde_json::Value::from(env!("CARGO_PKG_VERSION")),
+ );
+ let bytes = serde_json::to_vec(document)?;
+ #[cfg(any(test, feature = "test-hooks"))]
+ {
+ let failpoint = self
+ .data_dir_format_write_failpoint
+ .swap(0, Ordering::AcqRel);
+ xerj_common::fsio::write_file_durable_with_hook(&path, &bytes, |stage| {
+ use xerj_common::fsio::DurableWriteStage;
+ let injected_stage = match failpoint {
+ x if x == DataDirFormatWriteFailpoint::BeforeTempWrite as u8 => {
+ Some(DurableWriteStage::BeforeTempWrite)
+ }
+ x if x == DataDirFormatWriteFailpoint::BeforeRename as u8 => {
+ Some(DurableWriteStage::BeforeRename)
+ }
+ x if x == DataDirFormatWriteFailpoint::BeforeParentFsync as u8 => {
+ Some(DurableWriteStage::BeforeParentFsync)
+ }
+ _ => None,
+ };
+ if injected_stage == Some(stage) {
+ Err(std::io::Error::other(
+ "injected data-directory marker persistence failure",
+ ))
+ } else {
+ Ok(())
+ }
+ })?;
+ }
+ #[cfg(not(any(test, feature = "test-hooks")))]
+ xerj_common::fsio::write_file_durable(&path, &bytes)?;
+ self.note_fts_v2_marker_durability_confirmed();
+ Ok(())
+ }
+
+ fn note_fts_v2_marker_durability_confirmed(&self) {
+ #[cfg(any(test, feature = "test-hooks"))]
+ self.fts_v2_marker_durability_confirmations
+ .fetch_add(1, Ordering::AcqRel);
+ self.fts_v2_marker_durability_confirmed
+ .store(true, Ordering::Release);
+ }
+
+ #[cfg(any(test, feature = "test-hooks"))]
+ pub fn set_data_dir_format_write_failpoint_for_test(
+ &self,
+ failpoint: DataDirFormatWriteFailpoint,
+ ) {
+ self.data_dir_format_write_failpoint
+ .store(failpoint as u8, Ordering::Release);
+ }
+
+ #[cfg(any(test, feature = "test-hooks"))]
+ pub fn fts_v2_marker_durability_confirmations_for_test(&self) -> u64 {
+ self.fts_v2_marker_durability_confirmations
+ .load(Ordering::Acquire)
+ }
+
+ #[cfg(any(test, feature = "test-hooks"))]
+ pub fn clear_fts_v2_marker_durability_confirmation_for_test(&self) {
+ self.fts_v2_marker_durability_confirmed
+ .store(false, Ordering::Release);
+ self.fts_v2_marker_durability_confirmations
+ .store(0, Ordering::Release);
+ }
+
// ── Segment version map rebuild ───────────────────────────────────────────
/// Rebuild the version map from all flushed segments on disk.
@@ -4136,6 +4334,48 @@ mod tests {
assert!(!complete.exists(), "orphan cleanup must remove `.complete`");
}
+ #[test]
+ fn fts_layout_discriminator_is_manifested_and_removed_with_its_segment() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ // `delete_segment_files` intentionally recognizes only the engine's
+ // canonical UUID-shaped segment prefix. Use a real-shaped id so this
+ // assertion exercises production cleanup rather than a no-op fixture.
+ let segment_id = "12345678-1234-1234-1234-123456789abc";
+ let segments = dir.path().join("segments");
+ for (role, bytes) in [
+ ("seg", b"segment".as_slice()),
+ ("sidx", b"sidx".as_slice()),
+ ("ids", b"ids".as_slice()),
+ ("fts-layout-v2", b"XERJ_FTS_FILENAME_LAYOUT_V2\n".as_slice()),
+ ] {
+ std::fs::write(segments.join(format!("{segment_id}.{role}")), bytes).unwrap();
+ }
+ let meta = SegmentMeta {
+ id: segment_id.to_owned(),
+ doc_count: 1,
+ size_bytes: 7,
+ min_seq_no: 1,
+ max_seq_no: 1,
+ created_at_ms: 0,
+ has_tombstones: false,
+ seg_path: format!("{segment_id}.seg"),
+ sidx_path: format!("{segment_id}.sidx"),
+ };
+ store.write_flush_completion_manifest(&meta).unwrap();
+ assert!(store.validate_flush_completion_manifest(segment_id, 1, 1, 1));
+
+ let (removed, _) = store.delete_segment_files(&[segment_id.to_owned()]);
+ assert_eq!(removed, 5, "segment family includes the discriminator");
+ assert!(std::fs::read_dir(&segments)
+ .unwrap()
+ .flatten()
+ .all(|entry| !entry
+ .file_name()
+ .to_string_lossy()
+ .starts_with(&format!("{segment_id}."))));
+ }
+
#[test]
fn artifact_crc_streams_large_inputs_in_bounded_chunks() {
struct TrackingReader {
@@ -5540,7 +5780,152 @@ mod tests {
assert!(marker.exists(), "open() must stamp {DATA_DIR_META_FILE}");
let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
- assert_eq!(meta.format_version, DATA_DIR_FORMAT_VERSION);
+ assert_eq!(meta.format_version, DATA_DIR_BASE_FORMAT_VERSION);
+ }
+
+ #[test]
+ fn ordinary_reopen_does_not_advance_the_baseline_format_marker() {
+ let dir = tempfile::tempdir().unwrap();
+ drop(open_test_store(dir.path()));
+ drop(open_test_store(dir.path()));
+
+ let marker = dir.path().join(DATA_DIR_META_FILE);
+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(marker).unwrap()).unwrap();
+ assert_eq!(meta.format_version, DATA_DIR_BASE_FORMAT_VERSION);
+ }
+
+ #[test]
+ fn encoded_fts_preflight_advances_marker_and_v1_refuses_it() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ let marker = dir.path().join(DATA_DIR_META_FILE);
+ let mut before: serde_json::Value =
+ serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ before["future_provenance"] = serde_json::json!({"keep": true});
+ std::fs::write(&marker, serde_json::to_vec(&before).unwrap()).unwrap();
+
+ store.ensure_fts_encoded_field_component_format().unwrap();
+ // The transition is idempotent and may be called by concurrent flush
+ // or merge preflights without changing its result.
+ store.ensure_fts_encoded_field_component_format().unwrap();
+
+ let marker_value: serde_json::Value =
+ serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ let meta: DataDirMeta = serde_json::from_value(marker_value.clone()).unwrap();
+ assert_eq!(
+ meta.format_version,
+ DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION
+ );
+ assert_eq!(marker_value["future_provenance"]["keep"], true);
+ assert!(IndexStore::check_data_dir_version_with_max(
+ dir.path(),
+ DATA_DIR_BASE_FORMAT_VERSION
+ )
+ .is_err());
+ assert!(IndexStore::check_data_dir_version(dir.path()).is_ok());
+
+ drop(store);
+ drop(open_test_store(dir.path()));
+ }
+
+ #[test]
+ fn failed_encoded_fts_marker_persistence_is_conservative_at_every_boundary() {
+ for (failpoint, expected_version) in [
+ (DataDirFormatWriteFailpoint::BeforeTempWrite, 1),
+ (DataDirFormatWriteFailpoint::BeforeRename, 1),
+ (DataDirFormatWriteFailpoint::BeforeParentFsync, 2),
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ store.set_data_dir_format_write_failpoint_for_test(failpoint);
+
+ assert!(store.ensure_fts_encoded_field_component_format().is_err());
+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 0);
+ let marker = dir.path().join(DATA_DIR_META_FILE);
+ let meta: DataDirMeta =
+ serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ assert_eq!(meta.format_version, expected_version, "{failpoint:?}");
+ assert!(IndexStore::check_data_dir_version(dir.path()).is_ok());
+ if expected_version == 2 {
+ assert!(IndexStore::check_data_dir_version_with_max(dir.path(), 1).is_err());
+ }
+
+ // In the post-rename case the v2 bytes are already visible, but
+ // retry must not treat visibility as durability. It crosses one
+ // platform-appropriate durable replacement confirmation before
+ // the store records success.
+ store.ensure_fts_encoded_field_component_format().unwrap();
+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 1);
+ store.ensure_fts_encoded_field_component_format().unwrap();
+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 1);
+ }
+ }
+
+ #[test]
+ fn concurrent_encoded_fts_preflights_are_idempotent() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ let barrier = Arc::new(std::sync::Barrier::new(9));
+ let mut workers = Vec::new();
+ for _ in 0..8 {
+ let store = Arc::clone(&store);
+ let barrier = Arc::clone(&barrier);
+ workers.push(std::thread::spawn(move || {
+ barrier.wait();
+ store.ensure_fts_encoded_field_component_format()
+ }));
+ }
+ barrier.wait();
+ for worker in workers {
+ worker.join().unwrap().unwrap();
+ }
+
+ let marker = dir.path().join(DATA_DIR_META_FILE);
+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(marker).unwrap()).unwrap();
+ assert_eq!(
+ meta.format_version,
+ DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION
+ );
+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 1);
+ }
+
+ #[test]
+ fn concurrent_retries_confirm_visible_unconfirmed_v2_marker_once() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ store.set_data_dir_format_write_failpoint_for_test(
+ DataDirFormatWriteFailpoint::BeforeParentFsync,
+ );
+ assert!(store.ensure_fts_encoded_field_component_format().is_err());
+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 0);
+
+ let marker = dir.path().join(DATA_DIR_META_FILE);
+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ assert_eq!(
+ meta.format_version, DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION,
+ "the injected post-rename failure must leave v2 visible but unconfirmed"
+ );
+
+ let barrier = Arc::new(std::sync::Barrier::new(9));
+ let mut workers = Vec::new();
+ for _ in 0..8 {
+ let store = Arc::clone(&store);
+ let barrier = Arc::clone(&barrier);
+ workers.push(std::thread::spawn(move || {
+ barrier.wait();
+ store.ensure_fts_encoded_field_component_format()
+ }));
+ }
+ barrier.wait();
+ for worker in workers {
+ worker.join().unwrap().unwrap();
+ }
+
+ assert_eq!(
+ store.fts_v2_marker_durability_confirmations_for_test(),
+ 1,
+ "the format mutex must serialize visible-marker durability confirmation"
+ );
}
/// THE UPGRADE PATH: an rc3-vintage data dir has NO format marker (rc3