AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(fts): keep case-aliased field sidecars from aborting publish (#2261)
xerj-org8920c06ce590
diff --git a/engine/crates/xerj-common/src/fsio.rs b/engine/crates/xerj-common/src/fsio.rsindex d408786b7..ad023928d 100644--- a/engine/crates/xerj-common/src/fsio.rs+++ b/engine/crates/xerj-common/src/fsio.rs@@ -151,9 +151,16 @@ pub fn stage_written_file(/// never torn, exactly as per-file durable writes did. What changes is the/// cost: two device flushes for the batch instead of two per file, which on/// Apple targets is the difference between ~250 cache drains per segment and-/// two. If a rename fails, the files already moved stay moved and the rest are-/// removed, which is also what a failure part-way through a loop of-/// [`write_file_durable`] calls left behind.+/// two.+///+/// If a rename fails, destinations already replaced by *this* call are+/// removed and the remaining temps are dropped. FTS side-cars are new files+/// under a fresh segment id, so that rollback leaves the directory empty of+/// this batch instead of a half-published family. A later directory fsync+/// failure does not roll those names back: the bytes are already complete,+/// and removing them would turn a durability-confirmation error into missing+/// files. Callers that replace a pre-existing file they still need must not+/// use this batch helper.pub fn publish_staged(dir: &Path, files: Vec<StagedFile>) -> std::io::Result<()> {if files.is_empty() {return Ok(());@@ -166,15 +173,32 @@ pub fn publish_staged(dir: &Path, files: Vec<StagedFile>) -> std::io::Result<()>}#[cfg(target_vendor = "apple")]sync_file_full(&std::fs::File::open(dir)?)?;+ let mut published_dests = Vec::with_capacity(files.len());for mut file in files {- replace_file_for_durable_write(&file.tmp, &file.dest)?;+ if let Err(error) = replace_file_for_durable_write(&file.tmp, &file.dest) {+ for dest in &published_dests {+ let _ = std::fs::remove_file(dest);+ }+ return Err(error);+ }file.published = true;+ published_dests.push(file.dest.clone());}#[cfg(not(windows))]fsync_dir(dir)?;Ok(())}+/// Windows `MoveFileExW` returns these when a scanner still has the temp+/// open at the moment of the replace. They are transient for the FTS publish+/// path (the handle is ours and already closed). `ERROR_FILE_NOT_FOUND` (2)+/// is not: retrying it hides a missing temp.+#[cfg(any(windows, test))]+fn replace_lock_should_retry(os_code: Option<i32>, retries_already: u32) -> bool {+ const REPLACE_LOCK_RETRIES: u32 = 6;+ retries_already < REPLACE_LOCK_RETRIES && matches!(os_code, Some(5 | 32 | 33))+}+/// Windows has no supported directory-flush primitive to call here:/// `File::open` on a/// directory fails with `ERROR_ACCESS_DENIED` (os error 5) because std@@ -272,20 +296,34 @@ fn replace_file_for_durable_write(source: &Path, destination: &Path) -> std::io:let source = wide_path(source)?;let destination = wide_path(destination)?;- // SAFETY: both buffers are NUL-terminated and live through the call.- // The flags request replacement plus synchronous persistence; no- // COPY_ALLOWED flag is present, and our temp file is same-directory.- let succeeded = unsafe {- MoveFileExW(- source.as_ptr(),- destination.as_ptr(),- MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,- )- };- if succeeded == 0 {- Err(std::io::Error::last_os_error())- } else {- Ok(())+ let mut retries_already = 0u32;+ loop {+ // SAFETY: both buffers are NUL-terminated and live through the call.+ // The flags request replacement plus synchronous persistence; no+ // COPY_ALLOWED flag is present, and our temp file is same-directory.+ let succeeded = unsafe {+ MoveFileExW(+ source.as_ptr(),+ destination.as_ptr(),+ MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,+ )+ };+ if succeeded != 0 {+ return Ok(());+ }+ let error = std::io::Error::last_os_error();+ if replace_lock_should_retry(error.raw_os_error(), retries_already) {+ retries_already += 1;+ // A filter driver (Defender, DLP) often locks a just-closed file+ // for a scan. Short backoff covers that without turning a+ // permanent access error into a long stall: six waits are+ // 10+20+…+60 ms.+ std::thread::sleep(std::time::Duration::from_millis(+ 10 * u64::from(retries_already),+ ));+ continue;+ }+ return Err(error);}}@@ -467,6 +505,46 @@ mod tests {assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);}+ #[test]+ fn replace_lock_retries_sharing_violations_but_not_a_missing_temp() {+ assert!(replace_lock_should_retry(Some(5), 0));+ assert!(replace_lock_should_retry(Some(32), 5));+ assert!(replace_lock_should_retry(Some(33), 0));+ assert!(!replace_lock_should_retry(Some(32), 6));+ assert!(!replace_lock_should_retry(Some(2), 0));+ assert!(!replace_lock_should_retry(None, 0));+ }++ #[test]+ fn publish_staged_rolls_back_a_partial_rename_and_a_later_publish_succeeds() {+ let dir = tempfile::tempdir().unwrap();+ let published = dir.path().join("a.post");+ let blocked = dir.path().join("b.meta");+ // A directory sitting on the destination makes the second replace fail+ // after the first has already moved into place.+ std::fs::create_dir(&blocked).unwrap();+ let staged = vec![+ stage_file(&published, b"alpha").unwrap(),+ stage_file(&blocked, b"beta").unwrap(),+ ];+ let error = publish_staged(dir.path(), staged).unwrap_err();+ assert_ne!(error.kind(), std::io::ErrorKind::InvalidInput);+ assert!(+ !published.exists(),+ "a dest renamed earlier in the batch must be removed"+ );+ assert!(blocked.is_dir());+ let names: Vec<_> = std::fs::read_dir(dir.path())+ .unwrap()+ .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())+ .collect();+ assert_eq!(names, vec!["b.meta".to_string()]);++ let staged = vec![stage_file(&published, b"alpha-again").unwrap()];+ publish_staged(dir.path(), staged).unwrap();+ assert_eq!(std::fs::read(&published).unwrap(), b"alpha-again");+ }+#[test]fn publish_staged_rejects_a_file_outside_the_directory() {let dir = tempfile::tempdir().unwrap();diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex f628d47c6..7c897dc3f 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2059,6 +2059,104 @@ mod flush_publication_recovery_tests {assert_eq!(result.hits[0].id, "survivor");}+ /// Issue #2254. `page_Id` is a portable literal under filename layout v2,+ /// so on NTFS it is the same file as `page_id`. The flush must publish+ /// layout v3, keep both fields searchable, and still open after restart.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn mixed_case_field_flush_publishes_v3_and_restarts_searchable() {+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config.clone()).unwrap();+ let mut schema = Schema::empty();+ schema+ .add_field(FieldConfig::new("page_Id", FieldType::Text))+ .unwrap();+ schema+ .add_field(FieldConfig::new("page_id", FieldType::Text))+ .unwrap();+ schema+ .add_field(FieldConfig::new("component", FieldType::Text))+ .unwrap();+ engine.create_index("case-field-publish", schema).unwrap();+ let idx = engine.get_index("case-field-publish").unwrap();+ idx.abort_background_tasks();+ assert_eq!(data_dir_format_version(&idx), 1);+ idx.index_document(+ Some("jbar".into()),+ json!({+ "page_Id": "1212561321588641792",+ "page_id": "other-page",+ "component": "JBar3d"+ }),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("jimg".into()),+ json!({"component": "JImg", "page_id": "img-only"}),+ )+ .await+ .unwrap();++ idx.flush().await.unwrap();+ assert_eq!(data_dir_format_version(&idx), 3);+ assert_eq!(idx.memtable.doc_count(), 0);+ let segment_names: Vec<String> = std::fs::read_dir(idx.data_dir.join("segments"))+ .unwrap()+ .flatten()+ .map(|entry| entry.file_name().to_string_lossy().into_owned())+ .collect();+ assert!(+ segment_names+ .iter()+ .any(|name| name.ends_with(".fts-layout-v3")),+ "{segment_names:?}"+ );+ assert!(segment_names+ .iter()+ .all(|name| !name.ends_with(".fts-layout-v2")));+ let mut case_keys = std::collections::HashSet::new();+ for name in segment_names.iter().filter(|name| name.ends_with(".fst")) {+ assert!(+ case_keys.insert(name.to_ascii_lowercase()),+ "case-folded FTS sidecar collision: {name}"+ );+ }++ let request = xerj_query::parse_request(&json!({+ "query": {"match": {"component": "JBar3d"}},+ "size": 10,+ "track_total_hits": true+ }))+ .unwrap();+ let result = idx.search(&request).await.unwrap();+ assert_eq!(result.total.value, 1);+ assert_eq!(result.hits[0].id, "jbar");+ let by_page = xerj_query::parse_request(&json!({+ "query": {"match": {"page_Id": "1212561321588641792"}},+ "size": 10+ }))+ .unwrap();+ let result = idx.search(&by_page).await.unwrap();+ assert_eq!(result.hits.len(), 1);+ assert_eq!(result.hits[0].id, "jbar");+ drop(idx);+ drop(engine);++ let restarted = crate::Engine::new(config).unwrap();+ let idx = restarted.get_index("case-field-publish").unwrap();+ idx.abort_background_tasks();+ assert_eq!(data_dir_format_version(&idx), 3);+ let result = idx.search(&request).await.unwrap();+ assert_eq!(result.total.value, 1);+ assert_eq!(result.hits[0].id, "jbar");+ let result = idx.search(&by_page).await.unwrap();+ assert_eq!(result.hits.len(), 1);+ assert_eq!(result.hits[0].id, "jbar");+ }+/// Build a one-date-field index whose `ts` field carries `precision`.async fn date_index(engine: &crate::Engine,@@ -13848,7 +13946,26 @@ impl Index {fts_writer.add_documents_parallel(&fts_input)}}- if fts_writer.uses_encoded_field_filename_components() {+ if fts_writer+ .uses_case_distinguished_field_filename_components()+ {+ if let Err(e) = store_for_task+ .ensure_fts_case_distinguished_field_component_format()+ {+ tracing::warn!(+ "merge: case-distinguished FTS filename format preflight failed: {e:#}"+ );+ return Ok(());+ }+ if let Err(e) = fts_writer+ .publish_case_distinguished_filename_layout()+ {+ tracing::warn!(+ "merge: FTS filename-layout publication failed: {e:#}"+ );+ return Ok(());+ }+ } else if fts_writer.uses_encoded_field_filename_components() {if let Err(e) = store_for_task.ensure_fts_encoded_field_component_format(){@@ -13867,7 +13984,7 @@ impl Index {}}if let Err(e) = fts_writer.finish() {- tracing::warn!("merge: FTS build failed: {e}");+ tracing::warn!("merge: FTS build failed: {e:#}");}Ok(())});@@ -34592,19 +34709,28 @@ async fn do_flush_shard(fts_writer.add_documents_parallel(&drained_fts_for_build);fts_add_us = t_add.elapsed().as_micros();let t_fin = std::time::Instant::now();- if fts_writer.uses_encoded_field_filename_components() {+ if fts_writer.uses_case_distinguished_field_filename_components() {+ store_for_warm.ensure_fts_case_distinguished_field_component_format()?;+ fts_writer+ .publish_case_distinguished_filename_layout()+ .map_err(|error| {+ xerj_storage::StorageError::Io(std::io::Error::other(format!(+ "FTS filename-layout publication failed: {error:#}"+ )))+ })?;+ } else if fts_writer.uses_encoded_field_filename_components() {store_for_warm.ensure_fts_encoded_field_component_format()?;fts_writer.publish_encoded_filename_layout().map_err(|error| {xerj_storage::StorageError::Io(std::io::Error::other(format!(- "FTS filename-layout publication failed: {error}"+ "FTS filename-layout publication failed: {error:#}")))})?;}fts_writer.finish().map_err(|error| {xerj_storage::StorageError::Io(std::io::Error::other(format!(- "FTS side-car build failed: {error}"+ "FTS side-car build failed: {error:#}")))})?;fts_finish_us = t_fin.elapsed().as_micros();diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rsindex e13939033..43b8ca36c 100644--- a/engine/crates/xerj-fts/src/index.rs+++ b/engine/crates/xerj-fts/src/index.rs@@ -11,10 +11,14 @@//! seg-<id>.<field-component>.meta — FieldStats + per-term metadata//! seg-<id>.<field-component>.norms — encoded per-doc field lengths//! seg-<id>.fts-layout-v2 — exact marker when components are encoded+//! seg-<id>.fts-layout-v3 — exact marker when a literal would+//! case-fold into another field on NTFS//! ```//!//! Short portable field names are used literally for on-disk compatibility;-//! all other logical field names map to a bounded digest component.+//! all other logical field names map to a bounded digest component. Layout v3+//! also digests a name whose Windows case-identity is not the name itself+//! (`page_Id`, `ı`, `ß`), so two fields cannot share one side-car file.//!//! The FST key is the term text (UTF-8 bytes, lexicographically sorted by construction).//! The FST output value is the byte offset in the `.post` file.@@ -61,25 +65,78 @@ const ENCODED_FIELD_COMPONENT_PREFIX: &str = "__xerj_fts_field_sha256_";const FTS_FILENAME_LAYOUT_V2_MARKER_SUFFIX: &str = "fts-layout-v2";const FTS_FILENAME_LAYOUT_V2_MARKER_BYTES: &[u8] = b"XERJ_FTS_FILENAME_LAYOUT_V2\n";+/// Layout v3. Same digest function as v2 for names v2 already hashed. Names+/// v2 kept as literals, but which NTFS case-folds onto a different literal,+/// are hashed too. The marker is the only authority: a v2 segment that stored+/// `page_Id` literally must still open as v2.+const FTS_FILENAME_LAYOUT_V3_MARKER_SUFFIX: &str = "fts-layout-v3";+const FTS_FILENAME_LAYOUT_V3_MARKER_BYTES: &[u8] = b"XERJ_FTS_FILENAME_LAYOUT_V3\n";++/// Which filename function a segment's side-cars were written with.+///+/// Chosen from the marker file alone. Existence of a side-car is never+/// evidence: a v1 literal can alias a v2 digest component.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum FilenameLayout {+ /// No marker. Paths are the raw field name when that stays inside the+ /// segment directory.+ Legacy,+ /// `fts-layout-v2`. Unsafe names are SHA-256 components; mixed-case+ /// portable names stay literal.+ Encoded,+ /// `fts-layout-v3`. v2's digests, plus a digest for any remaining literal+ /// whose Windows case-identity is not the field name itself.+ CaseDistinguished,+}+fn segment_filename_layout_v2_marker_path(segment_dir: &Path, segment_id: &str) -> PathBuf {segment_dir.join(format!("{segment_id}.{FTS_FILENAME_LAYOUT_V2_MARKER_SUFFIX}"))}-/// Absence means v1/raw. Exact magic means v2/encoded. Any other visible-/// marker is corruption. File-family existence is never layout evidence.-fn segment_uses_encoded_filename_layout(segment_dir: &Path, segment_id: &str) -> Result<bool> {- let marker = segment_filename_layout_v2_marker_path(segment_dir, segment_id);- match fs::read(&marker) {- Ok(bytes) if bytes == FTS_FILENAME_LAYOUT_V2_MARKER_BYTES => Ok(true),- Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", marker),- Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),+fn segment_filename_layout_v3_marker_path(segment_dir: &Path, segment_id: &str) -> PathBuf {+ segment_dir.join(format!(+ "{segment_id}.{FTS_FILENAME_LAYOUT_V3_MARKER_SUFFIX}"+ ))+}++enum LayoutMarkerRead {+ Absent,+ Exact,+}++fn read_layout_marker(path: &Path, expected: &[u8]) -> Result<LayoutMarkerRead> {+ match fs::read(path) {+ Ok(bytes) if bytes == expected => Ok(LayoutMarkerRead::Exact),+ Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", path),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(LayoutMarkerRead::Absent),Err(error) => Err(error)- .with_context(|| format!("reading FTS filename-layout discriminator {:?}", marker)),+ .with_context(|| format!("reading FTS filename-layout discriminator {:?}", path)),}}+/// v3 wins when its marker is present. A corrupt marker fails closed and does+/// not fall through to an older layout. File-family existence is never+/// layout evidence.+fn segment_filename_layout(segment_dir: &Path, segment_id: &str) -> Result<FilenameLayout> {+ let v3 = segment_filename_layout_v3_marker_path(segment_dir, segment_id);+ if matches!(+ read_layout_marker(&v3, FTS_FILENAME_LAYOUT_V3_MARKER_BYTES)?,+ LayoutMarkerRead::Exact+ ) {+ return Ok(FilenameLayout::CaseDistinguished);+ }+ let v2 = segment_filename_layout_v2_marker_path(segment_dir, segment_id);+ if matches!(+ read_layout_marker(&v2, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)?,+ LayoutMarkerRead::Exact+ ) {+ return Ok(FilenameLayout::Encoded);+ }+ Ok(FilenameLayout::Legacy)+}+/// Per-segment record of the analyzer each field's postings were built with/// (#937). Written by [`FtsIndexWriter::finish`] as JSON/// `{"fields":{"<field>":"<analyzer>"}}`, but ONLY when some field was@@ -143,13 +200,22 @@ fn is_windows_reserved_device_name(field: &str) -> bool {/// and overlong names can never create a child path or exceed a normal/// filesystem's component limit. Unicode, `@`, and internal spaces are valid/// portable literals; trailing spaces and dots are not portable to Windows.-/// Preserving literals also preserves v1's pre-existing limitation: distinct-/// portable names can alias on a filesystem that case-folds or normalizes-/// Unicode. Digest-derived names remain distinct by exact UTF-8 bytes modulo a-/// SHA-256 collision; eliminating literal aliases would require a new all-name-/// encoding or name table and is outside filename layout v2.+/// Preserving literals also preserves a limitation layout v2 left in place:+/// distinct portable names can alias on a filesystem that case-folds.+/// Layout v3 digests any literal whose Windows case-identity is not the name+/// itself, so `page_Id` and `page_id` (and the NTFS pairs `ı`/`i`, `ß`/`ss`,+/// `ſ`/`s`) cannot share a file. Digest-derived names remain distinct by+/// exact UTF-8 bytes modulo a SHA-256 collision. Unicode normalization+/// (NFC versus NFD) is still not encoded.fn field_file_component(field: &str) -> Cow<'_, str> {- let portable_literal = !field.is_empty()+ if portable_literal_field(field) {+ return Cow::Borrowed(field);+ }+ Cow::Owned(digest_field_component(field))+}++fn portable_literal_field(field: &str) -> bool {+ !field.is_empty()&& field.len() <= MAX_LITERAL_FIELD_COMPONENT_BYTES&& field != "."&& field != ".."@@ -163,14 +229,88 @@ fn field_file_component(field: &str) -> Cow<'_, str> {character,'/' | '\\' | '<' | '>' | ':' | '"' | '|' | '?' | '*')- });+ })+}- if portable_literal {- return Cow::Borrowed(field);+fn digest_field_component(field: &str) -> String {+ let digest = Sha256::digest(field.as_bytes());+ format!("{ENCODED_FIELD_COMPONENT_PREFIX}{digest:x}")+}++/// Layout v3 component. Unsafe names hash exactly as in v2 (the digest is of+/// the field's UTF-8 bytes, not of the v2 component). A portable literal is+/// hashed only when NTFS would treat it as the same file as a different+/// spelling.+fn field_file_component_v3(field: &str) -> Cow<'_, str> {+ if field_needs_case_distinguished_component(field) {+ Cow::Owned(digest_field_component(field))+ } else {+ field_file_component(field)}+}- let digest = Sha256::digest(field.as_bytes());- Cow::Owned(format!("{ENCODED_FIELD_COMPONENT_PREFIX}{digest:x}"))+/// `true` when v2 would keep `field` as a literal AND that literal's Windows+/// case-identity is a different string.+///+/// The identity is Unicode lowercase plus the NTFS upcase-table exceptions+/// that lowercase does not collapse: U+0131 `ı` → `i`, U+017F `ſ` → `s`,+/// U+212A `K` → `k`, U+00DF `ß` → `ss`. A name that is already that identity+/// (`page_id`, `i`, `ss`) stays a literal. Every other spelling of the same+/// identity is a digest, so the two cannot land on one NTFS file. Hashing the+/// exact bytes also keeps `Page` and `PAGE` apart from each other.+fn field_needs_case_distinguished_component(field: &str) -> bool {+ portable_literal_field(field) && !windows_case_identity_is_self(field)+}++fn windows_case_identity_is_self(field: &str) -> bool {+ if field.is_ascii() {+ return !field.bytes().any(|byte| byte.is_ascii_uppercase());+ }+ windows_case_identity(field) == field+}++fn windows_case_identity(field: &str) -> String {+ let mut identity = String::with_capacity(field.len());+ for character in field.chars() {+ push_windows_case_folded(character, &mut identity);+ }+ identity+}++fn push_windows_case_folded(character: char, out: &mut String) {+ match character {+ '\u{0131}' => out.push('i'),+ '\u{017f}' => out.push('s'),+ '\u{212a}' => out.push('k'),+ '\u{00df}' => out.push_str("ss"),+ other => {+ for lower in other.to_lowercase() {+ match lower {+ '\u{0131}' => out.push('i'),+ '\u{017f}' => out.push('s'),+ '\u{212a}' => out.push('k'),+ '\u{00df}' => out.push_str("ss"),+ folded => out.push(folded),+ }+ }+ }+ }+}++/// Temp path beside `path` whose name cannot collide with another field's+/// temp in this process, including on a case-folding volume. The FST temp+/// used to be `{segment}.{component}.fst.tmp` with no nonce: two components+/// that NTFS treats as one name shared that temp, the first rename consumed+/// it, and the second failed with `ERROR_FILE_NOT_FOUND` inside+/// `publishing FTS side-cars`.+fn unique_temp_sibling(path: &Path) -> PathBuf {+ static NONCE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);+ let nonce = NONCE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ let file_name = path+ .file_name()+ .map(|name| name.to_string_lossy().into_owned())+ .unwrap_or_else(|| "file".to_string());+ path.with_file_name(format!("{file_name}.tmp.{}.{nonce}", std::process::id()))}/// Derive one FTS side-car path from controlled components only.@@ -184,6 +324,24 @@ fn field_sidecar_path(segment_dir.join(format!("{segment_id}.{field_component}.{extension}"))}+fn field_sidecar_path_for_layout(+ segment_dir: &Path,+ segment_id: &str,+ field_name: &str,+ extension: &str,+ layout: FilenameLayout,+) -> PathBuf {+ match layout {+ FilenameLayout::CaseDistinguished => {+ let field_component = field_file_component_v3(field_name);+ segment_dir.join(format!("{segment_id}.{field_component}.{extension}"))+ }+ FilenameLayout::Encoded | FilenameLayout::Legacy => {+ field_sidecar_path(segment_dir, segment_id, field_name, extension)+ }+ }+}+/// Return the exact pre-fix raw path only when it remains one child of the/// segment directory on this host.///@@ -964,8 +1122,11 @@ fn index_field_values<'a>(/// ```text/// let mut writer = FtsIndexWriter::new(dir, segment_id, registry);/// for doc in docs { writer.add_document(doc_id, &fields); }-/// if writer.uses_encoded_field_filename_components() {-/// // The engine advances its data-directory format before this step.+/// if writer.uses_case_distinguished_field_filename_components() {+/// // The engine advances its data-directory format to v3 before this step.+/// writer.publish_case_distinguished_filename_layout()?;+/// } else if writer.uses_encoded_field_filename_components() {+/// // The engine advances its data-directory format to v2 before this step./// writer.publish_encoded_filename_layout()?;/// }/// writer.finish()?;@@ -976,7 +1137,8 @@ pub struct FtsIndexWriter {registry: Arc<AnalyzerRegistry>,/// Per-field: (config, postings_writer, field_stats, norms)fields: HashMap<String, FieldData>,- encoded_filename_layout_published: bool,+ /// Layout marker this writer has durably published, if any.+ published_filename_layout: Option<FilenameLayout>,/// Zstd effort for the `.meta` (ZFM4) and `.post` (ZPS1) envelopes./// [`ZSTD_DURABLE_LEVEL`] unless [`Self::with_zstd_level`] raised it.zstd_level: i32,@@ -1010,7 +1172,7 @@ impl FtsIndexWriter {segment_id: segment_id.into(),registry,fields: HashMap::new(),- encoded_filename_layout_published: false,+ published_filename_layout: None,zstd_level: ZSTD_DURABLE_LEVEL,}}@@ -1071,8 +1233,10 @@ impl FtsIndexWriter {}/// Whether finishing this writer can create a v2 digest-derived field- /// filename.+ /// filename (a name that is not a portable literal).///+ /// This is false for a mixed-case literal such as `page_Id`. That name+ /// needs [`Self::uses_case_distinguished_field_filename_components`]./// Engine callers use this after configuration/document ingestion and/// before [`Self::finish`] to durably advance the data-directory marker.pub fn uses_encoded_field_filename_components(&self) -> bool {@@ -1081,21 +1245,73 @@ impl FtsIndexWriter {.any(|field| matches!(field_file_component(field), Cow::Owned(_)))}+ /// Whether a portable literal in this writer would alias another field+ /// on a case-folding filesystem (`page_Id` / `page_id`, `ı` / `i`).+ pub fn uses_case_distinguished_field_filename_components(&self) -> bool {+ self.fields+ .keys()+ .any(|field| field_needs_case_distinguished_component(field))+ }++ fn required_filename_layout(&self) -> FilenameLayout {+ if self.uses_case_distinguished_field_filename_components() {+ FilenameLayout::CaseDistinguished+ } else if self.uses_encoded_field_filename_components() {+ FilenameLayout::Encoded+ } else {+ FilenameLayout::Legacy+ }+ }+/// Durably publish this segment's immutable v2 filename discriminator.////// The engine must durably advance the data-directory format first, call/// this method second, and call [`Self::finish`] last. `finish` refuses to/// emit encoded side-cars without this proof. An existing discriminator is- /// accepted only when it contains the exact v2 magic.+ /// accepted only when it contains the exact v2 magic. A writer that also+ /// needs layout v3 must call+ /// [`Self::publish_case_distinguished_filename_layout`] instead: a v2+ /// marker would keep `page_Id` literal.pub fn publish_encoded_filename_layout(&mut self) -> Result<()> {if !self.uses_encoded_field_filename_components() {return Ok(());}+ if self.uses_case_distinguished_field_filename_components() {+ bail!("case-distinguished FTS field filenames require filename layout v3, not v2");+ }+ self.publish_layout_marker(FilenameLayout::Encoded, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)+ }++ /// Durably publish this segment's immutable v3 filename discriminator.+ ///+ /// The engine must durably advance the data-directory format to v3 first.+ /// A v2 reader looks `page_Id` up as a literal and would silently skip the+ /// field, so the directory marker has to refuse that binary before any v3+ /// side-car exists. No-op when no field needs the distinction.+ pub fn publish_case_distinguished_filename_layout(&mut self) -> Result<()> {+ if !self.uses_case_distinguished_field_filename_components() {+ return Ok(());+ }+ self.publish_layout_marker(+ FilenameLayout::CaseDistinguished,+ FTS_FILENAME_LAYOUT_V3_MARKER_BYTES,+ )+ }++ fn publish_layout_marker(&mut self, layout: FilenameLayout, magic: &[u8]) -> Result<()> {fs::create_dir_all(&self.segment_dir).with_context(|| format!("creating segment dir {:?}", self.segment_dir))?;- let marker = segment_filename_layout_v2_marker_path(&self.segment_dir, &self.segment_id);+ let marker = match layout {+ FilenameLayout::Encoded => {+ segment_filename_layout_v2_marker_path(&self.segment_dir, &self.segment_id)+ }+ FilenameLayout::CaseDistinguished => {+ segment_filename_layout_v3_marker_path(&self.segment_dir, &self.segment_id)+ }+ FilenameLayout::Legacy => return Ok(()),+ };match fs::read(&marker) {- Ok(bytes) if bytes == FTS_FILENAME_LAYOUT_V2_MARKER_BYTES => {+ Ok(bytes) if bytes == magic => {// A previous attempt can have made the rename visible and// then failed its durability confirmation. A fresh writer has// no process-local proof, so confirm the directory entry@@ -1109,17 +1325,15 @@ impl FtsIndexWriter {// repairs a visible discriminator produced by an older build// whose Windows directory-sync shim was a no-op.#[cfg(windows)]- xerj_common::fsio::write_file_durable(&marker, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)- .with_context(|| {- format!("confirming FTS filename-layout discriminator {:?}", marker)- })?;+ xerj_common::fsio::write_file_durable(&marker, magic).with_context(|| {+ format!("confirming FTS filename-layout discriminator {:?}", marker)+ })?;}Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", marker),Err(error) if error.kind() == std::io::ErrorKind::NotFound => {- xerj_common::fsio::write_file_durable(&marker, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)- .with_context(|| {- format!("publishing FTS filename-layout discriminator {:?}", marker)- })?;+ xerj_common::fsio::write_file_durable(&marker, magic).with_context(|| {+ format!("publishing FTS filename-layout discriminator {:?}", marker)+ })?;}Err(error) => {return Err(error).with_context(|| {@@ -1127,7 +1341,7 @@ impl FtsIndexWriter {});}}- self.encoded_filename_layout_published = true;+ self.published_filename_layout = Some(layout);Ok(())}@@ -1383,9 +1597,11 @@ impl FtsIndexWriter {pub fn finish(self) -> Result<HashMap<String, FieldStats>> {use rayon::prelude::*;- if self.uses_encoded_field_filename_components() && !self.encoded_filename_layout_published- {- bail!("encoded FTS field filenames require a durable per-segment layout discriminator");+ let layout = self.required_filename_layout();+ if layout != FilenameLayout::Legacy && self.published_filename_layout != Some(layout) {+ bail!(+ "FTS filename layout {layout:?} requires a durable per-segment layout discriminator"+ );}fs::create_dir_all(&self.segment_dir)@@ -1451,6 +1667,7 @@ impl FtsIndexWriter {&field_name,field_data,zstd_level,+ layout,).with_context(|| format!("writing field '{}'", field_name))?;Ok((field_name, stats, files))@@ -1477,6 +1694,7 @@ impl FtsIndexWriter {field_name: &str,field_data: FieldData,zstd_level: i32,+ layout: FilenameLayout,) -> Result<Vec<xerj_common::fsio::StagedFile>> {let mut staged = Vec::with_capacity(4);// NOTE: `PathBuf::with_extension` replaces the final `.ext` in the path,@@ -1484,7 +1702,9 @@ impl FtsIndexWriter {// `with_extension("fst")` would strip `.field_name` and collapse every// field to the same file. The shared helper also keeps untrusted field// names from becoming paths.- let filename = |ext: &str| field_sidecar_path(segment_dir, segment_id, field_name, ext);+ let filename = |ext: &str| {+ field_sidecar_path_for_layout(segment_dir, segment_id, field_name, ext, layout)+ };let fst_path = filename("fst");let post_path = filename("post");let meta_path = filename("meta");@@ -1586,7 +1806,7 @@ impl FtsIndexWriter {// FST streams into a same-directory temp file; fsync + rename +// platform-specific durable replacement publishes it (RC4 W1 #10 —// see the postings note above).- let fst_tmp = filename("fst.tmp");+ let fst_tmp = unique_temp_sibling(&fst_path);let fst_file = BufWriter::new(File::create(&fst_tmp).with_context(|| format!("creating FST file {:?}", fst_tmp))?,);@@ -1921,8 +2141,13 @@ pub fn segments_indexed_field_names() -> Result<Vec<Vec<String>>> {let mut encoded: HashMap<String, &str> = HashMap::new();for name in known_fields {- if let Cow::Owned(component) = field_file_component(name) {- encoded.insert(component, name.as_str());+ // v2 and v3 hash unsafe names identically. v3 also hashes case-unstable+ // literals (`page_Id`) that v2 stored under the raw spelling, so a+ // directory scan has to recognize both components.+ for component in [field_file_component(name), field_file_component_v3(name)] {+ if let Cow::Owned(component) = component {+ encoded.insert(component, name.as_str());+ }}}@@ -2150,7 +2375,7 @@ impl FtsIndexReader {) -> Result<Self> {let segment_dir = segment_dir.as_ref().to_path_buf();let segment_id = segment_id.into();- let encoded_layout = segment_uses_encoded_filename_layout(&segment_dir, &segment_id)?;+ let layout = segment_filename_layout(&segment_dir, &segment_id)?;let mut fields = HashMap::new();for &field_name in field_names {@@ -2158,16 +2383,22 @@ impl FtsIndexReader {// Existence is deliberately irrelevant: otherwise a v1 field// whose literal name equals another field's digest component can// alias, and partial or stray files can alter layout selection.+ // A v2 marker keeps mixed-case literals (`page_Id.fst`) on the+ // v2 path; only a v3 marker looks up the case-distinguished digest.let filename = |ext: &str| -> Option<PathBuf> {- if encoded_layout {- Some(field_sidecar_path(- &segment_dir,- &segment_id,- field_name,- ext,- ))- } else {- legacy_field_sidecar_path(&segment_dir, &segment_id, field_name, ext)+ match layout {+ FilenameLayout::Legacy => {+ legacy_field_sidecar_path(&segment_dir, &segment_id, field_name, ext)+ }+ FilenameLayout::Encoded | FilenameLayout::CaseDistinguished => {+ Some(field_sidecar_path_for_layout(+ &segment_dir,+ &segment_id,+ field_name,+ ext,+ layout,+ ))+ }}};let Some(fst_path) = filename("fst") else {@@ -3764,4 +3995,211 @@ mod tests {.all(|entry| entry.file_type().unwrap().is_file()));assert!(!root.path().join("escape").exists());}++ #[test]+ fn case_aliased_fields_get_distinct_v3_components_and_round_trip() {+ for (left, right) in [("page_Id", "page_id"), ("ı", "i"), ("ß", "ss"), ("ſ", "s")] {+ assert_ne!(+ windows_case_identity(field_file_component_v3(left).as_ref()),+ windows_case_identity(field_file_component_v3(right).as_ref()),+ "{left:?} and {right:?} still share a Windows case key"+ );+ }+ // Layout v2 still aliases these. The marker, not file existence,+ // selects which component function the reader uses.+ assert_eq!(+ windows_case_identity(field_file_component("page_Id").as_ref()),+ windows_case_identity("page_id")+ );+ assert_eq!(field_file_component_v3("page_id").as_ref(), "page_id");+ assert!(field_file_component_v3("page_Id")+ .as_ref()+ .starts_with(ENCODED_FIELD_COMPONENT_PREFIX));+ assert_eq!(+ field_file_component("bad/field"),+ field_file_component_v3("bad/field"),+ "unsafe names must keep the v2 digest under v3"+ );+ assert_eq!(field_file_component_v3("配置.名字").as_ref(), "配置.名字");++ let dir = TempDir::new().unwrap();+ let mut plain = FtsIndexWriter::new(dir.path(), "seg-plain", make_registry());+ plain.add_document(+ 0,+ &[("page_id".to_owned(), FieldValues::from("only-lower"))]+ .into_iter()+ .collect(),+ );+ assert!(!plain.uses_case_distinguished_field_filename_components());+ plain.finish().unwrap();+ assert!(!segment_filename_layout_v3_marker_path(dir.path(), "seg-plain").exists());++ let document: HashMap<String, FieldValues> = [+ ("page_Id".to_owned(), FieldValues::from("upperneedle")),+ ("page_id".to_owned(), FieldValues::from("lowerneedle")),+ ("ı".to_owned(), FieldValues::from("dotlessneedle")),+ ("i".to_owned(), FieldValues::from("dottedneedle")),+ ]+ .into_iter()+ .collect();+ let mut refused = FtsIndexWriter::new(dir.path(), "seg-case", make_registry());+ refused.add_document(0, &document);+ assert!(refused.uses_case_distinguished_field_filename_components());+ assert!(refused.finish().is_err());++ let mut writer = FtsIndexWriter::new(dir.path(), "seg-case", make_registry());+ writer.add_document(0, &document);+ writer.publish_case_distinguished_filename_layout().unwrap();+ writer.finish().unwrap();+ assert!(segment_filename_layout_v3_marker_path(dir.path(), "seg-case").is_file());+ assert!(!segment_filename_layout_v2_marker_path(dir.path(), "seg-case").exists());++ let fields = ["page_Id", "page_id", "ı", "i"];+ let reader = FtsIndexReader::open(dir.path(), "seg-case", &fields).unwrap();+ assert!(reader.term_exists("page_Id", "upperneedle"));+ assert!(reader.term_exists("page_id", "lowerneedle"));+ assert!(!reader.term_exists("page_Id", "lowerneedle"));+ assert!(reader.term_exists("ı", "dotlessneedle"));+ assert!(reader.term_exists("i", "dottedneedle"));++ let known: Vec<String> = fields.iter().map(|field| (*field).to_owned()).collect();+ let listed = segment_indexed_field_names(dir.path(), "seg-case", &known).unwrap();+ assert_eq!(+ listed.iter().map(String::as_str).collect::<HashSet<_>>(),+ fields.into_iter().collect::<HashSet<_>>()+ );+ }++ #[test]+ fn v3_layout_reuses_v2_digests_for_unsafe_names() {+ let dir = TempDir::new().unwrap();+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-both", make_registry());+ let document: HashMap<String, FieldValues> = [+ ("bad/field".to_owned(), FieldValues::from("slashneedle")),+ ("page_Id".to_owned(), FieldValues::from("caseneedle")),+ ]+ .into_iter()+ .collect();+ writer.add_document(0, &document);+ writer.publish_case_distinguished_filename_layout().unwrap();+ writer.finish().unwrap();+ assert!(!segment_filename_layout_v2_marker_path(dir.path(), "seg-both").exists());+ assert!(field_sidecar_path(dir.path(), "seg-both", "bad/field", "fst").is_file());+ let reader =+ FtsIndexReader::open(dir.path(), "seg-both", &["bad/field", "page_Id"]).unwrap();+ assert!(reader.term_exists("bad/field", "slashneedle"));+ assert!(reader.term_exists("page_Id", "caseneedle"));+ }++ #[test]+ fn v2_and_legacy_segments_still_read_a_literal_mixed_case_field() {+ let root = TempDir::new().unwrap();+ let segment_dir = root.path().join("segments");+ fs::create_dir_all(&segment_dir).unwrap();+ for (segment_id, write_v2) in [("legacy-page", false), ("v2-page", true)] {+ write_legacy_raw_field_fixture(&segment_dir, segment_id, "page_Id");+ if write_v2 {+ fs::write(+ segment_filename_layout_v2_marker_path(&segment_dir, segment_id),+ FTS_FILENAME_LAYOUT_V2_MARKER_BYTES,+ )+ .unwrap();+ }+ let reader = FtsIndexReader::open(&segment_dir, segment_id, &["page_Id"]).unwrap();+ assert!(+ reader.term_exists("page_Id", "legacyneedle"),+ "{segment_id} dropped the literal mixed-case side-car"+ );+ }+ }++ #[test]+ fn corrupt_v3_marker_does_not_fall_through_to_a_literal_family() {+ let dir = TempDir::new().unwrap();+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-corrupt", make_registry());+ writer.add_document(+ 0,+ &[("title".to_owned(), FieldValues::from("needle"))]+ .into_iter()+ .collect(),+ );+ writer.finish().unwrap();+ fs::write(+ segment_filename_layout_v3_marker_path(dir.path(), "seg-corrupt"),+ b"not-a-layout-marker",+ )+ .unwrap();+ assert!(FtsIndexReader::open(dir.path(), "seg-corrupt", &["title"]).is_err());+ }++ /// The two JeecgBoot rows from issue #2254: `onl_drag_comp` JImg and+ /// `onl_drag_page_comp` JBar3d. Linux finishes them; the publish abort was+ /// the Windows rename of their side-cars. `page_Id` is the mixed-case+ /// column, and the config strings carry the URL and CJK that distinguished+ /// these rows from the rest of the dump.+ #[test]+ fn jeecg_jimg_and_jbar3d_rows_finish_for_standard_and_keyword() {+ let jimg = [+ ("comp_type", "JImg"),+ ("comp_name", "图片"),+ ("icon", "ion:image-sharp"),+ (+ "comp_config",+ r#"{"url":"http://api.jeecg.com/mock/42/nav"}"#,+ ),+ ("update_by", "admin"),+ ];+ let jbar = [+ ("component", "JBar3d"),+ ("page_Id", "1212561321588641792"),+ (+ "config",+ r#"{"filed":"维度","url":"http://api.jeecg.com/mock/33/chart","name":"苹果"}"#,+ ),+ ("create_by", "admin"),+ ];+ for analyzer in ["standard", "keyword"] {+ let dir = TempDir::new().unwrap();+ let segment = format!("jeecg-{analyzer}");+ let mut writer = FtsIndexWriter::new(dir.path(), &segment, make_registry());+ let config = FieldIndexConfig {+ analyzer: analyzer.to_owned(),+ store_positions: analyzer != "keyword",+ store_term_vectors: false,+ };+ for (name, _) in jimg.iter().chain(jbar.iter()) {+ writer.configure_field((*name).to_owned(), config.clone());+ }+ let to_doc = |rows: &[(&str, &str)]| -> HashMap<String, FieldValues> {+ rows.iter()+ .map(|(key, value)| ((*key).to_owned(), FieldValues::from(*value)))+ .collect()+ };+ writer.add_document(0, &to_doc(&jimg));+ writer.add_document(1, &to_doc(&jbar));+ assert!(writer.uses_case_distinguished_field_filename_components());+ writer.publish_case_distinguished_filename_layout().unwrap();+ writer.finish().expect(analyzer);+ let fields: Vec<&str> = jimg+ .iter()+ .chain(jbar.iter())+ .map(|(name, _)| *name)+ .collect();+ let reader = FtsIndexReader::open(dir.path(), &segment, &fields).unwrap();+ if analyzer == "standard" {+ assert!(reader.term_exists("comp_type", "jimg"));+ assert!(reader.term_exists("component", "jbar3d"));+ assert!(reader.term_exists("update_by", "admin"));+ assert!(reader.term_exists("comp_config", "nav"));+ assert!(reader.term_exists("config", "chart"));+ assert!(reader.term_exists("page_Id", "1212561321588641792"));+ } else {+ assert!(reader.term_exists("comp_type", "JImg"));+ assert!(reader.term_exists("component", "JBar3d"));+ assert!(reader.term_exists("page_Id", "1212561321588641792"));+ assert!(reader.term_exists("comp_name", "图片"));+ }+ assert!(segment_filename_layout_v3_marker_path(dir.path(), &segment).is_file());+ }+ }}diff --git a/engine/crates/xerj-storage/src/bundle.rs b/engine/crates/xerj-storage/src/bundle.rsindex 7a1bc836c..143aecede 100644--- a/engine/crates/xerj-storage/src/bundle.rs+++ b/engine/crates/xerj-storage/src/bundle.rs@@ -150,7 +150,7 @@ pub fn valid_flush_artifact_set(segment_id: &str, artifacts: &[(String, u64, u32let rest = &name[prefix.len()..];if matches!(rest,- "seg" | "sidx" | "ids" | "dv" | "fts-layout-v2" | "ftsan"+ "seg" | "sidx" | "ids" | "dv" | "fts-layout-v2" | "fts-layout-v3" | "ftsan") {if !roles.insert(rest) {return false;diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 737c31818..0a59e8bbc 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -39,10 +39,13 @@ use crate::{Result, SeqNo, StorageError};/// Highest on-disk data-directory format this binary can read.///-/// Version 2 reserves digest-derived FTS field filename components. A v2-/// binary can read v1 raw side-cars, but a v1 binary must refuse a directory-/// after the first v2 side-car can be created.-const DATA_DIR_FORMAT_VERSION: u32 = 2;+/// Version 2 reserves digest-derived FTS field filename components. Version 3+/// reserves digest components for field names whose Windows case-identity is+/// not the name itself (`page_Id` versus `page_id`, `ı` versus `i`, `ß`+/// versus `ss`). A v3 binary reads v1 raw side-cars and v2 encoded side-cars.+/// A v2 binary must refuse the directory once a v3 side-car can be created:+/// it would look `page_Id` up as a literal filename and silently skip it.+const DATA_DIR_FORMAT_VERSION: u32 = 3;/// Baseline written for fresh and pre-marker directories.///@@ -54,6 +57,10 @@ const DATA_DIR_BASE_FORMAT_VERSION: u32 = 1;/// First layout version that permits digest-derived FTS field components.const DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION: u32 = 2;+/// First layout version that digest-encodes field names which collide under+/// NTFS case folding but were portable literals in layout v2.+const DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION: u32 = 3;+#[cfg(any(test, feature = "test-hooks"))]#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum DataDirFormatWriteFailpoint {@@ -549,10 +556,18 @@ pub struct IndexStore {/// directory-sync shim. The format lock serializes the false -> true/// transition so concurrent flush/merge retries cannot bypass confirmation.fts_v2_marker_durability_confirmed: AtomicBool,+ /// Process-local proof that format version 3 has crossed the same+ /// durable-publication boundary as the v2 flag. A confirmed v2 marker is+ /// not proof of v3: the next case-distinguished field must still advance+ /// 2 → 3. Confirming v3 also latches the v2 flag so a later v2 preflight+ /// cannot rewrite the marker back down.+ fts_v3_marker_durability_confirmed: AtomicBool,#[cfg(any(test, feature = "test-hooks"))]data_dir_format_write_failpoint: std::sync::atomic::AtomicU8,#[cfg(any(test, feature = "test-hooks"))]fts_v2_marker_durability_confirmations: AtomicU64,+ #[cfg(any(test, feature = "test-hooks"))]+ fts_v3_marker_durability_confirmations: AtomicU64,/// Actual memtable shard count, derived at open time from/// `IndexStoreConfig.num_wal_shards.max(1).next_power_of_two()`.num_shards: usize,@@ -905,10 +920,13 @@ impl IndexStore {config,data_dir_format_lock: Mutex::new(()),fts_v2_marker_durability_confirmed: AtomicBool::new(false),+ fts_v3_marker_durability_confirmed: AtomicBool::new(false),#[cfg(any(test, feature = "test-hooks"))]data_dir_format_write_failpoint: std::sync::atomic::AtomicU8::new(0),#[cfg(any(test, feature = "test-hooks"))]fts_v2_marker_durability_confirmations: AtomicU64::new(0),+ #[cfg(any(test, feature = "test-hooks"))]+ fts_v3_marker_durability_confirmations: AtomicU64::new(0),num_shards,snapshot: ArcSwap::from_pointee(snapshot),wal_shards,@@ -1654,7 +1672,7 @@ impl IndexStore {/// Unlink every on-disk file belonging to the given segment ids — the/// primary `.seg` plus all side-cars (`.sidx`, `.ids`, `.dv`,/// `.<field>.post` / `.fst` / `.meta` / `.norms`) and the optional- /// `.fts-layout-v2` discriminator.+ /// `.fts-layout-v2` / `.fts-layout-v3` discriminator.////// Disk-space fix (2026-07): called by the engine merge task right/// after `apply_merge` commits, so merged-away input segments are@@ -4292,11 +4310,31 @@ impl IndexStore {/// and merge callers inside that one supported writer process; opening/// multiple raw `IndexStore`s on the same directory remains unsupported.pub fn ensure_fts_encoded_field_component_format(&self) -> Result<()> {+ self.ensure_fts_filename_component_format(DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION)+ }++ /// Durably fence a writer before it creates a case-distinguished FTS+ /// filename (`page_Id` and the other NTFS case-fold literals layout v2+ /// kept). Advances the marker to v3 and does not return early on a+ /// previously confirmed v2 marker — that confirmation must not skip the+ /// 2 → 3 upgrade. Confirming v3 latches the v2 flag as well, so a later+ /// v2 preflight cannot rewrite the marker downwards.+ pub fn ensure_fts_case_distinguished_field_component_format(&self) -> Result<()> {+ self.ensure_fts_filename_component_format(+ DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION,+ )+ }++ fn ensure_fts_filename_component_format(&self, target: u32) -> Result<()> {let _format_guard = self.data_dir_format_lock.lock().unwrap();- if self- .fts_v2_marker_durability_confirmed- .load(Ordering::Acquire)- {+ let already = if target >= DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION {+ self.fts_v3_marker_durability_confirmed+ .load(Ordering::Acquire)+ } else {+ self.fts_v2_marker_durability_confirmed+ .load(Ordering::Acquire)+ };+ if already {return Ok(());}let path = Self::data_dir_meta_path(&self.data_dir);@@ -4328,12 +4366,14 @@ impl IndexStore {DATA_DIR_FORMAT_VERSION)));}- if meta.format_version >= DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION {+ if meta.format_version >= target {// Do not trust visibility as proof of durability. A previous- // attempt may have renamed v2 into place and then failed its- // durability confirmation. Re-establish that proof now, while- // still holding the transition lock, before permitting any- // encoded FTS filename to be created.+ // attempt may have renamed the marker into place and then+ // failed its durability confirmation. Re-establish that proof+ // now, while still holding the transition lock, before+ // permitting any encoded FTS filename to be created. A higher+ // version than `target` is confirmed and left in place — v2+ // preflight must not downgrade a v3 marker.#[cfg(not(windows))]if let Some(parent) = path.parent() {xerj_common::fsio::fsync_dir(parent)?;@@ -4345,7 +4385,7 @@ impl IndexStore {// Windows directory-sync shim was a no-op.#[cfg(windows)]xerj_common::fsio::write_file_durable(&path, &std::fs::read(&path)?)?;- self.note_fts_v2_marker_durability_confirmed();+ self.note_fts_filename_format_confirmed(target);return Ok(());}}@@ -4359,7 +4399,7 @@ impl IndexStore {})?;object.insert("format_version".to_string(),- serde_json::Value::from(DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION),+ serde_json::Value::from(target),);object.insert("xerj_version".to_string(),@@ -4396,10 +4436,27 @@ impl IndexStore {}#[cfg(not(any(test, feature = "test-hooks")))]xerj_common::fsio::write_file_durable(&path, &bytes)?;- self.note_fts_v2_marker_durability_confirmed();+ self.note_fts_filename_format_confirmed(target);Ok(())}+ fn note_fts_filename_format_confirmed(&self, target: u32) {+ if target >= DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION {+ #[cfg(any(test, feature = "test-hooks"))]+ self.fts_v3_marker_durability_confirmations+ .fetch_add(1, Ordering::AcqRel);+ self.fts_v3_marker_durability_confirmed+ .store(true, Ordering::Release);+ // Latch v2 without counting a v2 confirmation. The v2 counter is+ // the number of times the v2 preflight itself proved durability;+ // setting the flag stops that preflight from rewriting v3 as v2.+ self.fts_v2_marker_durability_confirmed+ .store(true, Ordering::Release);+ return;+ }+ self.note_fts_v2_marker_durability_confirmed();+ }+fn note_fts_v2_marker_durability_confirmed(&self) {#[cfg(any(test, feature = "test-hooks"))]self.fts_v2_marker_durability_confirmations@@ -4423,6 +4480,12 @@ impl IndexStore {.load(Ordering::Acquire)}+ #[cfg(any(test, feature = "test-hooks"))]+ pub fn fts_v3_marker_durability_confirmations_for_test(&self) -> u64 {+ self.fts_v3_marker_durability_confirmations+ .load(Ordering::Acquire)+ }+#[cfg(any(test, feature = "test-hooks"))]pub fn clear_fts_v2_marker_durability_confirmation_for_test(&self) {self.fts_v2_marker_durability_confirmed@@ -5593,6 +5656,35 @@ mod tests {.starts_with(&format!("{segment_id}."))));}+ #[test]+ fn fts_layout_v3_discriminator_is_a_valid_flush_artifact() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ let segment_id = "12345678-1234-1234-1234-123456789abd";+ let segments = dir.path().join("segments");+ for (role, bytes) in [+ ("seg", b"segment".as_slice()),+ ("sidx", b"sidx".as_slice()),+ ("ids", b"ids".as_slice()),+ ("fts-layout-v3", b"XERJ_FTS_FILENAME_LAYOUT_V3\n".as_slice()),+ ] {+ std::fs::write(segments.join(format!("{segment_id}.{role}")), bytes).unwrap();+ }+ let meta = SegmentMeta {+ id: segment_id.to_owned(),+ doc_count: 1,+ size_bytes: 7,+ min_seq_no: 1,+ max_seq_no: 1,+ created_at_ms: 0,+ has_tombstones: false,+ seg_path: format!("{segment_id}.seg"),+ sidx_path: format!("{segment_id}.sidx"),+ };+ store.write_flush_completion_manifest(&meta).unwrap();+ assert!(store.validate_flush_completion_manifest(segment_id, 1, 1, 1));+ }+#[test]fn artifact_crc_streams_large_inputs_in_bounded_chunks() {struct TrackingReader {@@ -7933,6 +8025,60 @@ mod tests {);}+ #[test]+ fn case_distinguished_fts_preflight_advances_to_v3_and_v2_does_not_downgrade() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ let marker = dir.path().join(DATA_DIR_META_FILE);++ store+ .ensure_fts_case_distinguished_field_component_format()+ .unwrap();+ store+ .ensure_fts_case_distinguished_field_component_format()+ .unwrap();+ assert_eq!(store.fts_v3_marker_durability_confirmations_for_test(), 1);++ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();+ assert_eq!(+ meta.format_version,+ DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION+ );+ assert!(IndexStore::check_data_dir_version_with_max(dir.path(), 2).is_err());+ assert!(IndexStore::check_data_dir_version(dir.path()).is_ok());++ store.ensure_fts_encoded_field_component_format().unwrap();+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();+ assert_eq!(+ meta.format_version, DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION,+ "a v2 preflight must not rewrite a v3 marker"+ );+ assert_eq!(+ store.fts_v2_marker_durability_confirmations_for_test(),+ 0,+ "latching the v2 flag from v3 is not a v2 confirmation"+ );+ }++ #[test]+ fn case_distinguished_fts_preflight_upgrades_a_confirmed_v2_marker() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ store.ensure_fts_encoded_field_component_format().unwrap();+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 1);++ store+ .ensure_fts_case_distinguished_field_component_format()+ .unwrap();+ let marker = dir.path().join(DATA_DIR_META_FILE);+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();+ assert_eq!(meta.format_version, 3);+ store.ensure_fts_encoded_field_component_format().unwrap();+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();+ assert_eq!(meta.format_version, 3);+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 1);+ }+/// THE UPGRADE PATH: an rc3-vintage data dir has NO format marker (rc3/// never wrote one). Reopening under this (rc4) binary must serve every/// doc AND stamp a marker so subsequent opens are versioned. This is the