Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(fts): keep case-aliased field sidecars from aborting publish (#2261)

xerj-org8920c06ce590
diff --git a/engine/crates/xerj-common/src/fsio.rs b/engine/crates/xerj-common/src/fsio.rs
index d408786b7..ad023928d 100644
--- a/engine/crates/xerj-common/src/fsio.rs
+++ b/engine/crates/xerj-common/src/fsio.rs
@@ -151,9 +151,16 @@ pub fn stage_written_file(
/// never torn, exactly as per-file durable writes did. What changes is the
/// cost: two device flushes for the batch instead of two per file, which on
/// Apple targets is the difference between ~250 cache drains per segment and
-/// two. If a rename fails, the files already moved stay moved and the rest are
-/// removed, which is also what a failure part-way through a loop of
-/// [`write_file_durable`] calls left behind.
+/// two.
+///
+/// If a rename fails, destinations already replaced by *this* call are
+/// removed and the remaining temps are dropped. FTS side-cars are new files
+/// under a fresh segment id, so that rollback leaves the directory empty of
+/// this batch instead of a half-published family. A later directory fsync
+/// failure does not roll those names back: the bytes are already complete,
+/// and removing them would turn a durability-confirmation error into missing
+/// files. Callers that replace a pre-existing file they still need must not
+/// use this batch helper.
pub fn publish_staged(dir: &Path, files: Vec<StagedFile>) -> std::io::Result<()> {
if files.is_empty() {
return Ok(());
@@ -166,15 +173,32 @@ pub fn publish_staged(dir: &Path, files: Vec<StagedFile>) -> std::io::Result<()>
}
#[cfg(target_vendor = "apple")]
sync_file_full(&std::fs::File::open(dir)?)?;
+ let mut published_dests = Vec::with_capacity(files.len());
for mut file in files {
- replace_file_for_durable_write(&file.tmp, &file.dest)?;
+ if let Err(error) = replace_file_for_durable_write(&file.tmp, &file.dest) {
+ for dest in &published_dests {
+ let _ = std::fs::remove_file(dest);
+ }
+ return Err(error);
+ }
file.published = true;
+ published_dests.push(file.dest.clone());
}
#[cfg(not(windows))]
fsync_dir(dir)?;
Ok(())
}
+/// Windows `MoveFileExW` returns these when a scanner still has the temp
+/// open at the moment of the replace. They are transient for the FTS publish
+/// path (the handle is ours and already closed). `ERROR_FILE_NOT_FOUND` (2)
+/// is not: retrying it hides a missing temp.
+#[cfg(any(windows, test))]
+fn replace_lock_should_retry(os_code: Option<i32>, retries_already: u32) -> bool {
+ const REPLACE_LOCK_RETRIES: u32 = 6;
+ retries_already < REPLACE_LOCK_RETRIES && matches!(os_code, Some(5 | 32 | 33))
+}
+
/// Windows has no supported directory-flush primitive to call here:
/// `File::open` on a
/// directory fails with `ERROR_ACCESS_DENIED` (os error 5) because std
@@ -272,20 +296,34 @@ fn replace_file_for_durable_write(source: &Path, destination: &Path) -> std::io:
let source = wide_path(source)?;
let destination = wide_path(destination)?;
- // SAFETY: both buffers are NUL-terminated and live through the call.
- // The flags request replacement plus synchronous persistence; no
- // COPY_ALLOWED flag is present, and our temp file is same-directory.
- let succeeded = unsafe {
- MoveFileExW(
- source.as_ptr(),
- destination.as_ptr(),
- MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
- )
- };
- if succeeded == 0 {
- Err(std::io::Error::last_os_error())
- } else {
- Ok(())
+ let mut retries_already = 0u32;
+ loop {
+ // SAFETY: both buffers are NUL-terminated and live through the call.
+ // The flags request replacement plus synchronous persistence; no
+ // COPY_ALLOWED flag is present, and our temp file is same-directory.
+ let succeeded = unsafe {
+ MoveFileExW(
+ source.as_ptr(),
+ destination.as_ptr(),
+ MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
+ )
+ };
+ if succeeded != 0 {
+ return Ok(());
+ }
+ let error = std::io::Error::last_os_error();
+ if replace_lock_should_retry(error.raw_os_error(), retries_already) {
+ retries_already += 1;
+ // A filter driver (Defender, DLP) often locks a just-closed file
+ // for a scan. Short backoff covers that without turning a
+ // permanent access error into a long stall: six waits are
+ // 10+20+…+60 ms.
+ std::thread::sleep(std::time::Duration::from_millis(
+ 10 * u64::from(retries_already),
+ ));
+ continue;
+ }
+ return Err(error);
}
}
@@ -467,6 +505,46 @@ mod tests {
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
}
+ #[test]
+ fn replace_lock_retries_sharing_violations_but_not_a_missing_temp() {
+ assert!(replace_lock_should_retry(Some(5), 0));
+ assert!(replace_lock_should_retry(Some(32), 5));
+ assert!(replace_lock_should_retry(Some(33), 0));
+ assert!(!replace_lock_should_retry(Some(32), 6));
+ assert!(!replace_lock_should_retry(Some(2), 0));
+ assert!(!replace_lock_should_retry(None, 0));
+ }
+
+ #[test]
+ fn publish_staged_rolls_back_a_partial_rename_and_a_later_publish_succeeds() {
+ let dir = tempfile::tempdir().unwrap();
+ let published = dir.path().join("a.post");
+ let blocked = dir.path().join("b.meta");
+ // A directory sitting on the destination makes the second replace fail
+ // after the first has already moved into place.
+ std::fs::create_dir(&blocked).unwrap();
+ let staged = vec![
+ stage_file(&published, b"alpha").unwrap(),
+ stage_file(&blocked, b"beta").unwrap(),
+ ];
+ let error = publish_staged(dir.path(), staged).unwrap_err();
+ assert_ne!(error.kind(), std::io::ErrorKind::InvalidInput);
+ assert!(
+ !published.exists(),
+ "a dest renamed earlier in the batch must be removed"
+ );
+ assert!(blocked.is_dir());
+ let names: Vec<_> = std::fs::read_dir(dir.path())
+ .unwrap()
+ .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
+ .collect();
+ assert_eq!(names, vec!["b.meta".to_string()]);
+
+ let staged = vec![stage_file(&published, b"alpha-again").unwrap()];
+ publish_staged(dir.path(), staged).unwrap();
+ assert_eq!(std::fs::read(&published).unwrap(), b"alpha-again");
+ }
+
#[test]
fn publish_staged_rejects_a_file_outside_the_directory() {
let dir = tempfile::tempdir().unwrap();
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index f628d47c6..7c897dc3f 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2059,6 +2059,104 @@ mod flush_publication_recovery_tests {
assert_eq!(result.hits[0].id, "survivor");
}
+ /// Issue #2254. `page_Id` is a portable literal under filename layout v2,
+ /// so on NTFS it is the same file as `page_id`. The flush must publish
+ /// layout v3, keep both fields searchable, and still open after restart.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn mixed_case_field_flush_publishes_v3_and_restarts_searchable() {
+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config.clone()).unwrap();
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new("page_Id", FieldType::Text))
+ .unwrap();
+ schema
+ .add_field(FieldConfig::new("page_id", FieldType::Text))
+ .unwrap();
+ schema
+ .add_field(FieldConfig::new("component", FieldType::Text))
+ .unwrap();
+ engine.create_index("case-field-publish", schema).unwrap();
+ let idx = engine.get_index("case-field-publish").unwrap();
+ idx.abort_background_tasks();
+ assert_eq!(data_dir_format_version(&idx), 1);
+ idx.index_document(
+ Some("jbar".into()),
+ json!({
+ "page_Id": "1212561321588641792",
+ "page_id": "other-page",
+ "component": "JBar3d"
+ }),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("jimg".into()),
+ json!({"component": "JImg", "page_id": "img-only"}),
+ )
+ .await
+ .unwrap();
+
+ idx.flush().await.unwrap();
+ assert_eq!(data_dir_format_version(&idx), 3);
+ assert_eq!(idx.memtable.doc_count(), 0);
+ let segment_names: Vec<String> = std::fs::read_dir(idx.data_dir.join("segments"))
+ .unwrap()
+ .flatten()
+ .map(|entry| entry.file_name().to_string_lossy().into_owned())
+ .collect();
+ assert!(
+ segment_names
+ .iter()
+ .any(|name| name.ends_with(".fts-layout-v3")),
+ "{segment_names:?}"
+ );
+ assert!(segment_names
+ .iter()
+ .all(|name| !name.ends_with(".fts-layout-v2")));
+ let mut case_keys = std::collections::HashSet::new();
+ for name in segment_names.iter().filter(|name| name.ends_with(".fst")) {
+ assert!(
+ case_keys.insert(name.to_ascii_lowercase()),
+ "case-folded FTS sidecar collision: {name}"
+ );
+ }
+
+ let request = xerj_query::parse_request(&json!({
+ "query": {"match": {"component": "JBar3d"}},
+ "size": 10,
+ "track_total_hits": true
+ }))
+ .unwrap();
+ let result = idx.search(&request).await.unwrap();
+ assert_eq!(result.total.value, 1);
+ assert_eq!(result.hits[0].id, "jbar");
+ let by_page = xerj_query::parse_request(&json!({
+ "query": {"match": {"page_Id": "1212561321588641792"}},
+ "size": 10
+ }))
+ .unwrap();
+ let result = idx.search(&by_page).await.unwrap();
+ assert_eq!(result.hits.len(), 1);
+ assert_eq!(result.hits[0].id, "jbar");
+ drop(idx);
+ drop(engine);
+
+ let restarted = crate::Engine::new(config).unwrap();
+ let idx = restarted.get_index("case-field-publish").unwrap();
+ idx.abort_background_tasks();
+ assert_eq!(data_dir_format_version(&idx), 3);
+ let result = idx.search(&request).await.unwrap();
+ assert_eq!(result.total.value, 1);
+ assert_eq!(result.hits[0].id, "jbar");
+ let result = idx.search(&by_page).await.unwrap();
+ assert_eq!(result.hits.len(), 1);
+ assert_eq!(result.hits[0].id, "jbar");
+ }
+
/// Build a one-date-field index whose `ts` field carries `precision`.
async fn date_index(
engine: &crate::Engine,
@@ -13848,7 +13946,26 @@ impl Index {
fts_writer.add_documents_parallel(&fts_input)
}
}
- if fts_writer.uses_encoded_field_filename_components() {
+ if fts_writer
+ .uses_case_distinguished_field_filename_components()
+ {
+ if let Err(e) = store_for_task
+ .ensure_fts_case_distinguished_field_component_format()
+ {
+ tracing::warn!(
+ "merge: case-distinguished FTS filename format preflight failed: {e:#}"
+ );
+ return Ok(());
+ }
+ if let Err(e) = fts_writer
+ .publish_case_distinguished_filename_layout()
+ {
+ tracing::warn!(
+ "merge: FTS filename-layout publication failed: {e:#}"
+ );
+ return Ok(());
+ }
+ } else if fts_writer.uses_encoded_field_filename_components() {
if let Err(e) = store_for_task
.ensure_fts_encoded_field_component_format()
{
@@ -13867,7 +13984,7 @@ impl Index {
}
}
if let Err(e) = fts_writer.finish() {
- tracing::warn!("merge: FTS build failed: {e}");
+ tracing::warn!("merge: FTS build failed: {e:#}");
}
Ok(())
});
@@ -34592,19 +34709,28 @@ async fn do_flush_shard(
fts_writer.add_documents_parallel(&drained_fts_for_build);
fts_add_us = t_add.elapsed().as_micros();
let t_fin = std::time::Instant::now();
- if fts_writer.uses_encoded_field_filename_components() {
+ if fts_writer.uses_case_distinguished_field_filename_components() {
+ store_for_warm.ensure_fts_case_distinguished_field_component_format()?;
+ fts_writer
+ .publish_case_distinguished_filename_layout()
+ .map_err(|error| {
+ xerj_storage::StorageError::Io(std::io::Error::other(format!(
+ "FTS filename-layout publication failed: {error:#}"
+ )))
+ })?;
+ } else if fts_writer.uses_encoded_field_filename_components() {
store_for_warm.ensure_fts_encoded_field_component_format()?;
fts_writer
.publish_encoded_filename_layout()
.map_err(|error| {
xerj_storage::StorageError::Io(std::io::Error::other(format!(
- "FTS filename-layout publication failed: {error}"
+ "FTS filename-layout publication failed: {error:#}"
)))
})?;
}
fts_writer.finish().map_err(|error| {
xerj_storage::StorageError::Io(std::io::Error::other(format!(
- "FTS side-car build failed: {error}"
+ "FTS side-car build failed: {error:#}"
)))
})?;
fts_finish_us = t_fin.elapsed().as_micros();
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index e13939033..43b8ca36c 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -11,10 +11,14 @@
//! seg-<id>.<field-component>.meta — FieldStats + per-term metadata
//! seg-<id>.<field-component>.norms — encoded per-doc field lengths
//! seg-<id>.fts-layout-v2 — exact marker when components are encoded
+//! seg-<id>.fts-layout-v3 — exact marker when a literal would
+//! case-fold into another field on NTFS
//! ```
//!
//! Short portable field names are used literally for on-disk compatibility;
-//! all other logical field names map to a bounded digest component.
+//! all other logical field names map to a bounded digest component. Layout v3
+//! also digests a name whose Windows case-identity is not the name itself
+//! (`page_Id`, `ı`, `ß`), so two fields cannot share one side-car file.
//!
//! The FST key is the term text (UTF-8 bytes, lexicographically sorted by construction).
//! The FST output value is the byte offset in the `.post` file.
@@ -61,25 +65,78 @@ const ENCODED_FIELD_COMPONENT_PREFIX: &str = "__xerj_fts_field_sha256_";
const FTS_FILENAME_LAYOUT_V2_MARKER_SUFFIX: &str = "fts-layout-v2";
const FTS_FILENAME_LAYOUT_V2_MARKER_BYTES: &[u8] = b"XERJ_FTS_FILENAME_LAYOUT_V2\n";
+/// Layout v3. Same digest function as v2 for names v2 already hashed. Names
+/// v2 kept as literals, but which NTFS case-folds onto a different literal,
+/// are hashed too. The marker is the only authority: a v2 segment that stored
+/// `page_Id` literally must still open as v2.
+const FTS_FILENAME_LAYOUT_V3_MARKER_SUFFIX: &str = "fts-layout-v3";
+const FTS_FILENAME_LAYOUT_V3_MARKER_BYTES: &[u8] = b"XERJ_FTS_FILENAME_LAYOUT_V3\n";
+
+/// Which filename function a segment's side-cars were written with.
+///
+/// Chosen from the marker file alone. Existence of a side-car is never
+/// evidence: a v1 literal can alias a v2 digest component.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum FilenameLayout {
+ /// No marker. Paths are the raw field name when that stays inside the
+ /// segment directory.
+ Legacy,
+ /// `fts-layout-v2`. Unsafe names are SHA-256 components; mixed-case
+ /// portable names stay literal.
+ Encoded,
+ /// `fts-layout-v3`. v2's digests, plus a digest for any remaining literal
+ /// whose Windows case-identity is not the field name itself.
+ CaseDistinguished,
+}
+
fn segment_filename_layout_v2_marker_path(segment_dir: &Path, segment_id: &str) -> PathBuf {
segment_dir.join(format!(
"{segment_id}.{FTS_FILENAME_LAYOUT_V2_MARKER_SUFFIX}"
))
}
-/// Absence means v1/raw. Exact magic means v2/encoded. Any other visible
-/// marker is corruption. File-family existence is never layout evidence.
-fn segment_uses_encoded_filename_layout(segment_dir: &Path, segment_id: &str) -> Result<bool> {
- let marker = segment_filename_layout_v2_marker_path(segment_dir, segment_id);
- match fs::read(&marker) {
- Ok(bytes) if bytes == FTS_FILENAME_LAYOUT_V2_MARKER_BYTES => Ok(true),
- Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", marker),
- Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
+fn segment_filename_layout_v3_marker_path(segment_dir: &Path, segment_id: &str) -> PathBuf {
+ segment_dir.join(format!(
+ "{segment_id}.{FTS_FILENAME_LAYOUT_V3_MARKER_SUFFIX}"
+ ))
+}
+
+enum LayoutMarkerRead {
+ Absent,
+ Exact,
+}
+
+fn read_layout_marker(path: &Path, expected: &[u8]) -> Result<LayoutMarkerRead> {
+ match fs::read(path) {
+ Ok(bytes) if bytes == expected => Ok(LayoutMarkerRead::Exact),
+ Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", path),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(LayoutMarkerRead::Absent),
Err(error) => Err(error)
- .with_context(|| format!("reading FTS filename-layout discriminator {:?}", marker)),
+ .with_context(|| format!("reading FTS filename-layout discriminator {:?}", path)),
}
}
+/// v3 wins when its marker is present. A corrupt marker fails closed and does
+/// not fall through to an older layout. File-family existence is never
+/// layout evidence.
+fn segment_filename_layout(segment_dir: &Path, segment_id: &str) -> Result<FilenameLayout> {
+ let v3 = segment_filename_layout_v3_marker_path(segment_dir, segment_id);
+ if matches!(
+ read_layout_marker(&v3, FTS_FILENAME_LAYOUT_V3_MARKER_BYTES)?,
+ LayoutMarkerRead::Exact
+ ) {
+ return Ok(FilenameLayout::CaseDistinguished);
+ }
+ let v2 = segment_filename_layout_v2_marker_path(segment_dir, segment_id);
+ if matches!(
+ read_layout_marker(&v2, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)?,
+ LayoutMarkerRead::Exact
+ ) {
+ return Ok(FilenameLayout::Encoded);
+ }
+ Ok(FilenameLayout::Legacy)
+}
+
/// Per-segment record of the analyzer each field's postings were built with
/// (#937). Written by [`FtsIndexWriter::finish`] as JSON
/// `{"fields":{"<field>":"<analyzer>"}}`, but ONLY when some field was
@@ -143,13 +200,22 @@ fn is_windows_reserved_device_name(field: &str) -> bool {
/// and overlong names can never create a child path or exceed a normal
/// filesystem's component limit. Unicode, `@`, and internal spaces are valid
/// portable literals; trailing spaces and dots are not portable to Windows.
-/// Preserving literals also preserves v1's pre-existing limitation: distinct
-/// portable names can alias on a filesystem that case-folds or normalizes
-/// Unicode. Digest-derived names remain distinct by exact UTF-8 bytes modulo a
-/// SHA-256 collision; eliminating literal aliases would require a new all-name
-/// encoding or name table and is outside filename layout v2.
+/// Preserving literals also preserves a limitation layout v2 left in place:
+/// distinct portable names can alias on a filesystem that case-folds.
+/// Layout v3 digests any literal whose Windows case-identity is not the name
+/// itself, so `page_Id` and `page_id` (and the NTFS pairs `ı`/`i`, `ß`/`ss`,
+/// `ſ`/`s`) cannot share a file. Digest-derived names remain distinct by
+/// exact UTF-8 bytes modulo a SHA-256 collision. Unicode normalization
+/// (NFC versus NFD) is still not encoded.
fn field_file_component(field: &str) -> Cow<'_, str> {
- let portable_literal = !field.is_empty()
+ if portable_literal_field(field) {
+ return Cow::Borrowed(field);
+ }
+ Cow::Owned(digest_field_component(field))
+}
+
+fn portable_literal_field(field: &str) -> bool {
+ !field.is_empty()
&& field.len() <= MAX_LITERAL_FIELD_COMPONENT_BYTES
&& field != "."
&& field != ".."
@@ -163,14 +229,88 @@ fn field_file_component(field: &str) -> Cow<'_, str> {
character,
'/' | '\\' | '<' | '>' | ':' | '"' | '|' | '?' | '*'
)
- });
+ })
+}
- if portable_literal {
- return Cow::Borrowed(field);
+fn digest_field_component(field: &str) -> String {
+ let digest = Sha256::digest(field.as_bytes());
+ format!("{ENCODED_FIELD_COMPONENT_PREFIX}{digest:x}")
+}
+
+/// Layout v3 component. Unsafe names hash exactly as in v2 (the digest is of
+/// the field's UTF-8 bytes, not of the v2 component). A portable literal is
+/// hashed only when NTFS would treat it as the same file as a different
+/// spelling.
+fn field_file_component_v3(field: &str) -> Cow<'_, str> {
+ if field_needs_case_distinguished_component(field) {
+ Cow::Owned(digest_field_component(field))
+ } else {
+ field_file_component(field)
}
+}
- let digest = Sha256::digest(field.as_bytes());
- Cow::Owned(format!("{ENCODED_FIELD_COMPONENT_PREFIX}{digest:x}"))
+/// `true` when v2 would keep `field` as a literal AND that literal's Windows
+/// case-identity is a different string.
+///
+/// The identity is Unicode lowercase plus the NTFS upcase-table exceptions
+/// that lowercase does not collapse: U+0131 `ı` → `i`, U+017F `ſ` → `s`,
+/// U+212A `K` → `k`, U+00DF `ß` → `ss`. A name that is already that identity
+/// (`page_id`, `i`, `ss`) stays a literal. Every other spelling of the same
+/// identity is a digest, so the two cannot land on one NTFS file. Hashing the
+/// exact bytes also keeps `Page` and `PAGE` apart from each other.
+fn field_needs_case_distinguished_component(field: &str) -> bool {
+ portable_literal_field(field) && !windows_case_identity_is_self(field)
+}
+
+fn windows_case_identity_is_self(field: &str) -> bool {
+ if field.is_ascii() {
+ return !field.bytes().any(|byte| byte.is_ascii_uppercase());
+ }
+ windows_case_identity(field) == field
+}
+
+fn windows_case_identity(field: &str) -> String {
+ let mut identity = String::with_capacity(field.len());
+ for character in field.chars() {
+ push_windows_case_folded(character, &mut identity);
+ }
+ identity
+}
+
+fn push_windows_case_folded(character: char, out: &mut String) {
+ match character {
+ '\u{0131}' => out.push('i'),
+ '\u{017f}' => out.push('s'),
+ '\u{212a}' => out.push('k'),
+ '\u{00df}' => out.push_str("ss"),
+ other => {
+ for lower in other.to_lowercase() {
+ match lower {
+ '\u{0131}' => out.push('i'),
+ '\u{017f}' => out.push('s'),
+ '\u{212a}' => out.push('k'),
+ '\u{00df}' => out.push_str("ss"),
+ folded => out.push(folded),
+ }
+ }
+ }
+ }
+}
+
+/// Temp path beside `path` whose name cannot collide with another field's
+/// temp in this process, including on a case-folding volume. The FST temp
+/// used to be `{segment}.{component}.fst.tmp` with no nonce: two components
+/// that NTFS treats as one name shared that temp, the first rename consumed
+/// it, and the second failed with `ERROR_FILE_NOT_FOUND` inside
+/// `publishing FTS side-cars`.
+fn unique_temp_sibling(path: &Path) -> PathBuf {
+ static NONCE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
+ let nonce = NONCE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ let file_name = path
+ .file_name()
+ .map(|name| name.to_string_lossy().into_owned())
+ .unwrap_or_else(|| "file".to_string());
+ path.with_file_name(format!("{file_name}.tmp.{}.{nonce}", std::process::id()))
}
/// Derive one FTS side-car path from controlled components only.
@@ -184,6 +324,24 @@ fn field_sidecar_path(
segment_dir.join(format!("{segment_id}.{field_component}.{extension}"))
}
+fn field_sidecar_path_for_layout(
+ segment_dir: &Path,
+ segment_id: &str,
+ field_name: &str,
+ extension: &str,
+ layout: FilenameLayout,
+) -> PathBuf {
+ match layout {
+ FilenameLayout::CaseDistinguished => {
+ let field_component = field_file_component_v3(field_name);
+ segment_dir.join(format!("{segment_id}.{field_component}.{extension}"))
+ }
+ FilenameLayout::Encoded | FilenameLayout::Legacy => {
+ field_sidecar_path(segment_dir, segment_id, field_name, extension)
+ }
+ }
+}
+
/// Return the exact pre-fix raw path only when it remains one child of the
/// segment directory on this host.
///
@@ -964,8 +1122,11 @@ fn index_field_values<'a>(
/// ```text
/// let mut writer = FtsIndexWriter::new(dir, segment_id, registry);
/// for doc in docs { writer.add_document(doc_id, &fields); }
-/// if writer.uses_encoded_field_filename_components() {
-/// // The engine advances its data-directory format before this step.
+/// if writer.uses_case_distinguished_field_filename_components() {
+/// // The engine advances its data-directory format to v3 before this step.
+/// writer.publish_case_distinguished_filename_layout()?;
+/// } else if writer.uses_encoded_field_filename_components() {
+/// // The engine advances its data-directory format to v2 before this step.
/// writer.publish_encoded_filename_layout()?;
/// }
/// writer.finish()?;
@@ -976,7 +1137,8 @@ pub struct FtsIndexWriter {
registry: Arc<AnalyzerRegistry>,
/// Per-field: (config, postings_writer, field_stats, norms)
fields: HashMap<String, FieldData>,
- encoded_filename_layout_published: bool,
+ /// Layout marker this writer has durably published, if any.
+ published_filename_layout: Option<FilenameLayout>,
/// Zstd effort for the `.meta` (ZFM4) and `.post` (ZPS1) envelopes.
/// [`ZSTD_DURABLE_LEVEL`] unless [`Self::with_zstd_level`] raised it.
zstd_level: i32,
@@ -1010,7 +1172,7 @@ impl FtsIndexWriter {
segment_id: segment_id.into(),
registry,
fields: HashMap::new(),
- encoded_filename_layout_published: false,
+ published_filename_layout: None,
zstd_level: ZSTD_DURABLE_LEVEL,
}
}
@@ -1071,8 +1233,10 @@ impl FtsIndexWriter {
}
/// Whether finishing this writer can create a v2 digest-derived field
- /// filename.
+ /// filename (a name that is not a portable literal).
///
+ /// This is false for a mixed-case literal such as `page_Id`. That name
+ /// needs [`Self::uses_case_distinguished_field_filename_components`].
/// Engine callers use this after configuration/document ingestion and
/// before [`Self::finish`] to durably advance the data-directory marker.
pub fn uses_encoded_field_filename_components(&self) -> bool {
@@ -1081,21 +1245,73 @@ impl FtsIndexWriter {
.any(|field| matches!(field_file_component(field), Cow::Owned(_)))
}
+ /// Whether a portable literal in this writer would alias another field
+ /// on a case-folding filesystem (`page_Id` / `page_id`, `ı` / `i`).
+ pub fn uses_case_distinguished_field_filename_components(&self) -> bool {
+ self.fields
+ .keys()
+ .any(|field| field_needs_case_distinguished_component(field))
+ }
+
+ fn required_filename_layout(&self) -> FilenameLayout {
+ if self.uses_case_distinguished_field_filename_components() {
+ FilenameLayout::CaseDistinguished
+ } else if self.uses_encoded_field_filename_components() {
+ FilenameLayout::Encoded
+ } else {
+ FilenameLayout::Legacy
+ }
+ }
+
/// Durably publish this segment's immutable v2 filename discriminator.
///
/// The engine must durably advance the data-directory format first, call
/// this method second, and call [`Self::finish`] last. `finish` refuses to
/// emit encoded side-cars without this proof. An existing discriminator is
- /// accepted only when it contains the exact v2 magic.
+ /// accepted only when it contains the exact v2 magic. A writer that also
+ /// needs layout v3 must call
+ /// [`Self::publish_case_distinguished_filename_layout`] instead: a v2
+ /// marker would keep `page_Id` literal.
pub fn publish_encoded_filename_layout(&mut self) -> Result<()> {
if !self.uses_encoded_field_filename_components() {
return Ok(());
}
+ if self.uses_case_distinguished_field_filename_components() {
+ bail!("case-distinguished FTS field filenames require filename layout v3, not v2");
+ }
+ self.publish_layout_marker(FilenameLayout::Encoded, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)
+ }
+
+ /// Durably publish this segment's immutable v3 filename discriminator.
+ ///
+ /// The engine must durably advance the data-directory format to v3 first.
+ /// A v2 reader looks `page_Id` up as a literal and would silently skip the
+ /// field, so the directory marker has to refuse that binary before any v3
+ /// side-car exists. No-op when no field needs the distinction.
+ pub fn publish_case_distinguished_filename_layout(&mut self) -> Result<()> {
+ if !self.uses_case_distinguished_field_filename_components() {
+ return Ok(());
+ }
+ self.publish_layout_marker(
+ FilenameLayout::CaseDistinguished,
+ FTS_FILENAME_LAYOUT_V3_MARKER_BYTES,
+ )
+ }
+
+ fn publish_layout_marker(&mut self, layout: FilenameLayout, magic: &[u8]) -> Result<()> {
fs::create_dir_all(&self.segment_dir)
.with_context(|| format!("creating segment dir {:?}", self.segment_dir))?;
- let marker = segment_filename_layout_v2_marker_path(&self.segment_dir, &self.segment_id);
+ let marker = match layout {
+ FilenameLayout::Encoded => {
+ segment_filename_layout_v2_marker_path(&self.segment_dir, &self.segment_id)
+ }
+ FilenameLayout::CaseDistinguished => {
+ segment_filename_layout_v3_marker_path(&self.segment_dir, &self.segment_id)
+ }
+ FilenameLayout::Legacy => return Ok(()),
+ };
match fs::read(&marker) {
- Ok(bytes) if bytes == FTS_FILENAME_LAYOUT_V2_MARKER_BYTES => {
+ Ok(bytes) if bytes == magic => {
// A previous attempt can have made the rename visible and
// then failed its durability confirmation. A fresh writer has
// no process-local proof, so confirm the directory entry
@@ -1109,17 +1325,15 @@ impl FtsIndexWriter {
// repairs a visible discriminator produced by an older build
// whose Windows directory-sync shim was a no-op.
#[cfg(windows)]
- xerj_common::fsio::write_file_durable(&marker, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)
- .with_context(|| {
- format!("confirming FTS filename-layout discriminator {:?}", marker)
- })?;
+ xerj_common::fsio::write_file_durable(&marker, magic).with_context(|| {
+ format!("confirming FTS filename-layout discriminator {:?}", marker)
+ })?;
}
Ok(_) => bail!("corrupt FTS filename-layout discriminator {:?}", marker),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
- xerj_common::fsio::write_file_durable(&marker, FTS_FILENAME_LAYOUT_V2_MARKER_BYTES)
- .with_context(|| {
- format!("publishing FTS filename-layout discriminator {:?}", marker)
- })?;
+ xerj_common::fsio::write_file_durable(&marker, magic).with_context(|| {
+ format!("publishing FTS filename-layout discriminator {:?}", marker)
+ })?;
}
Err(error) => {
return Err(error).with_context(|| {
@@ -1127,7 +1341,7 @@ impl FtsIndexWriter {
});
}
}
- self.encoded_filename_layout_published = true;
+ self.published_filename_layout = Some(layout);
Ok(())
}
@@ -1383,9 +1597,11 @@ impl FtsIndexWriter {
pub fn finish(self) -> Result<HashMap<String, FieldStats>> {
use rayon::prelude::*;
- if self.uses_encoded_field_filename_components() && !self.encoded_filename_layout_published
- {
- bail!("encoded FTS field filenames require a durable per-segment layout discriminator");
+ let layout = self.required_filename_layout();
+ if layout != FilenameLayout::Legacy && self.published_filename_layout != Some(layout) {
+ bail!(
+ "FTS filename layout {layout:?} requires a durable per-segment layout discriminator"
+ );
}
fs::create_dir_all(&self.segment_dir)
@@ -1451,6 +1667,7 @@ impl FtsIndexWriter {
&field_name,
field_data,
zstd_level,
+ layout,
)
.with_context(|| format!("writing field '{}'", field_name))?;
Ok((field_name, stats, files))
@@ -1477,6 +1694,7 @@ impl FtsIndexWriter {
field_name: &str,
field_data: FieldData,
zstd_level: i32,
+ layout: FilenameLayout,
) -> Result<Vec<xerj_common::fsio::StagedFile>> {
let mut staged = Vec::with_capacity(4);
// NOTE: `PathBuf::with_extension` replaces the final `.ext` in the path,
@@ -1484,7 +1702,9 @@ impl FtsIndexWriter {
// `with_extension("fst")` would strip `.field_name` and collapse every
// field to the same file. The shared helper also keeps untrusted field
// names from becoming paths.
- let filename = |ext: &str| field_sidecar_path(segment_dir, segment_id, field_name, ext);
+ let filename = |ext: &str| {
+ field_sidecar_path_for_layout(segment_dir, segment_id, field_name, ext, layout)
+ };
let fst_path = filename("fst");
let post_path = filename("post");
let meta_path = filename("meta");
@@ -1586,7 +1806,7 @@ impl FtsIndexWriter {
// FST streams into a same-directory temp file; fsync + rename +
// platform-specific durable replacement publishes it (RC4 W1 #10 —
// see the postings note above).
- let fst_tmp = filename("fst.tmp");
+ let fst_tmp = unique_temp_sibling(&fst_path);
let fst_file = BufWriter::new(
File::create(&fst_tmp).with_context(|| format!("creating FST file {:?}", fst_tmp))?,
);
@@ -1921,8 +2141,13 @@ pub fn segments_indexed_field_names(
) -> Result<Vec<Vec<String>>> {
let mut encoded: HashMap<String, &str> = HashMap::new();
for name in known_fields {
- if let Cow::Owned(component) = field_file_component(name) {
- encoded.insert(component, name.as_str());
+ // v2 and v3 hash unsafe names identically. v3 also hashes case-unstable
+ // literals (`page_Id`) that v2 stored under the raw spelling, so a
+ // directory scan has to recognize both components.
+ for component in [field_file_component(name), field_file_component_v3(name)] {
+ if let Cow::Owned(component) = component {
+ encoded.insert(component, name.as_str());
+ }
}
}
@@ -2150,7 +2375,7 @@ impl FtsIndexReader {
) -> Result<Self> {
let segment_dir = segment_dir.as_ref().to_path_buf();
let segment_id = segment_id.into();
- let encoded_layout = segment_uses_encoded_filename_layout(&segment_dir, &segment_id)?;
+ let layout = segment_filename_layout(&segment_dir, &segment_id)?;
let mut fields = HashMap::new();
for &field_name in field_names {
@@ -2158,16 +2383,22 @@ impl FtsIndexReader {
// Existence is deliberately irrelevant: otherwise a v1 field
// whose literal name equals another field's digest component can
// alias, and partial or stray files can alter layout selection.
+ // A v2 marker keeps mixed-case literals (`page_Id.fst`) on the
+ // v2 path; only a v3 marker looks up the case-distinguished digest.
let filename = |ext: &str| -> Option<PathBuf> {
- if encoded_layout {
- Some(field_sidecar_path(
- &segment_dir,
- &segment_id,
- field_name,
- ext,
- ))
- } else {
- legacy_field_sidecar_path(&segment_dir, &segment_id, field_name, ext)
+ match layout {
+ FilenameLayout::Legacy => {
+ legacy_field_sidecar_path(&segment_dir, &segment_id, field_name, ext)
+ }
+ FilenameLayout::Encoded | FilenameLayout::CaseDistinguished => {
+ Some(field_sidecar_path_for_layout(
+ &segment_dir,
+ &segment_id,
+ field_name,
+ ext,
+ layout,
+ ))
+ }
}
};
let Some(fst_path) = filename("fst") else {
@@ -3764,4 +3995,211 @@ mod tests {
.all(|entry| entry.file_type().unwrap().is_file()));
assert!(!root.path().join("escape").exists());
}
+
+ #[test]
+ fn case_aliased_fields_get_distinct_v3_components_and_round_trip() {
+ for (left, right) in [("page_Id", "page_id"), ("ı", "i"), ("ß", "ss"), ("ſ", "s")] {
+ assert_ne!(
+ windows_case_identity(field_file_component_v3(left).as_ref()),
+ windows_case_identity(field_file_component_v3(right).as_ref()),
+ "{left:?} and {right:?} still share a Windows case key"
+ );
+ }
+ // Layout v2 still aliases these. The marker, not file existence,
+ // selects which component function the reader uses.
+ assert_eq!(
+ windows_case_identity(field_file_component("page_Id").as_ref()),
+ windows_case_identity("page_id")
+ );
+ assert_eq!(field_file_component_v3("page_id").as_ref(), "page_id");
+ assert!(field_file_component_v3("page_Id")
+ .as_ref()
+ .starts_with(ENCODED_FIELD_COMPONENT_PREFIX));
+ assert_eq!(
+ field_file_component("bad/field"),
+ field_file_component_v3("bad/field"),
+ "unsafe names must keep the v2 digest under v3"
+ );
+ assert_eq!(field_file_component_v3("配置.名字").as_ref(), "配置.名字");
+
+ let dir = TempDir::new().unwrap();
+ let mut plain = FtsIndexWriter::new(dir.path(), "seg-plain", make_registry());
+ plain.add_document(
+ 0,
+ &[("page_id".to_owned(), FieldValues::from("only-lower"))]
+ .into_iter()
+ .collect(),
+ );
+ assert!(!plain.uses_case_distinguished_field_filename_components());
+ plain.finish().unwrap();
+ assert!(!segment_filename_layout_v3_marker_path(dir.path(), "seg-plain").exists());
+
+ let document: HashMap<String, FieldValues> = [
+ ("page_Id".to_owned(), FieldValues::from("upperneedle")),
+ ("page_id".to_owned(), FieldValues::from("lowerneedle")),
+ ("ı".to_owned(), FieldValues::from("dotlessneedle")),
+ ("i".to_owned(), FieldValues::from("dottedneedle")),
+ ]
+ .into_iter()
+ .collect();
+ let mut refused = FtsIndexWriter::new(dir.path(), "seg-case", make_registry());
+ refused.add_document(0, &document);
+ assert!(refused.uses_case_distinguished_field_filename_components());
+ assert!(refused.finish().is_err());
+
+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-case", make_registry());
+ writer.add_document(0, &document);
+ writer.publish_case_distinguished_filename_layout().unwrap();
+ writer.finish().unwrap();
+ assert!(segment_filename_layout_v3_marker_path(dir.path(), "seg-case").is_file());
+ assert!(!segment_filename_layout_v2_marker_path(dir.path(), "seg-case").exists());
+
+ let fields = ["page_Id", "page_id", "ı", "i"];
+ let reader = FtsIndexReader::open(dir.path(), "seg-case", &fields).unwrap();
+ assert!(reader.term_exists("page_Id", "upperneedle"));
+ assert!(reader.term_exists("page_id", "lowerneedle"));
+ assert!(!reader.term_exists("page_Id", "lowerneedle"));
+ assert!(reader.term_exists("ı", "dotlessneedle"));
+ assert!(reader.term_exists("i", "dottedneedle"));
+
+ let known: Vec<String> = fields.iter().map(|field| (*field).to_owned()).collect();
+ let listed = segment_indexed_field_names(dir.path(), "seg-case", &known).unwrap();
+ assert_eq!(
+ listed.iter().map(String::as_str).collect::<HashSet<_>>(),
+ fields.into_iter().collect::<HashSet<_>>()
+ );
+ }
+
+ #[test]
+ fn v3_layout_reuses_v2_digests_for_unsafe_names() {
+ let dir = TempDir::new().unwrap();
+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-both", make_registry());
+ let document: HashMap<String, FieldValues> = [
+ ("bad/field".to_owned(), FieldValues::from("slashneedle")),
+ ("page_Id".to_owned(), FieldValues::from("caseneedle")),
+ ]
+ .into_iter()
+ .collect();
+ writer.add_document(0, &document);
+ writer.publish_case_distinguished_filename_layout().unwrap();
+ writer.finish().unwrap();
+ assert!(!segment_filename_layout_v2_marker_path(dir.path(), "seg-both").exists());
+ assert!(field_sidecar_path(dir.path(), "seg-both", "bad/field", "fst").is_file());
+ let reader =
+ FtsIndexReader::open(dir.path(), "seg-both", &["bad/field", "page_Id"]).unwrap();
+ assert!(reader.term_exists("bad/field", "slashneedle"));
+ assert!(reader.term_exists("page_Id", "caseneedle"));
+ }
+
+ #[test]
+ fn v2_and_legacy_segments_still_read_a_literal_mixed_case_field() {
+ let root = TempDir::new().unwrap();
+ let segment_dir = root.path().join("segments");
+ fs::create_dir_all(&segment_dir).unwrap();
+ for (segment_id, write_v2) in [("legacy-page", false), ("v2-page", true)] {
+ write_legacy_raw_field_fixture(&segment_dir, segment_id, "page_Id");
+ if write_v2 {
+ fs::write(
+ segment_filename_layout_v2_marker_path(&segment_dir, segment_id),
+ FTS_FILENAME_LAYOUT_V2_MARKER_BYTES,
+ )
+ .unwrap();
+ }
+ let reader = FtsIndexReader::open(&segment_dir, segment_id, &["page_Id"]).unwrap();
+ assert!(
+ reader.term_exists("page_Id", "legacyneedle"),
+ "{segment_id} dropped the literal mixed-case side-car"
+ );
+ }
+ }
+
+ #[test]
+ fn corrupt_v3_marker_does_not_fall_through_to_a_literal_family() {
+ let dir = TempDir::new().unwrap();
+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-corrupt", make_registry());
+ writer.add_document(
+ 0,
+ &[("title".to_owned(), FieldValues::from("needle"))]
+ .into_iter()
+ .collect(),
+ );
+ writer.finish().unwrap();
+ fs::write(
+ segment_filename_layout_v3_marker_path(dir.path(), "seg-corrupt"),
+ b"not-a-layout-marker",
+ )
+ .unwrap();
+ assert!(FtsIndexReader::open(dir.path(), "seg-corrupt", &["title"]).is_err());
+ }
+
+ /// The two JeecgBoot rows from issue #2254: `onl_drag_comp` JImg and
+ /// `onl_drag_page_comp` JBar3d. Linux finishes them; the publish abort was
+ /// the Windows rename of their side-cars. `page_Id` is the mixed-case
+ /// column, and the config strings carry the URL and CJK that distinguished
+ /// these rows from the rest of the dump.
+ #[test]
+ fn jeecg_jimg_and_jbar3d_rows_finish_for_standard_and_keyword() {
+ let jimg = [
+ ("comp_type", "JImg"),
+ ("comp_name", "图片"),
+ ("icon", "ion:image-sharp"),
+ (
+ "comp_config",
+ r#"{"url":"http://api.jeecg.com/mock/42/nav"}"#,
+ ),
+ ("update_by", "admin"),
+ ];
+ let jbar = [
+ ("component", "JBar3d"),
+ ("page_Id", "1212561321588641792"),
+ (
+ "config",
+ r#"{"filed":"维度","url":"http://api.jeecg.com/mock/33/chart","name":"苹果"}"#,
+ ),
+ ("create_by", "admin"),
+ ];
+ for analyzer in ["standard", "keyword"] {
+ let dir = TempDir::new().unwrap();
+ let segment = format!("jeecg-{analyzer}");
+ let mut writer = FtsIndexWriter::new(dir.path(), &segment, make_registry());
+ let config = FieldIndexConfig {
+ analyzer: analyzer.to_owned(),
+ store_positions: analyzer != "keyword",
+ store_term_vectors: false,
+ };
+ for (name, _) in jimg.iter().chain(jbar.iter()) {
+ writer.configure_field((*name).to_owned(), config.clone());
+ }
+ let to_doc = |rows: &[(&str, &str)]| -> HashMap<String, FieldValues> {
+ rows.iter()
+ .map(|(key, value)| ((*key).to_owned(), FieldValues::from(*value)))
+ .collect()
+ };
+ writer.add_document(0, &to_doc(&jimg));
+ writer.add_document(1, &to_doc(&jbar));
+ assert!(writer.uses_case_distinguished_field_filename_components());
+ writer.publish_case_distinguished_filename_layout().unwrap();
+ writer.finish().expect(analyzer);
+ let fields: Vec<&str> = jimg
+ .iter()
+ .chain(jbar.iter())
+ .map(|(name, _)| *name)
+ .collect();
+ let reader = FtsIndexReader::open(dir.path(), &segment, &fields).unwrap();
+ if analyzer == "standard" {
+ assert!(reader.term_exists("comp_type", "jimg"));
+ assert!(reader.term_exists("component", "jbar3d"));
+ assert!(reader.term_exists("update_by", "admin"));
+ assert!(reader.term_exists("comp_config", "nav"));
+ assert!(reader.term_exists("config", "chart"));
+ assert!(reader.term_exists("page_Id", "1212561321588641792"));
+ } else {
+ assert!(reader.term_exists("comp_type", "JImg"));
+ assert!(reader.term_exists("component", "JBar3d"));
+ assert!(reader.term_exists("page_Id", "1212561321588641792"));
+ assert!(reader.term_exists("comp_name", "图片"));
+ }
+ assert!(segment_filename_layout_v3_marker_path(dir.path(), &segment).is_file());
+ }
+ }
}
diff --git a/engine/crates/xerj-storage/src/bundle.rs b/engine/crates/xerj-storage/src/bundle.rs
index 7a1bc836c..143aecede 100644
--- a/engine/crates/xerj-storage/src/bundle.rs
+++ b/engine/crates/xerj-storage/src/bundle.rs
@@ -150,7 +150,7 @@ pub fn valid_flush_artifact_set(segment_id: &str, artifacts: &[(String, u64, u32
let rest = &name[prefix.len()..];
if matches!(
rest,
- "seg" | "sidx" | "ids" | "dv" | "fts-layout-v2" | "ftsan"
+ "seg" | "sidx" | "ids" | "dv" | "fts-layout-v2" | "fts-layout-v3" | "ftsan"
) {
if !roles.insert(rest) {
return false;
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 737c31818..0a59e8bbc 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -39,10 +39,13 @@ use crate::{Result, SeqNo, StorageError};
/// Highest on-disk data-directory format this binary can read.
///
-/// Version 2 reserves digest-derived FTS field filename components. A v2
-/// binary can read v1 raw side-cars, but a v1 binary must refuse a directory
-/// after the first v2 side-car can be created.
-const DATA_DIR_FORMAT_VERSION: u32 = 2;
+/// Version 2 reserves digest-derived FTS field filename components. Version 3
+/// reserves digest components for field names whose Windows case-identity is
+/// not the name itself (`page_Id` versus `page_id`, `ı` versus `i`, `ß`
+/// versus `ss`). A v3 binary reads v1 raw side-cars and v2 encoded side-cars.
+/// A v2 binary must refuse the directory once a v3 side-car can be created:
+/// it would look `page_Id` up as a literal filename and silently skip it.
+const DATA_DIR_FORMAT_VERSION: u32 = 3;
/// Baseline written for fresh and pre-marker directories.
///
@@ -54,6 +57,10 @@ const DATA_DIR_BASE_FORMAT_VERSION: u32 = 1;
/// First layout version that permits digest-derived FTS field components.
const DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION: u32 = 2;
+/// First layout version that digest-encodes field names which collide under
+/// NTFS case folding but were portable literals in layout v2.
+const DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION: u32 = 3;
+
#[cfg(any(test, feature = "test-hooks"))]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DataDirFormatWriteFailpoint {
@@ -549,10 +556,18 @@ pub struct IndexStore {
/// directory-sync shim. The format lock serializes the false -> true
/// transition so concurrent flush/merge retries cannot bypass confirmation.
fts_v2_marker_durability_confirmed: AtomicBool,
+ /// Process-local proof that format version 3 has crossed the same
+ /// durable-publication boundary as the v2 flag. A confirmed v2 marker is
+ /// not proof of v3: the next case-distinguished field must still advance
+ /// 2 → 3. Confirming v3 also latches the v2 flag so a later v2 preflight
+ /// cannot rewrite the marker back down.
+ fts_v3_marker_durability_confirmed: AtomicBool,
#[cfg(any(test, feature = "test-hooks"))]
data_dir_format_write_failpoint: std::sync::atomic::AtomicU8,
#[cfg(any(test, feature = "test-hooks"))]
fts_v2_marker_durability_confirmations: AtomicU64,
+ #[cfg(any(test, feature = "test-hooks"))]
+ fts_v3_marker_durability_confirmations: AtomicU64,
/// Actual memtable shard count, derived at open time from
/// `IndexStoreConfig.num_wal_shards.max(1).next_power_of_two()`.
num_shards: usize,
@@ -905,10 +920,13 @@ impl IndexStore {
config,
data_dir_format_lock: Mutex::new(()),
fts_v2_marker_durability_confirmed: AtomicBool::new(false),
+ fts_v3_marker_durability_confirmed: AtomicBool::new(false),
#[cfg(any(test, feature = "test-hooks"))]
data_dir_format_write_failpoint: std::sync::atomic::AtomicU8::new(0),
#[cfg(any(test, feature = "test-hooks"))]
fts_v2_marker_durability_confirmations: AtomicU64::new(0),
+ #[cfg(any(test, feature = "test-hooks"))]
+ fts_v3_marker_durability_confirmations: AtomicU64::new(0),
num_shards,
snapshot: ArcSwap::from_pointee(snapshot),
wal_shards,
@@ -1654,7 +1672,7 @@ impl IndexStore {
/// Unlink every on-disk file belonging to the given segment ids — the
/// primary `.seg` plus all side-cars (`.sidx`, `.ids`, `.dv`,
/// `.<field>.post` / `.fst` / `.meta` / `.norms`) and the optional
- /// `.fts-layout-v2` discriminator.
+ /// `.fts-layout-v2` / `.fts-layout-v3` discriminator.
///
/// Disk-space fix (2026-07): called by the engine merge task right
/// after `apply_merge` commits, so merged-away input segments are
@@ -4292,11 +4310,31 @@ impl IndexStore {
/// and merge callers inside that one supported writer process; opening
/// multiple raw `IndexStore`s on the same directory remains unsupported.
pub fn ensure_fts_encoded_field_component_format(&self) -> Result<()> {
+ self.ensure_fts_filename_component_format(DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION)
+ }
+
+ /// Durably fence a writer before it creates a case-distinguished FTS
+ /// filename (`page_Id` and the other NTFS case-fold literals layout v2
+ /// kept). Advances the marker to v3 and does not return early on a
+ /// previously confirmed v2 marker — that confirmation must not skip the
+ /// 2 → 3 upgrade. Confirming v3 latches the v2 flag as well, so a later
+ /// v2 preflight cannot rewrite the marker downwards.
+ pub fn ensure_fts_case_distinguished_field_component_format(&self) -> Result<()> {
+ self.ensure_fts_filename_component_format(
+ DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION,
+ )
+ }
+
+ fn ensure_fts_filename_component_format(&self, target: u32) -> Result<()> {
let _format_guard = self.data_dir_format_lock.lock().unwrap();
- if self
- .fts_v2_marker_durability_confirmed
- .load(Ordering::Acquire)
- {
+ let already = if target >= DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION {
+ self.fts_v3_marker_durability_confirmed
+ .load(Ordering::Acquire)
+ } else {
+ self.fts_v2_marker_durability_confirmed
+ .load(Ordering::Acquire)
+ };
+ if already {
return Ok(());
}
let path = Self::data_dir_meta_path(&self.data_dir);
@@ -4328,12 +4366,14 @@ impl IndexStore {
DATA_DIR_FORMAT_VERSION
)));
}
- if meta.format_version >= DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION {
+ if meta.format_version >= target {
// Do not trust visibility as proof of durability. A previous
- // attempt may have renamed v2 into place and then failed its
- // durability confirmation. Re-establish that proof now, while
- // still holding the transition lock, before permitting any
- // encoded FTS filename to be created.
+ // attempt may have renamed the marker into place and then
+ // failed its durability confirmation. Re-establish that proof
+ // now, while still holding the transition lock, before
+ // permitting any encoded FTS filename to be created. A higher
+ // version than `target` is confirmed and left in place — v2
+ // preflight must not downgrade a v3 marker.
#[cfg(not(windows))]
if let Some(parent) = path.parent() {
xerj_common::fsio::fsync_dir(parent)?;
@@ -4345,7 +4385,7 @@ impl IndexStore {
// Windows directory-sync shim was a no-op.
#[cfg(windows)]
xerj_common::fsio::write_file_durable(&path, &std::fs::read(&path)?)?;
- self.note_fts_v2_marker_durability_confirmed();
+ self.note_fts_filename_format_confirmed(target);
return Ok(());
}
}
@@ -4359,7 +4399,7 @@ impl IndexStore {
})?;
object.insert(
"format_version".to_string(),
- serde_json::Value::from(DATA_DIR_FTS_ENCODED_FIELD_COMPONENT_VERSION),
+ serde_json::Value::from(target),
);
object.insert(
"xerj_version".to_string(),
@@ -4396,10 +4436,27 @@ impl IndexStore {
}
#[cfg(not(any(test, feature = "test-hooks")))]
xerj_common::fsio::write_file_durable(&path, &bytes)?;
- self.note_fts_v2_marker_durability_confirmed();
+ self.note_fts_filename_format_confirmed(target);
Ok(())
}
+ fn note_fts_filename_format_confirmed(&self, target: u32) {
+ if target >= DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION {
+ #[cfg(any(test, feature = "test-hooks"))]
+ self.fts_v3_marker_durability_confirmations
+ .fetch_add(1, Ordering::AcqRel);
+ self.fts_v3_marker_durability_confirmed
+ .store(true, Ordering::Release);
+ // Latch v2 without counting a v2 confirmation. The v2 counter is
+ // the number of times the v2 preflight itself proved durability;
+ // setting the flag stops that preflight from rewriting v3 as v2.
+ self.fts_v2_marker_durability_confirmed
+ .store(true, Ordering::Release);
+ return;
+ }
+ self.note_fts_v2_marker_durability_confirmed();
+ }
+
fn note_fts_v2_marker_durability_confirmed(&self) {
#[cfg(any(test, feature = "test-hooks"))]
self.fts_v2_marker_durability_confirmations
@@ -4423,6 +4480,12 @@ impl IndexStore {
.load(Ordering::Acquire)
}
+ #[cfg(any(test, feature = "test-hooks"))]
+ pub fn fts_v3_marker_durability_confirmations_for_test(&self) -> u64 {
+ self.fts_v3_marker_durability_confirmations
+ .load(Ordering::Acquire)
+ }
+
#[cfg(any(test, feature = "test-hooks"))]
pub fn clear_fts_v2_marker_durability_confirmation_for_test(&self) {
self.fts_v2_marker_durability_confirmed
@@ -5593,6 +5656,35 @@ mod tests {
.starts_with(&format!("{segment_id}."))));
}
+ #[test]
+ fn fts_layout_v3_discriminator_is_a_valid_flush_artifact() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ let segment_id = "12345678-1234-1234-1234-123456789abd";
+ let segments = dir.path().join("segments");
+ for (role, bytes) in [
+ ("seg", b"segment".as_slice()),
+ ("sidx", b"sidx".as_slice()),
+ ("ids", b"ids".as_slice()),
+ ("fts-layout-v3", b"XERJ_FTS_FILENAME_LAYOUT_V3\n".as_slice()),
+ ] {
+ std::fs::write(segments.join(format!("{segment_id}.{role}")), bytes).unwrap();
+ }
+ let meta = SegmentMeta {
+ id: segment_id.to_owned(),
+ doc_count: 1,
+ size_bytes: 7,
+ min_seq_no: 1,
+ max_seq_no: 1,
+ created_at_ms: 0,
+ has_tombstones: false,
+ seg_path: format!("{segment_id}.seg"),
+ sidx_path: format!("{segment_id}.sidx"),
+ };
+ store.write_flush_completion_manifest(&meta).unwrap();
+ assert!(store.validate_flush_completion_manifest(segment_id, 1, 1, 1));
+ }
+
#[test]
fn artifact_crc_streams_large_inputs_in_bounded_chunks() {
struct TrackingReader {
@@ -7933,6 +8025,60 @@ mod tests {
);
}
+ #[test]
+ fn case_distinguished_fts_preflight_advances_to_v3_and_v2_does_not_downgrade() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ let marker = dir.path().join(DATA_DIR_META_FILE);
+
+ store
+ .ensure_fts_case_distinguished_field_component_format()
+ .unwrap();
+ store
+ .ensure_fts_case_distinguished_field_component_format()
+ .unwrap();
+ assert_eq!(store.fts_v3_marker_durability_confirmations_for_test(), 1);
+
+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ assert_eq!(
+ meta.format_version,
+ DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION
+ );
+ assert!(IndexStore::check_data_dir_version_with_max(dir.path(), 2).is_err());
+ assert!(IndexStore::check_data_dir_version(dir.path()).is_ok());
+
+ store.ensure_fts_encoded_field_component_format().unwrap();
+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ assert_eq!(
+ meta.format_version, DATA_DIR_FTS_CASE_DISTINGUISHED_FIELD_COMPONENT_VERSION,
+ "a v2 preflight must not rewrite a v3 marker"
+ );
+ assert_eq!(
+ store.fts_v2_marker_durability_confirmations_for_test(),
+ 0,
+ "latching the v2 flag from v3 is not a v2 confirmation"
+ );
+ }
+
+ #[test]
+ fn case_distinguished_fts_preflight_upgrades_a_confirmed_v2_marker() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ store.ensure_fts_encoded_field_component_format().unwrap();
+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 1);
+
+ store
+ .ensure_fts_case_distinguished_field_component_format()
+ .unwrap();
+ let marker = dir.path().join(DATA_DIR_META_FILE);
+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ assert_eq!(meta.format_version, 3);
+ store.ensure_fts_encoded_field_component_format().unwrap();
+ let meta: DataDirMeta = serde_json::from_slice(&std::fs::read(&marker).unwrap()).unwrap();
+ assert_eq!(meta.format_version, 3);
+ assert_eq!(store.fts_v2_marker_durability_confirmations_for_test(), 1);
+ }
+
/// THE UPGRADE PATH: an rc3-vintage data dir has NO format marker (rc3
/// never wrote one). Reopening under this (rc4) binary must serve every
/// doc AND stamp a marker so subsequent opens are versioned. This is the