AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): stop grouping 0-byte files as content duplicates
snmsoodand48a4748e0d2
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rsindex 2907ae21d..2755c1496 100644--- a/engine/crates/xerj-autoindex/src/content.rs+++ b/engine/crates/xerj-autoindex/src/content.rs@@ -310,6 +310,30 @@ fn resolve_with_digests((files[left].is_symlink, files[left].rel_id.as_str()).cmp(&(files[right].is_symlink, files[right].rel_id.as_str()))});+ // #2449: a 0-byte file has no content to share, so a zero-byte group is+ // not a duplicate group. Every empty file in a tree (`.gitkeep`,+ // `__init__.py`, placeholder `.ts` files) hashes to the same digest, so+ // without this every one of them but the first was reported as a+ // "content duplicate" of an unrelated path — pure noise in the plan, and+ // on a resume it also diverged from the fresh plan, which drops aliases+ // whose canonical is junk (an empty file is junk: `binary content+ // (empty)`). Each empty file keeps its own identity instead, so it is+ // reported once, by path, for what it is.+ //+ // The first file keeps the historical content key; the rest get a+ // path-derived one, because equal bytes would otherwise give them equal+ // ids. `-empty-` rather than `-collision-` on purpose: these are not+ // unequal files that collided, they are files with nothing to dedupe.+ if files[indices[0]].size == 0 {+ for &index in &indices[1..] {+ resolved_keys[index] = format!(+ "{}-empty-{:032x}",+ digests[index],+ xxhash_rust::xxh3::xxh3_128(files[index].rel_id.as_bytes())+ );+ }+ continue;+ }// A digest match is still only a candidate: verify exact bytes.let mut representatives: Vec<usize> = Vec::new();for index in indices {@@ -580,6 +604,48 @@ mod tests {assert_eq!(inv.keys[0], inv.duplicates[0].file_key);}+ /// #2449: every 0-byte file hashes to the same digest, so all of them but+ /// the first were reported as "byte-identical content duplicates" of an+ /// unrelated path. A placeholder file duplicates nothing — it has no+ /// content — so a zero-byte group must not group at all.+ #[test]+ fn zero_byte_files_are_not_content_duplicates_of_each_other() {+ let dir = tempfile::tempdir().unwrap();+ for name in ["pkg/__init__.py", "src/placeholder.ts", "empty.json"] {+ let path = dir.path().join(name);+ fs::create_dir_all(path.parent().unwrap()).unwrap();+ fs::write(&path, b"").unwrap();+ }+ let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();+ assert_eq!(inv.files.len(), 3, "every empty file stays its own entry");+ assert!(+ inv.duplicates.is_empty(),+ "an empty file duplicates nothing: {:?}",+ inv.duplicates.iter().map(|d| &d.rel).collect::<Vec<_>>()+ );+ let unique: std::collections::HashSet<&String> = inv.keys.iter().collect();+ assert_eq!(+ unique.len(),+ inv.keys.len(),+ "empty files must still get distinct ids: {:?}",+ inv.keys+ );+ }++ /// The carve-out is only for 0 bytes: equal non-empty files still group.+ #[test]+ fn a_zero_byte_file_does_not_stop_equal_non_empty_files_from_grouping() {+ let dir = tempfile::tempdir().unwrap();+ fs::write(dir.path().join("empty.txt"), b"").unwrap();+ fs::write(dir.path().join("a.txt"), b"same records\n").unwrap();+ fs::write(dir.path().join("b.txt"), b"same records\n").unwrap();+ let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();+ assert_eq!(inv.files.len(), 2);+ assert_eq!(inv.duplicates.len(), 1);+ assert_eq!(inv.duplicates[0].rel, "b.txt");+ assert_eq!(inv.duplicates[0].duplicate_of, "a.txt");+ }+#[test]fn same_prefix_and_size_but_different_tail_get_distinct_durable_keys() {let dir = tempfile::tempdir().unwrap();