Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

vecdb: hand-roll CRC32 in-module, drop crc32fast dependency

laurenspriemce89cf6e34f4
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 309fb08e40..f07475e5be 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2663,7 +2663,6 @@ version = "0.0.0"
dependencies = [
"anyhow",
"ash",
- "crc32fast",
"ente-assets",
"ente-image",
"fast_image_resize",
diff --git a/rust/crates/ml/Cargo.toml b/rust/crates/ml/Cargo.toml
index e13c70c9fc..67de8babf3 100644
--- a/rust/crates/ml/Cargo.toml
+++ b/rust/crates/ml/Cargo.toml
@@ -8,7 +8,6 @@ ml-assets = []
usearch = ["dep:usearch"]
[dependencies]
-crc32fast = "1.5"
ente-assets.workspace = true
ente-image.workspace = true
fast_image_resize = { version = "5.5.0", features = ["image", "rayon"] }
diff --git a/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.md b/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.md
index a75ed79dc7..b86f52fe01 100644
--- a/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.md
+++ b/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.md
@@ -17,6 +17,6 @@ We replaced usearch (C++ under the hood, broken on the napi desktop toolchain, r
- Single metric (inner product on L2-normalized vectors, distance = 1 βˆ’ dot) and single scalar (f32) are hardcoded; the file header carries scalar/metric tags and the distance kernel sits behind one internal seam so smaller scalars (f16/i8) can be added without a format break.
- Keys are strings (non-empty UTF-8, ≀ 256 bytes), not integers: file/face/cluster IDs are used directly, which clears the way to retire the four AUTOINCREMENT key-mapping tables the app kept in sqlite solely for usearch's u64 keys once vecdb replaces usearch. Internally keys are interned to u32 slots; the graph, arena, and log replay speak slots.
- Vectors live on disk as raw little-endian f32 payloads in the framed log records, and in memory in a chunked, 32-byte-aligned arena (chunks of SIMD lanes; free-list slot reuse; growth never reallocates existing chunks). Memory-mapping the log was rejected: it breaks wasm compatibility, fights rename-based compaction on Windows, and framed records give no alignment guarantee.
-- vecdb is layered as an IO-free core (graph, arena, SIMD kernels β€” wasm32-compatible) beneath a native-only storage layer (fs, fsync, locking). The writer lock is std file locking (`File::try_lock`) on the `<path>.lock` sidecar β€” no `libc`, no `windows-sys` β€” and is held continuously across compaction, so there is never an unlock window; locking the sidecar rather than the log keeps Windows whole-file lock semantics away from readers and log renames. With `wide` and `crc32fast` already in the dependency tree, vecdb adds zero new Cargo.lock entries.
+- vecdb is layered as an IO-free core (graph, arena, SIMD kernels β€” wasm32-compatible) beneath a native-only storage layer (fs, fsync, locking). The writer lock is std file locking (`File::try_lock`) on the `<path>.lock` sidecar β€” no `libc`, no `windows-sys` β€” and is held continuously across compaction, so there is never an unlock window; locking the sidecar rather than the log keeps Windows whole-file lock semantics away from readers and log renames. vecdb's only dependency is `wide` (already in the tree); locking is std file locking and CRC32 is hand-rolled in-module, so vecdb adds zero new Cargo.lock entries.
- usearch stays in the tree, feature-gated, while vecdb runs as a parallel engine for internal users; removal is a separate later decision.
- Derived-state failures degrade instead of failing writes: snapshot and compaction errors are logged as warnings and never fail an acknowledged write, open never fails for snapshot reasons, and only an explicit `flush()` surfaces snapshot errors to the caller.
diff --git a/rust/crates/ml/src/vecdb/crc.rs b/rust/crates/ml/src/vecdb/crc.rs
new file mode 100644
index 0000000000..831703900b
--- /dev/null
+++ b/rust/crates/ml/src/vecdb/crc.rs
@@ -0,0 +1,151 @@
+const POLYNOMIAL: u32 = 0xEDB8_8320;
+const INITIAL_STATE: u32 = 0xFFFF_FFFF;
+
+const TABLES: [[u32; 256]; 8] = build_tables();
+
+const fn build_tables() -> [[u32; 256]; 8] {
+ let mut tables = [[0u32; 256]; 8];
+ let mut byte = 0usize;
+ while byte < 256 {
+ let mut crc = byte as u32;
+ let mut bit = 0;
+ while bit < 8 {
+ crc = (crc >> 1) ^ (POLYNOMIAL * (crc & 1));
+ bit += 1;
+ }
+ tables[0][byte] = crc;
+ byte += 1;
+ }
+ let mut byte = 0usize;
+ while byte < 256 {
+ let mut crc = tables[0][byte];
+ let mut stride = 1;
+ while stride < 8 {
+ crc = tables[0][(crc & 0xFF) as usize] ^ (crc >> 8);
+ tables[stride][byte] = crc;
+ stride += 1;
+ }
+ byte += 1;
+ }
+ tables
+}
+
+pub(crate) fn crc32(bytes: &[u8]) -> u32 {
+ update_state(INITIAL_STATE, bytes) ^ INITIAL_STATE
+}
+
+pub(crate) struct Crc32 {
+ state: u32,
+}
+
+impl Crc32 {
+ pub(crate) fn new() -> Self {
+ Self {
+ state: INITIAL_STATE,
+ }
+ }
+
+ pub(crate) fn update(&mut self, bytes: &[u8]) {
+ self.state = update_state(self.state, bytes);
+ }
+
+ pub(crate) fn finalize(self) -> u32 {
+ self.state ^ INITIAL_STATE
+ }
+}
+
+fn update_state(state: u32, bytes: &[u8]) -> u32 {
+ let (chunks, tail) = bytes.as_chunks::<8>();
+ let mut state = state;
+ for chunk in chunks {
+ let low = state ^ u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
+ state = TABLES[7][(low & 0xFF) as usize]
+ ^ TABLES[6][((low >> 8) & 0xFF) as usize]
+ ^ TABLES[5][((low >> 16) & 0xFF) as usize]
+ ^ TABLES[4][(low >> 24) as usize]
+ ^ TABLES[3][chunk[4] as usize]
+ ^ TABLES[2][chunk[5] as usize]
+ ^ TABLES[1][chunk[6] as usize]
+ ^ TABLES[0][chunk[7] as usize];
+ }
+ for &byte in tail {
+ state = TABLES[0][((state ^ byte as u32) & 0xFF) as usize] ^ (state >> 8);
+ }
+ state
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::kernel::splitmix64;
+ use super::*;
+
+ fn bitwise_crc32(bytes: &[u8]) -> u32 {
+ let mut crc = INITIAL_STATE;
+ for &byte in bytes {
+ crc ^= byte as u32;
+ for _ in 0..8 {
+ crc = (crc >> 1) ^ (POLYNOMIAL * (crc & 1));
+ }
+ }
+ crc ^ INITIAL_STATE
+ }
+
+ fn byte_at_a_time(bytes: &[u8]) -> u32 {
+ let mut state = INITIAL_STATE;
+ for &byte in bytes {
+ state = TABLES[0][((state ^ byte as u32) & 0xFF) as usize] ^ (state >> 8);
+ }
+ state ^ INITIAL_STATE
+ }
+
+ fn seeded_bytes(seed: u64, len: usize) -> Vec<u8> {
+ let mut state = seed;
+ (0..len).map(|_| splitmix64(&mut state) as u8).collect()
+ }
+
+ #[test]
+ fn matches_the_standard_check_value() {
+ assert_eq!(crc32(b"123456789"), 0xCBF4_3926);
+ }
+
+ #[test]
+ fn empty_input_hashes_to_zero() {
+ assert_eq!(crc32(&[]), 0);
+ assert_eq!(Crc32::new().finalize(), 0);
+ }
+
+ #[test]
+ fn agrees_with_bitwise_reference_across_lengths_and_offsets() {
+ let buffer = seeded_bytes(0x00C1_0000, 608);
+ for len in 0..600usize {
+ for offset in 0..8usize {
+ let slice = &buffer[offset..offset + len];
+ assert_eq!(
+ crc32(slice),
+ bitwise_crc32(slice),
+ "len {len} offset {offset}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn agrees_with_byte_at_a_time_table_walk() {
+ for seed in 0..8u64 {
+ let bytes = seeded_bytes(0x00C2_0000 + seed, 517);
+ assert_eq!(crc32(&bytes), byte_at_a_time(&bytes));
+ }
+ }
+
+ #[test]
+ fn split_updates_match_one_shot_at_every_boundary() {
+ let bytes = seeded_bytes(0x00C3_0000, 100);
+ let whole = crc32(&bytes);
+ for split in 0..=bytes.len() {
+ let mut hasher = Crc32::new();
+ hasher.update(&bytes[..split]);
+ hasher.update(&bytes[split..]);
+ assert_eq!(hasher.finalize(), whole, "split at {split}");
+ }
+ }
+}
diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rs
index c4996b4847..6a43b7c072 100644
--- a/rust/crates/ml/src/vecdb/log.rs
+++ b/rust/crates/ml/src/vecdb/log.rs
@@ -6,6 +6,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use super::VecDbError;
use super::arena::{MAX_KEY_BYTES, validate_key};
+use super::crc::{Crc32, crc32};
use super::kernel::{LANE_WIDTH, splitmix64};
pub(crate) const HEADER_LEN: usize = 32;
@@ -280,7 +281,7 @@ impl LogScanner<'_> {
stored_crc_bytes[2],
stored_crc_bytes[3],
]);
- let mut hasher = crc32fast::Hasher::new();
+ let mut hasher = Crc32::new();
hasher.update(&prefix);
hasher.update(body);
if hasher.finalize() != stored_crc {
@@ -338,7 +339,7 @@ fn encode_header(dims: u32, generation: &[u8; 16]) -> [u8; HEADER_LEN] {
bytes[7] = METRIC_TAG_INNER_PRODUCT;
bytes[8..12].copy_from_slice(&dims.to_le_bytes());
bytes[12..28].copy_from_slice(generation);
- let crc = crc32fast::hash(&bytes[0..28]);
+ let crc = crc32(&bytes[0..28]);
bytes[28..32].copy_from_slice(&crc.to_le_bytes());
bytes
}
@@ -351,7 +352,7 @@ fn decode_header(bytes: &[u8; HEADER_LEN], expected_dims: usize) -> Result<[u8;
)));
}
let stored_crc = u32::from_le_bytes([bytes[28], bytes[29], bytes[30], bytes[31]]);
- let computed_crc = crc32fast::hash(&bytes[0..28]);
+ let computed_crc = crc32(&bytes[0..28]);
if stored_crc != computed_crc {
return Err(VecDbError::Corrupt(format!(
"log header crc mismatch: stored {stored_crc:08x}, computed {computed_crc:08x}"
@@ -422,7 +423,7 @@ fn encode_record_into(buffer: &mut Vec<u8>, entry: &LogEntry<'_>) {
for value in payload {
buffer.extend_from_slice(&value.to_le_bytes());
}
- let crc = crc32fast::hash(&buffer[start..]);
+ let crc = crc32(&buffer[start..]);
buffer.extend_from_slice(&crc.to_le_bytes());
}
@@ -550,7 +551,7 @@ mod tests {
bytes.extend_from_slice(&key_len.to_le_bytes());
bytes.extend_from_slice(key);
bytes.extend_from_slice(payload);
- let crc = crc32fast::hash(&bytes);
+ let crc = crc32(&bytes);
bytes.extend_from_slice(&crc.to_le_bytes());
bytes
}
@@ -562,7 +563,7 @@ mod tests {
}
fn refresh_crc(bytes: &mut [u8; HEADER_LEN]) {
- let crc = crc32fast::hash(&bytes[0..28]);
+ let crc = crc32(&bytes[0..28]);
bytes[28..32].copy_from_slice(&crc.to_le_bytes());
}
diff --git a/rust/crates/ml/src/vecdb/mod.rs b/rust/crates/ml/src/vecdb/mod.rs
index 6133839200..9d5f8c2f10 100644
--- a/rust/crates/ml/src/vecdb/mod.rs
+++ b/rust/crates/ml/src/vecdb/mod.rs
@@ -3,6 +3,7 @@ use std::path::{Path, PathBuf};
use thiserror::Error;
mod arena;
+mod crc;
mod graph;
mod kernel;
mod lock;
diff --git a/rust/crates/ml/src/vecdb/snapshot.rs b/rust/crates/ml/src/vecdb/snapshot.rs
index 74fc978410..b5602c7f1a 100644
--- a/rust/crates/ml/src/vecdb/snapshot.rs
+++ b/rust/crates/ml/src/vecdb/snapshot.rs
@@ -3,6 +3,7 @@ use std::io::{ErrorKind, Write};
use std::path::{Path, PathBuf};
use super::VecDbError;
+use super::crc::crc32;
use super::graph::{Graph, GraphNodeParts};
use super::log::{remove_if_present, rename_with_windows_fallback, sync_parent_dir};
@@ -92,7 +93,7 @@ fn encode_snapshot(generation: &[u8; 16], covered_log_offset: u64, graph: &Graph
}
}
}
- let crc = crc32fast::hash(&bytes[MAGIC.len()..]);
+ let crc = crc32(&bytes[MAGIC.len()..]);
bytes.extend_from_slice(&crc.to_le_bytes());
bytes
}
@@ -118,7 +119,7 @@ fn decode_snapshot(
bytes[body_end + 2],
bytes[body_end + 3],
]);
- let computed_crc = crc32fast::hash(&bytes[MAGIC.len()..body_end]);
+ let computed_crc = crc32(&bytes[MAGIC.len()..body_end]);
if stored_crc != computed_crc {
return Err(format!(
"crc mismatch: stored {stored_crc:08x}, computed {computed_crc:08x}"
@@ -315,7 +316,7 @@ mod tests {
fn refresh_crc(bytes: &mut [u8]) {
let body_end = bytes.len() - CRC_LEN;
- let crc = crc32fast::hash(&bytes[MAGIC.len()..body_end]);
+ let crc = crc32(&bytes[MAGIC.len()..body_end]);
bytes[body_end..].copy_from_slice(&crc.to_le_bytes());
}
@@ -588,7 +589,7 @@ mod tests {
bytes.extend_from_slice(&slot.to_le_bytes());
bytes.push(0);
bytes.extend_from_slice(&0u16.to_le_bytes());
- let crc = crc32fast::hash(&bytes[MAGIC.len()..]);
+ let crc = crc32(&bytes[MAGIC.len()..]);
bytes.extend_from_slice(&crc.to_le_bytes());
let loaded = decode_snapshot(&bytes, &golden_generation(), 300).unwrap();
assert_eq!(loaded.entry_point, Some(slot));