π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: hand-roll CRC32 in-module, drop crc32fast dependency
laurenspriemce89cf6e34f4
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 309fb08e40..f07475e5be 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2663,7 +2663,6 @@ version = "0.0.0"dependencies = ["anyhow","ash",- "crc32fast","ente-assets","ente-image","fast_image_resize",diff --git a/rust/crates/ml/Cargo.toml b/rust/crates/ml/Cargo.tomlindex e13c70c9fc..67de8babf3 100644--- a/rust/crates/ml/Cargo.toml+++ b/rust/crates/ml/Cargo.toml@@ -8,7 +8,6 @@ ml-assets = []usearch = ["dep:usearch"][dependencies]-crc32fast = "1.5"ente-assets.workspace = trueente-image.workspace = truefast_image_resize = { version = "5.5.0", features = ["image", "rayon"] }diff --git a/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.md b/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.mdindex a75ed79dc7..b86f52fe01 100644--- a/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.md+++ b/rust/crates/ml/docs/adr/0001-vecdb-durable-log-plus-derived-graph.md@@ -17,6 +17,6 @@ We replaced usearch (C++ under the hood, broken on the napi desktop toolchain, r- Single metric (inner product on L2-normalized vectors, distance = 1 β dot) and single scalar (f32) are hardcoded; the file header carries scalar/metric tags and the distance kernel sits behind one internal seam so smaller scalars (f16/i8) can be added without a format break.- Keys are strings (non-empty UTF-8, β€ 256 bytes), not integers: file/face/cluster IDs are used directly, which clears the way to retire the four AUTOINCREMENT key-mapping tables the app kept in sqlite solely for usearch's u64 keys once vecdb replaces usearch. Internally keys are interned to u32 slots; the graph, arena, and log replay speak slots.- Vectors live on disk as raw little-endian f32 payloads in the framed log records, and in memory in a chunked, 32-byte-aligned arena (chunks of SIMD lanes; free-list slot reuse; growth never reallocates existing chunks). Memory-mapping the log was rejected: it breaks wasm compatibility, fights rename-based compaction on Windows, and framed records give no alignment guarantee.-- vecdb is layered as an IO-free core (graph, arena, SIMD kernels β wasm32-compatible) beneath a native-only storage layer (fs, fsync, locking). The writer lock is std file locking (`File::try_lock`) on the `<path>.lock` sidecar β no `libc`, no `windows-sys` β and is held continuously across compaction, so there is never an unlock window; locking the sidecar rather than the log keeps Windows whole-file lock semantics away from readers and log renames. With `wide` and `crc32fast` already in the dependency tree, vecdb adds zero new Cargo.lock entries.+- vecdb is layered as an IO-free core (graph, arena, SIMD kernels β wasm32-compatible) beneath a native-only storage layer (fs, fsync, locking). The writer lock is std file locking (`File::try_lock`) on the `<path>.lock` sidecar β no `libc`, no `windows-sys` β and is held continuously across compaction, so there is never an unlock window; locking the sidecar rather than the log keeps Windows whole-file lock semantics away from readers and log renames. vecdb's only dependency is `wide` (already in the tree); locking is std file locking and CRC32 is hand-rolled in-module, so vecdb adds zero new Cargo.lock entries.- usearch stays in the tree, feature-gated, while vecdb runs as a parallel engine for internal users; removal is a separate later decision.- Derived-state failures degrade instead of failing writes: snapshot and compaction errors are logged as warnings and never fail an acknowledged write, open never fails for snapshot reasons, and only an explicit `flush()` surfaces snapshot errors to the caller.diff --git a/rust/crates/ml/src/vecdb/crc.rs b/rust/crates/ml/src/vecdb/crc.rsnew file mode 100644index 0000000000..831703900b--- /dev/null+++ b/rust/crates/ml/src/vecdb/crc.rs@@ -0,0 +1,151 @@+const POLYNOMIAL: u32 = 0xEDB8_8320;+const INITIAL_STATE: u32 = 0xFFFF_FFFF;++const TABLES: [[u32; 256]; 8] = build_tables();++const fn build_tables() -> [[u32; 256]; 8] {+ let mut tables = [[0u32; 256]; 8];+ let mut byte = 0usize;+ while byte < 256 {+ let mut crc = byte as u32;+ let mut bit = 0;+ while bit < 8 {+ crc = (crc >> 1) ^ (POLYNOMIAL * (crc & 1));+ bit += 1;+ }+ tables[0][byte] = crc;+ byte += 1;+ }+ let mut byte = 0usize;+ while byte < 256 {+ let mut crc = tables[0][byte];+ let mut stride = 1;+ while stride < 8 {+ crc = tables[0][(crc & 0xFF) as usize] ^ (crc >> 8);+ tables[stride][byte] = crc;+ stride += 1;+ }+ byte += 1;+ }+ tables+}++pub(crate) fn crc32(bytes: &[u8]) -> u32 {+ update_state(INITIAL_STATE, bytes) ^ INITIAL_STATE+}++pub(crate) struct Crc32 {+ state: u32,+}++impl Crc32 {+ pub(crate) fn new() -> Self {+ Self {+ state: INITIAL_STATE,+ }+ }++ pub(crate) fn update(&mut self, bytes: &[u8]) {+ self.state = update_state(self.state, bytes);+ }++ pub(crate) fn finalize(self) -> u32 {+ self.state ^ INITIAL_STATE+ }+}++fn update_state(state: u32, bytes: &[u8]) -> u32 {+ let (chunks, tail) = bytes.as_chunks::<8>();+ let mut state = state;+ for chunk in chunks {+ let low = state ^ u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);+ state = TABLES[7][(low & 0xFF) as usize]+ ^ TABLES[6][((low >> 8) & 0xFF) as usize]+ ^ TABLES[5][((low >> 16) & 0xFF) as usize]+ ^ TABLES[4][(low >> 24) as usize]+ ^ TABLES[3][chunk[4] as usize]+ ^ TABLES[2][chunk[5] as usize]+ ^ TABLES[1][chunk[6] as usize]+ ^ TABLES[0][chunk[7] as usize];+ }+ for &byte in tail {+ state = TABLES[0][((state ^ byte as u32) & 0xFF) as usize] ^ (state >> 8);+ }+ state+}++#[cfg(test)]+mod tests {+ use super::super::kernel::splitmix64;+ use super::*;++ fn bitwise_crc32(bytes: &[u8]) -> u32 {+ let mut crc = INITIAL_STATE;+ for &byte in bytes {+ crc ^= byte as u32;+ for _ in 0..8 {+ crc = (crc >> 1) ^ (POLYNOMIAL * (crc & 1));+ }+ }+ crc ^ INITIAL_STATE+ }++ fn byte_at_a_time(bytes: &[u8]) -> u32 {+ let mut state = INITIAL_STATE;+ for &byte in bytes {+ state = TABLES[0][((state ^ byte as u32) & 0xFF) as usize] ^ (state >> 8);+ }+ state ^ INITIAL_STATE+ }++ fn seeded_bytes(seed: u64, len: usize) -> Vec<u8> {+ let mut state = seed;+ (0..len).map(|_| splitmix64(&mut state) as u8).collect()+ }++ #[test]+ fn matches_the_standard_check_value() {+ assert_eq!(crc32(b"123456789"), 0xCBF4_3926);+ }++ #[test]+ fn empty_input_hashes_to_zero() {+ assert_eq!(crc32(&[]), 0);+ assert_eq!(Crc32::new().finalize(), 0);+ }++ #[test]+ fn agrees_with_bitwise_reference_across_lengths_and_offsets() {+ let buffer = seeded_bytes(0x00C1_0000, 608);+ for len in 0..600usize {+ for offset in 0..8usize {+ let slice = &buffer[offset..offset + len];+ assert_eq!(+ crc32(slice),+ bitwise_crc32(slice),+ "len {len} offset {offset}"+ );+ }+ }+ }++ #[test]+ fn agrees_with_byte_at_a_time_table_walk() {+ for seed in 0..8u64 {+ let bytes = seeded_bytes(0x00C2_0000 + seed, 517);+ assert_eq!(crc32(&bytes), byte_at_a_time(&bytes));+ }+ }++ #[test]+ fn split_updates_match_one_shot_at_every_boundary() {+ let bytes = seeded_bytes(0x00C3_0000, 100);+ let whole = crc32(&bytes);+ for split in 0..=bytes.len() {+ let mut hasher = Crc32::new();+ hasher.update(&bytes[..split]);+ hasher.update(&bytes[split..]);+ assert_eq!(hasher.finalize(), whole, "split at {split}");+ }+ }+}diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rsindex c4996b4847..6a43b7c072 100644--- a/rust/crates/ml/src/vecdb/log.rs+++ b/rust/crates/ml/src/vecdb/log.rs@@ -6,6 +6,7 @@ use std::time::{SystemTime, UNIX_EPOCH};use super::VecDbError;use super::arena::{MAX_KEY_BYTES, validate_key};+use super::crc::{Crc32, crc32};use super::kernel::{LANE_WIDTH, splitmix64};pub(crate) const HEADER_LEN: usize = 32;@@ -280,7 +281,7 @@ impl LogScanner<'_> {stored_crc_bytes[2],stored_crc_bytes[3],]);- let mut hasher = crc32fast::Hasher::new();+ let mut hasher = Crc32::new();hasher.update(&prefix);hasher.update(body);if hasher.finalize() != stored_crc {@@ -338,7 +339,7 @@ fn encode_header(dims: u32, generation: &[u8; 16]) -> [u8; HEADER_LEN] {bytes[7] = METRIC_TAG_INNER_PRODUCT;bytes[8..12].copy_from_slice(&dims.to_le_bytes());bytes[12..28].copy_from_slice(generation);- let crc = crc32fast::hash(&bytes[0..28]);+ let crc = crc32(&bytes[0..28]);bytes[28..32].copy_from_slice(&crc.to_le_bytes());bytes}@@ -351,7 +352,7 @@ fn decode_header(bytes: &[u8; HEADER_LEN], expected_dims: usize) -> Result<[u8;)));}let stored_crc = u32::from_le_bytes([bytes[28], bytes[29], bytes[30], bytes[31]]);- let computed_crc = crc32fast::hash(&bytes[0..28]);+ let computed_crc = crc32(&bytes[0..28]);if stored_crc != computed_crc {return Err(VecDbError::Corrupt(format!("log header crc mismatch: stored {stored_crc:08x}, computed {computed_crc:08x}"@@ -422,7 +423,7 @@ fn encode_record_into(buffer: &mut Vec<u8>, entry: &LogEntry<'_>) {for value in payload {buffer.extend_from_slice(&value.to_le_bytes());}- let crc = crc32fast::hash(&buffer[start..]);+ let crc = crc32(&buffer[start..]);buffer.extend_from_slice(&crc.to_le_bytes());}@@ -550,7 +551,7 @@ mod tests {bytes.extend_from_slice(&key_len.to_le_bytes());bytes.extend_from_slice(key);bytes.extend_from_slice(payload);- let crc = crc32fast::hash(&bytes);+ let crc = crc32(&bytes);bytes.extend_from_slice(&crc.to_le_bytes());bytes}@@ -562,7 +563,7 @@ mod tests {}fn refresh_crc(bytes: &mut [u8; HEADER_LEN]) {- let crc = crc32fast::hash(&bytes[0..28]);+ let crc = crc32(&bytes[0..28]);bytes[28..32].copy_from_slice(&crc.to_le_bytes());}diff --git a/rust/crates/ml/src/vecdb/mod.rs b/rust/crates/ml/src/vecdb/mod.rsindex 6133839200..9d5f8c2f10 100644--- a/rust/crates/ml/src/vecdb/mod.rs+++ b/rust/crates/ml/src/vecdb/mod.rs@@ -3,6 +3,7 @@ use std::path::{Path, PathBuf};use thiserror::Error;mod arena;+mod crc;mod graph;mod kernel;mod lock;diff --git a/rust/crates/ml/src/vecdb/snapshot.rs b/rust/crates/ml/src/vecdb/snapshot.rsindex 74fc978410..b5602c7f1a 100644--- a/rust/crates/ml/src/vecdb/snapshot.rs+++ b/rust/crates/ml/src/vecdb/snapshot.rs@@ -3,6 +3,7 @@ use std::io::{ErrorKind, Write};use std::path::{Path, PathBuf};use super::VecDbError;+use super::crc::crc32;use super::graph::{Graph, GraphNodeParts};use super::log::{remove_if_present, rename_with_windows_fallback, sync_parent_dir};@@ -92,7 +93,7 @@ fn encode_snapshot(generation: &[u8; 16], covered_log_offset: u64, graph: &Graph}}}- let crc = crc32fast::hash(&bytes[MAGIC.len()..]);+ let crc = crc32(&bytes[MAGIC.len()..]);bytes.extend_from_slice(&crc.to_le_bytes());bytes}@@ -118,7 +119,7 @@ fn decode_snapshot(bytes[body_end + 2],bytes[body_end + 3],]);- let computed_crc = crc32fast::hash(&bytes[MAGIC.len()..body_end]);+ let computed_crc = crc32(&bytes[MAGIC.len()..body_end]);if stored_crc != computed_crc {return Err(format!("crc mismatch: stored {stored_crc:08x}, computed {computed_crc:08x}"@@ -315,7 +316,7 @@ mod tests {fn refresh_crc(bytes: &mut [u8]) {let body_end = bytes.len() - CRC_LEN;- let crc = crc32fast::hash(&bytes[MAGIC.len()..body_end]);+ let crc = crc32(&bytes[MAGIC.len()..body_end]);bytes[body_end..].copy_from_slice(&crc.to_le_bytes());}@@ -588,7 +589,7 @@ mod tests {bytes.extend_from_slice(&slot.to_le_bytes());bytes.push(0);bytes.extend_from_slice(&0u16.to_le_bytes());- let crc = crc32fast::hash(&bytes[MAGIC.len()..]);+ let crc = crc32(&bytes[MAGIC.len()..]);bytes.extend_from_slice(&crc.to_le_bytes());let loaded = decode_snapshot(&bytes, &golden_generation(), 300).unwrap();assert_eq!(loaded.entry_point, Some(slot));