Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Move vecdb into its own crate

laurenspriem0dc528709462
diff --git a/.github/scripts/ci/paths.json b/.github/scripts/ci/paths.json
index 533dfda147..abaae036c2 100644
--- a/.github/scripts/ci/paths.json
+++ b/.github/scripts/ci/paths.json
@@ -19,6 +19,7 @@
"rust/bindings/napi/**",
"rust/crates/image/**",
"rust/crates/ml/**",
+ "rust/crates/vecdb/**",
"rust/tools/codegen/**"
],
"docs-verify-build": [".github/workflows/docs-verify-build.yml", "docs/**"],
@@ -64,6 +65,7 @@
"rust/crates/locker/**",
"rust/crates/ml/**",
"rust/crates/photos/**",
+ "rust/crates/vecdb/**",
"rust/tools/codegen/**"
],
"mobile-podfile-lock": [
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 650b173887..20b7cc6510 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2784,6 +2784,7 @@ dependencies = [
"ente-location",
"ente-ml",
"ente-photos",
+ "ente-vecdb",
"flutter_rust_bridge",
"log",
]
@@ -2903,6 +2904,15 @@ dependencies = [
"log",
]
+[[package]]
+name = "ente-vecdb"
+version = "0.0.0"
+dependencies = [
+ "log",
+ "tempfile",
+ "thiserror 2.0.18",
+]
+
[[package]]
name = "ente-wasm-lib"
version = "0.0.0"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 3a8c67e29e..fc9c2b55ed 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -39,6 +39,7 @@ members = [
"crates/photos",
"crates/space",
"crates/test-support",
+ "crates/vecdb",
"tools/codegen",
"tools/location-dataset",
]
@@ -73,6 +74,7 @@ ente-photos = { path = "crates/photos" }
ente-space = { path = "crates/space" }
ente-test-support = { path = "crates/test-support" }
ente-uniffi-log = { path = "bindings/uniffi/log" }
+ente-vecdb = { path = "crates/vecdb" }
ente-wasm-lib = { path = "bindings/wasm/lib" }
fast_image_resize = { version = "5.5.0", features = ["image"] }
flate2 = "1.1.9"
@@ -153,6 +155,8 @@ unwrap_used = "deny"
[profile.dev.package.argon2]
opt-level = 3
-# Unoptimized HNSW construction makes the vecdb recall fixtures dominate test runtime.
[profile.dev.package.ente-ml]
opt-level = 3
+
+[profile.dev.package.ente-vecdb]
+opt-level = 3
diff --git a/rust/bindings/frb/photos/Cargo.toml b/rust/bindings/frb/photos/Cargo.toml
index 216fcf747a..ac1b6f50cb 100644
--- a/rust/bindings/frb/photos/Cargo.toml
+++ b/rust/bindings/frb/photos/Cargo.toml
@@ -15,6 +15,7 @@ ente-image.workspace = true
ente-location.workspace = true
ente-ml = { workspace = true, features = ["usearch"] }
ente-photos.workspace = true
+ente-vecdb.workspace = true
flutter_rust_bridge.workspace = true
log.workspace = true
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rs
index e7400b5b4a..b927005996 100644
--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs
+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs
@@ -1,6 +1,6 @@
use std::path::Path;
-use ente_ml::vecdb;
+use ente_vecdb as vecdb;
use flutter_rust_bridge::frb;
#[derive(Clone, Debug)]
diff --git a/rust/checks/lint-exceptions/check.mjs b/rust/checks/lint-exceptions/check.mjs
index 5ee1c9bba7..7b6f813f76 100644
--- a/rust/checks/lint-exceptions/check.mjs
+++ b/rust/checks/lint-exceptions/check.mjs
@@ -11,7 +11,7 @@ for (const path of files()) {
const lints = expectedLints(attribute.text);
if (attribute.inner) {
if (
- path !== "rust/crates/ml/examples/vecdb_bench.rs" ||
+ path !== "rust/crates/vecdb/examples/vecdb_bench.rs" ||
lints?.length !== 1 ||
lints[0] !== "clippy::expect_used"
) {
diff --git a/rust/checks/lint-exceptions/test.mjs b/rust/checks/lint-exceptions/test.mjs
index 9301ed2130..31fac57a00 100644
--- a/rust/checks/lint-exceptions/test.mjs
+++ b/rust/checks/lint-exceptions/test.mjs
@@ -38,14 +38,14 @@ assert.equal(result.status, 0, result.stderr);
assert.equal(result.stderr, "");
result = run(
- "rust/crates/ml/examples/vecdb_bench.rs",
+ "rust/crates/vecdb/examples/vecdb_bench.rs",
'#![expect(clippy::expect_used, reason = "fixture")]\nfn main() {}\n',
);
assert.equal(result.status, 0, result.stderr);
assert.equal(result.stderr, "");
result = run(
- "rust/crates/ml/examples/vecdb_bench.rs",
+ "rust/crates/vecdb/examples/vecdb_bench.rs",
'#![expect(clippy::expect_used, clippy::unwrap_used, reason = "fixture")]\nfn main() {}\n',
);
assert.equal(result.status, 1, result.stderr);
diff --git a/rust/crates/ml/src/lib.rs b/rust/crates/ml/src/lib.rs
index 6cd23fb566..d38adfa9cb 100644
--- a/rust/crates/ml/src/lib.rs
+++ b/rust/crates/ml/src/lib.rs
@@ -16,6 +16,5 @@ mod preprocess;
mod runtime;
pub mod scan;
pub mod types;
-pub mod vecdb;
#[cfg(feature = "usearch")]
pub mod vector_db;
diff --git a/rust/crates/vecdb/Cargo.toml b/rust/crates/vecdb/Cargo.toml
new file mode 100644
index 0000000000..fb2aa32bcf
--- /dev/null
+++ b/rust/crates/vecdb/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "ente-vecdb"
+edition.workspace = true
+
+[dependencies]
+log.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+
+[lints]
+workspace = true
diff --git a/rust/crates/ml/src/vecdb/README.md b/rust/crates/vecdb/README.md
similarity index 96%
rename from rust/crates/ml/src/vecdb/README.md
rename to rust/crates/vecdb/README.md
index fd2caab838..cc477df47b 100644
--- a/rust/crates/ml/src/vecdb/README.md
+++ b/rust/crates/vecdb/README.md
@@ -5,7 +5,7 @@ optional and default to i8 and cosine. These settings are persisted at creation
remain unchanged through reopen, compaction, and reset.
```rust
-use ente_ml::vecdb::{DistanceMetric, StorageKind, VecDb};
+use ente_vecdb::{DistanceMetric, StorageKind, VecDb};
let index = VecDb::open(path, 512, None, None)?;
let reader = VecDb::open_read_only(path, 512, None, None)?;
diff --git a/rust/crates/ml/examples/vecdb_bench.rs b/rust/crates/vecdb/examples/vecdb_bench.rs
similarity index 99%
rename from rust/crates/ml/examples/vecdb_bench.rs
rename to rust/crates/vecdb/examples/vecdb_bench.rs
index d73c44d071..429206c4b5 100644
--- a/rust/crates/ml/examples/vecdb_bench.rs
+++ b/rust/crates/vecdb/examples/vecdb_bench.rs
@@ -7,9 +7,7 @@ use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
-use ente_ml::vecdb::{
- AttrValue, Attribute, DistanceMetric, Match, SearchParams, StorageKind, VecDb,
-};
+use ente_vecdb::{AttrValue, Attribute, DistanceMetric, Match, SearchParams, StorageKind, VecDb};
const SEED: u64 = 0xE47E_0000_0000_0001;
const LATENT_DIMS: usize = 24;
@@ -172,7 +170,7 @@ fn parse_scales(list: &str) -> Vec<usize> {
fn usage_exit() -> ! {
eprintln!(
- "usage: cargo run -p ente-ml --example vecdb_bench --release -- \
+ "usage: cargo run -p ente-vecdb --example vecdb_bench --release -- \
[--scales 10000,100000] [--dims 512] [--attrs] [--storage f32|i8] [--metric cosine|ip]"
);
std::process::exit(2);
diff --git a/rust/crates/ml/src/vecdb/arena.rs b/rust/crates/vecdb/src/arena.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/arena.rs
rename to rust/crates/vecdb/src/arena.rs
diff --git a/rust/crates/ml/src/vecdb/crc.rs b/rust/crates/vecdb/src/crc.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/crc.rs
rename to rust/crates/vecdb/src/crc.rs
diff --git a/rust/crates/ml/src/vecdb/graph.rs b/rust/crates/vecdb/src/graph.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/graph.rs
rename to rust/crates/vecdb/src/graph.rs
diff --git a/rust/crates/ml/src/vecdb/kernel.rs b/rust/crates/vecdb/src/kernel.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/kernel.rs
rename to rust/crates/vecdb/src/kernel.rs
diff --git a/rust/crates/ml/src/vecdb/mod.rs b/rust/crates/vecdb/src/lib.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/mod.rs
rename to rust/crates/vecdb/src/lib.rs
diff --git a/rust/crates/ml/src/vecdb/lock.rs b/rust/crates/vecdb/src/lock.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/lock.rs
rename to rust/crates/vecdb/src/lock.rs
diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/vecdb/src/log.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/log.rs
rename to rust/crates/vecdb/src/log.rs
diff --git a/rust/crates/ml/src/vecdb/snapshot.rs b/rust/crates/vecdb/src/snapshot.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/snapshot.rs
rename to rust/crates/vecdb/src/snapshot.rs
diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/vecdb/src/store.rs
similarity index 100%
rename from rust/crates/ml/src/vecdb/store.rs
rename to rust/crates/vecdb/src/store.rs