Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

bench: drop the usearch comparison

laurenspriembeb556c0165e
diff --git a/rust/crates/ml/examples/vecdb_bench.rs b/rust/crates/ml/examples/vecdb_bench.rs
index 57f4ff255e..6f654bf200 100644
--- a/rust/crates/ml/examples/vecdb_bench.rs
+++ b/rust/crates/ml/examples/vecdb_bench.rs
@@ -19,7 +19,6 @@ const DEFAULT_DIMS: usize = 512;
struct Config {
scales: Vec<usize>,
dims: usize,
- compare_usearch: bool,
}
struct BenchData {
@@ -62,29 +61,23 @@ fn main() {
config.dims, QUERY_COUNT, SEED
);
println!(
- "scales: {} usearch comparison: {}",
+ "scales: {}",
config
.scales
.iter()
.map(|scale| scale.to_string())
.collect::<Vec<_>>()
- .join(", "),
- if config.compare_usearch {
- "enabled"
- } else {
- "disabled"
- }
+ .join(", ")
);
let temp_root = tempfile::TempDir::new().expect("create bench temp dir");
for &scale in &config.scales {
- run_scale(scale, config.dims, config.compare_usearch, temp_root.path());
+ run_scale(scale, config.dims, temp_root.path());
}
}
fn parse_args() -> Config {
let mut scales = parse_scales(DEFAULT_SCALES);
let mut dims = DEFAULT_DIMS;
- let mut compare_usearch = false;
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
@@ -94,7 +87,6 @@ fn parse_args() -> Config {
.parse()
.unwrap_or_else(|_| usage_exit())
}
- "--compare-usearch" => compare_usearch = true,
_ => usage_exit(),
}
}
@@ -102,18 +94,7 @@ fn parse_args() -> Config {
eprintln!("--dims must be a nonzero multiple of 8");
std::process::exit(2);
}
- if compare_usearch && !cfg!(feature = "usearch") {
- eprintln!(
- "--compare-usearch requires a build with the usearch feature: cargo run -p ente-ml \
- --example vecdb_bench --release --features usearch -- --compare-usearch"
- );
- std::process::exit(2);
- }
- Config {
- scales,
- dims,
- compare_usearch,
- }
+ Config { scales, dims }
}
fn required_value(value: Option<String>) -> String {
@@ -136,13 +117,13 @@ fn parse_scales(list: &str) -> Vec<usize> {
fn usage_exit() -> ! {
eprintln!(
- "usage: cargo run -p ente-ml --example vecdb_bench --release [--features usearch] -- \
- [--scales 10000,100000] [--dims 512] [--compare-usearch]"
+ "usage: cargo run -p ente-ml --example vecdb_bench --release -- \
+ [--scales 10000,100000] [--dims 512]"
);
std::process::exit(2);
}
-fn run_scale(scale: usize, dims: usize, compare_usearch: bool, temp_root: &Path) {
+fn run_scale(scale: usize, dims: usize, temp_root: &Path) {
let clusters = (scale / 150).max(1) as u64;
println!();
println!("=== scale {scale} dims {dims} clusters {clusters} ===");
@@ -152,9 +133,6 @@ fn run_scale(scale: usize, dims: usize, compare_usearch: bool, temp_root: &Path)
std::fs::create_dir_all(&dir).expect("create bench dir");
let vecdb = run_vecdb(&data, dims, &dir, scale);
print_vecdb_report(&vecdb);
- if compare_usearch {
- run_comparison(&data, dims, &dir, scale, &vecdb);
- }
let _ = std::fs::remove_dir_all(&dir);
}
@@ -673,391 +651,3 @@ fn fmt_bytes(bytes: u64) -> String {
format!("{:.2} MB", value / (1024.0 * 1024.0))
}
}
-
-#[cfg(not(feature = "usearch"))]
-fn run_comparison(
- _data: &BenchData,
- _dims: usize,
- _dir: &Path,
- _scale: usize,
- _vecdb: &VecdbReport,
-) {
-}
-
-#[cfg(feature = "usearch")]
-fn run_comparison(data: &BenchData, dims: usize, dir: &Path, scale: usize, vecdb: &VecdbReport) {
- let usearch = usearch_bench::run(data, dims, dir, scale, vecdb.threshold);
- usearch_bench::print_report(&usearch);
- usearch_bench::print_side_by_side(vecdb, &usearch, scale);
-}
-
-#[cfg(feature = "usearch")]
-mod usearch_bench {
- use std::collections::HashSet;
- use std::path::Path;
- use std::time::{Duration, Instant};
-
- use ente_ml::vector_db::VectorDB;
- use usearch::{Index, IndexOptions, MetricKind, ScalarKind};
-
- use super::{
- BenchData, QUERY_COUNT, SEARCH_K, SINGLE_ADD_COUNT, VecdbReport, WARMUP_QUERIES, fmt_bytes,
- fmt_duration, per_op_text, row,
- };
-
- const THRESHOLD_STEP_COUNTS: [usize; 5] = [200, 500, 2000, 5000, 10000];
-
- pub struct UsearchReport {
- legacy: Option<(Duration, usize)>,
- pure_add_total: Duration,
- pure_add_count: usize,
- save: Duration,
- approx_total: Duration,
- exact_total: Duration,
- recall: f64,
- threshold: f32,
- threshold_total: Duration,
- threshold_avg_hits: f64,
- filtered_total: Duration,
- file_bytes: u64,
- memory_bytes: u64,
- }
-
- pub fn run(
- data: &BenchData,
- dims: usize,
- dir: &Path,
- scale: usize,
- threshold: f32,
- ) -> UsearchReport {
- let options = IndexOptions {
- dimensions: dims,
- metric: MetricKind::IP,
- quantization: ScalarKind::F32,
- ..IndexOptions::default()
- };
- let index = Index::new(&options).expect("create usearch index");
- index
- .reserve(data.entries.len() + 1)
- .expect("usearch reserve");
- eprintln!("[scale {scale}] usearch pure adds");
- let started = Instant::now();
- for (slot, (_, vector)) in data.entries.iter().enumerate() {
- index.add(slot as u64, vector).expect("usearch add");
- }
- let pure_add_total = started.elapsed();
- index
- .add(data.entries.len() as u64, &data.probe)
- .expect("usearch probe add");
- let save_path = dir.join("bench.usearch");
- let save_str = save_path.to_str().expect("usearch save path").to_string();
- let started = Instant::now();
- index.save(&save_str).expect("usearch save");
- let save = started.elapsed();
- let file_bytes = std::fs::metadata(&save_path)
- .map(|meta| meta.len())
- .unwrap_or(0);
- let memory_bytes = index.memory_usage() as u64;
-
- eprintln!("[scale {scale}] usearch searches");
- for query in data.queries.iter().take(WARMUP_QUERIES) {
- index.search(query, SEARCH_K).expect("usearch warmup");
- }
- let started = Instant::now();
- let approx: Vec<Vec<u64>> = data
- .queries
- .iter()
- .map(|query| index.search(query, SEARCH_K).expect("usearch search").keys)
- .collect();
- let approx_total = started.elapsed();
- for query in data.queries.iter().take(WARMUP_QUERIES) {
- index
- .exact_search(query, SEARCH_K)
- .expect("usearch exact warmup");
- }
- let started = Instant::now();
- let exact: Vec<Vec<u64>> = data
- .queries
- .iter()
- .map(|query| {
- index
- .exact_search(query, SEARCH_K)
- .expect("usearch exact search")
- .keys
- })
- .collect();
- let exact_total = started.elapsed();
- let recall = key_recall(&exact, &approx);
-
- for query in data.queries.iter().take(WARMUP_QUERIES) {
- threshold_search(&index, query, threshold);
- }
- let started = Instant::now();
- let threshold_hits: Vec<usize> = data
- .queries
- .iter()
- .map(|query| threshold_search(&index, query, threshold).len())
- .collect();
- let threshold_total = started.elapsed();
- let threshold_avg_hits =
- threshold_hits.iter().sum::<usize>() as f64 / threshold_hits.len().max(1) as f64;
-
- for query in data.queries.iter().take(WARMUP_QUERIES) {
- filtered_search(&index, query, &data.allowed_indices);
- }
- let started = Instant::now();
- for query in &data.queries {
- filtered_search(&index, query, &data.allowed_indices);
- }
- let filtered_total = started.elapsed();
-
- let legacy = if scale <= 10_000 {
- eprintln!("[scale {scale}] usearch legacy save-per-add");
- let legacy_path = dir.join("legacy.usearch");
- let mut wrapper =
- VectorDB::new(legacy_path.to_str().expect("legacy usearch path"), dims)
- .expect("legacy usearch wrapper");
- let count = SINGLE_ADD_COUNT.min(data.entries.len());
- let started = Instant::now();
- for (slot, (_, vector)) in data.entries.iter().take(count).enumerate() {
- wrapper
- .add_vector(slot as u64, vector)
- .expect("legacy usearch add");
- }
- Some((started.elapsed(), count))
- } else {
- None
- };
-
- UsearchReport {
- legacy,
- pure_add_total,
- pure_add_count: data.entries.len(),
- save,
- approx_total,
- exact_total,
- recall,
- threshold,
- threshold_total,
- threshold_avg_hits,
- filtered_total,
- file_bytes,
- memory_bytes,
- }
- }
-
- fn threshold_search(index: &Index, query: &[f32], max_distance: f32) -> Vec<u64> {
- let index_size = index.size();
- if index_size == 0 {
- return Vec::new();
- }
- let mut previous_count = 0usize;
- for step_count in THRESHOLD_STEP_COUNTS {
- let count = step_count.min(index_size);
- if count <= previous_count {
- continue;
- }
- previous_count = count;
- let matches = index
- .search(query, count)
- .expect("usearch threshold search");
- let should_expand = count < index_size
- && matches
- .distances
- .last()
- .is_some_and(|distance| *distance <= max_distance);
- if should_expand {
- continue;
- }
- return keys_within(matches.keys, &matches.distances, max_distance);
- }
- let matches = index
- .search(query, index_size)
- .expect("usearch threshold search");
- keys_within(matches.keys, &matches.distances, max_distance)
- }
-
- fn keys_within(mut keys: Vec<u64>, distances: &[f32], max_distance: f32) -> Vec<u64> {
- let keep = distances.partition_point(|distance| *distance <= max_distance);
- keys.truncate(keep);
- keys
- }
-
- fn filtered_search(index: &Index, query: &[f32], allowed_indices: &[usize]) -> Vec<u64> {
- let allowed: HashSet<u64> = allowed_indices.iter().map(|index| *index as u64).collect();
- index
- .filtered_search(query, SEARCH_K, |key| allowed.contains(&key))
- .expect("usearch filtered search")
- .keys
- }
-
- fn key_recall(truth: &[Vec<u64>], found: &[Vec<u64>]) -> f64 {
- let mut hits = 0usize;
- let mut total = 0usize;
- for (expected, actual) in truth.iter().zip(found) {
- let wanted: HashSet<u64> = expected.iter().copied().collect();
- total += wanted.len();
- hits += actual.iter().filter(|key| wanted.contains(key)).count();
- }
- if total == 0 {
- return 1.0;
- }
- hits as f64 / total as f64
- }
-
- pub fn print_report(report: &UsearchReport) {
- println!(" usearch");
- match report.legacy {
- Some((total, count)) => row(
- "legacy add (save per add)",
- format!("{count} ops"),
- total,
- per_op_text(total, count, "op"),
- ),
- None => println!(
- " {:<44} skipped at this scale (would take hours)",
- "legacy add (save per add)"
- ),
- }
- row(
- "pure add (in-memory, no save)",
- format!("{} vecs", report.pure_add_count),
- report.pure_add_total,
- per_op_text(report.pure_add_total, report.pure_add_count, "vec"),
- );
- row(
- "one save at full scale",
- "-".to_string(),
- report.save,
- String::new(),
- );
- row(
- "search approx k=10",
- format!("{QUERY_COUNT} queries"),
- report.approx_total,
- per_op_text(report.approx_total, QUERY_COUNT, "query"),
- );
- row(
- "search exact k=10",
- format!("{QUERY_COUNT} queries"),
- report.exact_total,
- per_op_text(report.exact_total, QUERY_COUNT, "query"),
- );
- println!(
- " {:<44} {:.4}",
- "recall@10 (approx vs exact)", report.recall
- );
- row(
- &format!(
- "search approx threshold d<={:.4} (stepped)",
- report.threshold
- ),
- format!("{QUERY_COUNT} queries"),
- report.threshold_total,
- per_op_text(report.threshold_total, QUERY_COUNT, "query"),
- );
- println!(
- " {:<44} {:.1}",
- "avg hits per threshold query", report.threshold_avg_hits
- );
- row(
- "search approx k=10 filtered (10% allowed)",
- format!("{QUERY_COUNT} queries"),
- report.filtered_total,
- per_op_text(report.filtered_total, QUERY_COUNT, "query"),
- );
- println!(
- " {:<44} file {} reported mem {}",
- "sizes at full scale",
- fmt_bytes(report.file_bytes),
- fmt_bytes(report.memory_bytes)
- );
- }
-
- pub fn print_side_by_side(vecdb: &VecdbReport, usearch: &UsearchReport, scale: usize) {
- println!(" side by side (scale {scale})");
- pair("metric", "vecdb".to_string(), "usearch".to_string());
- pair(
- "durable add per op",
- per_op_text(vecdb.single_total, vecdb.single_count, "op"),
- usearch
- .legacy
- .map(|(total, count)| per_op_text(total, count, "op"))
- .unwrap_or_else(|| "- (skipped)".to_string()),
- );
- pair(
- "bulk/pure add per vec",
- per_op_text(vecdb.bulk_total, vecdb.bulk_count, "vec"),
- per_op_text(usearch.pure_add_total, usearch.pure_add_count, "vec"),
- );
- pair(
- "full build wall time",
- fmt_duration(vecdb.single_total + vecdb.bulk_total),
- fmt_duration(usearch.pure_add_total),
- );
- pair(
- "approx search per query",
- per_op_text(vecdb.approx_total, QUERY_COUNT, "query"),
- per_op_text(usearch.approx_total, QUERY_COUNT, "query"),
- );
- pair(
- "recall@10 vs own exact",
- format!("{:.4}", vecdb.recall),
- format!("{:.4}", usearch.recall),
- );
- pair(
- "exact search per query",
- per_op_text(vecdb.exact_total, QUERY_COUNT, "query"),
- per_op_text(usearch.exact_total, QUERY_COUNT, "query"),
- );
- pair(
- "threshold search per query",
- per_op_text(vecdb.threshold_total, QUERY_COUNT, "query"),
- per_op_text(usearch.threshold_total, QUERY_COUNT, "query"),
- );
- pair(
- "threshold avg hits",
- format!("{:.1}", vecdb.threshold_avg_hits),
- format!("{:.1}", usearch.threshold_avg_hits),
- );
- pair(
- "filtered search per query",
- per_op_text(vecdb.filtered_total, QUERY_COUNT, "query"),
- per_op_text(usearch.filtered_total, QUERY_COUNT, "query"),
- );
- pair(
- "full save / snapshot",
- fmt_duration(vecdb.snapshot_write),
- fmt_duration(usearch.save),
- );
- pair(
- "on-disk bytes",
- fmt_bytes(vecdb.log_bytes + vecdb.snapshot_bytes),
- fmt_bytes(usearch.file_bytes),
- );
- pair(
- "approx memory",
- fmt_bytes(vecdb.memory_bytes as u64),
- fmt_bytes(usearch.memory_bytes),
- );
- println!(
- " note: vecdb adds are durable (fsync per op/batch, periodic snapshots); usearch \
- pure adds are in-memory only"
- );
- println!(
- " note: usearch legacy save-per-add rewrites the whole index file per add, \
- without fsync"
- );
- println!(
- " note: vecdb on-disk = log + graph snapshot; usearch on-disk = single index file"
- );
- println!(
- " note: threshold uses vecdb's ~1% distance on both engines (usearch via legacy \
- stepped expansion); filtered restricts to the same seeded 10% key subset, k=10"
- );
- }
-
- fn pair(label: &str, vecdb_value: String, usearch_value: String) {
- println!(" {label:<28} {vecdb_value:>18} {usearch_value:>18}");
- }
-}