Skip to content

web search done right

implement upsert for more hyperloglog sizes

Mikkel Denkerc97541fa1f34
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rs
index 78ebd0a8..8c17cc4a 100644
--- a/crates/bloom/src/lib.rs
+++ b/crates/bloom/src/lib.rs
@@ -67,6 +67,12 @@ impl U64BloomFilter {
}
}
+ pub fn fill(&mut self) {
+ for i in 0..self.bit_vec.len() {
+ self.bit_vec.set(i, true);
+ }
+ }
+
fn hash(item: u64) -> usize {
item.wrapping_mul(LARGE_PRIME) as usize
}
diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rs
index f18442d9..757390e5 100644
--- a/crates/core/src/ampc/dht/upsert.rs
+++ b/crates/core/src/ampc/dht/upsert.rs
@@ -37,60 +37,55 @@ pub trait UpsertFn {
#[enum_dispatch(UpsertFn)]
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub enum UpsertEnum {
+ HyperLogLog2Upsert,
+ HyperLogLog4Upsert,
+ HyperLogLog8Upsert,
+ HyperLogLog16Upsert,
+ HyperLogLog32Upsert,
HyperLogLog64Upsert,
HyperLogLog128Upsert,
U64Add,
F64Add,
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct HyperLogLog64Upsert;
-
-impl UpsertFn for HyperLogLog64Upsert {
- fn upsert(&self, old: Value, new: Value) -> Value {
- let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<64>, _>(
- old.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
- let (new, _) = bincode::decode_from_slice::<HyperLogLog<64>, _>(
- new.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
-
- old.merge(&new);
-
- bincode::encode_to_vec(&old, bincode::config::standard())
- .unwrap()
- .into()
- }
-}
+macro_rules! hyperloglog_upsert {
+ ($name:ident => $n:expr) => {
+ #[derive(
+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,
+ )]
+ pub struct $name;
+
+ impl UpsertFn for $name {
+ fn upsert(&self, old: Value, new: Value) -> Value {
+ let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(
+ old.as_bytes(),
+ bincode::config::standard(),
+ )
+ .unwrap();
+ let (new, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(
+ new.as_bytes(),
+ bincode::config::standard(),
+ )
+ .unwrap();
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct HyperLogLog128Upsert;
+ old.merge(&new);
-impl UpsertFn for HyperLogLog128Upsert {
- fn upsert(&self, old: Value, new: Value) -> Value {
- let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<128>, _>(
- old.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
- let (new, _) = bincode::decode_from_slice::<HyperLogLog<128>, _>(
- new.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
-
- old.merge(&new);
-
- bincode::encode_to_vec(&old, bincode::config::standard())
- .unwrap()
- .into()
- }
+ bincode::encode_to_vec(&old, bincode::config::standard())
+ .unwrap()
+ .into()
+ }
+ }
+ };
}
+hyperloglog_upsert!(HyperLogLog2Upsert => 2);
+hyperloglog_upsert!(HyperLogLog4Upsert => 4);
+hyperloglog_upsert!(HyperLogLog8Upsert => 8);
+hyperloglog_upsert!(HyperLogLog16Upsert => 16);
+hyperloglog_upsert!(HyperLogLog32Upsert => 32);
+hyperloglog_upsert!(HyperLogLog64Upsert => 64);
+hyperloglog_upsert!(HyperLogLog128Upsert => 128);
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct U64Add;
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
index 169a0c8a..95e3a4f8 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
@@ -46,9 +46,7 @@ impl CentralityWorker {
let num_nodes = graph.estimate_num_nodes() as u64;
let mut changed_nodes = U64BloomFilter::new(num_nodes, 0.05);
- for node in graph.nodes() {
- changed_nodes.insert(node.as_u64());
- }
+ changed_nodes.fill();
Self {
shard,
@@ -62,9 +60,7 @@ impl CentralityWorker {
pub fn setup_changed_nodes(&self, upper_bound_num_nodes: u64) {
let mut new_changed_nodes = U64BloomFilter::new(upper_bound_num_nodes, 0.05);
- for node in self.graph.nodes() {
- new_changed_nodes.insert(node.as_u64());
- }
+ new_changed_nodes.fill();
*self.changed_nodes.lock().unwrap() = new_changed_nodes;
}
diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rs
index 248bdba0..052bc228 100644
--- a/crates/core/src/entrypoint/centrality.rs
+++ b/crates/core/src/entrypoint/centrality.rs
@@ -82,7 +82,7 @@ impl Centrality {
{
approx_rank.insert(node, rank as u64).unwrap();
- if approx_rank.uncommitted_inserts() > 1_000_000 {
+ if approx_rank.uncommitted_inserts() > 100_000_000 {
approx_rank.commit().unwrap();
}