web search done right
implement upsert for more hyperloglog sizes
Mikkel Denkerc97541fa1f34
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rsindex 78ebd0a8..8c17cc4a 100644--- a/crates/bloom/src/lib.rs+++ b/crates/bloom/src/lib.rs@@ -67,6 +67,12 @@ impl U64BloomFilter {}}+ pub fn fill(&mut self) {+ for i in 0..self.bit_vec.len() {+ self.bit_vec.set(i, true);+ }+ }+fn hash(item: u64) -> usize {item.wrapping_mul(LARGE_PRIME) as usize}diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rsindex f18442d9..757390e5 100644--- a/crates/core/src/ampc/dht/upsert.rs+++ b/crates/core/src/ampc/dht/upsert.rs@@ -37,60 +37,55 @@ pub trait UpsertFn {#[enum_dispatch(UpsertFn)]#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub enum UpsertEnum {+ HyperLogLog2Upsert,+ HyperLogLog4Upsert,+ HyperLogLog8Upsert,+ HyperLogLog16Upsert,+ HyperLogLog32Upsert,HyperLogLog64Upsert,HyperLogLog128Upsert,U64Add,F64Add,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct HyperLogLog64Upsert;--impl UpsertFn for HyperLogLog64Upsert {- fn upsert(&self, old: Value, new: Value) -> Value {- let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<64>, _>(- old.as_bytes(),- bincode::config::standard(),- )- .unwrap();- let (new, _) = bincode::decode_from_slice::<HyperLogLog<64>, _>(- new.as_bytes(),- bincode::config::standard(),- )- .unwrap();-- old.merge(&new);-- bincode::encode_to_vec(&old, bincode::config::standard())- .unwrap()- .into()- }-}+macro_rules! hyperloglog_upsert {+ ($name:ident => $n:expr) => {+ #[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+ )]+ pub struct $name;++ impl UpsertFn for $name {+ fn upsert(&self, old: Value, new: Value) -> Value {+ let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(+ old.as_bytes(),+ bincode::config::standard(),+ )+ .unwrap();+ let (new, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(+ new.as_bytes(),+ bincode::config::standard(),+ )+ .unwrap();-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct HyperLogLog128Upsert;+ old.merge(&new);-impl UpsertFn for HyperLogLog128Upsert {- fn upsert(&self, old: Value, new: Value) -> Value {- let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<128>, _>(- old.as_bytes(),- bincode::config::standard(),- )- .unwrap();- let (new, _) = bincode::decode_from_slice::<HyperLogLog<128>, _>(- new.as_bytes(),- bincode::config::standard(),- )- .unwrap();-- old.merge(&new);-- bincode::encode_to_vec(&old, bincode::config::standard())- .unwrap()- .into()- }+ bincode::encode_to_vec(&old, bincode::config::standard())+ .unwrap()+ .into()+ }+ }+ };}+hyperloglog_upsert!(HyperLogLog2Upsert => 2);+hyperloglog_upsert!(HyperLogLog4Upsert => 4);+hyperloglog_upsert!(HyperLogLog8Upsert => 8);+hyperloglog_upsert!(HyperLogLog16Upsert => 16);+hyperloglog_upsert!(HyperLogLog32Upsert => 32);+hyperloglog_upsert!(HyperLogLog64Upsert => 64);+hyperloglog_upsert!(HyperLogLog128Upsert => 128);+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct U64Add;diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rsindex 169a0c8a..95e3a4f8 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs@@ -46,9 +46,7 @@ impl CentralityWorker {let num_nodes = graph.estimate_num_nodes() as u64;let mut changed_nodes = U64BloomFilter::new(num_nodes, 0.05);- for node in graph.nodes() {- changed_nodes.insert(node.as_u64());- }+ changed_nodes.fill();Self {shard,@@ -62,9 +60,7 @@ impl CentralityWorker {pub fn setup_changed_nodes(&self, upper_bound_num_nodes: u64) {let mut new_changed_nodes = U64BloomFilter::new(upper_bound_num_nodes, 0.05);- for node in self.graph.nodes() {- new_changed_nodes.insert(node.as_u64());- }+ new_changed_nodes.fill();*self.changed_nodes.lock().unwrap() = new_changed_nodes;}diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rsindex 248bdba0..052bc228 100644--- a/crates/core/src/entrypoint/centrality.rs+++ b/crates/core/src/entrypoint/centrality.rs@@ -82,7 +82,7 @@ impl Centrality {{approx_rank.insert(node, rank as u64).unwrap();- if approx_rank.uncommitted_inserts() > 1_000_000 {+ if approx_rank.uncommitted_inserts() > 100_000_000 {approx_rank.commit().unwrap();}