web search done right
reduce memory usage and serialization/deserialization in dht by storing keys and values as enums instead of vec<u8>
Mikkel Denkerf7a354572e48
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rsindex 8c17cc4a..81dbc86d 100644--- a/crates/bloom/src/lib.rs+++ b/crates/bloom/src/lib.rs@@ -47,7 +47,16 @@ fn num_hashes(num_bits: u64, estimated_items: u64) -> u64 {(((num_bits as f64) / estimated_items as f64 * 2.0_f64.ln()).ceil() as u64).max(1)}-#[derive(Clone, bincode::Encode, bincode::Decode, Debug, serde::Serialize, serde::Deserialize)]+#[derive(+ Clone,+ bincode::Encode,+ bincode::Decode,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ PartialEq,+ Eq,+)]pub struct U64BloomFilter {#[bincode(with_serde)]bit_vec: BitVec,diff --git a/crates/core/src/ampc/dht/client.rs b/crates/core/src/ampc/dht/client.rsindex a448a9e1..919c8dd8 100644--- a/crates/core/src/ampc/dht/client.rs+++ b/crates/core/src/ampc/dht/client.rs@@ -27,9 +27,11 @@ use std::{use crate::{distributed::member::ShardId, Result};use super::{+ key::{Key, KeyTrait},network::api,- store::{Key, Table, Value},+ store::Table,upsert::UpsertEnum,+ value::Value,UpsertAction,};@@ -301,14 +303,14 @@ impl Client {}pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {- self.shard_for_key(key.as_bytes())?.get(table, key).await+ self.shard_for_key(&key.as_bytes())?.get(table, key).await}pub async fn batch_get(&self, table: Table, keys: Vec<Key>) -> Result<Vec<(Key, Value)>> {let mut shard_keys: BTreeMap<ShardId, Vec<Key>> = BTreeMap::new();for key in keys {- let shard = self.shard_id_for_key(key.as_bytes())?;+ let shard = self.shard_id_for_key(&key.as_bytes())?;shard_keys.entry(*shard).or_default().push(key);}@@ -326,7 +328,7 @@ impl Client {}pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {- self.shard_for_key(key.as_bytes())?+ self.shard_for_key(&key.as_bytes())?.set(table, key, value).await}@@ -335,7 +337,7 @@ impl Client {let mut shard_values: BTreeMap<ShardId, Vec<(Key, Value)>> = BTreeMap::new();for (key, value) in values {- let shard = self.shard_id_for_key(key.as_bytes())?;+ let shard = self.shard_id_for_key(&key.as_bytes())?;shard_values.entry(*shard).or_default().push((key, value));}@@ -367,7 +369,7 @@ impl Client {key: Key,value: Value,) -> Result<UpsertAction> {- self.shard_for_key(key.as_bytes())?+ self.shard_for_key(&key.as_bytes())?.upsert(table, upsert, key, value).await}@@ -381,7 +383,7 @@ impl Client {let mut shard_values: BTreeMap<ShardId, Vec<(Key, Value)>> = BTreeMap::new();for (key, value) in values {- let shard = self.shard_id_for_key(key.as_bytes())?;+ let shard = self.shard_id_for_key(&key.as_bytes())?;shard_values.entry(*shard).or_default().push((key, value));}diff --git a/crates/core/src/ampc/dht/key.rs b/crates/core/src/ampc/dht/key.rsnew file mode 100644index 00000000..4f1acaa3--- /dev/null+++ b/crates/core/src/ampc/dht/key.rs@@ -0,0 +1,113 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::distributed::member::ShardId;+use crate::webgraph::NodeID;++pub trait KeyTrait: TryFrom<Key> + Into<Key> {+ fn as_bytes(&self) -> Vec<u8>;+}++impl KeyTrait for String {+ fn as_bytes(&self) -> Vec<u8> {+ self.as_bytes().to_vec()+ }+}++impl KeyTrait for NodeID {+ fn as_bytes(&self) -> Vec<u8> {+ self.as_u64().to_le_bytes().to_vec()+ }+}++type Unit = ();+impl KeyTrait for Unit {+ fn as_bytes(&self) -> Vec<u8> {+ vec![]+ }+}++impl KeyTrait for ShardId {+ fn as_bytes(&self) -> Vec<u8> {+ self.as_u64().to_le_bytes().to_vec()+ }+}++type U64 = u64;+impl KeyTrait for U64 {+ fn as_bytes(&self) -> Vec<u8> {+ self.to_le_bytes().to_vec()+ }+}++#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialOrd,+ Ord,+ PartialEq,+ Eq,+)]+pub enum Key {+ String(String),+ NodeID(NodeID),+ Unit(Unit),+ ShardId(ShardId),+ U64(U64),+}++impl KeyTrait for Key {+ fn as_bytes(&self) -> Vec<u8> {+ match self {+ Key::String(key) => KeyTrait::as_bytes(key),+ Key::NodeID(key) => KeyTrait::as_bytes(key),+ Key::Unit(key) => KeyTrait::as_bytes(key),+ Key::ShardId(key) => KeyTrait::as_bytes(key),+ Key::U64(key) => KeyTrait::as_bytes(key),+ }+ }+}++macro_rules! impl_from_to_key {+ ($key:ty, $variant:ident) => {+ impl From<$key> for Key {+ fn from(key: $key) -> Self {+ Key::$variant(key)+ }+ }++ impl TryFrom<Key> for $key {+ type Error = anyhow::Error;++ fn try_from(key: Key) -> Result<Self, Self::Error> {+ match key {+ Key::$variant(key) => Ok(key),+ _ => anyhow::bail!("Key is not of type {}", stringify!($key)),+ }+ }+ }+ };+}++impl_from_to_key!(String, String);+impl_from_to_key!(NodeID, NodeID);+impl_from_to_key!(Unit, Unit);+impl_from_to_key!(ShardId, ShardId);+impl_from_to_key!(U64, U64);diff --git a/crates/core/src/ampc/dht/mod.rs b/crates/core/src/ampc/dht/mod.rsindex 11fcf2e7..9498ef8b 100644--- a/crates/core/src/ampc/dht/mod.rs+++ b/crates/core/src/ampc/dht/mod.rs@@ -27,10 +27,12 @@//! Heavily inspired by https://github.com/datafuselabs/openraft/blob/main/examples/raft-kv-memstore/mod client;+pub mod key;pub mod log_store;pub mod network;pub mod store;pub mod upsert;+pub mod value;use network::api::{AllTables, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Set, Upsert,@@ -41,6 +43,9 @@ use std::io::Cursor;use openraft::TokioRuntime;+pub use key::{Key, KeyTrait};+pub use value::{Value, ValueTrait};+pub use self::network::Server;pub use network::api::RemoteClient;@@ -48,7 +53,7 @@ pub use network::raft::RemoteClient as RaftClient;pub use crate::distributed::member::ShardId;pub use client::{Client, Shard};-pub use store::{Key, Table, Value};+pub use store::Table;pub use upsert::*;pub type NodeId = u64;@@ -132,6 +137,7 @@ raft_sonic_request_response!(#[cfg(test)]pub mod tests {+ use super::{key::Key, value::Value};use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};use tokio::sync::Mutex;use tracing_test::traced_test;@@ -218,54 +224,54 @@ pub mod tests {c1.set(table.clone(),- "hello".as_bytes().to_vec().into(),- "world".as_bytes().to_vec().into(),+ "hello".to_string().into(),+ "world".to_string().into(),).await?;- let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;- assert_eq!(res, Some("world".as_bytes().into()));+ let res = c1.get(table.clone(), "hello".to_string().into()).await?;+ assert_eq!(res, Some("world".to_string().into()));c2.set(table.clone(),- "ensure-linearized-read".as_bytes().into(),- vec![].into(),+ "ensure-linearized-read".to_string().into(),+ ().into(),).await?;tokio::time::sleep(std::time::Duration::from_secs(1)).await;- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;- assert_eq!(res, Some("world".as_bytes().into()));+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;+ assert_eq!(res, Some("world".to_string().into()));c2.set(table.clone(),- "hello".as_bytes().into(),- "world2".as_bytes().into(),+ "hello".to_string().into(),+ "world2".to_string().into(),).await?;c1.set(table.clone(),- "ensure-linearized-read".as_bytes().into(),- vec![].into(),+ "ensure-linearized-read".to_string().into(),+ ().into(),).await?;tokio::time::sleep(std::time::Duration::from_secs(1)).await;- let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;- assert_eq!(res, Some("world2".as_bytes().into()));+ let res = c1.get(table.clone(), "hello".to_string().into()).await?;+ assert_eq!(res, Some("world2".to_string().into()));let res = c1.batch_get(table.clone(), vec![]).await?;assert!(res.is_empty());let res = c1- .batch_get(table.clone(), vec!["hello".as_bytes().into()])+ .batch_get(table.clone(), vec!["hello".to_string().into()]).await?;assert!(res.len() == 1);let res = c1- .batch_get(table.clone(), vec!["non-existent-key".as_bytes().into()])+ .batch_get(table.clone(), vec!["non-existent-key".to_string().into()]).await?;assert!(res.is_empty());@@ -316,29 +322,29 @@ pub mod tests {c1.set(table.clone(),- "hello".as_bytes().into(),- "world".as_bytes().into(),+ "hello".to_string().into(),+ "world".to_string().into(),).await?;c2.set(table.clone(),- "ensure-linearized-read".as_bytes().into(),- vec![].into(),+ "ensure-linearized-read".to_string().into(),+ ().into(),).await?;tokio::time::sleep(std::time::Duration::from_secs(1)).await;- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;- assert_eq!(res, Some("world".as_bytes().into()));+ assert_eq!(res, Some("world".to_string().into()));rc1.join(3, addr3).await?;tokio::time::sleep(std::time::Duration::from_secs(1)).await; // join is not blockinglet c3 = RemoteClient::new(addr3);- let res = c3.get(table.clone(), "hello".as_bytes().into()).await?;- assert_eq!(res, Some("world".as_bytes().into()));+ let res = c3.get(table.clone(), "hello".to_string().into()).await?;+ assert_eq!(res, Some("world".to_string().into()));Ok(())}@@ -379,16 +385,16 @@ pub mod tests {client.set(table.clone(),- "hello".as_bytes().into(),- "world".as_bytes().into(),+ "hello".to_string().into(),+ "world".to_string().into(),).await?;client.set(table.clone(),- "hello2".as_bytes().into(),- "world2".as_bytes().into(),+ "hello2".to_string().into(),+ "world2".to_string().into(),).await?;@@ -406,8 +412,8 @@ pub mod tests {assert_eq!(res,vec![- ("hello".as_bytes().into(), "world".as_bytes().into()),- ("hello2".as_bytes().into(), "world2".as_bytes().into()),+ ("hello".to_string().into(), "world".to_string().into()),+ ("hello2".to_string().into(), "world2".to_string().into()),]);@@ -418,12 +424,7 @@ pub mod tests {.batch_set(table.clone(),(0..N)- .map(|i| {- (- i.to_be_bytes().as_ref().into(),- i.to_be_bytes().as_ref().into(),- )- })+ .map(|i| (i.into(), i.into())).collect::<Vec<(Key, Value)>>(),).await?;@@ -441,13 +442,8 @@ pub mod tests {assert_eq!(res.len(), N as usize);for (i, r) in res.into_iter().enumerate() {- assert_eq!(- r,- (- i.to_be_bytes().as_ref().into(),- i.to_be_bytes().as_ref().into(),- )- );+ let i = i as u64;+ assert_eq!(r, (i.into(), i.into(),));}Ok(())@@ -496,13 +492,13 @@ pub mod tests {c1.set(table.clone(),- "hello".as_bytes().into(),- "world".as_bytes().into(),+ "hello".to_string().into(),+ "world".to_string().into(),).await?;- let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;- assert_eq!(res, Some("world".as_bytes().into()));+ let res = c1.get(table.clone(), "hello".to_string().into()).await?;+ assert_eq!(res, Some("world".to_string().into()));// crash node 2tracing::info!("crashing node 2");@@ -521,8 +517,8 @@ pub mod tests {let c2 = RemoteClient::new(addr2);- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;- assert_eq!(res, Some("world".as_bytes().into()));+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;+ assert_eq!(res, Some("world".to_string().into()));// crash node 2 againtracing::info!("crashing node 2 again");@@ -531,8 +527,8 @@ pub mod tests {c1.set(table.clone(),- "hello".as_bytes().into(),- "world2".as_bytes().into(),+ "hello".to_string().into(),+ "world2".to_string().into(),).await?;@@ -547,8 +543,8 @@ pub mod tests {let c2 = RemoteClient::new(addr2);- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;- assert_eq!(res, Some("world2".as_bytes().into()));+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;+ assert_eq!(res, Some("world2".to_string().into()));Ok(())}@@ -564,7 +560,7 @@ pub mod tests {Arbitrary,)]enum Action {- Set { key: Vec<u8>, value: Vec<u8> },+ Set { key: String, value: String },// get actions[prev_key % actions.len()]// if actions[prev_key % actions.len()] is a get, then get a non-existent keyGet { prev_key: usize },@@ -576,7 +572,7 @@ pub mod tests {#[test]#[traced_test]fn proptest_chaos(actions: Vec<Action>) {- let ground_truth = Arc::new(Mutex::new(BTreeMap::<Vec<u8>, Vec<u8>>::new()));+ let ground_truth = Arc::new(Mutex::new(BTreeMap::<String, String>::new()));tokio::runtime::Builder::new_multi_thread().enable_all()@@ -640,23 +636,23 @@ pub mod tests {}Action::Get { prev_key } => {let client = clients.choose(&mut rand::thread_rng()).unwrap();- client.set(table.clone(), b"ensure-linearized-read".to_vec().into(), vec![].into()).await.unwrap();+ client.set(table.clone(), "ensure-linearized-read".to_string().into(), ().into()).await.unwrap();let key = if i == 0 {- b"non-existent-key".to_vec()+ "non-existent-key".to_string()} else {match shared_actions[prev_key % i] {Action::Set { ref key, .. } => {key.clone()},- Action::Get { .. } => b"non-existent-key".to_vec(),+ Action::Get { .. } => "non-existent-key".to_string(),}};let res = client.get(table.clone(), key.clone().into()).await.unwrap();let expected = ground_truth.lock().await.get(&key).cloned();- assert_eq!(res.map(|v| v.as_bytes().to_vec()), expected);+ assert_eq!(res.map(|v| v.try_into().unwrap()), expected);}}}diff --git a/crates/core/src/ampc/dht/network/api.rs b/crates/core/src/ampc/dht/network/api.rsindex 575ef91a..3e645682 100644--- a/crates/core/src/ampc/dht/network/api.rs+++ b/crates/core/src/ampc/dht/network/api.rs@@ -23,9 +23,7 @@ use std::{use crate::{ampc::dht::{- store::{Key, Table, Value},- upsert::UpsertEnum,- BasicNode, UpsertAction,+ key::Key, store::Table, upsert::UpsertEnum, value::Value, BasicNode, UpsertAction,},distributed::retry_strategy::RandomBackoff,Result,diff --git a/crates/core/src/ampc/dht/store.rs b/crates/core/src/ampc/dht/store.rsindex 62a4bec7..1ffde379 100644--- a/crates/core/src/ampc/dht/store.rs+++ b/crates/core/src/ampc/dht/store.rs@@ -36,8 +36,10 @@ use tokio::sync::RwLock;use crate::ampc::dht::network::api;+use super::key::Key;use super::upsert::UpsertEnum;use super::upsert::UpsertFn;+use super::value::Value;use super::BasicNode;use super::NodeId;use super::TypeConfig;@@ -77,74 +79,6 @@ impl From<&str> for Table {}}-#[derive(- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- Debug,- Clone,- PartialOrd,- Ord,- PartialEq,- Eq,-)]-#[serde(transparent)]-pub struct Key(Vec<u8>);--impl Key {- pub fn as_bytes(&self) -> &[u8] {- &self.0- }-}--impl From<Vec<u8>> for Key {- fn from(v: Vec<u8>) -> Self {- Self(v)- }-}--impl From<&[u8]> for Key {- fn from(v: &[u8]) -> Self {- Self(v.to_vec())- }-}--#[derive(- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- Debug,- Clone,- PartialEq,- Eq,-)]-#[serde(transparent)]-pub struct Value(Vec<u8>);--impl Value {- pub fn as_bytes(&self) -> &[u8] {- &self.0- }-- pub fn as_bytes_mut(&mut self) -> &mut [u8] {- &mut self.0- }-}--impl From<Vec<u8>> for Value {- fn from(v: Vec<u8>) -> Self {- Self(v)- }-}--impl From<&[u8]> for Value {- fn from(v: &[u8]) -> Self {- Self(v.to_vec())- }-}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Default, Clone,)]diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rsindex 17e70b5f..69864c75 100644--- a/crates/core/src/ampc/dht/upsert.rs+++ b/crates/core/src/ampc/dht/upsert.rs@@ -16,8 +16,6 @@use enum_dispatch::enum_dispatch;-use crate::{hyperloglog::HyperLogLog, kahan_sum::KahanSum};-use super::Value;#[derive(@@ -37,8 +35,6 @@ pub trait UpsertFn {#[enum_dispatch(UpsertFn)]#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub enum UpsertEnum {- HyperLogLog2Upsert,- HyperLogLog4Upsert,HyperLogLog8Upsert,HyperLogLog16Upsert,HyperLogLog32Upsert,@@ -50,8 +46,18 @@ pub enum UpsertEnum {KahanSumAdd,}+macro_rules! unwrap_value {+ ($value:expr, $variant:ident) => {+ if let Value::$variant(value) = $value {+ value+ } else {+ panic!("Expected {}", stringify!($variant));+ }+ };+}+macro_rules! hyperloglog_upsert {- ($name:ident => $n:expr) => {+ ($name:ident, $variant:ident) => {#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,)]@@ -59,50 +65,32 @@ macro_rules! hyperloglog_upsert {impl UpsertFn for $name {fn upsert(&self, old: Value, new: Value) -> Value {- let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(- old.as_bytes(),- bincode::config::standard(),- )- .unwrap();- let (new, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(- new.as_bytes(),- bincode::config::standard(),- )- .unwrap();+ let mut old = unwrap_value!(old, $variant);+ let new = unwrap_value!(new, $variant);old.merge(&new);- bincode::encode_to_vec(&old, bincode::config::standard())- .unwrap()- .into()+ Value::$variant(old)}}};}-hyperloglog_upsert!(HyperLogLog2Upsert => 2);-hyperloglog_upsert!(HyperLogLog4Upsert => 4);-hyperloglog_upsert!(HyperLogLog8Upsert => 8);-hyperloglog_upsert!(HyperLogLog16Upsert => 16);-hyperloglog_upsert!(HyperLogLog32Upsert => 32);-hyperloglog_upsert!(HyperLogLog64Upsert => 64);-hyperloglog_upsert!(HyperLogLog128Upsert => 128);+hyperloglog_upsert!(HyperLogLog8Upsert, HyperLogLog8);+hyperloglog_upsert!(HyperLogLog16Upsert, HyperLogLog16);+hyperloglog_upsert!(HyperLogLog32Upsert, HyperLogLog32);+hyperloglog_upsert!(HyperLogLog64Upsert, HyperLogLog64);+hyperloglog_upsert!(HyperLogLog128Upsert, HyperLogLog128);#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct U64Add;impl UpsertFn for U64Add {fn upsert(&self, old: Value, new: Value) -> Value {- let (old, _) =- bincode::decode_from_slice::<u64, _>(old.as_bytes(), bincode::config::standard())- .unwrap();- let (new, _) =- bincode::decode_from_slice::<u64, _>(new.as_bytes(), bincode::config::standard())- .unwrap();-- bincode::encode_to_vec(old + new, bincode::config::standard())- .unwrap()- .into()+ let old = unwrap_value!(old, U64);+ let new = unwrap_value!(new, U64);++ Value::U64(old + new)}}@@ -111,16 +99,10 @@ pub struct F32Add;impl UpsertFn for F32Add {fn upsert(&self, old: Value, new: Value) -> Value {- let (old, _) =- bincode::decode_from_slice::<f32, _>(old.as_bytes(), bincode::config::standard())- .unwrap();- let (new, _) =- bincode::decode_from_slice::<f32, _>(new.as_bytes(), bincode::config::standard())- .unwrap();-- bincode::encode_to_vec(old + new, bincode::config::standard())- .unwrap()- .into()+ let old = unwrap_value!(old, F32);+ let new = unwrap_value!(new, F32);++ Value::F32(old + new)}}@@ -129,16 +111,10 @@ pub struct F64Add;impl UpsertFn for F64Add {fn upsert(&self, old: Value, new: Value) -> Value {- let (old, _) =- bincode::decode_from_slice::<f64, _>(old.as_bytes(), bincode::config::standard())- .unwrap();- let (new, _) =- bincode::decode_from_slice::<f64, _>(new.as_bytes(), bincode::config::standard())- .unwrap();-- bincode::encode_to_vec(old + new, bincode::config::standard())- .unwrap()- .into()+ let old = unwrap_value!(old, F64);+ let new = unwrap_value!(new, F64);++ Value::F64(old + new)}}@@ -147,19 +123,11 @@ pub struct KahanSumAdd;impl UpsertFn for KahanSumAdd {fn upsert(&self, old: Value, new: Value) -> Value {- let (old, _) =- bincode::decode_from_slice::<KahanSum, _>(old.as_bytes(), bincode::config::standard())- .unwrap();- let (new, _) =- bincode::decode_from_slice::<KahanSum, _>(new.as_bytes(), bincode::config::standard())- .unwrap();+ let old = unwrap_value!(old, KahanSum);+ let new = unwrap_value!(new, KahanSum);let new: f64 = new.into();- let res: KahanSum = old + new;-- bincode::encode_to_vec(res, bincode::config::standard())- .unwrap()- .into()+ Value::KahanSum(old + new)}}diff --git a/crates/core/src/ampc/dht/value.rs b/crates/core/src/ampc/dht/value.rsnew file mode 100644index 00000000..6736f197--- /dev/null+++ b/crates/core/src/ampc/dht/value.rs@@ -0,0 +1,112 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use bloom::U64BloomFilter;++use crate::kahan_sum::KahanSum;++pub trait ValueTrait: TryFrom<Value> + Into<Value> {}++impl ValueTrait for String {}++type ApproxHarmonicMeta = crate::entrypoint::ampc::approximated_harmonic_centrality::Meta;+impl ValueTrait for ApproxHarmonicMeta {}++type F32 = f32;+impl ValueTrait for F32 {}++type F64 = f64;+impl ValueTrait for F64 {}++type U64 = u64;+impl ValueTrait for U64 {}++impl ValueTrait for KahanSum {}++type HyperLogLog8 = crate::hyperloglog::HyperLogLog<8>;+impl ValueTrait for HyperLogLog8 {}+type HyperLogLog16 = crate::hyperloglog::HyperLogLog<16>;+impl ValueTrait for HyperLogLog16 {}+type HyperLogLog32 = crate::hyperloglog::HyperLogLog<32>;+impl ValueTrait for HyperLogLog32 {}+type HyperLogLog64 = crate::hyperloglog::HyperLogLog<64>;+impl ValueTrait for HyperLogLog64 {}+type HyperLogLog128 = crate::hyperloglog::HyperLogLog<128>;+impl ValueTrait for HyperLogLog128 {}++type HarmonicMeta = crate::entrypoint::ampc::harmonic_centrality::Meta;+impl ValueTrait for HarmonicMeta {}++impl ValueTrait for U64BloomFilter {}++type Unit = ();+impl ValueTrait for Unit {}++#[derive(+ serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone, PartialEq,+)]+pub enum Value {+ String(String),+ ApproxHarmonicMeta(ApproxHarmonicMeta),+ F32(F32),+ F64(F64),+ U64(U64),+ KahanSum(KahanSum),+ HyperLogLog8(HyperLogLog8),+ HyperLogLog16(HyperLogLog16),+ HyperLogLog32(HyperLogLog32),+ HyperLogLog64(HyperLogLog64),+ HyperLogLog128(HyperLogLog128),+ HarmonicMeta(HarmonicMeta),+ U64BloomFilter(U64BloomFilter),+ Unit(Unit),+}++macro_rules! impl_from_to_value {+ ($val:ty, $variant:ident) => {+ impl From<$val> for Value {+ fn from(val: $val) -> Self {+ Value::$variant(val)+ }+ }++ impl TryFrom<Value> for $val {+ type Error = anyhow::Error;++ fn try_from(val: Value) -> Result<Self, Self::Error> {+ match val {+ Value::$variant(val) => Ok(val),+ _ => anyhow::bail!("Value is not of type {}", stringify!($val)),+ }+ }+ }+ };+}++impl_from_to_value!(String, String);+impl_from_to_value!(ApproxHarmonicMeta, ApproxHarmonicMeta);+impl_from_to_value!(F32, F32);+impl_from_to_value!(F64, F64);+impl_from_to_value!(U64, U64);+impl_from_to_value!(KahanSum, KahanSum);+impl_from_to_value!(HyperLogLog8, HyperLogLog8);+impl_from_to_value!(HyperLogLog16, HyperLogLog16);+impl_from_to_value!(HyperLogLog32, HyperLogLog32);+impl_from_to_value!(HyperLogLog64, HyperLogLog64);+impl_from_to_value!(HyperLogLog128, HyperLogLog128);+impl_from_to_value!(HarmonicMeta, HarmonicMeta);+impl_from_to_value!(U64BloomFilter, U64BloomFilter);+impl_from_to_value!(Unit, Unit);diff --git a/crates/core/src/ampc/dht_conn.rs b/crates/core/src/ampc/dht_conn.rsindex b6b4d9e1..42e76ab7 100644--- a/crates/core/src/ampc/dht_conn.rs+++ b/crates/core/src/ampc/dht_conn.rs@@ -17,7 +17,7 @@use crate::block_on;use std::{collections::BTreeMap, net::SocketAddr, pin::Pin};-use super::dht::{self, upsert::UpsertEnum, UpsertAction};+use super::dht::{self, upsert::UpsertEnum, KeyTrait, UpsertAction, ValueTrait};use crate::Result;@@ -62,6 +62,7 @@ macro_rules! impl_dht_tables {};}+use anyhow::anyhow;use futures::{Stream, StreamExt};pub(crate) use impl_dht_tables;@@ -95,13 +96,53 @@ impl Table {}}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]+#[derive(Debug)]pub struct DefaultDhtTable<K, V> {table: Table,client: dht::Client,_maker: std::marker::PhantomData<(K, V)>,}+impl<K, V> bincode::Encode for DefaultDhtTable<K, V> {+ fn encode<E: bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> std::prelude::v1::Result<(), bincode::error::EncodeError> {+ self.table.encode(encoder)?;+ self.client.encode(encoder)+ }+}++impl<K, V> bincode::Decode for DefaultDhtTable<K, V> {+ fn decode<D: bincode::de::Decoder>(+ decoder: &mut D,+ ) -> std::prelude::v1::Result<Self, bincode::error::DecodeError> {+ let table = Table::decode(decoder)?;+ let client = dht::Client::decode(decoder)?;++ Ok(Self {+ table,+ client,+ _maker: std::marker::PhantomData,+ })+ }+}++impl<'de, K, V> bincode::BorrowDecode<'de> for DefaultDhtTable<K, V> {+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(+ decoder: &mut D,+ ) -> std::prelude::v1::Result<Self, bincode::error::DecodeError> {+ let table = Table::borrow_decode(decoder)?;+ let client = dht::Client::borrow_decode(decoder)?;++ Ok(Self {+ table,+ client,+ _maker: std::marker::PhantomData,+ })+ }+}+impl<K, V> Clone for DefaultDhtTable<K, V> {fn clone(&self) -> Self {Self {@@ -113,8 +154,8 @@ impl<K, V> Clone for DefaultDhtTable<K, V> {}impl<K, V> DefaultDhtTable<K, V>where- K: bincode::Encode + bincode::Decode,- V: bincode::Encode + bincode::Decode,+ K: KeyTrait,+ V: ValueTrait,{pub fn new<S: AsRef<str>>(members: &[(dht::ShardId, SocketAddr)], prefix: S) -> Self {Self {@@ -130,43 +171,37 @@ where}pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {- type Key: bincode::Encode + bincode::Decode;- type Value: bincode::Encode + bincode::Decode;+ type Key: KeyTrait;+ type Value: ValueTrait;fn client(&self) -> &dht::Client;fn table(&self) -> &Table;fn next(&self) -> Self;fn get(&self, key: Self::Key) -> Option<Self::Value> {- let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();-block_on(self.client().get(self.table().dht(), key.into())).unwrap().map(|v| {- let (k, _) =- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).unwrap();- k+ Self::Value::try_from(v)+ .map_err(|_| anyhow!("unexpected value type in DHT table get"))+ .unwrap()})}fn batch_get(&self, keys: Vec<Self::Key>) -> Vec<(Self::Key, Self::Value)> {- let keys: Vec<dht::Key> = keys- .into_iter()- .map(|k| {- bincode::encode_to_vec(&k, bincode::config::standard())- .unwrap()- .into()- })- .collect::<Vec<_>>();+ let keys: Vec<dht::Key> = keys.into_iter().map(|k| k.into()).collect::<Vec<_>>();let values = block_on(self.client().batch_get(self.table().dht(), keys)).unwrap();values.into_iter().map(|(k, v)| {- let (k, _) =- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).unwrap();- let (v, _) =- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).unwrap();+ let k = Self::Key::try_from(k)+ .map_err(|_| anyhow!("unexpected key type in DHT table batch-get"))+ .unwrap();++ let v = Self::Value::try_from(v)+ .map_err(|_| anyhow!("unexpected value type in DHT table batch-get"))+ .unwrap();(k, v)})@@ -174,9 +209,6 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {}fn set(&self, key: Self::Key, value: Self::Value) {- let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();- let value = bincode::encode_to_vec(&value, bincode::config::standard()).unwrap();-block_on(self.client().set(self.table().dht(), key.into(), value.into()),@@ -187,16 +219,7 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {fn batch_set(&self, pairs: Vec<(Self::Key, Self::Value)>) {let pairs: Vec<(dht::Key, dht::Value)> = pairs.into_iter()- .map(|(k, v)| {- (- bincode::encode_to_vec(&k, bincode::config::standard())- .unwrap()- .into(),- bincode::encode_to_vec(&v, bincode::config::standard())- .unwrap()- .into(),- )- })+ .map(|(k, v)| (k.into(), v.into())).collect();block_on(self.client().batch_set(self.table().dht(), pairs)).unwrap();@@ -212,9 +235,6 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {key: Self::Key,value: Self::Value,) -> UpsertAction {- let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();- let value = bincode::encode_to_vec(&value, bincode::config::standard()).unwrap();-block_on(self.client().upsert(self.table().dht(), upsert, key.into(), value.into()),@@ -229,16 +249,7 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {) -> Vec<(Self::Key, UpsertAction)> {let pairs: Vec<(dht::Key, dht::Value)> = pairs.into_iter()- .map(|(k, v)| {- (- bincode::encode_to_vec(&k, bincode::config::standard())- .unwrap()- .into(),- bincode::encode_to_vec(&v, bincode::config::standard())- .unwrap()- .into(),- )- })+ .map(|(k, v)| (k.into(), v.into())).collect();block_on(@@ -248,8 +259,10 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {.unwrap().into_iter().map(|(k, did_upsert)| {- let (k, _) =- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).unwrap();+ let k = k+ .try_into()+ .map_err(|_| anyhow!("unexpected key type in DHT table batch-upsert"))+ .unwrap();(k, did_upsert)}).collect()@@ -274,10 +287,13 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {fn iter(&self) -> impl Iterator<Item = (Self::Key, Self::Value)> + '_ {self.raw_iter().map(|(key, value)| {- let (key, _) =- bincode::decode_from_slice(key.as_bytes(), bincode::config::standard()).unwrap();- let (value, _) =- bincode::decode_from_slice(value.as_bytes(), bincode::config::standard()).unwrap();+ let key = Self::Key::try_from(key)+ .map_err(|_| anyhow!("unexpected key type in DHT table iter"))+ .unwrap();++ let value = Self::Value::try_from(value)+ .map_err(|_| anyhow!("unexpected value type in DHT table iter"))+ .unwrap();(key, value)})@@ -339,8 +355,9 @@ whereimpl<K, V> DhtTable for DefaultDhtTable<K, V>where- K: bincode::Encode + bincode::Decode,- V: bincode::Encode + bincode::Decode,+ K: KeyTrait,++ V: ValueTrait,{type Key = K;type Value = V;@@ -418,32 +435,10 @@ mod tests {use super::*;- #[derive(- Debug,- Clone,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- PartialEq,- Eq,- PartialOrd,- Ord,- )]- struct Id(u64);- #[derive(- Debug,- Clone,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- PartialEq,- Eq,- )]- struct Counter(u64);-- #[derive(Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+ type Id = u64;+ type Counter = u64;++ #[derive(Clone, bincode::Encode, bincode::Decode)]struct Tables {id: DefaultDhtTable<Id, Counter>,}@@ -492,38 +487,26 @@ mod tests {id: DefaultDhtTable::new(&[(1.into(), addr)], "id"),};- tables.id.set(Id(0), Counter(0));+ tables.id.set(0, 0);- assert_eq!(tables.id.get(Id(0)), Some(Counter(0)));+ assert_eq!(tables.id.get(0), Some(0));- tables- .id- .batch_set(vec![(Id(1), Counter(0)), (Id(2), Counter(0))]);+ tables.id.batch_set(vec![(1, 0), (2, 0)]);- let mut res = tables.id.batch_get(vec![Id(1), Id(2)]);+ let mut res = tables.id.batch_get(vec![1, 2]);res.sort_by(|(a, _), (b, _)| a.cmp(b));- assert_eq!(res, vec![(Id(1), Counter(0)), (Id(2), Counter(0))]);+ assert_eq!(res, vec![(1, 0), (2, 0)]);- tables.id.upsert(upsert::U64Add, Id(0), Counter(1));- assert_eq!(tables.id.get(Id(0)), Some(Counter(1)));+ tables.id.upsert(upsert::U64Add, 0, 1);+ assert_eq!(tables.id.get(0), Some(1));- tables.id.batch_upsert(- upsert::U64Add,- vec![(Id(1), Counter(1)), (Id(2), Counter(1))],- );+ tables.id.batch_upsert(upsert::U64Add, vec![(1, 1), (2, 1)]);- let mut res = tables.id.batch_get(vec![Id(0), Id(1), Id(2)]);+ let mut res = tables.id.batch_get(vec![0, 1, 2]);res.sort_by(|(a, _), (b, _)| a.cmp(b));- assert_eq!(- res,- vec![- (Id(0), Counter(1)),- (Id(1), Counter(1)),- (Id(2), Counter(1))- ]- );+ assert_eq!(res, vec![(0, 1), (1, 1), (2, 1)]);Ok(())}diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex aad69587..0483ff9d 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -38,6 +38,10 @@ impl ShardId {pub fn new(id: u64) -> Self {Self(id)}++ pub fn as_u64(&self) -> u64 {+ self.0+ }}impl From<u64> for ShardId {diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rsindex bbf62fed..d8578f8d 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs@@ -29,13 +29,22 @@ use crate::{use self::mapper::ApproxCentralityMapper;use self::worker::{ApproxCentralityWorker, RemoteApproxCentralityWorker};-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialEq,+ Eq,+)]pub struct Meta {round: u64,num_samples_per_worker: u64,}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]pub struct ApproxCentralityTables {centrality: DefaultDhtTable<webgraph::NodeID, f32>,meta: DefaultDhtTable<(), Meta>,diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rsindex a4922891..90b06feb 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs@@ -29,13 +29,22 @@ pub use coordinator::{CentralityFinish, CentralitySetup};pub use mapper::CentralityMapper;pub use worker::{CentralityWorker, RemoteCentralityWorker};-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialEq,+ Eq,+)]pub struct Meta {round_had_changes: bool,upper_bound_num_nodes: u64,}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]pub struct CentralityTables {counters: DefaultDhtTable<webgraph::NodeID, HyperLogLog<64>>,meta: DefaultDhtTable<(), Meta>,@@ -54,7 +63,7 @@ impl_dht_tables!([counters, meta, centrality, changed_nodes]);-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]pub struct CentralityJob {shard: ShardId,}diff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rsindex 06126748..5a60d43f 100644--- a/crates/core/src/hyperloglog.rs+++ b/crates/core/src/hyperloglog.rs@@ -4325,7 +4325,9 @@ impl HyperLogLogHasher for StableHasher {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Eq,+)]pub struct HyperLogLog<const N: usize, H = FastHasher>whereH: HyperLogLogHasher,@@ -4334,6 +4336,12 @@ where_hasher: PhantomData<H>,}+impl<H: HyperLogLogHasher, const N: usize> PartialEq for HyperLogLog<N, H> {+ fn eq(&self, other: &Self) -> bool {+ self.registers == other.registers+ }+}+impl<H: HyperLogLogHasher, const N: usize> Default for HyperLogLog<N, H> {fn default() -> Self {Self {diff --git a/crates/core/src/kahan_sum.rs b/crates/core/src/kahan_sum.rsindex 7c73d272..e40119ae 100644--- a/crates/core/src/kahan_sum.rs+++ b/crates/core/src/kahan_sum.rs@@ -25,6 +25,7 @@ use std::ops::{Add, AddAssign};serde::Deserialize,bincode::Encode,bincode::Decode,+ PartialEq,)]pub struct KahanSum {sum: f64,