Skip to content

web search done right

reduce memory usage and serialization/deserialization in dht by storing keys and values as enums instead of vec<u8>

Mikkel Denkerf7a354572e48
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rs
index 8c17cc4a..81dbc86d 100644
--- a/crates/bloom/src/lib.rs
+++ b/crates/bloom/src/lib.rs
@@ -47,7 +47,16 @@ fn num_hashes(num_bits: u64, estimated_items: u64) -> u64 {
(((num_bits as f64) / estimated_items as f64 * 2.0_f64.ln()).ceil() as u64).max(1)
}
-#[derive(Clone, bincode::Encode, bincode::Decode, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(
+ Clone,
+ bincode::Encode,
+ bincode::Decode,
+ Debug,
+ serde::Serialize,
+ serde::Deserialize,
+ PartialEq,
+ Eq,
+)]
pub struct U64BloomFilter {
#[bincode(with_serde)]
bit_vec: BitVec,
diff --git a/crates/core/src/ampc/dht/client.rs b/crates/core/src/ampc/dht/client.rs
index a448a9e1..919c8dd8 100644
--- a/crates/core/src/ampc/dht/client.rs
+++ b/crates/core/src/ampc/dht/client.rs
@@ -27,9 +27,11 @@ use std::{
use crate::{distributed::member::ShardId, Result};
use super::{
+ key::{Key, KeyTrait},
network::api,
- store::{Key, Table, Value},
+ store::Table,
upsert::UpsertEnum,
+ value::Value,
UpsertAction,
};
@@ -301,14 +303,14 @@ impl Client {
}
pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {
- self.shard_for_key(key.as_bytes())?.get(table, key).await
+ self.shard_for_key(&key.as_bytes())?.get(table, key).await
}
pub async fn batch_get(&self, table: Table, keys: Vec<Key>) -> Result<Vec<(Key, Value)>> {
let mut shard_keys: BTreeMap<ShardId, Vec<Key>> = BTreeMap::new();
for key in keys {
- let shard = self.shard_id_for_key(key.as_bytes())?;
+ let shard = self.shard_id_for_key(&key.as_bytes())?;
shard_keys.entry(*shard).or_default().push(key);
}
@@ -326,7 +328,7 @@ impl Client {
}
pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {
- self.shard_for_key(key.as_bytes())?
+ self.shard_for_key(&key.as_bytes())?
.set(table, key, value)
.await
}
@@ -335,7 +337,7 @@ impl Client {
let mut shard_values: BTreeMap<ShardId, Vec<(Key, Value)>> = BTreeMap::new();
for (key, value) in values {
- let shard = self.shard_id_for_key(key.as_bytes())?;
+ let shard = self.shard_id_for_key(&key.as_bytes())?;
shard_values.entry(*shard).or_default().push((key, value));
}
@@ -367,7 +369,7 @@ impl Client {
key: Key,
value: Value,
) -> Result<UpsertAction> {
- self.shard_for_key(key.as_bytes())?
+ self.shard_for_key(&key.as_bytes())?
.upsert(table, upsert, key, value)
.await
}
@@ -381,7 +383,7 @@ impl Client {
let mut shard_values: BTreeMap<ShardId, Vec<(Key, Value)>> = BTreeMap::new();
for (key, value) in values {
- let shard = self.shard_id_for_key(key.as_bytes())?;
+ let shard = self.shard_id_for_key(&key.as_bytes())?;
shard_values.entry(*shard).or_default().push((key, value));
}
diff --git a/crates/core/src/ampc/dht/key.rs b/crates/core/src/ampc/dht/key.rs
new file mode 100644
index 00000000..4f1acaa3
--- /dev/null
+++ b/crates/core/src/ampc/dht/key.rs
@@ -0,0 +1,113 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::distributed::member::ShardId;
+use crate::webgraph::NodeID;
+
+pub trait KeyTrait: TryFrom<Key> + Into<Key> {
+ fn as_bytes(&self) -> Vec<u8>;
+}
+
+impl KeyTrait for String {
+ fn as_bytes(&self) -> Vec<u8> {
+ self.as_bytes().to_vec()
+ }
+}
+
+impl KeyTrait for NodeID {
+ fn as_bytes(&self) -> Vec<u8> {
+ self.as_u64().to_le_bytes().to_vec()
+ }
+}
+
+type Unit = ();
+impl KeyTrait for Unit {
+ fn as_bytes(&self) -> Vec<u8> {
+ vec![]
+ }
+}
+
+impl KeyTrait for ShardId {
+ fn as_bytes(&self) -> Vec<u8> {
+ self.as_u64().to_le_bytes().to_vec()
+ }
+}
+
+type U64 = u64;
+impl KeyTrait for U64 {
+ fn as_bytes(&self) -> Vec<u8> {
+ self.to_le_bytes().to_vec()
+ }
+}
+
+#[derive(
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Debug,
+ Clone,
+ PartialOrd,
+ Ord,
+ PartialEq,
+ Eq,
+)]
+pub enum Key {
+ String(String),
+ NodeID(NodeID),
+ Unit(Unit),
+ ShardId(ShardId),
+ U64(U64),
+}
+
+impl KeyTrait for Key {
+ fn as_bytes(&self) -> Vec<u8> {
+ match self {
+ Key::String(key) => KeyTrait::as_bytes(key),
+ Key::NodeID(key) => KeyTrait::as_bytes(key),
+ Key::Unit(key) => KeyTrait::as_bytes(key),
+ Key::ShardId(key) => KeyTrait::as_bytes(key),
+ Key::U64(key) => KeyTrait::as_bytes(key),
+ }
+ }
+}
+
+macro_rules! impl_from_to_key {
+ ($key:ty, $variant:ident) => {
+ impl From<$key> for Key {
+ fn from(key: $key) -> Self {
+ Key::$variant(key)
+ }
+ }
+
+ impl TryFrom<Key> for $key {
+ type Error = anyhow::Error;
+
+ fn try_from(key: Key) -> Result<Self, Self::Error> {
+ match key {
+ Key::$variant(key) => Ok(key),
+ _ => anyhow::bail!("Key is not of type {}", stringify!($key)),
+ }
+ }
+ }
+ };
+}
+
+impl_from_to_key!(String, String);
+impl_from_to_key!(NodeID, NodeID);
+impl_from_to_key!(Unit, Unit);
+impl_from_to_key!(ShardId, ShardId);
+impl_from_to_key!(U64, U64);
diff --git a/crates/core/src/ampc/dht/mod.rs b/crates/core/src/ampc/dht/mod.rs
index 11fcf2e7..9498ef8b 100644
--- a/crates/core/src/ampc/dht/mod.rs
+++ b/crates/core/src/ampc/dht/mod.rs
@@ -27,10 +27,12 @@
//! Heavily inspired by https://github.com/datafuselabs/openraft/blob/main/examples/raft-kv-memstore/
mod client;
+pub mod key;
pub mod log_store;
pub mod network;
pub mod store;
pub mod upsert;
+pub mod value;
use network::api::{
AllTables, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Set, Upsert,
@@ -41,6 +43,9 @@ use std::io::Cursor;
use openraft::TokioRuntime;
+pub use key::{Key, KeyTrait};
+pub use value::{Value, ValueTrait};
+
pub use self::network::Server;
pub use network::api::RemoteClient;
@@ -48,7 +53,7 @@ pub use network::raft::RemoteClient as RaftClient;
pub use crate::distributed::member::ShardId;
pub use client::{Client, Shard};
-pub use store::{Key, Table, Value};
+pub use store::Table;
pub use upsert::*;
pub type NodeId = u64;
@@ -132,6 +137,7 @@ raft_sonic_request_response!(
#[cfg(test)]
pub mod tests {
+ use super::{key::Key, value::Value};
use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};
use tokio::sync::Mutex;
use tracing_test::traced_test;
@@ -218,54 +224,54 @@ pub mod tests {
c1.set(
table.clone(),
- "hello".as_bytes().to_vec().into(),
- "world".as_bytes().to_vec().into(),
+ "hello".to_string().into(),
+ "world".to_string().into(),
)
.await?;
- let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;
- assert_eq!(res, Some("world".as_bytes().into()));
+ let res = c1.get(table.clone(), "hello".to_string().into()).await?;
+ assert_eq!(res, Some("world".to_string().into()));
c2.set(
table.clone(),
- "ensure-linearized-read".as_bytes().into(),
- vec![].into(),
+ "ensure-linearized-read".to_string().into(),
+ ().into(),
)
.await?;
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;
- assert_eq!(res, Some("world".as_bytes().into()));
+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;
+ assert_eq!(res, Some("world".to_string().into()));
c2.set(
table.clone(),
- "hello".as_bytes().into(),
- "world2".as_bytes().into(),
+ "hello".to_string().into(),
+ "world2".to_string().into(),
)
.await?;
c1.set(
table.clone(),
- "ensure-linearized-read".as_bytes().into(),
- vec![].into(),
+ "ensure-linearized-read".to_string().into(),
+ ().into(),
)
.await?;
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
- let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;
- assert_eq!(res, Some("world2".as_bytes().into()));
+ let res = c1.get(table.clone(), "hello".to_string().into()).await?;
+ assert_eq!(res, Some("world2".to_string().into()));
let res = c1.batch_get(table.clone(), vec![]).await?;
assert!(res.is_empty());
let res = c1
- .batch_get(table.clone(), vec!["hello".as_bytes().into()])
+ .batch_get(table.clone(), vec!["hello".to_string().into()])
.await?;
assert!(res.len() == 1);
let res = c1
- .batch_get(table.clone(), vec!["non-existent-key".as_bytes().into()])
+ .batch_get(table.clone(), vec!["non-existent-key".to_string().into()])
.await?;
assert!(res.is_empty());
@@ -316,29 +322,29 @@ pub mod tests {
c1.set(
table.clone(),
- "hello".as_bytes().into(),
- "world".as_bytes().into(),
+ "hello".to_string().into(),
+ "world".to_string().into(),
)
.await?;
c2.set(
table.clone(),
- "ensure-linearized-read".as_bytes().into(),
- vec![].into(),
+ "ensure-linearized-read".to_string().into(),
+ ().into(),
)
.await?;
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;
+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;
- assert_eq!(res, Some("world".as_bytes().into()));
+ assert_eq!(res, Some("world".to_string().into()));
rc1.join(3, addr3).await?;
tokio::time::sleep(std::time::Duration::from_secs(1)).await; // join is not blocking
let c3 = RemoteClient::new(addr3);
- let res = c3.get(table.clone(), "hello".as_bytes().into()).await?;
- assert_eq!(res, Some("world".as_bytes().into()));
+ let res = c3.get(table.clone(), "hello".to_string().into()).await?;
+ assert_eq!(res, Some("world".to_string().into()));
Ok(())
}
@@ -379,16 +385,16 @@ pub mod tests {
client
.set(
table.clone(),
- "hello".as_bytes().into(),
- "world".as_bytes().into(),
+ "hello".to_string().into(),
+ "world".to_string().into(),
)
.await?;
client
.set(
table.clone(),
- "hello2".as_bytes().into(),
- "world2".as_bytes().into(),
+ "hello2".to_string().into(),
+ "world2".to_string().into(),
)
.await?;
@@ -406,8 +412,8 @@ pub mod tests {
assert_eq!(
res,
vec![
- ("hello".as_bytes().into(), "world".as_bytes().into()),
- ("hello2".as_bytes().into(), "world2".as_bytes().into()),
+ ("hello".to_string().into(), "world".to_string().into()),
+ ("hello2".to_string().into(), "world2".to_string().into()),
]
);
@@ -418,12 +424,7 @@ pub mod tests {
.batch_set(
table.clone(),
(0..N)
- .map(|i| {
- (
- i.to_be_bytes().as_ref().into(),
- i.to_be_bytes().as_ref().into(),
- )
- })
+ .map(|i| (i.into(), i.into()))
.collect::<Vec<(Key, Value)>>(),
)
.await?;
@@ -441,13 +442,8 @@ pub mod tests {
assert_eq!(res.len(), N as usize);
for (i, r) in res.into_iter().enumerate() {
- assert_eq!(
- r,
- (
- i.to_be_bytes().as_ref().into(),
- i.to_be_bytes().as_ref().into(),
- )
- );
+ let i = i as u64;
+ assert_eq!(r, (i.into(), i.into(),));
}
Ok(())
@@ -496,13 +492,13 @@ pub mod tests {
c1.set(
table.clone(),
- "hello".as_bytes().into(),
- "world".as_bytes().into(),
+ "hello".to_string().into(),
+ "world".to_string().into(),
)
.await?;
- let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;
- assert_eq!(res, Some("world".as_bytes().into()));
+ let res = c1.get(table.clone(), "hello".to_string().into()).await?;
+ assert_eq!(res, Some("world".to_string().into()));
// crash node 2
tracing::info!("crashing node 2");
@@ -521,8 +517,8 @@ pub mod tests {
let c2 = RemoteClient::new(addr2);
- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;
- assert_eq!(res, Some("world".as_bytes().into()));
+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;
+ assert_eq!(res, Some("world".to_string().into()));
// crash node 2 again
tracing::info!("crashing node 2 again");
@@ -531,8 +527,8 @@ pub mod tests {
c1.set(
table.clone(),
- "hello".as_bytes().into(),
- "world2".as_bytes().into(),
+ "hello".to_string().into(),
+ "world2".to_string().into(),
)
.await?;
@@ -547,8 +543,8 @@ pub mod tests {
let c2 = RemoteClient::new(addr2);
- let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;
- assert_eq!(res, Some("world2".as_bytes().into()));
+ let res = c2.get(table.clone(), "hello".to_string().into()).await?;
+ assert_eq!(res, Some("world2".to_string().into()));
Ok(())
}
@@ -564,7 +560,7 @@ pub mod tests {
Arbitrary,
)]
enum Action {
- Set { key: Vec<u8>, value: Vec<u8> },
+ Set { key: String, value: String },
// get actions[prev_key % actions.len()]
// if actions[prev_key % actions.len()] is a get, then get a non-existent key
Get { prev_key: usize },
@@ -576,7 +572,7 @@ pub mod tests {
#[test]
#[traced_test]
fn proptest_chaos(actions: Vec<Action>) {
- let ground_truth = Arc::new(Mutex::new(BTreeMap::<Vec<u8>, Vec<u8>>::new()));
+ let ground_truth = Arc::new(Mutex::new(BTreeMap::<String, String>::new()));
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -640,23 +636,23 @@ pub mod tests {
}
Action::Get { prev_key } => {
let client = clients.choose(&mut rand::thread_rng()).unwrap();
- client.set(table.clone(), b"ensure-linearized-read".to_vec().into(), vec![].into()).await.unwrap();
+ client.set(table.clone(), "ensure-linearized-read".to_string().into(), ().into()).await.unwrap();
let key = if i == 0 {
- b"non-existent-key".to_vec()
+ "non-existent-key".to_string()
} else {
match shared_actions[prev_key % i] {
Action::Set { ref key, .. } => {
key.clone()
},
- Action::Get { .. } => b"non-existent-key".to_vec(),
+ Action::Get { .. } => "non-existent-key".to_string(),
}
};
let res = client.get(table.clone(), key.clone().into()).await.unwrap();
let expected = ground_truth.lock().await.get(&key).cloned();
- assert_eq!(res.map(|v| v.as_bytes().to_vec()), expected);
+ assert_eq!(res.map(|v| v.try_into().unwrap()), expected);
}
}
}
diff --git a/crates/core/src/ampc/dht/network/api.rs b/crates/core/src/ampc/dht/network/api.rs
index 575ef91a..3e645682 100644
--- a/crates/core/src/ampc/dht/network/api.rs
+++ b/crates/core/src/ampc/dht/network/api.rs
@@ -23,9 +23,7 @@ use std::{
use crate::{
ampc::dht::{
- store::{Key, Table, Value},
- upsert::UpsertEnum,
- BasicNode, UpsertAction,
+ key::Key, store::Table, upsert::UpsertEnum, value::Value, BasicNode, UpsertAction,
},
distributed::retry_strategy::RandomBackoff,
Result,
diff --git a/crates/core/src/ampc/dht/store.rs b/crates/core/src/ampc/dht/store.rs
index 62a4bec7..1ffde379 100644
--- a/crates/core/src/ampc/dht/store.rs
+++ b/crates/core/src/ampc/dht/store.rs
@@ -36,8 +36,10 @@ use tokio::sync::RwLock;
use crate::ampc::dht::network::api;
+use super::key::Key;
use super::upsert::UpsertEnum;
use super::upsert::UpsertFn;
+use super::value::Value;
use super::BasicNode;
use super::NodeId;
use super::TypeConfig;
@@ -77,74 +79,6 @@ impl From<&str> for Table {
}
}
-#[derive(
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- Debug,
- Clone,
- PartialOrd,
- Ord,
- PartialEq,
- Eq,
-)]
-#[serde(transparent)]
-pub struct Key(Vec<u8>);
-
-impl Key {
- pub fn as_bytes(&self) -> &[u8] {
- &self.0
- }
-}
-
-impl From<Vec<u8>> for Key {
- fn from(v: Vec<u8>) -> Self {
- Self(v)
- }
-}
-
-impl From<&[u8]> for Key {
- fn from(v: &[u8]) -> Self {
- Self(v.to_vec())
- }
-}
-
-#[derive(
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- Debug,
- Clone,
- PartialEq,
- Eq,
-)]
-#[serde(transparent)]
-pub struct Value(Vec<u8>);
-
-impl Value {
- pub fn as_bytes(&self) -> &[u8] {
- &self.0
- }
-
- pub fn as_bytes_mut(&mut self) -> &mut [u8] {
- &mut self.0
- }
-}
-
-impl From<Vec<u8>> for Value {
- fn from(v: Vec<u8>) -> Self {
- Self(v)
- }
-}
-
-impl From<&[u8]> for Value {
- fn from(v: &[u8]) -> Self {
- Self(v.to_vec())
- }
-}
-
#[derive(
serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Default, Clone,
)]
diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rs
index 17e70b5f..69864c75 100644
--- a/crates/core/src/ampc/dht/upsert.rs
+++ b/crates/core/src/ampc/dht/upsert.rs
@@ -16,8 +16,6 @@
use enum_dispatch::enum_dispatch;
-use crate::{hyperloglog::HyperLogLog, kahan_sum::KahanSum};
-
use super::Value;
#[derive(
@@ -37,8 +35,6 @@ pub trait UpsertFn {
#[enum_dispatch(UpsertFn)]
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub enum UpsertEnum {
- HyperLogLog2Upsert,
- HyperLogLog4Upsert,
HyperLogLog8Upsert,
HyperLogLog16Upsert,
HyperLogLog32Upsert,
@@ -50,8 +46,18 @@ pub enum UpsertEnum {
KahanSumAdd,
}
+macro_rules! unwrap_value {
+ ($value:expr, $variant:ident) => {
+ if let Value::$variant(value) = $value {
+ value
+ } else {
+ panic!("Expected {}", stringify!($variant));
+ }
+ };
+}
+
macro_rules! hyperloglog_upsert {
- ($name:ident => $n:expr) => {
+ ($name:ident, $variant:ident) => {
#[derive(
Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,
)]
@@ -59,50 +65,32 @@ macro_rules! hyperloglog_upsert {
impl UpsertFn for $name {
fn upsert(&self, old: Value, new: Value) -> Value {
- let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(
- old.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
- let (new, _) = bincode::decode_from_slice::<HyperLogLog<$n>, _>(
- new.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
+ let mut old = unwrap_value!(old, $variant);
+ let new = unwrap_value!(new, $variant);
old.merge(&new);
- bincode::encode_to_vec(&old, bincode::config::standard())
- .unwrap()
- .into()
+ Value::$variant(old)
}
}
};
}
-hyperloglog_upsert!(HyperLogLog2Upsert => 2);
-hyperloglog_upsert!(HyperLogLog4Upsert => 4);
-hyperloglog_upsert!(HyperLogLog8Upsert => 8);
-hyperloglog_upsert!(HyperLogLog16Upsert => 16);
-hyperloglog_upsert!(HyperLogLog32Upsert => 32);
-hyperloglog_upsert!(HyperLogLog64Upsert => 64);
-hyperloglog_upsert!(HyperLogLog128Upsert => 128);
+hyperloglog_upsert!(HyperLogLog8Upsert, HyperLogLog8);
+hyperloglog_upsert!(HyperLogLog16Upsert, HyperLogLog16);
+hyperloglog_upsert!(HyperLogLog32Upsert, HyperLogLog32);
+hyperloglog_upsert!(HyperLogLog64Upsert, HyperLogLog64);
+hyperloglog_upsert!(HyperLogLog128Upsert, HyperLogLog128);
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct U64Add;
impl UpsertFn for U64Add {
fn upsert(&self, old: Value, new: Value) -> Value {
- let (old, _) =
- bincode::decode_from_slice::<u64, _>(old.as_bytes(), bincode::config::standard())
- .unwrap();
- let (new, _) =
- bincode::decode_from_slice::<u64, _>(new.as_bytes(), bincode::config::standard())
- .unwrap();
-
- bincode::encode_to_vec(old + new, bincode::config::standard())
- .unwrap()
- .into()
+ let old = unwrap_value!(old, U64);
+ let new = unwrap_value!(new, U64);
+
+ Value::U64(old + new)
}
}
@@ -111,16 +99,10 @@ pub struct F32Add;
impl UpsertFn for F32Add {
fn upsert(&self, old: Value, new: Value) -> Value {
- let (old, _) =
- bincode::decode_from_slice::<f32, _>(old.as_bytes(), bincode::config::standard())
- .unwrap();
- let (new, _) =
- bincode::decode_from_slice::<f32, _>(new.as_bytes(), bincode::config::standard())
- .unwrap();
-
- bincode::encode_to_vec(old + new, bincode::config::standard())
- .unwrap()
- .into()
+ let old = unwrap_value!(old, F32);
+ let new = unwrap_value!(new, F32);
+
+ Value::F32(old + new)
}
}
@@ -129,16 +111,10 @@ pub struct F64Add;
impl UpsertFn for F64Add {
fn upsert(&self, old: Value, new: Value) -> Value {
- let (old, _) =
- bincode::decode_from_slice::<f64, _>(old.as_bytes(), bincode::config::standard())
- .unwrap();
- let (new, _) =
- bincode::decode_from_slice::<f64, _>(new.as_bytes(), bincode::config::standard())
- .unwrap();
-
- bincode::encode_to_vec(old + new, bincode::config::standard())
- .unwrap()
- .into()
+ let old = unwrap_value!(old, F64);
+ let new = unwrap_value!(new, F64);
+
+ Value::F64(old + new)
}
}
@@ -147,19 +123,11 @@ pub struct KahanSumAdd;
impl UpsertFn for KahanSumAdd {
fn upsert(&self, old: Value, new: Value) -> Value {
- let (old, _) =
- bincode::decode_from_slice::<KahanSum, _>(old.as_bytes(), bincode::config::standard())
- .unwrap();
- let (new, _) =
- bincode::decode_from_slice::<KahanSum, _>(new.as_bytes(), bincode::config::standard())
- .unwrap();
+ let old = unwrap_value!(old, KahanSum);
+ let new = unwrap_value!(new, KahanSum);
let new: f64 = new.into();
- let res: KahanSum = old + new;
-
- bincode::encode_to_vec(res, bincode::config::standard())
- .unwrap()
- .into()
+ Value::KahanSum(old + new)
}
}
diff --git a/crates/core/src/ampc/dht/value.rs b/crates/core/src/ampc/dht/value.rs
new file mode 100644
index 00000000..6736f197
--- /dev/null
+++ b/crates/core/src/ampc/dht/value.rs
@@ -0,0 +1,112 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use bloom::U64BloomFilter;
+
+use crate::kahan_sum::KahanSum;
+
+pub trait ValueTrait: TryFrom<Value> + Into<Value> {}
+
+impl ValueTrait for String {}
+
+type ApproxHarmonicMeta = crate::entrypoint::ampc::approximated_harmonic_centrality::Meta;
+impl ValueTrait for ApproxHarmonicMeta {}
+
+type F32 = f32;
+impl ValueTrait for F32 {}
+
+type F64 = f64;
+impl ValueTrait for F64 {}
+
+type U64 = u64;
+impl ValueTrait for U64 {}
+
+impl ValueTrait for KahanSum {}
+
+type HyperLogLog8 = crate::hyperloglog::HyperLogLog<8>;
+impl ValueTrait for HyperLogLog8 {}
+type HyperLogLog16 = crate::hyperloglog::HyperLogLog<16>;
+impl ValueTrait for HyperLogLog16 {}
+type HyperLogLog32 = crate::hyperloglog::HyperLogLog<32>;
+impl ValueTrait for HyperLogLog32 {}
+type HyperLogLog64 = crate::hyperloglog::HyperLogLog<64>;
+impl ValueTrait for HyperLogLog64 {}
+type HyperLogLog128 = crate::hyperloglog::HyperLogLog<128>;
+impl ValueTrait for HyperLogLog128 {}
+
+type HarmonicMeta = crate::entrypoint::ampc::harmonic_centrality::Meta;
+impl ValueTrait for HarmonicMeta {}
+
+impl ValueTrait for U64BloomFilter {}
+
+type Unit = ();
+impl ValueTrait for Unit {}
+
+#[derive(
+ serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone, PartialEq,
+)]
+pub enum Value {
+ String(String),
+ ApproxHarmonicMeta(ApproxHarmonicMeta),
+ F32(F32),
+ F64(F64),
+ U64(U64),
+ KahanSum(KahanSum),
+ HyperLogLog8(HyperLogLog8),
+ HyperLogLog16(HyperLogLog16),
+ HyperLogLog32(HyperLogLog32),
+ HyperLogLog64(HyperLogLog64),
+ HyperLogLog128(HyperLogLog128),
+ HarmonicMeta(HarmonicMeta),
+ U64BloomFilter(U64BloomFilter),
+ Unit(Unit),
+}
+
+macro_rules! impl_from_to_value {
+ ($val:ty, $variant:ident) => {
+ impl From<$val> for Value {
+ fn from(val: $val) -> Self {
+ Value::$variant(val)
+ }
+ }
+
+ impl TryFrom<Value> for $val {
+ type Error = anyhow::Error;
+
+ fn try_from(val: Value) -> Result<Self, Self::Error> {
+ match val {
+ Value::$variant(val) => Ok(val),
+ _ => anyhow::bail!("Value is not of type {}", stringify!($val)),
+ }
+ }
+ }
+ };
+}
+
+impl_from_to_value!(String, String);
+impl_from_to_value!(ApproxHarmonicMeta, ApproxHarmonicMeta);
+impl_from_to_value!(F32, F32);
+impl_from_to_value!(F64, F64);
+impl_from_to_value!(U64, U64);
+impl_from_to_value!(KahanSum, KahanSum);
+impl_from_to_value!(HyperLogLog8, HyperLogLog8);
+impl_from_to_value!(HyperLogLog16, HyperLogLog16);
+impl_from_to_value!(HyperLogLog32, HyperLogLog32);
+impl_from_to_value!(HyperLogLog64, HyperLogLog64);
+impl_from_to_value!(HyperLogLog128, HyperLogLog128);
+impl_from_to_value!(HarmonicMeta, HarmonicMeta);
+impl_from_to_value!(U64BloomFilter, U64BloomFilter);
+impl_from_to_value!(Unit, Unit);
diff --git a/crates/core/src/ampc/dht_conn.rs b/crates/core/src/ampc/dht_conn.rs
index b6b4d9e1..42e76ab7 100644
--- a/crates/core/src/ampc/dht_conn.rs
+++ b/crates/core/src/ampc/dht_conn.rs
@@ -17,7 +17,7 @@
use crate::block_on;
use std::{collections::BTreeMap, net::SocketAddr, pin::Pin};
-use super::dht::{self, upsert::UpsertEnum, UpsertAction};
+use super::dht::{self, upsert::UpsertEnum, KeyTrait, UpsertAction, ValueTrait};
use crate::Result;
@@ -62,6 +62,7 @@ macro_rules! impl_dht_tables {
};
}
+use anyhow::anyhow;
use futures::{Stream, StreamExt};
pub(crate) use impl_dht_tables;
@@ -95,13 +96,53 @@ impl Table {
}
}
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]
+#[derive(Debug)]
pub struct DefaultDhtTable<K, V> {
table: Table,
client: dht::Client,
_maker: std::marker::PhantomData<(K, V)>,
}
+impl<K, V> bincode::Encode for DefaultDhtTable<K, V> {
+ fn encode<E: bincode::enc::Encoder>(
+ &self,
+ encoder: &mut E,
+ ) -> std::prelude::v1::Result<(), bincode::error::EncodeError> {
+ self.table.encode(encoder)?;
+ self.client.encode(encoder)
+ }
+}
+
+impl<K, V> bincode::Decode for DefaultDhtTable<K, V> {
+ fn decode<D: bincode::de::Decoder>(
+ decoder: &mut D,
+ ) -> std::prelude::v1::Result<Self, bincode::error::DecodeError> {
+ let table = Table::decode(decoder)?;
+ let client = dht::Client::decode(decoder)?;
+
+ Ok(Self {
+ table,
+ client,
+ _maker: std::marker::PhantomData,
+ })
+ }
+}
+
+impl<'de, K, V> bincode::BorrowDecode<'de> for DefaultDhtTable<K, V> {
+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(
+ decoder: &mut D,
+ ) -> std::prelude::v1::Result<Self, bincode::error::DecodeError> {
+ let table = Table::borrow_decode(decoder)?;
+ let client = dht::Client::borrow_decode(decoder)?;
+
+ Ok(Self {
+ table,
+ client,
+ _maker: std::marker::PhantomData,
+ })
+ }
+}
+
impl<K, V> Clone for DefaultDhtTable<K, V> {
fn clone(&self) -> Self {
Self {
@@ -113,8 +154,8 @@ impl<K, V> Clone for DefaultDhtTable<K, V> {
}
impl<K, V> DefaultDhtTable<K, V>
where
- K: bincode::Encode + bincode::Decode,
- V: bincode::Encode + bincode::Decode,
+ K: KeyTrait,
+ V: ValueTrait,
{
pub fn new<S: AsRef<str>>(members: &[(dht::ShardId, SocketAddr)], prefix: S) -> Self {
Self {
@@ -130,43 +171,37 @@ where
}
pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
- type Key: bincode::Encode + bincode::Decode;
- type Value: bincode::Encode + bincode::Decode;
+ type Key: KeyTrait;
+ type Value: ValueTrait;
fn client(&self) -> &dht::Client;
fn table(&self) -> &Table;
fn next(&self) -> Self;
fn get(&self, key: Self::Key) -> Option<Self::Value> {
- let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();
-
block_on(self.client().get(self.table().dht(), key.into()))
.unwrap()
.map(|v| {
- let (k, _) =
- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).unwrap();
- k
+ Self::Value::try_from(v)
+ .map_err(|_| anyhow!("unexpected value type in DHT table get"))
+ .unwrap()
})
}
fn batch_get(&self, keys: Vec<Self::Key>) -> Vec<(Self::Key, Self::Value)> {
- let keys: Vec<dht::Key> = keys
- .into_iter()
- .map(|k| {
- bincode::encode_to_vec(&k, bincode::config::standard())
- .unwrap()
- .into()
- })
- .collect::<Vec<_>>();
+ let keys: Vec<dht::Key> = keys.into_iter().map(|k| k.into()).collect::<Vec<_>>();
let values = block_on(self.client().batch_get(self.table().dht(), keys)).unwrap();
values
.into_iter()
.map(|(k, v)| {
- let (k, _) =
- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).unwrap();
- let (v, _) =
- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).unwrap();
+ let k = Self::Key::try_from(k)
+ .map_err(|_| anyhow!("unexpected key type in DHT table batch-get"))
+ .unwrap();
+
+ let v = Self::Value::try_from(v)
+ .map_err(|_| anyhow!("unexpected value type in DHT table batch-get"))
+ .unwrap();
(k, v)
})
@@ -174,9 +209,6 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
}
fn set(&self, key: Self::Key, value: Self::Value) {
- let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();
- let value = bincode::encode_to_vec(&value, bincode::config::standard()).unwrap();
-
block_on(
self.client()
.set(self.table().dht(), key.into(), value.into()),
@@ -187,16 +219,7 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
fn batch_set(&self, pairs: Vec<(Self::Key, Self::Value)>) {
let pairs: Vec<(dht::Key, dht::Value)> = pairs
.into_iter()
- .map(|(k, v)| {
- (
- bincode::encode_to_vec(&k, bincode::config::standard())
- .unwrap()
- .into(),
- bincode::encode_to_vec(&v, bincode::config::standard())
- .unwrap()
- .into(),
- )
- })
+ .map(|(k, v)| (k.into(), v.into()))
.collect();
block_on(self.client().batch_set(self.table().dht(), pairs)).unwrap();
@@ -212,9 +235,6 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
key: Self::Key,
value: Self::Value,
) -> UpsertAction {
- let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();
- let value = bincode::encode_to_vec(&value, bincode::config::standard()).unwrap();
-
block_on(
self.client()
.upsert(self.table().dht(), upsert, key.into(), value.into()),
@@ -229,16 +249,7 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
) -> Vec<(Self::Key, UpsertAction)> {
let pairs: Vec<(dht::Key, dht::Value)> = pairs
.into_iter()
- .map(|(k, v)| {
- (
- bincode::encode_to_vec(&k, bincode::config::standard())
- .unwrap()
- .into(),
- bincode::encode_to_vec(&v, bincode::config::standard())
- .unwrap()
- .into(),
- )
- })
+ .map(|(k, v)| (k.into(), v.into()))
.collect();
block_on(
@@ -248,8 +259,10 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
.unwrap()
.into_iter()
.map(|(k, did_upsert)| {
- let (k, _) =
- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).unwrap();
+ let k = k
+ .try_into()
+ .map_err(|_| anyhow!("unexpected key type in DHT table batch-upsert"))
+ .unwrap();
(k, did_upsert)
})
.collect()
@@ -274,10 +287,13 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
fn iter(&self) -> impl Iterator<Item = (Self::Key, Self::Value)> + '_ {
self.raw_iter().map(|(key, value)| {
- let (key, _) =
- bincode::decode_from_slice(key.as_bytes(), bincode::config::standard()).unwrap();
- let (value, _) =
- bincode::decode_from_slice(value.as_bytes(), bincode::config::standard()).unwrap();
+ let key = Self::Key::try_from(key)
+ .map_err(|_| anyhow!("unexpected key type in DHT table iter"))
+ .unwrap();
+
+ let value = Self::Value::try_from(value)
+ .map_err(|_| anyhow!("unexpected value type in DHT table iter"))
+ .unwrap();
(key, value)
})
@@ -339,8 +355,9 @@ where
impl<K, V> DhtTable for DefaultDhtTable<K, V>
where
- K: bincode::Encode + bincode::Decode,
- V: bincode::Encode + bincode::Decode,
+ K: KeyTrait,
+
+ V: ValueTrait,
{
type Key = K;
type Value = V;
@@ -418,32 +435,10 @@ mod tests {
use super::*;
- #[derive(
- Debug,
- Clone,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- PartialEq,
- Eq,
- PartialOrd,
- Ord,
- )]
- struct Id(u64);
- #[derive(
- Debug,
- Clone,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- PartialEq,
- Eq,
- )]
- struct Counter(u64);
-
- #[derive(Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+ type Id = u64;
+ type Counter = u64;
+
+ #[derive(Clone, bincode::Encode, bincode::Decode)]
struct Tables {
id: DefaultDhtTable<Id, Counter>,
}
@@ -492,38 +487,26 @@ mod tests {
id: DefaultDhtTable::new(&[(1.into(), addr)], "id"),
};
- tables.id.set(Id(0), Counter(0));
+ tables.id.set(0, 0);
- assert_eq!(tables.id.get(Id(0)), Some(Counter(0)));
+ assert_eq!(tables.id.get(0), Some(0));
- tables
- .id
- .batch_set(vec![(Id(1), Counter(0)), (Id(2), Counter(0))]);
+ tables.id.batch_set(vec![(1, 0), (2, 0)]);
- let mut res = tables.id.batch_get(vec![Id(1), Id(2)]);
+ let mut res = tables.id.batch_get(vec![1, 2]);
res.sort_by(|(a, _), (b, _)| a.cmp(b));
- assert_eq!(res, vec![(Id(1), Counter(0)), (Id(2), Counter(0))]);
+ assert_eq!(res, vec![(1, 0), (2, 0)]);
- tables.id.upsert(upsert::U64Add, Id(0), Counter(1));
- assert_eq!(tables.id.get(Id(0)), Some(Counter(1)));
+ tables.id.upsert(upsert::U64Add, 0, 1);
+ assert_eq!(tables.id.get(0), Some(1));
- tables.id.batch_upsert(
- upsert::U64Add,
- vec![(Id(1), Counter(1)), (Id(2), Counter(1))],
- );
+ tables.id.batch_upsert(upsert::U64Add, vec![(1, 1), (2, 1)]);
- let mut res = tables.id.batch_get(vec![Id(0), Id(1), Id(2)]);
+ let mut res = tables.id.batch_get(vec![0, 1, 2]);
res.sort_by(|(a, _), (b, _)| a.cmp(b));
- assert_eq!(
- res,
- vec![
- (Id(0), Counter(1)),
- (Id(1), Counter(1)),
- (Id(2), Counter(1))
- ]
- );
+ assert_eq!(res, vec![(0, 1), (1, 1), (2, 1)]);
Ok(())
}
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index aad69587..0483ff9d 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -38,6 +38,10 @@ impl ShardId {
pub fn new(id: u64) -> Self {
Self(id)
}
+
+ pub fn as_u64(&self) -> u64 {
+ self.0
+ }
}
impl From<u64> for ShardId {
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
index bbf62fed..d8578f8d 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
@@ -29,13 +29,22 @@ use crate::{
use self::mapper::ApproxCentralityMapper;
use self::worker::{ApproxCentralityWorker, RemoteApproxCentralityWorker};
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+#[derive(
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Debug,
+ Clone,
+ PartialEq,
+ Eq,
+)]
pub struct Meta {
round: u64,
num_samples_per_worker: u64,
}
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct ApproxCentralityTables {
centrality: DefaultDhtTable<webgraph::NodeID, f32>,
meta: DefaultDhtTable<(), Meta>,
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
index a4922891..90b06feb 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
@@ -29,13 +29,22 @@ pub use coordinator::{CentralityFinish, CentralitySetup};
pub use mapper::CentralityMapper;
pub use worker::{CentralityWorker, RemoteCentralityWorker};
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+#[derive(
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Debug,
+ Clone,
+ PartialEq,
+ Eq,
+)]
pub struct Meta {
round_had_changes: bool,
upper_bound_num_nodes: u64,
}
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct CentralityTables {
counters: DefaultDhtTable<webgraph::NodeID, HyperLogLog<64>>,
meta: DefaultDhtTable<(), Meta>,
@@ -54,7 +63,7 @@ impl_dht_tables!(
[counters, meta, centrality, changed_nodes]
);
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct CentralityJob {
shard: ShardId,
}
diff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rs
index 06126748..5a60d43f 100644
--- a/crates/core/src/hyperloglog.rs
+++ b/crates/core/src/hyperloglog.rs
@@ -4325,7 +4325,9 @@ impl HyperLogLogHasher for StableHasher {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(
+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Eq,
+)]
pub struct HyperLogLog<const N: usize, H = FastHasher>
where
H: HyperLogLogHasher,
@@ -4334,6 +4336,12 @@ where
_hasher: PhantomData<H>,
}
+impl<H: HyperLogLogHasher, const N: usize> PartialEq for HyperLogLog<N, H> {
+ fn eq(&self, other: &Self) -> bool {
+ self.registers == other.registers
+ }
+}
+
impl<H: HyperLogLogHasher, const N: usize> Default for HyperLogLog<N, H> {
fn default() -> Self {
Self {
diff --git a/crates/core/src/kahan_sum.rs b/crates/core/src/kahan_sum.rs
index 7c73d272..e40119ae 100644
--- a/crates/core/src/kahan_sum.rs
+++ b/crates/core/src/kahan_sum.rs
@@ -25,6 +25,7 @@ use std::ops::{Add, AddAssign};
serde::Deserialize,
bincode::Encode,
bincode::Decode,
+ PartialEq,
)]
pub struct KahanSum {
sum: f64,