Skip to content

web search done right

refactor kv trait out from webgraph

Mikkel Denkerd19597bebac7
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
new file mode 100644
index 00000000..1b3be914
--- /dev/null
+++ b/src/kv/mod.rs
@@ -0,0 +1,45 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{de::DeserializeOwned, Serialize};
+
+pub mod rocksdb_store;
+pub mod sled_store;
+
+pub trait Kv<K, V>
+where
+ K: Serialize + DeserializeOwned,
+ V: Serialize + DeserializeOwned,
+{
+ fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>>;
+ fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>);
+ fn flush(&self);
+ fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a>;
+
+ fn get(&self, key: &K) -> Option<V> {
+ let key_bytes = bincode::serialize(key).expect("failed to serialize key");
+
+ self.get_raw(&key_bytes)
+ .map(|bytes| bincode::deserialize(&bytes).expect("failed to deserialize stored value"))
+ }
+
+ fn insert(&self, key: K, value: V) {
+ let key_bytes = bincode::serialize(&key).expect("failed to serialize key");
+ let val_bytes = bincode::serialize(&value).expect("failed to serialize value");
+
+ self.insert_raw(key_bytes, val_bytes)
+ }
+}
diff --git a/src/kv/rocksdb_store.rs b/src/kv/rocksdb_store.rs
new file mode 100644
index 00000000..c6f02dc2
--- /dev/null
+++ b/src/kv/rocksdb_store.rs
@@ -0,0 +1,94 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{fs, marker::PhantomData};
+
+use rocksdb::{DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, SingleThreaded, DB};
+use serde::{de::DeserializeOwned, Serialize};
+
+use crate::kv::Kv;
+
+pub struct RocksDbStore {}
+
+impl RocksDbStore {
+ pub fn open<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>
+ where
+ P: AsRef<std::path::Path>,
+ K: Serialize + DeserializeOwned + 'static,
+ V: Serialize + DeserializeOwned + 'static,
+ {
+ if !path.as_ref().exists() {
+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");
+ }
+
+ Box::new(DB::open_default(path).expect("unable to open rocks db"))
+ }
+}
+
+impl<K, V> Kv<K, V> for rocksdb::DB
+where
+ K: Serialize + DeserializeOwned + 'static,
+ V: Serialize + DeserializeOwned + 'static,
+{
+ fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {
+ self.get(key).expect("failed to retrieve key")
+ }
+
+ fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {
+ self.put(key, value).expect("failed to insert value");
+ }
+
+ fn flush(&self) {
+ self.flush().expect("failed to flush");
+ }
+
+ fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {
+ let iter = self.iterator(IteratorMode::Start);
+
+ Box::new(IntoIter {
+ inner: iter,
+ key: Default::default(),
+ value: Default::default(),
+ })
+ }
+}
+
+pub struct IntoIter<'a, K, V>
+where
+ K: Serialize + DeserializeOwned,
+ V: Serialize + DeserializeOwned,
+{
+ inner: DBIteratorWithThreadMode<'a, DBWithThreadMode<SingleThreaded>>,
+ key: PhantomData<K>,
+ value: PhantomData<V>,
+}
+
+impl<'a, K, V> Iterator for IntoIter<'a, K, V>
+where
+ K: Serialize + DeserializeOwned,
+ V: Serialize + DeserializeOwned,
+{
+ type Item = (K, V);
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.inner.next().map(|(key_bytes, value_bytes)| {
+ (
+ bincode::deserialize(&key_bytes).expect("Failed to deserialize key"),
+ bincode::deserialize(&value_bytes).expect("Failed to deserialize value"),
+ )
+ })
+ }
+}
diff --git a/src/kv/sled_store.rs b/src/kv/sled_store.rs
new file mode 100644
index 00000000..25e12aa3
--- /dev/null
+++ b/src/kv/sled_store.rs
@@ -0,0 +1,80 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::marker::PhantomData;
+
+use nom::AsBytes;
+use serde::{de::DeserializeOwned, Serialize};
+
+use crate::kv::Kv;
+
+pub struct SledStore {}
+
+impl<K, V> Kv<K, V> for sled::Tree
+where
+ K: Serialize + DeserializeOwned + 'static,
+ V: Serialize + DeserializeOwned + 'static,
+{
+ fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {
+ self.get(key)
+ .expect("failed to retrieve key")
+ .map(|v| v.as_bytes().to_vec())
+ }
+
+ fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {
+ self.insert(key, value).expect("failed to insert value");
+ }
+
+ fn flush(&self) {
+ self.flush().expect("failed to flush");
+ }
+
+ fn iter(&self) -> Box<dyn Iterator<Item = (K, V)>> {
+ Box::new(IntoIter {
+ inner: self.iter(),
+ key: Default::default(),
+ value: Default::default(),
+ })
+ }
+}
+
+pub struct IntoIter<K, V>
+where
+ K: Serialize + DeserializeOwned,
+ V: Serialize + DeserializeOwned,
+{
+ inner: sled::Iter,
+ key: PhantomData<K>,
+ value: PhantomData<V>,
+}
+
+impl<K, V> Iterator for IntoIter<K, V>
+where
+ K: Serialize + DeserializeOwned,
+ V: Serialize + DeserializeOwned,
+{
+ type Item = (K, V);
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.inner.next().map(|res| {
+ let (key_bytes, value_bytes) = res.expect("Failed to get next record from sled tree");
+ (
+ bincode::deserialize(&key_bytes).expect("Failed to deserialize key"),
+ bincode::deserialize(&value_bytes).expect("Failed to deserialize value"),
+ )
+ })
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 59de805c..dc7ef3e2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -30,6 +30,8 @@ mod autosuggest;
mod directory;
mod exponential_backoff;
mod frontend;
+mod image_store;
+mod kv;
pub mod query;
pub mod ranking;
mod schema;