web search done right
refactor kv trait out from webgraph
Mikkel Denkerd19597bebac7
diff --git a/src/kv/mod.rs b/src/kv/mod.rsnew file mode 100644index 00000000..1b3be914--- /dev/null+++ b/src/kv/mod.rs@@ -0,0 +1,45 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use serde::{de::DeserializeOwned, Serialize};++pub mod rocksdb_store;+pub mod sled_store;++pub trait Kv<K, V>+where+ K: Serialize + DeserializeOwned,+ V: Serialize + DeserializeOwned,+{+ fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>>;+ fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>);+ fn flush(&self);+ fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a>;++ fn get(&self, key: &K) -> Option<V> {+ let key_bytes = bincode::serialize(key).expect("failed to serialize key");++ self.get_raw(&key_bytes)+ .map(|bytes| bincode::deserialize(&bytes).expect("failed to deserialize stored value"))+ }++ fn insert(&self, key: K, value: V) {+ let key_bytes = bincode::serialize(&key).expect("failed to serialize key");+ let val_bytes = bincode::serialize(&value).expect("failed to serialize value");++ self.insert_raw(key_bytes, val_bytes)+ }+}diff --git a/src/kv/rocksdb_store.rs b/src/kv/rocksdb_store.rsnew file mode 100644index 00000000..c6f02dc2--- /dev/null+++ b/src/kv/rocksdb_store.rs@@ -0,0 +1,94 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{fs, marker::PhantomData};++use rocksdb::{DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, SingleThreaded, DB};+use serde::{de::DeserializeOwned, Serialize};++use crate::kv::Kv;++pub struct RocksDbStore {}++impl RocksDbStore {+ pub fn open<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>+ where+ P: AsRef<std::path::Path>,+ K: Serialize + DeserializeOwned + 'static,+ V: Serialize + DeserializeOwned + 'static,+ {+ if !path.as_ref().exists() {+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");+ }++ Box::new(DB::open_default(path).expect("unable to open rocks db"))+ }+}++impl<K, V> Kv<K, V> for rocksdb::DB+where+ K: Serialize + DeserializeOwned + 'static,+ V: Serialize + DeserializeOwned + 'static,+{+ fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {+ self.get(key).expect("failed to retrieve key")+ }++ fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {+ self.put(key, value).expect("failed to insert value");+ }++ fn flush(&self) {+ self.flush().expect("failed to flush");+ }++ fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {+ let iter = self.iterator(IteratorMode::Start);++ Box::new(IntoIter {+ inner: iter,+ key: Default::default(),+ value: Default::default(),+ })+ }+}++pub struct IntoIter<'a, K, V>+where+ K: Serialize + DeserializeOwned,+ V: Serialize + DeserializeOwned,+{+ inner: DBIteratorWithThreadMode<'a, DBWithThreadMode<SingleThreaded>>,+ key: PhantomData<K>,+ value: PhantomData<V>,+}++impl<'a, K, V> Iterator for IntoIter<'a, K, V>+where+ K: Serialize + DeserializeOwned,+ V: Serialize + DeserializeOwned,+{+ type Item = (K, V);++ fn next(&mut self) -> Option<Self::Item> {+ self.inner.next().map(|(key_bytes, value_bytes)| {+ (+ bincode::deserialize(&key_bytes).expect("Failed to deserialize key"),+ bincode::deserialize(&value_bytes).expect("Failed to deserialize value"),+ )+ })+ }+}diff --git a/src/kv/sled_store.rs b/src/kv/sled_store.rsnew file mode 100644index 00000000..25e12aa3--- /dev/null+++ b/src/kv/sled_store.rs@@ -0,0 +1,80 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::marker::PhantomData;++use nom::AsBytes;+use serde::{de::DeserializeOwned, Serialize};++use crate::kv::Kv;++pub struct SledStore {}++impl<K, V> Kv<K, V> for sled::Tree+where+ K: Serialize + DeserializeOwned + 'static,+ V: Serialize + DeserializeOwned + 'static,+{+ fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {+ self.get(key)+ .expect("failed to retrieve key")+ .map(|v| v.as_bytes().to_vec())+ }++ fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {+ self.insert(key, value).expect("failed to insert value");+ }++ fn flush(&self) {+ self.flush().expect("failed to flush");+ }++ fn iter(&self) -> Box<dyn Iterator<Item = (K, V)>> {+ Box::new(IntoIter {+ inner: self.iter(),+ key: Default::default(),+ value: Default::default(),+ })+ }+}++pub struct IntoIter<K, V>+where+ K: Serialize + DeserializeOwned,+ V: Serialize + DeserializeOwned,+{+ inner: sled::Iter,+ key: PhantomData<K>,+ value: PhantomData<V>,+}++impl<K, V> Iterator for IntoIter<K, V>+where+ K: Serialize + DeserializeOwned,+ V: Serialize + DeserializeOwned,+{+ type Item = (K, V);++ fn next(&mut self) -> Option<Self::Item> {+ self.inner.next().map(|res| {+ let (key_bytes, value_bytes) = res.expect("Failed to get next record from sled tree");+ (+ bincode::deserialize(&key_bytes).expect("Failed to deserialize key"),+ bincode::deserialize(&value_bytes).expect("Failed to deserialize value"),+ )+ })+ }+}diff --git a/src/lib.rs b/src/lib.rsindex 59de805c..dc7ef3e2 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -30,6 +30,8 @@ mod autosuggest;mod directory;mod exponential_backoff;mod frontend;+mod image_store;+mod kv;pub mod query;pub mod ranking;mod schema;