web search done right
improve rocksdb kv trait and properly add segments to meta when merging webgraphs
Mikkel Denkera73e6ea51c78
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rsindex 3174befe..507fe190 100644--- a/core/src/entity_index/mod.rs+++ b/core/src/entity_index/mod.rs@@ -176,7 +176,9 @@ impl EntityIndex {tantivy::Index::create_in_dir(&tv_path, schema.clone())?};- let attribute_occurrences = RocksDbStore::open(path.as_ref().join("attribute_occurrences"));+ let attribute_occurrences = Box::new(RocksDbStore::open(+ path.as_ref().join("attribute_occurrences"),+ ));let stopwords: HashSet<String> = include_str!("../../stopwords/English.txt").lines()diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex cd29d257..f680c98d 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -257,6 +257,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {if let Err(err) = index.insert(webpage) {warn!("{:?}", err);+ panic!();}}diff --git a/core/src/image_store.rs b/core/src/image_store.rsindex f5d1e8a2..69f4a54c 100644--- a/core/src/image_store.rs+++ b/core/src/image_store.rs@@ -92,7 +92,7 @@ impl BaseImageStore {}fn open_with_filters<P: AsRef<Path>>(path: P, filters: Vec<Box<dyn ImageFilter>>) -> Self {- let store = RocksDbStore::open(path);+ let store = Box::new(RocksDbStore::open(path));Self { store, filters }}diff --git a/core/src/kv/mod.rs b/core/src/kv/mod.rsindex d7884a38..402dd49e 100644--- a/core/src/kv/mod.rs+++ b/core/src/kv/mod.rs@@ -14,9 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::hash::Hash;-use std::{collections::HashMap, sync::RwLock};-use serde::{de::DeserializeOwned, Serialize};pub mod rocksdb_store;@@ -53,55 +50,4 @@ whereself.insert_raw(key_bytes, val_bytes);}-- fn load_in_memory(&self) -> Memory<K, V>- where- K: Eq + Hash,- {- let mut map = HashMap::new();-- for (k, v) in self.iter() {- map.insert(k, v);- }-- Memory(RwLock::new(map))- }-}--pub struct Memory<K, V>(RwLock<HashMap<K, V>>);--impl<K, V> Kv<K, V> for Memory<K, V>-where- K: Serialize + DeserializeOwned + Send + Sync + Eq + Hash + Clone,- V: Serialize + DeserializeOwned + Send + Sync + Clone,-{- fn get_raw(&self, _key: &[u8]) -> Option<Vec<u8>> {- unimplemented!()- }-- fn insert_raw(&self, _key: Vec<u8>, _value: Vec<u8>) {- unimplemented!()- }-- fn delete_raw(&mut self, _key: &[u8]) {- unimplemented!()- }-- fn flush(&self) {}-- fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {- unimplemented!()- }-- fn get(&self, key: &K) -> Option<V> {- self.0.read().unwrap().get(key).cloned()- }-- fn insert(&self, key: K, val: V) {- self.0.write().unwrap().insert(key, val);- }-- fn delete(&mut self, key: &K) {- self.0.write().unwrap().remove(key);- }}diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rsindex d089d492..90f26af4 100644--- a/core/src/kv/rocksdb_store.rs+++ b/core/src/kv/rocksdb_store.rs@@ -17,51 +17,74 @@use std::{fs, marker::PhantomData};use rocksdb::{- DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, Options, SingleThreaded, DB,+ BlockBasedOptions, DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, Options,+ SingleThreaded, DB,};use serde::{de::DeserializeOwned, Serialize};use crate::kv::Kv;-pub struct RocksDbStore {}+pub struct RocksDbStore {+ db: DB,+ _cache: rocksdb::Cache,+}impl RocksDbStore {- pub fn open<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>+ pub fn open<P>(path: P) -> SelfwhereP: AsRef<std::path::Path>,- K: Serialize + DeserializeOwned + 'static,- V: Serialize + DeserializeOwned + 'static,{if !path.as_ref().exists() {fs::create_dir_all(path.as_ref()).expect("faild to create dir");}let mut options = Options::default();-options.create_if_missing(true);- options.set_max_open_files(1);- options.set_max_file_opening_threads(1);- options.optimize_for_point_lookup(512 * 1024 * 1024); // 512 MB- Box::new(DB::open(&options, path).expect("unable to open rocks db"))+ options.set_max_background_jobs(8);+ options.increase_parallelism(8);+ options.set_max_subcompactions(8);++ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb++ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)+ options.set_level_compaction_dynamic_level_bytes(true);+ options.set_bytes_per_sync(1048576);+ let mut block_options = BlockBasedOptions::default();+ block_options.set_block_size(16 * 1024);+ block_options.set_format_version(5);+ block_options.set_cache_index_and_filter_blocks(true);+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);++ block_options.set_ribbon_filter(10.0);++ options.set_block_based_table_factory(&block_options);+ options.set_row_cache(&cache);+ options.set_compression_type(rocksdb::DBCompressionType::Lz4);++ options.optimize_for_point_lookup(512); // 512 MB++ let db = DB::open(&options, path).expect("unable to open rocks db");++ Self { db, _cache: cache }}}-impl<K, V> Kv<K, V> for rocksdb::DB+impl<K, V> Kv<K, V> for RocksDbStorewhereK: Serialize + DeserializeOwned + 'static,V: Serialize + DeserializeOwned + 'static,{fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {- self.get(key).expect("failed to retrieve key")+ self.db.get(key).expect("failed to retrieve key")}fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {- self.put(key, value).expect("failed to insert value");+ self.db.put(key, value).expect("failed to insert value");}fn flush(&self) {- if let Err(err) = self.flush() {+ if let Err(err) = self.db.flush() {match err.kind() {rocksdb::ErrorKind::NotSupported => {}_ => panic!("failed to flush: {err:?}"),@@ -70,7 +93,7 @@ where}fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {- let iter = self.iterator(IteratorMode::Start);+ let iter = self.db.iterator(IteratorMode::Start);Box::new(IntoIter {inner: iter,@@ -80,7 +103,7 @@ where}fn delete_raw(&mut self, key: &[u8]) {- rocksdb::DB::delete(self, key).unwrap();+ self.db.delete(key).unwrap();}}diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 8474f9d7..649fc14e 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -35,7 +35,7 @@ pub struct IndexerCentralityStore {impl IndexerCentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {- harmonic: RocksDbStore::open(path.as_ref().join("harmonic")),+ harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),}}}@@ -78,7 +78,7 @@ pub struct CentralityStore {impl CentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {- harmonic: RocksDbStore::open(path.as_ref().join("harmonic")),+ harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity")).ok().unwrap_or_default(),diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rsindex 26171229..5ab10e53 100644--- a/core/src/subdomain_count.rs+++ b/core/src/subdomain_count.rs@@ -29,7 +29,7 @@ pub struct SubdomainCounter {impl SubdomainCounter {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {- inner: RocksDbStore::open(path),+ inner: Box::new(RocksDbStore::open(path)),}}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 0acb8d9b..816a5f9a 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -556,6 +556,11 @@ impl Webgraph {Meta::open(meta_path)}+ fn save_metadata(&mut self) {+ let path = Path::new(&self.path).join("metadata.json");+ self.meta.save(path);+ }+fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {fs::create_dir_all(&path).unwrap();let meta = Self::meta(&path);@@ -592,6 +597,7 @@ impl Webgraph {self.segments.push(Segment::open(new_path, id));}+ self.save_metadata();self.id2node.flush();}@@ -800,6 +806,8 @@ mod test {graph.merge(other);}+ dbg!(&graph.path);+assert_eq!(graph.distances(Node::from("A")).get(&Node::from("H")),Some(&7)diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 4f11cf82..4a91c885 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -147,6 +147,7 @@ impl EdgeStoreWriter {pub struct EdgeStore {reversed: bool,ranges: rocksdb::DB, // column[nodes] = nodeid -> (start, end); column[labels] = nodeid -> (start, end)+ _cache: rocksdb::Cache,edge_labels_file: File,edge_labels_len: usize,edge_labels: Mmap,@@ -225,6 +226,7 @@ impl EdgeStore {Self {reversed,ranges,+ _cache: cache,edge_labels,edge_labels_len,edge_labels_file,@@ -333,6 +335,12 @@ impl EdgeStore {fn flush(&mut self) {self.ranges.flush().unwrap();+ self.ranges+ .flush_cf(self.ranges.cf_handle("nodes").unwrap())+ .unwrap();+ self.ranges+ .flush_cf(self.ranges.cf_handle("labels").unwrap())+ .unwrap();self.edge_nodes_file.flush().unwrap();self.edge_labels_file.flush().unwrap();