Skip to content

web search done right

improve rocksdb kv trait and properly add segments to meta when merging webgraphs

Mikkel Denkera73e6ea51c78
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index 3174befe..507fe190 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -176,7 +176,9 @@ impl EntityIndex {
tantivy::Index::create_in_dir(&tv_path, schema.clone())?
};
- let attribute_occurrences = RocksDbStore::open(path.as_ref().join("attribute_occurrences"));
+ let attribute_occurrences = Box::new(RocksDbStore::open(
+ path.as_ref().join("attribute_occurrences"),
+ ));
let stopwords: HashSet<String> = include_str!("../../stopwords/English.txt")
.lines()
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index cd29d257..f680c98d 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -257,6 +257,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
if let Err(err) = index.insert(webpage) {
warn!("{:?}", err);
+ panic!();
}
}
diff --git a/core/src/image_store.rs b/core/src/image_store.rs
index f5d1e8a2..69f4a54c 100644
--- a/core/src/image_store.rs
+++ b/core/src/image_store.rs
@@ -92,7 +92,7 @@ impl BaseImageStore {
}
fn open_with_filters<P: AsRef<Path>>(path: P, filters: Vec<Box<dyn ImageFilter>>) -> Self {
- let store = RocksDbStore::open(path);
+ let store = Box::new(RocksDbStore::open(path));
Self { store, filters }
}
diff --git a/core/src/kv/mod.rs b/core/src/kv/mod.rs
index d7884a38..402dd49e 100644
--- a/core/src/kv/mod.rs
+++ b/core/src/kv/mod.rs
@@ -14,9 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::hash::Hash;
-use std::{collections::HashMap, sync::RwLock};
-
use serde::{de::DeserializeOwned, Serialize};
pub mod rocksdb_store;
@@ -53,55 +50,4 @@ where
self.insert_raw(key_bytes, val_bytes);
}
-
- fn load_in_memory(&self) -> Memory<K, V>
- where
- K: Eq + Hash,
- {
- let mut map = HashMap::new();
-
- for (k, v) in self.iter() {
- map.insert(k, v);
- }
-
- Memory(RwLock::new(map))
- }
-}
-
-pub struct Memory<K, V>(RwLock<HashMap<K, V>>);
-
-impl<K, V> Kv<K, V> for Memory<K, V>
-where
- K: Serialize + DeserializeOwned + Send + Sync + Eq + Hash + Clone,
- V: Serialize + DeserializeOwned + Send + Sync + Clone,
-{
- fn get_raw(&self, _key: &[u8]) -> Option<Vec<u8>> {
- unimplemented!()
- }
-
- fn insert_raw(&self, _key: Vec<u8>, _value: Vec<u8>) {
- unimplemented!()
- }
-
- fn delete_raw(&mut self, _key: &[u8]) {
- unimplemented!()
- }
-
- fn flush(&self) {}
-
- fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {
- unimplemented!()
- }
-
- fn get(&self, key: &K) -> Option<V> {
- self.0.read().unwrap().get(key).cloned()
- }
-
- fn insert(&self, key: K, val: V) {
- self.0.write().unwrap().insert(key, val);
- }
-
- fn delete(&mut self, key: &K) {
- self.0.write().unwrap().remove(key);
- }
}
diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rs
index d089d492..90f26af4 100644
--- a/core/src/kv/rocksdb_store.rs
+++ b/core/src/kv/rocksdb_store.rs
@@ -17,51 +17,74 @@
use std::{fs, marker::PhantomData};
use rocksdb::{
- DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, Options, SingleThreaded, DB,
+ BlockBasedOptions, DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, Options,
+ SingleThreaded, DB,
};
use serde::{de::DeserializeOwned, Serialize};
use crate::kv::Kv;
-pub struct RocksDbStore {}
+pub struct RocksDbStore {
+ db: DB,
+ _cache: rocksdb::Cache,
+}
impl RocksDbStore {
- pub fn open<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>
+ pub fn open<P>(path: P) -> Self
where
P: AsRef<std::path::Path>,
- K: Serialize + DeserializeOwned + 'static,
- V: Serialize + DeserializeOwned + 'static,
{
if !path.as_ref().exists() {
fs::create_dir_all(path.as_ref()).expect("faild to create dir");
}
let mut options = Options::default();
-
options.create_if_missing(true);
- options.set_max_open_files(1);
- options.set_max_file_opening_threads(1);
- options.optimize_for_point_lookup(512 * 1024 * 1024); // 512 MB
- Box::new(DB::open(&options, path).expect("unable to open rocks db"))
+ options.set_max_background_jobs(8);
+ options.increase_parallelism(8);
+ options.set_max_subcompactions(8);
+
+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb
+
+ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
+ options.set_level_compaction_dynamic_level_bytes(true);
+ options.set_bytes_per_sync(1048576);
+ let mut block_options = BlockBasedOptions::default();
+ block_options.set_block_size(16 * 1024);
+ block_options.set_format_version(5);
+ block_options.set_cache_index_and_filter_blocks(true);
+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
+
+ block_options.set_ribbon_filter(10.0);
+
+ options.set_block_based_table_factory(&block_options);
+ options.set_row_cache(&cache);
+ options.set_compression_type(rocksdb::DBCompressionType::Lz4);
+
+ options.optimize_for_point_lookup(512); // 512 MB
+
+ let db = DB::open(&options, path).expect("unable to open rocks db");
+
+ Self { db, _cache: cache }
}
}
-impl<K, V> Kv<K, V> for rocksdb::DB
+impl<K, V> Kv<K, V> for RocksDbStore
where
K: Serialize + DeserializeOwned + 'static,
V: Serialize + DeserializeOwned + 'static,
{
fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {
- self.get(key).expect("failed to retrieve key")
+ self.db.get(key).expect("failed to retrieve key")
}
fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {
- self.put(key, value).expect("failed to insert value");
+ self.db.put(key, value).expect("failed to insert value");
}
fn flush(&self) {
- if let Err(err) = self.flush() {
+ if let Err(err) = self.db.flush() {
match err.kind() {
rocksdb::ErrorKind::NotSupported => {}
_ => panic!("failed to flush: {err:?}"),
@@ -70,7 +93,7 @@ where
}
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {
- let iter = self.iterator(IteratorMode::Start);
+ let iter = self.db.iterator(IteratorMode::Start);
Box::new(IntoIter {
inner: iter,
@@ -80,7 +103,7 @@ where
}
fn delete_raw(&mut self, key: &[u8]) {
- rocksdb::DB::delete(self, key).unwrap();
+ self.db.delete(key).unwrap();
}
}
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 8474f9d7..649fc14e 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -35,7 +35,7 @@ pub struct IndexerCentralityStore {
impl IndexerCentralityStore {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
- harmonic: RocksDbStore::open(path.as_ref().join("harmonic")),
+ harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),
}
}
}
@@ -78,7 +78,7 @@ pub struct CentralityStore {
impl CentralityStore {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
- harmonic: RocksDbStore::open(path.as_ref().join("harmonic")),
+ harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),
inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
.ok()
.unwrap_or_default(),
diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rs
index 26171229..5ab10e53 100644
--- a/core/src/subdomain_count.rs
+++ b/core/src/subdomain_count.rs
@@ -29,7 +29,7 @@ pub struct SubdomainCounter {
impl SubdomainCounter {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
- inner: RocksDbStore::open(path),
+ inner: Box::new(RocksDbStore::open(path)),
}
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 0acb8d9b..816a5f9a 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -556,6 +556,11 @@ impl Webgraph {
Meta::open(meta_path)
}
+ fn save_metadata(&mut self) {
+ let path = Path::new(&self.path).join("metadata.json");
+ self.meta.save(path);
+ }
+
fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {
fs::create_dir_all(&path).unwrap();
let meta = Self::meta(&path);
@@ -592,6 +597,7 @@ impl Webgraph {
self.segments.push(Segment::open(new_path, id));
}
+ self.save_metadata();
self.id2node.flush();
}
@@ -800,6 +806,8 @@ mod test {
graph.merge(other);
}
+ dbg!(&graph.path);
+
assert_eq!(
graph.distances(Node::from("A")).get(&Node::from("H")),
Some(&7)
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 4f11cf82..4a91c885 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -147,6 +147,7 @@ impl EdgeStoreWriter {
pub struct EdgeStore {
reversed: bool,
ranges: rocksdb::DB, // column[nodes] = nodeid -> (start, end); column[labels] = nodeid -> (start, end)
+ _cache: rocksdb::Cache,
edge_labels_file: File,
edge_labels_len: usize,
edge_labels: Mmap,
@@ -225,6 +226,7 @@ impl EdgeStore {
Self {
reversed,
ranges,
+ _cache: cache,
edge_labels,
edge_labels_len,
edge_labels_file,
@@ -333,6 +335,12 @@ impl EdgeStore {
fn flush(&mut self) {
self.ranges.flush().unwrap();
+ self.ranges
+ .flush_cf(self.ranges.cf_handle("nodes").unwrap())
+ .unwrap();
+ self.ranges
+ .flush_cf(self.ranges.cf_handle("labels").unwrap())
+ .unwrap();
self.edge_nodes_file.flush().unwrap();
self.edge_labels_file.flush().unwrap();