Skip to content

web search done right

bump webgraph rocksdb to use never version for format (enabling all the nice improvements)

Mikkel Denker778db90cda1f
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 90f6f2d4..16709ceb 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -475,6 +475,7 @@ impl Webgraph {
match (self.commit_mode, self.segments.first_mut()) {
(CommitMode::SingleSegment, Some(segment)) => {
segment.add(live_segment);
+ segment.flush();
}
_ => {
let segment = live_segment.commit(Path::new(&self.path).join("segments"));
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 43806816..ac903622 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -129,7 +129,7 @@ impl StoredSegment {
}
}
- fn flush(&mut self) {
+ pub fn flush(&mut self) {
self.full_adjacency.flush();
self.full_reversed_adjacency.flush();
self.small_adjacency.flush();
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 6b6a2956..6008d84d 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -30,16 +30,13 @@ where
{
pub fn open<P: AsRef<Path>>(path: P) -> Self {
let mut options = rocksdb::Options::default();
- options.set_max_open_files(-1);
+ options.set_max_open_files(1);
options.create_if_missing(true);
- options.increase_parallelism(8);
- options.set_max_background_jobs(8);
- options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB memtable
- options.set_max_write_buffer_number(2);
let mut block_options = BlockBasedOptions::default();
- block_options.set_ribbon_filter(5.0);
+ block_options.set_ribbon_filter(10.0);
+ block_options.set_format_version(5);
block_options.disable_cache();
@@ -112,10 +109,7 @@ where
}
pub fn keys(&self) -> impl Iterator<Item = K> + '_ {
- let mut read_opts = rocksdb::ReadOptions::default();
-
- read_opts.set_readahead_size(4_194_304); // 4 MB
- read_opts.set_pin_data(true);
+ let read_opts = rocksdb::ReadOptions::default();
self.db
.iterator_opt(rocksdb::IteratorMode::Start, read_opts)
@@ -126,9 +120,7 @@ where
}
pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
- let mut read_opts = rocksdb::ReadOptions::default();
-
- read_opts.set_readahead_size(4_194_304); // 4 MB
+ let read_opts = rocksdb::ReadOptions::default();
self.db
.iterator_opt(rocksdb::IteratorMode::Start, read_opts)