Skip to content

web search done right

increase webgraph batch sizes

Mikkel Denkerf0be91bcc357
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 50579369..40ec824f 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -250,10 +250,10 @@ impl UrlStateDbShard {
options.set_block_based_table_factory(&block_options);
options.set_max_background_jobs(8);
options.increase_parallelism(8);
+ options.set_max_subcompactions(8);
options.set_write_buffer_size(512 * 1024 * 1024);
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
- options.set_max_subcompactions(8);
options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
options.set_compression_type(rocksdb::DBCompressionType::None);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 4bf0f2e6..4ef97bc0 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -35,7 +35,7 @@ mod store;
use self::segment::StoredSegment;
pub const MAX_LABEL_LENGTH: usize = 1024;
-const MAX_BATCH_SIZE: usize = 1_000;
+const MAX_BATCH_SIZE: usize = 50_000;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NodeID(u128);
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 1fe89414..02eaa474 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -21,7 +21,7 @@ use rocksdb::BlockBasedOptions;
use super::{Edge, NodeID};
-const MAX_BATCH_SIZE: usize = 1_000;
+const MAX_BATCH_SIZE: usize = 50_000;
pub struct EdgeStore<L> {
reversed: bool,
@@ -44,6 +44,10 @@ where
options.set_block_based_table_factory(&block_options);
options.set_optimize_filters_for_hits(true);
+ options.set_max_background_jobs(8);
+ options.increase_parallelism(8);
+ options.set_max_subcompactions(8);
+
options.set_compression_type(rocksdb::DBCompressionType::Lz4);
options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);