web search done right
increase webgraph batch sizes
Mikkel Denkerf0be91bcc357
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 50579369..40ec824f 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -250,10 +250,10 @@ impl UrlStateDbShard {options.set_block_based_table_factory(&block_options);options.set_max_background_jobs(8);options.increase_parallelism(8);+ options.set_max_subcompactions(8);options.set_write_buffer_size(512 * 1024 * 1024);options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);- options.set_max_subcompactions(8);options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);options.set_compression_type(rocksdb::DBCompressionType::None);diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 4bf0f2e6..4ef97bc0 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -35,7 +35,7 @@ mod store;use self::segment::StoredSegment;pub const MAX_LABEL_LENGTH: usize = 1024;-const MAX_BATCH_SIZE: usize = 1_000;+const MAX_BATCH_SIZE: usize = 50_000;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct NodeID(u128);diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 1fe89414..02eaa474 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -21,7 +21,7 @@ use rocksdb::BlockBasedOptions;use super::{Edge, NodeID};-const MAX_BATCH_SIZE: usize = 1_000;+const MAX_BATCH_SIZE: usize = 50_000;pub struct EdgeStore<L> {reversed: bool,@@ -44,6 +44,10 @@ whereoptions.set_block_based_table_factory(&block_options);options.set_optimize_filters_for_hits(true);+ options.set_max_background_jobs(8);+ options.increase_parallelism(8);+ options.set_max_subcompactions(8);+options.set_compression_type(rocksdb::DBCompressionType::Lz4);options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);