web search done right
Decrease webgraph memory consumption when building. I think the write too many (and too big) write buffers got allocated, which caused OOM.
Mikkel Denkerf45074619dba
diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rsindex fa730d0e..3dceac08 100644--- a/core/src/api/webgraph.rs+++ b/core/src/api/webgraph.rs@@ -35,7 +35,7 @@ impl RemoteWebgraph {Self { cluster }}- async fn host(&self) -> SocketAddr {+ async fn host(&self) -> Option<SocketAddr> {self.cluster.members().await@@ -44,7 +44,6 @@ impl RemoteWebgraph {Service::Webgraph { host } => Some(host),_ => None,})- .unwrap()}}@@ -68,7 +67,11 @@ pub async fn similar_sites(extract::Json(params): extract::Json<SimilarSitesParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {state.counters.explore_counter.inc();- let host = state.remote_webgraph.host().await;+ let host = state+ .remote_webgraph+ .host()+ .await+ .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;let retry = ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200))@@ -124,7 +127,11 @@ pub async fn knows_site(extract::State(state): extract::State<Arc<State>>,extract::Query(params): extract::Query<KnowsSiteParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {- let host = state.remote_webgraph.host().await;+ let host = state+ .remote_webgraph+ .host()+ .await+ .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;let retry = ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200))diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex fb6b02c5..90f6f2d4 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -35,6 +35,8 @@ mod store;use self::segment::{LiveSegment, StoredSegment};use self::store::Store;+pub const MAX_LABEL_LENGTH: usize = 1024;+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct NodeID(u128);@@ -444,6 +446,7 @@ impl Webgraph {pub fn insert(&mut self, from: Node, to: Node, label: String) {let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));+ let label = label.chars().take(MAX_LABEL_LENGTH).collect::<String>();self.live_segment.insert(from_id, to_id, label);}@@ -963,4 +966,23 @@ mod test {assert_eq!(graph.segments.len(), 1);verify_graph(&graph);}++ #[test]+ fn cap_label_length() {+ let mut graph = WebgraphBuilder::new_memory().open();++ graph.insert(+ Node::from("A"),+ Node::from("B"),+ "a".repeat(MAX_LABEL_LENGTH + 1),+ );++ graph.commit();++ assert_eq!(graph.segments.len(), 1);+ assert_eq!(+ graph.outgoing_edges(Node::from("A"))[0].label,+ "a".repeat(MAX_LABEL_LENGTH)+ );+ }}diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 360bfa21..6b6a2956 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -30,24 +30,24 @@ where{pub fn open<P: AsRef<Path>>(path: P) -> Self {let mut options = rocksdb::Options::default();- options.set_max_open_files(8);+ options.set_max_open_files(-1);options.create_if_missing(true);options.increase_parallelism(8);- options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable- options.set_max_write_buffer_number(8);+ options.set_max_background_jobs(8);+ options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB memtable+ options.set_max_write_buffer_number(2);let mut block_options = BlockBasedOptions::default();block_options.set_ribbon_filter(5.0);block_options.disable_cache();- block_options.set_block_size(128 * 1024); // 128 KB block size-options.set_block_based_table_factory(&block_options);options.set_optimize_filters_for_hits(true);options.set_compression_type(rocksdb::DBCompressionType::Lz4);+ options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();@@ -73,7 +73,10 @@ wherelet key_bytes = bincode::serialize(key).unwrap();let value_bytes = bincode::serialize(value).unwrap();- self.db.put(key_bytes, value_bytes).unwrap();+ let mut options = rocksdb::WriteOptions::default();+ options.disable_wal(true);++ self.db.put_opt(key_bytes, value_bytes, &options).unwrap();}pub fn batch_put<'a>(&'a self, it: impl Iterator<Item = (&'a K, &'a V)>) {@@ -86,7 +89,10 @@ wherebatch.put(key_bytes, value_bytes);}- self.db.write(batch).unwrap();+ let mut options = rocksdb::WriteOptions::default();+ options.disable_wal(true);++ self.db.write_opt(batch, &options).unwrap();}pub fn batch_put_owned(&self, it: impl Iterator<Item = (K, V)>) {@@ -99,7 +105,10 @@ wherebatch.put(key_bytes, value_bytes);}- self.db.write(batch).unwrap();+ let mut options = rocksdb::WriteOptions::default();+ options.disable_wal(true);++ self.db.write_opt(batch, &options).unwrap();}pub fn keys(&self) -> impl Iterator<Item = K> + '_ {