Skip to content

web search done right

Decrease webgraph memory consumption when building. I think the write too many (and too big) write buffers got allocated, which caused OOM.

Mikkel Denkerf45074619dba
diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rs
index fa730d0e..3dceac08 100644
--- a/core/src/api/webgraph.rs
+++ b/core/src/api/webgraph.rs
@@ -35,7 +35,7 @@ impl RemoteWebgraph {
Self { cluster }
}
- async fn host(&self) -> SocketAddr {
+ async fn host(&self) -> Option<SocketAddr> {
self.cluster
.members()
.await
@@ -44,7 +44,6 @@ impl RemoteWebgraph {
Service::Webgraph { host } => Some(host),
_ => None,
})
- .unwrap()
}
}
@@ -68,7 +67,11 @@ pub async fn similar_sites(
extract::Json(params): extract::Json<SimilarSitesParams>,
) -> std::result::Result<impl IntoResponse, StatusCode> {
state.counters.explore_counter.inc();
- let host = state.remote_webgraph.host().await;
+ let host = state
+ .remote_webgraph
+ .host()
+ .await
+ .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;
let retry = ExponentialBackoff::from_millis(30)
.with_limit(Duration::from_millis(200))
@@ -124,7 +127,11 @@ pub async fn knows_site(
extract::State(state): extract::State<Arc<State>>,
extract::Query(params): extract::Query<KnowsSiteParams>,
) -> std::result::Result<impl IntoResponse, StatusCode> {
- let host = state.remote_webgraph.host().await;
+ let host = state
+ .remote_webgraph
+ .host()
+ .await
+ .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;
let retry = ExponentialBackoff::from_millis(30)
.with_limit(Duration::from_millis(200))
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index fb6b02c5..90f6f2d4 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -35,6 +35,8 @@ mod store;
use self::segment::{LiveSegment, StoredSegment};
use self::store::Store;
+pub const MAX_LABEL_LENGTH: usize = 1024;
+
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NodeID(u128);
@@ -444,6 +446,7 @@ impl Webgraph {
pub fn insert(&mut self, from: Node, to: Node, label: String) {
let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));
+ let label = label.chars().take(MAX_LABEL_LENGTH).collect::<String>();
self.live_segment.insert(from_id, to_id, label);
}
@@ -963,4 +966,23 @@ mod test {
assert_eq!(graph.segments.len(), 1);
verify_graph(&graph);
}
+
+ #[test]
+ fn cap_label_length() {
+ let mut graph = WebgraphBuilder::new_memory().open();
+
+ graph.insert(
+ Node::from("A"),
+ Node::from("B"),
+ "a".repeat(MAX_LABEL_LENGTH + 1),
+ );
+
+ graph.commit();
+
+ assert_eq!(graph.segments.len(), 1);
+ assert_eq!(
+ graph.outgoing_edges(Node::from("A"))[0].label,
+ "a".repeat(MAX_LABEL_LENGTH)
+ );
+ }
}
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 360bfa21..6b6a2956 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -30,24 +30,24 @@ where
{
pub fn open<P: AsRef<Path>>(path: P) -> Self {
let mut options = rocksdb::Options::default();
- options.set_max_open_files(8);
+ options.set_max_open_files(-1);
options.create_if_missing(true);
options.increase_parallelism(8);
- options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable
- options.set_max_write_buffer_number(8);
+ options.set_max_background_jobs(8);
+ options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB memtable
+ options.set_max_write_buffer_number(2);
let mut block_options = BlockBasedOptions::default();
block_options.set_ribbon_filter(5.0);
block_options.disable_cache();
- block_options.set_block_size(128 * 1024); // 128 KB block size
-
options.set_block_based_table_factory(&block_options);
options.set_optimize_filters_for_hits(true);
options.set_compression_type(rocksdb::DBCompressionType::Lz4);
+ options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();
@@ -73,7 +73,10 @@ where
let key_bytes = bincode::serialize(key).unwrap();
let value_bytes = bincode::serialize(value).unwrap();
- self.db.put(key_bytes, value_bytes).unwrap();
+ let mut options = rocksdb::WriteOptions::default();
+ options.disable_wal(true);
+
+ self.db.put_opt(key_bytes, value_bytes, &options).unwrap();
}
pub fn batch_put<'a>(&'a self, it: impl Iterator<Item = (&'a K, &'a V)>) {
@@ -86,7 +89,10 @@ where
batch.put(key_bytes, value_bytes);
}
- self.db.write(batch).unwrap();
+ let mut options = rocksdb::WriteOptions::default();
+ options.disable_wal(true);
+
+ self.db.write_opt(batch, &options).unwrap();
}
pub fn batch_put_owned(&self, it: impl Iterator<Item = (K, V)>) {
@@ -99,7 +105,10 @@ where
batch.put(key_bytes, value_bytes);
}
- self.db.write(batch).unwrap();
+ let mut options = rocksdb::WriteOptions::default();
+ options.disable_wal(true);
+
+ self.db.write_opt(batch, &options).unwrap();
}
pub fn keys(&self) -> impl Iterator<Item = K> + '_ {