web search done right
Improve webgraph write performance. We can remove all 'small_adjacency' databases and simply not deserialize the labels if we don't need the m. This will increase the IO a bit for get queries as we now have to read the label bytes into memory no matter what, but seing as full_adjacency for the current webgraph is only 10% larger than small_adjacen cy, I don't think this will be too much of a performance hit. On the other hand, removing small_adjacency will approximately half the write amplification.
Mikkel Denker0f27a61a8be0
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rsindex 73d735fb..05c57624 100644--- a/core/benches/search-preindexed-optic.rs+++ b/core/benches/search-preindexed-optic.rs@@ -30,7 +30,7 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();let searcher = LocalSearcher::new(index);- let optic = include_str!("../../optics/testcases/discussions.optic");+ let optic = include_str!("../../optics/testcases/samples/discussions.optic");for _ in 0..10_000 {bench!("the", searcher, optic, c);diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 40ec824f..8b12ebb1 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -240,21 +240,28 @@ impl UrlStateDbShard {options.create_if_missing(true);- let mut block_options = rocksdb::BlockBasedOptions::default();- block_options.set_ribbon_filter(10.0);- block_options.set_format_version(5);-- let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024)?; // 1GB- block_options.set_block_cache(&cache);-- options.set_block_based_table_factory(&block_options);options.set_max_background_jobs(8);options.increase_parallelism(8);options.set_max_subcompactions(8);options.set_write_buffer_size(512 * 1024 * 1024);options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);++ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)+ options.set_level_compaction_dynamic_level_bytes(true);+ options.set_bytes_per_sync(1048576);+ let mut block_options = rocksdb::BlockBasedOptions::default();+ let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024)?; // 1GB+ block_options.set_block_cache(&cache);+ block_options.set_ribbon_filter(10.0);+ block_options.set_format_version(5);+ block_options.set_block_size(16 * 1024);+ block_options.set_cache_index_and_filter_blocks(true);+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);++ options.set_block_based_table_factory(&block_options);+ options.set_optimize_filters_for_hits(true);+options.set_compression_type(rocksdb::DBCompressionType::None);let db = rocksdb::DB::open(&options, path.as_ref().join("urls"))?;diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 4ef97bc0..81fcb225 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -76,10 +76,38 @@ impl intmap::Key for NodeID {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]+pub trait EdgeLabel+where+ Self: Send + Sync + Sized,+{+ fn to_bytes(&self) -> anyhow::Result<Vec<u8>>;+ fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self>;+}++impl EdgeLabel for String {+ fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {+ Ok(self.as_bytes().to_vec())+ }++ fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self> {+ Ok(String::from_utf8(bytes.to_vec())?)+ }+}++impl EdgeLabel for () {+ fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {+ Ok(Vec::new())+ }++ fn from_bytes(_bytes: &[u8]) -> anyhow::Result<Self> {+ Ok(())+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct Edge<L>where- L: Send + Sync,+ L: EdgeLabel,{pub from: NodeID,pub to: NodeID,@@ -240,7 +268,7 @@ pub trait ShortestPaths {fn dijkstra<F1, F2, L>(source: Node, node_edges: F1, edge_node: F2) -> BTreeMap<NodeID, u8>where- L: Send + Sync,+ L: EdgeLabel,F1: Fn(NodeID) -> Vec<Edge<L>>,F2: Fn(&Edge<L>) -> NodeID,{@@ -598,7 +626,7 @@ impl Webgraph {fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>where- L: Send + Sync,+ L: EdgeLabel,F1: Sized + Sync + Fn(&StoredSegment) -> Vec<Edge<L>>,F2: Fn(&mut Vec<Edge<L>>),{diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex c8b9548e..17c0a1a1 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -18,16 +18,12 @@ use std::path::{Path, PathBuf};use super::{store::EdgeStore, Edge, NodeID};-const FULL_ADJACENCY_STORE: &str = "full_adjacency";-const FULL_REVERSED_ADJACENCY_STORE: &str = "full_reversed_adjacency";-const SMALL_ADJACENCY_STORE: &str = "small_adjacency";-const SMALL_REVERSED_ADJACENCY_STORE: &str = "small_reversed_adjacency";+const ADJACENCY_STORE: &str = "adjacency";+const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";pub struct StoredSegment {- full_adjacency: EdgeStore<String>,- full_reversed_adjacency: EdgeStore<String>,- small_adjacency: EdgeStore<()>,- small_reversed_adjacency: EdgeStore<()>,+ full_adjacency: EdgeStore,+ full_reversed_adjacency: EdgeStore,id: String,folder_path: String,}@@ -36,25 +32,14 @@ impl StoredSegment {pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {StoredSegment {full_adjacency: EdgeStore::open(- folder_path.as_ref().join(&id).join(FULL_ADJACENCY_STORE),+ folder_path.as_ref().join(&id).join(ADJACENCY_STORE),false,),full_reversed_adjacency: EdgeStore::open(folder_path.as_ref().join(&id)- .join(FULL_REVERSED_ADJACENCY_STORE),- true,- ),- small_adjacency: EdgeStore::open(- folder_path.as_ref().join(&id).join(SMALL_ADJACENCY_STORE),- false,- ),- small_reversed_adjacency: EdgeStore::open(- folder_path- .as_ref()- .join(&id)- .join(SMALL_REVERSED_ADJACENCY_STORE),+ .join(REVERSED_ADJACENCY_STORE),true,),folder_path: folder_path@@ -68,7 +53,7 @@ impl StoredSegment {}pub fn estimate_num_nodes(&self) -> usize {- self.small_adjacency.estimate_len()+ self.full_adjacency.estimate_len()}pub fn outgoing_edges_with_label(@@ -79,7 +64,7 @@ impl StoredSegment {}pub fn outgoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {- self.small_adjacency.get(*node)+ self.full_adjacency.get(*node)}pub fn ingoing_edges_with_label(@@ -90,14 +75,12 @@ impl StoredSegment {}pub fn ingoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {- self.small_reversed_adjacency.get(*node)+ self.full_reversed_adjacency.get(*node)}pub fn flush(&mut self) {self.full_adjacency.flush();self.full_reversed_adjacency.flush();- self.small_adjacency.flush();- self.small_reversed_adjacency.flush();}pub fn id(&self) -> String {@@ -109,16 +92,13 @@ impl StoredSegment {}pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {- self.small_adjacency.iter()+ self.full_adjacency.iter()}fn merge_with(&self, other: &StoredSegment) {self.full_adjacency.merge_with(&other.full_adjacency);self.full_reversed_adjacency.merge_with(&other.full_reversed_adjacency);- self.small_adjacency.merge_with(&other.small_adjacency);- self.small_reversed_adjacency- .merge_with(&other.small_reversed_adjacency);}pub fn merge(mut segments: Vec<StoredSegment>) -> Self {@@ -139,20 +119,8 @@ impl StoredSegment {}pub fn insert(&mut self, edges: &[Edge<String>]) {- let small_edges: Vec<_> = edges- .iter()- .map(|e| Edge {- from: e.from,- to: e.to,- label: (),- })- .collect();-- self.full_adjacency.put(edges.iter().cloned());- self.full_reversed_adjacency.put(edges.iter().cloned());- self.small_adjacency.put(small_edges.iter().cloned());- self.small_reversed_adjacency- .put(small_edges.iter().cloned());+ self.full_adjacency.put(edges.iter());+ self.full_reversed_adjacency.put(edges.iter());}}diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 02eaa474..4c2a129e 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -19,48 +19,45 @@ use std::path::Path;use hashbrown::HashSet;use rocksdb::BlockBasedOptions;-use super::{Edge, NodeID};+use super::{Edge, EdgeLabel, NodeID};const MAX_BATCH_SIZE: usize = 50_000;-pub struct EdgeStore<L> {+pub struct EdgeStore {reversed: bool,db: rocksdb::DB,- _phantom: std::marker::PhantomData<L>,}-impl<L> EdgeStore<L>-where- L: serde::Serialize + serde::de::DeserializeOwned + Send + Sync,-{+impl EdgeStore {pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {let mut options = rocksdb::Options::default();options.create_if_missing(true);- let mut block_options = BlockBasedOptions::default();- block_options.set_ribbon_filter(10.0);- block_options.set_format_version(5);-- options.set_block_based_table_factory(&block_options);- options.set_optimize_filters_for_hits(true);-options.set_max_background_jobs(8);options.increase_parallelism(8);options.set_max_subcompactions(8);+ options.set_allow_mmap_reads(true);+ options.set_allow_mmap_writes(true);++ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)+ options.set_level_compaction_dynamic_level_bytes(true);+ options.set_bytes_per_sync(1048576);+ let mut block_options = BlockBasedOptions::default();+ block_options.set_block_size(16 * 1024);+ block_options.set_format_version(5);+ block_options.set_cache_index_and_filter_blocks(true);+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);++ options.set_block_based_table_factory(&block_options);options.set_compression_type(rocksdb::DBCompressionType::Lz4);- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();- Self {- db,- reversed,- _phantom: std::marker::PhantomData,- }+ Self { db, reversed }}- pub fn get(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {+ pub fn get<L: EdgeLabel>(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {let prefix_bytes = node.bit_128().to_be_bytes();let suffix = 0u128.to_be_bytes();@@ -85,7 +82,7 @@ wherelet suffix = u128::from_be_bytes(key[16..32].try_into().unwrap());- let label = bincode::deserialize(&value).ok()?;+ let label = L::from_bytes(&value).ok()?;if self.reversed {Some(Edge {@@ -103,7 +100,7 @@ where})}- pub fn put(&self, edges: impl Iterator<Item = Edge<L>>) {+ pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a Edge<L>>) {let mut batch = rocksdb::WriteBatch::default();let mut batch_keys = HashSet::new();@@ -128,7 +125,7 @@ wherebatch_keys.insert(key_bytes.clone());- let value_bytes = bincode::serialize(&edge.label).unwrap();+ let value_bytes = edge.label.to_bytes().unwrap();batch.put(key_bytes, value_bytes);@@ -142,7 +139,7 @@ whereself.db.write_opt(batch, &opts).unwrap();}- pub fn iter(&self) -> impl Iterator<Item = Edge<L>> + '_ + Send + Sync {+ pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = Edge<L>> + '_ + Send + Sync {let read_opts = rocksdb::ReadOptions::default();self.db@@ -168,7 +165,7 @@ where(from, to)};- let label = bincode::deserialize(&val).unwrap();+ let label = L::from_bytes(&val).unwrap();Edge { from, to, label }})@@ -185,8 +182,19 @@ where.unwrap_or(0) as usize}- pub fn merge_with(&self, other: &EdgeStore<L>) {- self.put(other.iter());+ pub fn merge_with(&self, other: &EdgeStore) {+ let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);++ for edge in other.iter::<String>() {+ batch.push(edge);++ if batch.len() >= MAX_BATCH_SIZE {+ self.put(batch.iter());+ batch.clear();+ }+ }++ self.put(batch.iter());}}@@ -196,8 +204,7 @@ mod tests {#[test]fn test_insert() {- let kv: EdgeStore<String> =- EdgeStore::open(crate::gen_temp_path().join("test-segment"), false);+ let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), false);let e = Edge {from: NodeID(0),@@ -205,7 +212,7 @@ mod tests {label: "test".to_string(),};- kv.put(vec![e.clone()].into_iter());+ kv.put([e.clone()].iter());kv.flush();@@ -214,15 +221,14 @@ mod tests {assert_eq!(edges.len(), 1);assert_eq!(&edges[0], &e);- let edges: Vec<_> = kv.get(NodeID(1)).collect();+ let edges: Vec<Edge<String>> = kv.get(NodeID(1)).collect();assert_eq!(edges.len(), 0);}#[test]fn test_reversed() {- let kv: EdgeStore<String> =- EdgeStore::open(crate::gen_temp_path().join("test-segment"), true);+ let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), true);let e = Edge {from: NodeID(0),@@ -230,15 +236,15 @@ mod tests {label: "test".to_string(),};- kv.put(vec![e.clone()].into_iter());+ kv.put([e.clone()].iter());kv.flush();- let edges: Vec<_> = kv.get(NodeID(0)).collect();+ let edges: Vec<Edge<String>> = kv.get(NodeID(0)).collect();assert_eq!(edges.len(), 0);- let edges: Vec<_> = kv.get(NodeID(1)).collect();+ let edges: Vec<Edge<String>> = kv.get(NodeID(1)).collect();assert_eq!(edges.len(), 1);assert_eq!(&edges[0], &e);