Skip to content

web search done right

Improve webgraph write performance. We can remove all 'small_adjacency' databases and simply not deserialize the labels if we don't need the m. This will increase the IO a bit for get queries as we now have to read the label bytes into memory no matter what, but seing as full_adjacency for the current webgraph is only 10% larger than small_adjacen cy, I don't think this will be too much of a performance hit. On the other hand, removing small_adjacency will approximately half the write amplification.

Mikkel Denker0f27a61a8be0
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rs
index 73d735fb..05c57624 100644
--- a/core/benches/search-preindexed-optic.rs
+++ b/core/benches/search-preindexed-optic.rs
@@ -30,7 +30,7 @@ macro_rules! bench {
pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
let searcher = LocalSearcher::new(index);
- let optic = include_str!("../../optics/testcases/discussions.optic");
+ let optic = include_str!("../../optics/testcases/samples/discussions.optic");
for _ in 0..10_000 {
bench!("the", searcher, optic, c);
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 40ec824f..8b12ebb1 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -240,21 +240,28 @@ impl UrlStateDbShard {
options.create_if_missing(true);
- let mut block_options = rocksdb::BlockBasedOptions::default();
- block_options.set_ribbon_filter(10.0);
- block_options.set_format_version(5);
-
- let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024)?; // 1GB
- block_options.set_block_cache(&cache);
-
- options.set_block_based_table_factory(&block_options);
options.set_max_background_jobs(8);
options.increase_parallelism(8);
options.set_max_subcompactions(8);
options.set_write_buffer_size(512 * 1024 * 1024);
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
+
+ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
+ options.set_level_compaction_dynamic_level_bytes(true);
+ options.set_bytes_per_sync(1048576);
+ let mut block_options = rocksdb::BlockBasedOptions::default();
+ let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024)?; // 1GB
+ block_options.set_block_cache(&cache);
+ block_options.set_ribbon_filter(10.0);
+ block_options.set_format_version(5);
+ block_options.set_block_size(16 * 1024);
+ block_options.set_cache_index_and_filter_blocks(true);
+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
+
+ options.set_block_based_table_factory(&block_options);
+ options.set_optimize_filters_for_hits(true);
+
options.set_compression_type(rocksdb::DBCompressionType::None);
let db = rocksdb::DB::open(&options, path.as_ref().join("urls"))?;
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 4ef97bc0..81fcb225 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -76,10 +76,38 @@ impl intmap::Key for NodeID {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
+pub trait EdgeLabel
+where
+ Self: Send + Sync + Sized,
+{
+ fn to_bytes(&self) -> anyhow::Result<Vec<u8>>;
+ fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self>;
+}
+
+impl EdgeLabel for String {
+ fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
+ Ok(self.as_bytes().to_vec())
+ }
+
+ fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self> {
+ Ok(String::from_utf8(bytes.to_vec())?)
+ }
+}
+
+impl EdgeLabel for () {
+ fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
+ Ok(Vec::new())
+ }
+
+ fn from_bytes(_bytes: &[u8]) -> anyhow::Result<Self> {
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Edge<L>
where
- L: Send + Sync,
+ L: EdgeLabel,
{
pub from: NodeID,
pub to: NodeID,
@@ -240,7 +268,7 @@ pub trait ShortestPaths {
fn dijkstra<F1, F2, L>(source: Node, node_edges: F1, edge_node: F2) -> BTreeMap<NodeID, u8>
where
- L: Send + Sync,
+ L: EdgeLabel,
F1: Fn(NodeID) -> Vec<Edge<L>>,
F2: Fn(&Edge<L>) -> NodeID,
{
@@ -598,7 +626,7 @@ impl Webgraph {
fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>
where
- L: Send + Sync,
+ L: EdgeLabel,
F1: Sized + Sync + Fn(&StoredSegment) -> Vec<Edge<L>>,
F2: Fn(&mut Vec<Edge<L>>),
{
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index c8b9548e..17c0a1a1 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -18,16 +18,12 @@ use std::path::{Path, PathBuf};
use super::{store::EdgeStore, Edge, NodeID};
-const FULL_ADJACENCY_STORE: &str = "full_adjacency";
-const FULL_REVERSED_ADJACENCY_STORE: &str = "full_reversed_adjacency";
-const SMALL_ADJACENCY_STORE: &str = "small_adjacency";
-const SMALL_REVERSED_ADJACENCY_STORE: &str = "small_reversed_adjacency";
+const ADJACENCY_STORE: &str = "adjacency";
+const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";
pub struct StoredSegment {
- full_adjacency: EdgeStore<String>,
- full_reversed_adjacency: EdgeStore<String>,
- small_adjacency: EdgeStore<()>,
- small_reversed_adjacency: EdgeStore<()>,
+ full_adjacency: EdgeStore,
+ full_reversed_adjacency: EdgeStore,
id: String,
folder_path: String,
}
@@ -36,25 +32,14 @@ impl StoredSegment {
pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
StoredSegment {
full_adjacency: EdgeStore::open(
- folder_path.as_ref().join(&id).join(FULL_ADJACENCY_STORE),
+ folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
false,
),
full_reversed_adjacency: EdgeStore::open(
folder_path
.as_ref()
.join(&id)
- .join(FULL_REVERSED_ADJACENCY_STORE),
- true,
- ),
- small_adjacency: EdgeStore::open(
- folder_path.as_ref().join(&id).join(SMALL_ADJACENCY_STORE),
- false,
- ),
- small_reversed_adjacency: EdgeStore::open(
- folder_path
- .as_ref()
- .join(&id)
- .join(SMALL_REVERSED_ADJACENCY_STORE),
+ .join(REVERSED_ADJACENCY_STORE),
true,
),
folder_path: folder_path
@@ -68,7 +53,7 @@ impl StoredSegment {
}
pub fn estimate_num_nodes(&self) -> usize {
- self.small_adjacency.estimate_len()
+ self.full_adjacency.estimate_len()
}
pub fn outgoing_edges_with_label(
@@ -79,7 +64,7 @@ impl StoredSegment {
}
pub fn outgoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {
- self.small_adjacency.get(*node)
+ self.full_adjacency.get(*node)
}
pub fn ingoing_edges_with_label(
@@ -90,14 +75,12 @@ impl StoredSegment {
}
pub fn ingoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {
- self.small_reversed_adjacency.get(*node)
+ self.full_reversed_adjacency.get(*node)
}
pub fn flush(&mut self) {
self.full_adjacency.flush();
self.full_reversed_adjacency.flush();
- self.small_adjacency.flush();
- self.small_reversed_adjacency.flush();
}
pub fn id(&self) -> String {
@@ -109,16 +92,13 @@ impl StoredSegment {
}
pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {
- self.small_adjacency.iter()
+ self.full_adjacency.iter()
}
fn merge_with(&self, other: &StoredSegment) {
self.full_adjacency.merge_with(&other.full_adjacency);
self.full_reversed_adjacency
.merge_with(&other.full_reversed_adjacency);
- self.small_adjacency.merge_with(&other.small_adjacency);
- self.small_reversed_adjacency
- .merge_with(&other.small_reversed_adjacency);
}
pub fn merge(mut segments: Vec<StoredSegment>) -> Self {
@@ -139,20 +119,8 @@ impl StoredSegment {
}
pub fn insert(&mut self, edges: &[Edge<String>]) {
- let small_edges: Vec<_> = edges
- .iter()
- .map(|e| Edge {
- from: e.from,
- to: e.to,
- label: (),
- })
- .collect();
-
- self.full_adjacency.put(edges.iter().cloned());
- self.full_reversed_adjacency.put(edges.iter().cloned());
- self.small_adjacency.put(small_edges.iter().cloned());
- self.small_reversed_adjacency
- .put(small_edges.iter().cloned());
+ self.full_adjacency.put(edges.iter());
+ self.full_reversed_adjacency.put(edges.iter());
}
}
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 02eaa474..4c2a129e 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -19,48 +19,45 @@ use std::path::Path;
use hashbrown::HashSet;
use rocksdb::BlockBasedOptions;
-use super::{Edge, NodeID};
+use super::{Edge, EdgeLabel, NodeID};
const MAX_BATCH_SIZE: usize = 50_000;
-pub struct EdgeStore<L> {
+pub struct EdgeStore {
reversed: bool,
db: rocksdb::DB,
- _phantom: std::marker::PhantomData<L>,
}
-impl<L> EdgeStore<L>
-where
- L: serde::Serialize + serde::de::DeserializeOwned + Send + Sync,
-{
+impl EdgeStore {
pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {
let mut options = rocksdb::Options::default();
options.create_if_missing(true);
- let mut block_options = BlockBasedOptions::default();
- block_options.set_ribbon_filter(10.0);
- block_options.set_format_version(5);
-
- options.set_block_based_table_factory(&block_options);
- options.set_optimize_filters_for_hits(true);
-
options.set_max_background_jobs(8);
options.increase_parallelism(8);
options.set_max_subcompactions(8);
+ options.set_allow_mmap_reads(true);
+ options.set_allow_mmap_writes(true);
+
+ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
+ options.set_level_compaction_dynamic_level_bytes(true);
+ options.set_bytes_per_sync(1048576);
+ let mut block_options = BlockBasedOptions::default();
+ block_options.set_block_size(16 * 1024);
+ block_options.set_format_version(5);
+ block_options.set_cache_index_and_filter_blocks(true);
+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
+
+ options.set_block_based_table_factory(&block_options);
options.set_compression_type(rocksdb::DBCompressionType::Lz4);
- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();
- Self {
- db,
- reversed,
- _phantom: std::marker::PhantomData,
- }
+ Self { db, reversed }
}
- pub fn get(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {
+ pub fn get<L: EdgeLabel>(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {
let prefix_bytes = node.bit_128().to_be_bytes();
let suffix = 0u128.to_be_bytes();
@@ -85,7 +82,7 @@ where
let suffix = u128::from_be_bytes(key[16..32].try_into().unwrap());
- let label = bincode::deserialize(&value).ok()?;
+ let label = L::from_bytes(&value).ok()?;
if self.reversed {
Some(Edge {
@@ -103,7 +100,7 @@ where
})
}
- pub fn put(&self, edges: impl Iterator<Item = Edge<L>>) {
+ pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a Edge<L>>) {
let mut batch = rocksdb::WriteBatch::default();
let mut batch_keys = HashSet::new();
@@ -128,7 +125,7 @@ where
batch_keys.insert(key_bytes.clone());
- let value_bytes = bincode::serialize(&edge.label).unwrap();
+ let value_bytes = edge.label.to_bytes().unwrap();
batch.put(key_bytes, value_bytes);
@@ -142,7 +139,7 @@ where
self.db.write_opt(batch, &opts).unwrap();
}
- pub fn iter(&self) -> impl Iterator<Item = Edge<L>> + '_ + Send + Sync {
+ pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = Edge<L>> + '_ + Send + Sync {
let read_opts = rocksdb::ReadOptions::default();
self.db
@@ -168,7 +165,7 @@ where
(from, to)
};
- let label = bincode::deserialize(&val).unwrap();
+ let label = L::from_bytes(&val).unwrap();
Edge { from, to, label }
})
@@ -185,8 +182,19 @@ where
.unwrap_or(0) as usize
}
- pub fn merge_with(&self, other: &EdgeStore<L>) {
- self.put(other.iter());
+ pub fn merge_with(&self, other: &EdgeStore) {
+ let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
+
+ for edge in other.iter::<String>() {
+ batch.push(edge);
+
+ if batch.len() >= MAX_BATCH_SIZE {
+ self.put(batch.iter());
+ batch.clear();
+ }
+ }
+
+ self.put(batch.iter());
}
}
@@ -196,8 +204,7 @@ mod tests {
#[test]
fn test_insert() {
- let kv: EdgeStore<String> =
- EdgeStore::open(crate::gen_temp_path().join("test-segment"), false);
+ let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), false);
let e = Edge {
from: NodeID(0),
@@ -205,7 +212,7 @@ mod tests {
label: "test".to_string(),
};
- kv.put(vec![e.clone()].into_iter());
+ kv.put([e.clone()].iter());
kv.flush();
@@ -214,15 +221,14 @@ mod tests {
assert_eq!(edges.len(), 1);
assert_eq!(&edges[0], &e);
- let edges: Vec<_> = kv.get(NodeID(1)).collect();
+ let edges: Vec<Edge<String>> = kv.get(NodeID(1)).collect();
assert_eq!(edges.len(), 0);
}
#[test]
fn test_reversed() {
- let kv: EdgeStore<String> =
- EdgeStore::open(crate::gen_temp_path().join("test-segment"), true);
+ let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), true);
let e = Edge {
from: NodeID(0),
@@ -230,15 +236,15 @@ mod tests {
label: "test".to_string(),
};
- kv.put(vec![e.clone()].into_iter());
+ kv.put([e.clone()].iter());
kv.flush();
- let edges: Vec<_> = kv.get(NodeID(0)).collect();
+ let edges: Vec<Edge<String>> = kv.get(NodeID(0)).collect();
assert_eq!(edges.len(), 0);
- let edges: Vec<_> = kv.get(NodeID(1)).collect();
+ let edges: Vec<Edge<String>> = kv.get(NodeID(1)).collect();
assert_eq!(edges.len(), 1);
assert_eq!(&edges[0], &e);