web search done right
Improve read performance for webgraph. We have split adjacency into a small_adjacency and full_adjacency (same for reverse) rocksdb databases. This allows us to read edge labels when we need them, which will increase the read performance for the webgraph edges at the expense of storage usage.
Mikkel Denkerd9b0fd745086
diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 476842df..920e73ab 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -127,4 +127,8 @@ harness = false[[bench]]name = "crawl-coordinator"+harness = false++[[bench]]+name = "similar-sites"harness = false\ No newline at end of filediff --git a/core/benches/similar-sites.rs b/core/benches/similar-sites.rsnew file mode 100644index 00000000..d9ecd090--- /dev/null+++ b/core/benches/similar-sites.rs@@ -0,0 +1,26 @@+use std::sync::Arc;++use criterion::{criterion_group, criterion_main, Criterion};+use stract::{+ ranking::inbound_similarity::InboundSimilarity, similar_sites::SimilarSitesFinder,+ webgraph::WebgraphBuilder,+};++const WEBGRAPH_PATH: &str = "data/webgraph";+const INBOUND_SIMILARITY_PATH: &str = "data/centrality/inbound_similarity";++pub fn criterion_benchmark(c: &mut Criterion) {+ let webgraph = Arc::new(WebgraphBuilder::new(WEBGRAPH_PATH).open());+ let inbound = InboundSimilarity::open(INBOUND_SIMILARITY_PATH).unwrap();++ let finder = SimilarSitesFinder::new(webgraph, inbound);++ for _ in 0..10 {+ c.bench_function("similar_sites", |b| {+ b.iter(|| finder.find_similar_sites(&["google.com".to_string()], 100))+ });+ }+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/core/src/main.rs b/core/src/main.rsindex c93edd28..f95a06a2 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -47,7 +47,7 @@ enum Commands {options: AliceOptions,},- /// Build the search index.+ /// Build an index.Indexer {#[clap(subcommand)]options: IndexingOptions,@@ -119,39 +119,48 @@ enum AliceOptions {#[derive(Subcommand)]enum CentralityType {+ /// Calculate all the supported centrality metrics.All,+ /// Calculate harmonic centrality. Harmonic centrality is the sum of the inverse distance from all+ /// other nodes to a given node. It is a measure of how central a node (webpage) is in a graph.Harmonic,+ /// Create an index that can be used to quickly calculate the inbound link similarity between two nodes.+ /// This is used to calculate the similarity between two webpages.Similarity,}#[derive(Subcommand)]enum WebgraphOptions {- Create {- config_path: String,- },+ /// Create a new webgraph.+ Create { config_path: String },++ /// Merge multiple webgraphs into a single graph with the specified number of segments.+ /// Can also be used to merge segments of a single webgraph.Merge {#[clap(required = true)]paths: Vec<String>,#[clap(long)]num_segments: Option<usize>,},- Server {- config_path: String,- },++ /// Deploy the webgraph server. The webgraph server is responsible for serving the webgraph to the search servers.+ /// This is e.g. used to find similar sites etc.+ Server { config_path: String },}#[derive(Subcommand)]enum IndexingOptions {- Search {- config_path: String,- },+ /// Create the search index.+ Search { config_path: String },++ /// Create the entity index. Used in the sidebar of the search UI.Entity {wikipedia_dump_path: String,output_path: String,},- Merge {- indexes: Vec<String>,- },++ /// Merge multiple search indexes into a single index.+ Merge { indexes: Vec<String> },}fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 9e7a72ca..ea1e9789 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -43,11 +43,16 @@ impl From<u64> for NodeID {}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]-pub(crate) struct StoredEdge {+pub(crate) struct FullStoredEdge {other: SegmentNodeID,label: String,}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]+pub(crate) struct SmallStoredEdge {+ other: SegmentNodeID,+}+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]pub struct Node {pub name: String,diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex ac74f098..9dac1fc3 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -21,7 +21,17 @@ use std::{path::Path,};-use super::{open_bin, save_bin, store::Store, Edge, Loaded, NodeID, StoredEdge};+use super::{+ open_bin, save_bin, store::Store, Edge, FullStoredEdge, Loaded, NodeID, SmallStoredEdge,+};++const FULL_ADJACENCY_STORE: &str = "full_adjacency";+const FULL_REVERSED_ADJACENCY_STORE: &str = "full_reversed_adjacency";+const SMALL_ADJACENCY_STORE: &str = "small_adjacency";+const SMALL_REVERSED_ADJACENCY_STORE: &str = "small_reversed_adjacency";+const ID_MAPPING_STORE: &str = "id_mapping";+const REV_ID_MAPPING_STORE: &str = "rev_id_mapping";+const META_STORE: &str = "meta.bin";#[derive(Debug,@@ -51,12 +61,15 @@ struct MergePacket {next_segment_node_id: SegmentNodeID,new_id_mapping: Store<NodeID, SegmentNodeID>,new_rev_id_mapping: Store<SegmentNodeID, NodeID>,- new_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,+ new_full_adjacency: Store<SegmentNodeID, HashSet<FullStoredEdge>>,+ new_small_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,}pub struct StoredSegment {- adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,- reversed_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,+ full_adjacency: Store<SegmentNodeID, HashSet<FullStoredEdge>>,+ full_reversed_adjacency: Store<SegmentNodeID, HashSet<FullStoredEdge>>,+ small_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,+ small_reversed_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,id_mapping: Store<NodeID, SegmentNodeID>,rev_id_mapping: Store<SegmentNodeID, NodeID>,meta: Meta,@@ -67,11 +80,17 @@ pub struct StoredSegment {impl StoredSegment {pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {StoredSegment {- adjacency: Store::open(folder_path.as_ref().join("adjacency")),- reversed_adjacency: Store::open(folder_path.as_ref().join("reversed_adjacency")),- id_mapping: Store::open(folder_path.as_ref().join("id_mapping")),- rev_id_mapping: Store::open(folder_path.as_ref().join("rev_id_mapping")),- meta: open_bin(folder_path.as_ref().join("meta.bin")),+ full_adjacency: Store::open(folder_path.as_ref().join(FULL_ADJACENCY_STORE)),+ full_reversed_adjacency: Store::open(+ folder_path.as_ref().join(FULL_REVERSED_ADJACENCY_STORE),+ ),+ small_adjacency: Store::open(folder_path.as_ref().join(SMALL_ADJACENCY_STORE)),+ small_reversed_adjacency: Store::open(+ folder_path.as_ref().join(SMALL_REVERSED_ADJACENCY_STORE),+ ),+ id_mapping: Store::open(folder_path.as_ref().join(ID_MAPPING_STORE)),+ rev_id_mapping: Store::open(folder_path.as_ref().join(REV_ID_MAPPING_STORE)),+ meta: open_bin(folder_path.as_ref().join(META_STORE)),folder_path: folder_path.as_ref().as_os_str()@@ -97,26 +116,29 @@ impl StoredSegment {pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {self.id_mapping(node).and_then(|segment_id| {- self.adjacency.get(&segment_id).map(|edges| {- edges- .into_iter()- .map(move |edge| {- if load_label {- Edge {- from: *node,- to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),- label: Loaded::Some(edge.label),- }- } else {- Edge {- from: *node,- to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),- label: Loaded::NotYet,- }- }- })- .collect()- })+ if load_label {+ self.full_adjacency.get(&segment_id).map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: *node,+ to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),+ label: Loaded::Some(edge.label),+ })+ .collect()+ })+ } else {+ self.small_adjacency.get(&segment_id).map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: *node,+ to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),+ label: Loaded::NotYet,+ })+ .collect()+ })+ }}).unwrap_or_default()}@@ -124,41 +146,40 @@ impl StoredSegment {pub fn ingoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {self.id_mapping(node).and_then(|segment_id| {- self.reversed_adjacency.get(&segment_id).map(|edges| {- edges- .into_iter()- .map(move |edge| {- if load_label {- Edge {- from: self- .rev_id_mapping(&SegmentNodeID(edge.other.0))- .unwrap(),- to: *node,- label: Loaded::Some(edge.label),- }- } else {- Edge {- from: self- .rev_id_mapping(&SegmentNodeID(edge.other.0))- .unwrap(),- to: *node,- label: Loaded::NotYet,- }- }- })- .collect()- })+ if load_label {+ self.full_reversed_adjacency.get(&segment_id).map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),+ to: *node,+ label: Loaded::Some(edge.label),+ })+ .collect()+ })+ } else {+ self.small_reversed_adjacency.get(&segment_id).map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),+ to: *node,+ label: Loaded::NotYet,+ })+ .collect()+ })+ }}).unwrap_or_default()}fn flush(&mut self) {- self.adjacency.flush();- self.reversed_adjacency.flush();+ self.full_adjacency.flush();+ self.full_reversed_adjacency.flush();self.id_mapping.flush();self.rev_id_mapping.flush();- save_bin(&self.meta, self.path().join("meta.bin"));+ save_bin(&self.meta, self.path().join(META_STORE));}pub fn id(&self) -> String {@@ -170,22 +191,24 @@ impl StoredSegment {}pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {- self.adjacency.iter().flat_map(move |(node_id, edges)| {- let from = self.rev_id_mapping(&node_id).unwrap();-- edges.into_iter().map(move |stored_edge| Edge {- from,- to: self- .rev_id_mapping(&SegmentNodeID(stored_edge.other.0))- .unwrap(),- label: Loaded::NotYet,+ self.full_adjacency+ .iter()+ .flat_map(move |(node_id, edges)| {+ let from = self.rev_id_mapping(&node_id).unwrap();++ edges.into_iter().map(move |stored_edge| Edge {+ from,+ to: self+ .rev_id_mapping(&SegmentNodeID(stored_edge.other.0))+ .unwrap(),+ label: Loaded::NotYet,+ })})- })}fn merge_adjacency<F1>(edges_fn: F1, packet: &mut MergePacket, segments: &[Self])where- F1: Fn(&Self, &NodeID) -> Option<HashSet<StoredEdge>>,+ F1: Fn(&Self, &NodeID) -> Option<HashSet<FullStoredEdge>>,{if segments.is_empty() {return;@@ -240,7 +263,7 @@ impl StoredSegment {let new_segment_node_id = packet.new_id_mapping.get(&node).unwrap();- let new_edge = StoredEdge {+ let new_edge = FullStoredEdge {other: new_segment_node_id,label: edge.label,};@@ -254,7 +277,15 @@ impl StoredSegment {if !edges.is_empty() {let segment_node_id = packet.new_id_mapping.get(&node_id).unwrap();- packet.new_adjacency.put(&segment_node_id, &edges);+ let small_edges: HashSet<SmallStoredEdge> = edges+ .iter()+ .map(|edge| SmallStoredEdge { other: edge.other })+ .collect();++ packet.new_full_adjacency.put(&segment_node_id, &edges);+ packet+ .new_small_adjacency+ .put(&segment_node_id, &small_edges);}}}@@ -269,45 +300,54 @@ impl StoredSegment {let new_path = segments[0].path().parent().unwrap().join(&new_segment_id);- let new_adjacency = Store::open(new_path.join("adjacency"));+ let new_full_adjacency = Store::open(new_path.join(FULL_ADJACENCY_STORE));+ let new_small_adjacency = Store::open(new_path.join(SMALL_ADJACENCY_STORE));let next_segment_node_id = SegmentNodeID(0);- let new_id_mapping = Store::open(new_path.join("id_mapping"));- let new_rev_id_mapping = Store::open(new_path.join("rev_id_mapping"));+ let new_id_mapping = Store::open(new_path.join(ID_MAPPING_STORE));+ let new_rev_id_mapping = Store::open(new_path.join(REV_ID_MAPPING_STORE));let mut packet = MergePacket {next_segment_node_id,new_id_mapping,new_rev_id_mapping,- new_adjacency,+ new_full_adjacency,+ new_small_adjacency,};Self::merge_adjacency(|segment: &StoredSegment, node_id: &NodeID| {segment.id_mapping.get(node_id).map(|segment_node_id| {- segment.adjacency.get(&segment_node_id).unwrap_or_default()+ segment+ .full_adjacency+ .get(&segment_node_id)+ .unwrap_or_default()})},&mut packet,&segments,);- let new_adjacency = packet.new_adjacency;+ let new_adjacency = packet.new_full_adjacency;+ let new_small_adjacency = packet.new_small_adjacency;- let new_reversed_adjacency = Store::open(new_path.join("reversed_adjacency"));+ let new_full_reversed_adjacency = Store::open(new_path.join(FULL_REVERSED_ADJACENCY_STORE));+ let new_small_reversed_adjacency =+ Store::open(new_path.join(SMALL_REVERSED_ADJACENCY_STORE));let mut packet = MergePacket {next_segment_node_id: packet.next_segment_node_id,new_id_mapping: packet.new_id_mapping,new_rev_id_mapping: packet.new_rev_id_mapping,- new_adjacency: new_reversed_adjacency,+ new_full_adjacency: new_full_reversed_adjacency,+ new_small_adjacency: new_small_reversed_adjacency,};Self::merge_adjacency(|segment: &StoredSegment, node_id: &NodeID| {segment.id_mapping.get(node_id).map(|segment_node_id| {segment- .reversed_adjacency+ .full_reversed_adjacency.get(&segment_node_id).unwrap_or_default()})@@ -317,8 +357,10 @@ impl StoredSegment {);let mut res = Self {- adjacency: new_adjacency,- reversed_adjacency: packet.new_adjacency,+ full_adjacency: new_adjacency,+ small_adjacency: new_small_adjacency,+ full_reversed_adjacency: packet.new_full_adjacency,+ small_reversed_adjacency: packet.new_small_adjacency,id_mapping: packet.new_id_mapping,rev_id_mapping: packet.new_rev_id_mapping,meta: Meta {@@ -367,8 +409,8 @@ struct Meta {#[derive(Default)]pub struct LiveSegment {- adjacency: BTreeMap<SegmentNodeID, HashSet<StoredEdge>>,- reversed_adjacency: BTreeMap<SegmentNodeID, HashSet<StoredEdge>>,+ adjacency: BTreeMap<SegmentNodeID, HashSet<FullStoredEdge>>,+ reversed_adjacency: BTreeMap<SegmentNodeID, HashSet<FullStoredEdge>>,id_mapping: BTreeMap<NodeID, SegmentNodeID>,rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,next_id: u64,@@ -393,37 +435,75 @@ impl LiveSegment {let from = self.get_or_create_id(from);let to = self.get_or_create_id(to);- self.adjacency.entry(from).or_default().insert(StoredEdge {- other: to,- label: label.clone(),- });+ self.adjacency+ .entry(from)+ .or_default()+ .insert(FullStoredEdge {+ other: to,+ label: label.clone(),+ });self.reversed_adjacency.entry(to).or_default()- .insert(StoredEdge { other: from, label });+ .insert(FullStoredEdge { other: from, label });}pub fn commit<P: AsRef<Path>>(self, folder_path: P) -> StoredSegment {let segment_id = uuid::Uuid::new_v4().to_string();let path = folder_path.as_ref().join(&segment_id);- let adjacency = Store::open(path.join("adjacency"));- adjacency.batch_put(self.adjacency.iter());+ let small_adjacency: BTreeMap<SegmentNodeID, HashSet<SmallStoredEdge>> = self+ .adjacency+ .clone()+ .into_iter()+ .map(|(segment_node_id, edges)| {+ let edges = edges+ .into_iter()+ .map(|edge| SmallStoredEdge { other: edge.other })+ .collect();++ (segment_node_id, edges)+ })+ .collect();++ let small_rev_adjacency: BTreeMap<SegmentNodeID, HashSet<SmallStoredEdge>> = self+ .reversed_adjacency+ .clone()+ .into_iter()+ .map(|(segment_node_id, edges)| {+ let edges = edges+ .into_iter()+ .map(|edge| SmallStoredEdge { other: edge.other })+ .collect();++ (segment_node_id, edges)+ })+ .collect();++ let full_adjacency = Store::open(path.join(FULL_ADJACENCY_STORE));+ full_adjacency.batch_put(self.adjacency.iter());++ let full_reversed_adjacency = Store::open(path.join(FULL_REVERSED_ADJACENCY_STORE));+ full_reversed_adjacency.batch_put(self.reversed_adjacency.iter());- let reversed_adjacency = Store::open(path.join("reversed_adjacency"));+ let small_adjacency_store = Store::open(path.join(SMALL_ADJACENCY_STORE));+ small_adjacency_store.batch_put(small_adjacency.iter());- reversed_adjacency.batch_put(self.reversed_adjacency.iter());+ let small_rev_adjacency_store = Store::open(path.join(SMALL_REVERSED_ADJACENCY_STORE));+ small_rev_adjacency_store.batch_put(small_rev_adjacency.iter());- let id_mapping = Store::open(path.join("id_mapping"));+ let id_mapping = Store::open(path.join(ID_MAPPING_STORE));id_mapping.batch_put(self.id_mapping.iter());- let rev_id_mapping = Store::open(path.join("rev_id_mapping"));+ let rev_id_mapping = Store::open(path.join(REV_ID_MAPPING_STORE));rev_id_mapping.batch_put(self.rev_id_mapping.iter());let mut stored_segment = StoredSegment {- adjacency,- reversed_adjacency,+ small_adjacency: small_adjacency_store,+ small_reversed_adjacency: small_rev_adjacency_store,+ full_adjacency,+ full_reversed_adjacency,id_mapping,rev_id_mapping,meta: Meta {diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex e2dbe043..ea4e9911 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -36,11 +36,13 @@ whereoptions.set_max_write_buffer_number(8);let mut block_options = BlockBasedOptions::default();- block_options.set_bloom_filter(64.0, true);+ block_options.set_bloom_filter(128.0, true);- let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cache+ let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024).unwrap(); // 1024 MB cacheblock_options.set_block_cache(&cache);+ block_options.set_block_size(128 * 1024); // 128 KB block size+options.set_block_based_table_factory(&block_options);let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();@@ -54,8 +56,11 @@ wherepub fn get(&self, key: &K) -> Option<V> {let bytes = bincode::serialize(key).unwrap();+ let mut readopts = rocksdb::ReadOptions::default();+ readopts.set_readahead_size(4_194_304);+self.db- .get(bytes)+ .get_pinned_opt(bytes, &readopts).unwrap().map(|bytes| bincode::deserialize(&bytes).unwrap())}