web search done right
optimize webgraph for read after merges
Mikkel Denker8a302697d16e
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 102d87cc..245c785f 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -243,6 +243,9 @@ impl Webgraph {page_graph.merge(other_page)?;}+ host_graph.optimize_read();+ page_graph.optimize_read();+Ok(())}}diff --git a/crates/core/src/webgraph/id_node_db.rs b/crates/core/src/webgraph/id_node_db.rsindex 081e8b7c..865f9742 100644--- a/crates/core/src/webgraph/id_node_db.rs+++ b/crates/core/src/webgraph/id_node_db.rs@@ -54,18 +54,21 @@ impl Id2NodeDb {for (id, node) in iter {self.put(&id, &node);- if self.db.uncommitted_inserts() > 1_000_000 {+ if self.db.uncommitted_inserts() > 10_000_000 {self.flush();}}- if self.db.uncommitted_inserts() > 1_000_000 {+ if self.db.uncommitted_inserts() > 10_000_000 {self.flush();}}pub fn flush(&mut self) {self.db.commit().unwrap();+ }++ pub fn optimize_read(&mut self) {self.db.merge_all_segments().unwrap();}}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 925629e3..6ac45d2f 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -157,6 +157,14 @@ impl Webgraph {Ok(())}+ pub fn optimize_read(&mut self) {+ self.executor+ .map(|s| s.optimize_read(), self.segments.iter_mut())+ .unwrap();++ self.id2node.optimize_read();+ }+pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {let dedup = |edges: &mut Vec<Edge<String>>| {edges.sort_by_key(|e| e.from);diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex 50055101..b8cdf0b9 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -144,6 +144,11 @@ impl Segment {pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {self.adjacency.iter_without_label()}++ pub fn optimize_read(&mut self) {+ self.adjacency.optimize_read();+ self.reversed_adjacency.optimize_read();+ }}#[cfg(test)]diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 3f4f71f5..85c92465 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -53,6 +53,10 @@ impl PrefixDb {Self { db }}+ fn optimize_read(&mut self) {+ self.db.merge_all_segments().unwrap();+ }+fn insert(&mut self, node: &FullNodeID) {let key = [node.prefix.as_u64().to_le_bytes(),@@ -98,6 +102,11 @@ impl RangesDb {Self { nodes, labels }}++ pub fn optimize_read(&mut self) {+ self.nodes.merge_all_segments().unwrap();+ self.labels.merge_all_segments().unwrap();+ }}pub struct EdgeStore {@@ -154,6 +163,11 @@ impl EdgeStore {}}+ pub fn optimize_read(&mut self) {+ self.ranges.optimize_read();+ self.prefixes.optimize_read();+ }+/// Insert a batch of edges into the store./// The edges *must* have been de-duplicated by their from/to node./// I.e. if the store is not reversed, there should only ever be a single@@ -240,9 +254,10 @@ impl EdgeStore {|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },);batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });+ let batch_len = batch.len();s.put(&batch);batch.clear();- inserts_since_last_flush += 1;+ inserts_since_last_flush += batch_len;if inserts_since_last_flush >= 1_000_000 {s.flush();diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex 4166a53e..5b2f4280 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub const MAX_BATCH_SIZE: usize = 5_000_000;+pub const MAX_BATCH_SIZE: usize = 3_000_000;use std::{collections::BTreeSet,