Skip to content

web search done right

optimize webgraph for read after merges

Mikkel Denker8a302697d16e
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 102d87cc..245c785f 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -243,6 +243,9 @@ impl Webgraph {
page_graph.merge(other_page)?;
}
+ host_graph.optimize_read();
+ page_graph.optimize_read();
+
Ok(())
}
}
diff --git a/crates/core/src/webgraph/id_node_db.rs b/crates/core/src/webgraph/id_node_db.rs
index 081e8b7c..865f9742 100644
--- a/crates/core/src/webgraph/id_node_db.rs
+++ b/crates/core/src/webgraph/id_node_db.rs
@@ -54,18 +54,21 @@ impl Id2NodeDb {
for (id, node) in iter {
self.put(&id, &node);
- if self.db.uncommitted_inserts() > 1_000_000 {
+ if self.db.uncommitted_inserts() > 10_000_000 {
self.flush();
}
}
- if self.db.uncommitted_inserts() > 1_000_000 {
+ if self.db.uncommitted_inserts() > 10_000_000 {
self.flush();
}
}
pub fn flush(&mut self) {
self.db.commit().unwrap();
+ }
+
+ pub fn optimize_read(&mut self) {
self.db.merge_all_segments().unwrap();
}
}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 925629e3..6ac45d2f 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -157,6 +157,14 @@ impl Webgraph {
Ok(())
}
+ pub fn optimize_read(&mut self) {
+ self.executor
+ .map(|s| s.optimize_read(), self.segments.iter_mut())
+ .unwrap();
+
+ self.id2node.optimize_read();
+ }
+
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
let dedup = |edges: &mut Vec<Edge<String>>| {
edges.sort_by_key(|e| e.from);
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index 50055101..b8cdf0b9 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -144,6 +144,11 @@ impl Segment {
pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {
self.adjacency.iter_without_label()
}
+
+ pub fn optimize_read(&mut self) {
+ self.adjacency.optimize_read();
+ self.reversed_adjacency.optimize_read();
+ }
}
#[cfg(test)]
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 3f4f71f5..85c92465 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -53,6 +53,10 @@ impl PrefixDb {
Self { db }
}
+ fn optimize_read(&mut self) {
+ self.db.merge_all_segments().unwrap();
+ }
+
fn insert(&mut self, node: &FullNodeID) {
let key = [
node.prefix.as_u64().to_le_bytes(),
@@ -98,6 +102,11 @@ impl RangesDb {
Self { nodes, labels }
}
+
+ pub fn optimize_read(&mut self) {
+ self.nodes.merge_all_segments().unwrap();
+ self.labels.merge_all_segments().unwrap();
+ }
}
pub struct EdgeStore {
@@ -154,6 +163,11 @@ impl EdgeStore {
}
}
+ pub fn optimize_read(&mut self) {
+ self.ranges.optimize_read();
+ self.prefixes.optimize_read();
+ }
+
/// Insert a batch of edges into the store.
/// The edges *must* have been de-duplicated by their from/to node.
/// I.e. if the store is not reversed, there should only ever be a single
@@ -240,9 +254,10 @@ impl EdgeStore {
|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },
);
batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });
+ let batch_len = batch.len();
s.put(&batch);
batch.clear();
- inserts_since_last_flush += 1;
+ inserts_since_last_flush += batch_len;
if inserts_since_last_flush >= 1_000_000 {
s.flush();
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index 4166a53e..5b2f4280 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-pub const MAX_BATCH_SIZE: usize = 5_000_000;
+pub const MAX_BATCH_SIZE: usize = 3_000_000;
use std::{
collections::BTreeSet,