Skip to content

web search done right

no need to collect all page nodes when calculating centrality. there will be a few duplicates when iterating but that's okay as their hyperloglogs will stay the same

Mikkel Denker3e683b02f806
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rs
index 32c75d9c..fab31c7f 100644
--- a/crates/bloom/src/lib.rs
+++ b/crates/bloom/src/lib.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index 8def6de7..03b95077 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
-use indicatif::ParallelProgressIterator;
+use indicatif::ProgressIterator;
use itertools::Itertools;
use std::{cmp, collections::BTreeMap};
@@ -119,15 +119,17 @@ impl Mapper for ApproxCentralityMapper {
pool.install(|| match self {
ApproxCentralityMapper::InitCentrality => {
- let num_nodes = worker.graph().page_nodes().len();
- let nodes: Vec<_> = worker.graph().page_nodes().into_iter().collect();
- nodes
- .par_chunks(BATCH_SIZE)
- .progress_count(num_nodes as u64 / BATCH_SIZE as u64)
+ let num_nodes = worker.num_nodes();
+ worker
+ .graph()
+ .page_nodes()
+ .chunks(BATCH_SIZE)
+ .into_iter()
+ .progress_count(num_nodes / BATCH_SIZE as u64)
.for_each(|chunk| {
let pairs: Vec<_> = chunk
- .iter()
- .map(|node| (*node, KahanSum::from(0.0)))
+ .into_iter()
+ .map(|node| (node, KahanSum::from(0.0)))
.collect();
dht.next().centrality.batch_set(pairs)
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
index 46e3ba4d..ae67202c 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
@@ -21,6 +21,7 @@ use crate::{
member::{Service, ShardId},
sonic,
},
+ hyperloglog::HyperLogLog,
webgraph::{self, Webgraph},
Result,
};
@@ -32,13 +33,20 @@ use super::{impl_worker, ApproxCentralityJob, Message, RemoteWorker, Worker as _
pub struct ApproxCentralityWorker {
shard: ShardId,
graph: Arc<Webgraph>,
+ num_nodes: u64,
}
impl ApproxCentralityWorker {
pub fn new(graph: Webgraph, shard: ShardId) -> Self {
+ let num_nodes = graph
+ .page_nodes()
+ .map(|node| node.as_u128() as u64)
+ .collect::<HyperLogLog<2048>>()
+ .size() as u64;
Self {
graph: Arc::new(graph),
shard,
+ num_nodes,
}
}
@@ -49,6 +57,10 @@ impl ApproxCentralityWorker {
pub fn shard(&self) -> ShardId {
self.shard
}
+
+ pub fn num_nodes(&self) -> u64 {
+ self.num_nodes
+ }
}
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
@@ -58,7 +70,7 @@ impl Message<ApproxCentralityWorker> for NumNodes {
type Response = u64;
fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {
- worker.graph.page_nodes().len() as u64
+ worker.num_nodes
}
}
diff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rs
index 571b7a77..e7745b49 100644
--- a/crates/core/src/hyperloglog.rs
+++ b/crates/core/src/hyperloglog.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -4351,6 +4351,16 @@ impl<H: HyperLogLogHasher, const N: usize> Default for HyperLogLog<N, H> {
}
}
+impl<H: HyperLogLogHasher, const N: usize> FromIterator<u64> for HyperLogLog<N, H> {
+ fn from_iter<T: IntoIterator<Item = u64>>(iter: T) -> Self {
+ let mut hll = Self::default();
+ for item in iter {
+ hll.add(item);
+ }
+ hll
+ }
+}
+
impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {
#[inline]
fn am(&self) -> f64 {
diff --git a/crates/core/src/webgraph/centrality/approx_harmonic.rs b/crates/core/src/webgraph/centrality/approx_harmonic.rs
index c0d50a5f..0ce9a8e9 100644
--- a/crates/core/src/webgraph/centrality/approx_harmonic.rs
+++ b/crates/core/src/webgraph/centrality/approx_harmonic.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -22,6 +22,7 @@ use rayon::prelude::*;
use crate::{
config::WebgraphGranularity,
+ hyperloglog::HyperLogLog,
webgraph::{NodeID, ShortestPaths, Webgraph},
};
@@ -30,14 +31,18 @@ const EPSILON: f64 = 0.3;
// Approximate harmonic centrality by sampling O(log n / epsilon^2) nodes and
// computing single-source shortest paths from each of them.
//
-// Epsilong is set to 0.3.
+// Epsilon is set to 0.3.
pub struct ApproxHarmonic {
inner: speedy_kv::Db<NodeID, f64>,
}
impl ApproxHarmonic {
pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {
- let num_nodes = graph.page_nodes().len();
+ let num_nodes = graph
+ .page_nodes()
+ .map(|node| node.as_u128() as u64)
+ .collect::<HyperLogLog<2048>>()
+ .size() as u64;
tracing::info!("found approximately {} nodes in graph", num_nodes);
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 6ac81727..c32c4ff0 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -158,8 +158,8 @@ impl Webgraph {
self.store.iter_host_node_ids(0, u32::MAX).collect()
}
- pub fn page_nodes(&self) -> FxHashSet<NodeID> {
- self.store.iter_page_node_ids(0, u32::MAX).collect()
+ pub fn page_nodes(&self) -> impl Iterator<Item = NodeID> + '_ {
+ self.store.iter_page_node_ids(0, u32::MAX)
}
pub fn random_page_nodes_with_outgoing(&self, num: usize) -> Vec<NodeID> {