web search done right
no need to collect all page nodes when calculating centrality. there will be a few duplicates when iterating but that's okay as their hyperloglogs will stay the same
Mikkel Denker3e683b02f806
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rsindex 32c75d9c..fab31c7f 100644--- a/crates/bloom/src/lib.rs+++ b/crates/bloom/src/lib.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License asdiff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex 8def6de7..03b95077 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-use indicatif::ParallelProgressIterator;+use indicatif::ProgressIterator;use itertools::Itertools;use std::{cmp, collections::BTreeMap};@@ -119,15 +119,17 @@ impl Mapper for ApproxCentralityMapper {pool.install(|| match self {ApproxCentralityMapper::InitCentrality => {- let num_nodes = worker.graph().page_nodes().len();- let nodes: Vec<_> = worker.graph().page_nodes().into_iter().collect();- nodes- .par_chunks(BATCH_SIZE)- .progress_count(num_nodes as u64 / BATCH_SIZE as u64)+ let num_nodes = worker.num_nodes();+ worker+ .graph()+ .page_nodes()+ .chunks(BATCH_SIZE)+ .into_iter()+ .progress_count(num_nodes / BATCH_SIZE as u64).for_each(|chunk| {let pairs: Vec<_> = chunk- .iter()- .map(|node| (*node, KahanSum::from(0.0)))+ .into_iter()+ .map(|node| (node, KahanSum::from(0.0))).collect();dht.next().centrality.batch_set(pairs)diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rsindex 46e3ba4d..ae67202c 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs@@ -21,6 +21,7 @@ use crate::{member::{Service, ShardId},sonic,},+ hyperloglog::HyperLogLog,webgraph::{self, Webgraph},Result,};@@ -32,13 +33,20 @@ use super::{impl_worker, ApproxCentralityJob, Message, RemoteWorker, Worker as _pub struct ApproxCentralityWorker {shard: ShardId,graph: Arc<Webgraph>,+ num_nodes: u64,}impl ApproxCentralityWorker {pub fn new(graph: Webgraph, shard: ShardId) -> Self {+ let num_nodes = graph+ .page_nodes()+ .map(|node| node.as_u128() as u64)+ .collect::<HyperLogLog<2048>>()+ .size() as u64;Self {graph: Arc::new(graph),shard,+ num_nodes,}}@@ -49,6 +57,10 @@ impl ApproxCentralityWorker {pub fn shard(&self) -> ShardId {self.shard}++ pub fn num_nodes(&self) -> u64 {+ self.num_nodes+ }}#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]@@ -58,7 +70,7 @@ impl Message<ApproxCentralityWorker> for NumNodes {type Response = u64;fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {- worker.graph.page_nodes().len() as u64+ worker.num_nodes}}diff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rsindex 571b7a77..e7745b49 100644--- a/crates/core/src/hyperloglog.rs+++ b/crates/core/src/hyperloglog.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -4351,6 +4351,16 @@ impl<H: HyperLogLogHasher, const N: usize> Default for HyperLogLog<N, H> {}}+impl<H: HyperLogLogHasher, const N: usize> FromIterator<u64> for HyperLogLog<N, H> {+ fn from_iter<T: IntoIterator<Item = u64>>(iter: T) -> Self {+ let mut hll = Self::default();+ for item in iter {+ hll.add(item);+ }+ hll+ }+}+impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {#[inline]fn am(&self) -> f64 {diff --git a/crates/core/src/webgraph/centrality/approx_harmonic.rs b/crates/core/src/webgraph/centrality/approx_harmonic.rsindex c0d50a5f..0ce9a8e9 100644--- a/crates/core/src/webgraph/centrality/approx_harmonic.rs+++ b/crates/core/src/webgraph/centrality/approx_harmonic.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -22,6 +22,7 @@ use rayon::prelude::*;use crate::{config::WebgraphGranularity,+ hyperloglog::HyperLogLog,webgraph::{NodeID, ShortestPaths, Webgraph},};@@ -30,14 +31,18 @@ const EPSILON: f64 = 0.3;// Approximate harmonic centrality by sampling O(log n / epsilon^2) nodes and// computing single-source shortest paths from each of them.//-// Epsilong is set to 0.3.+// Epsilon is set to 0.3.pub struct ApproxHarmonic {inner: speedy_kv::Db<NodeID, f64>,}impl ApproxHarmonic {pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {- let num_nodes = graph.page_nodes().len();+ let num_nodes = graph+ .page_nodes()+ .map(|node| node.as_u128() as u64)+ .collect::<HyperLogLog<2048>>()+ .size() as u64;tracing::info!("found approximately {} nodes in graph", num_nodes);diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 6ac81727..c32c4ff0 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -158,8 +158,8 @@ impl Webgraph {self.store.iter_host_node_ids(0, u32::MAX).collect()}- pub fn page_nodes(&self) -> FxHashSet<NodeID> {- self.store.iter_page_node_ids(0, u32::MAX).collect()+ pub fn page_nodes(&self) -> impl Iterator<Item = NodeID> + '_ {+ self.store.iter_page_node_ids(0, u32::MAX)}pub fn random_page_nodes_with_outgoing(&self, num: usize) -> Vec<NodeID> {