web search done right
calculate harmonic centrality using hyperball
Mikkel Denker769890a64bc3
diff --git a/src/hyperloglog.rs b/src/hyperloglog.rsindex 811ff243..5e701120 100644--- a/src/hyperloglog.rs+++ b/src/hyperloglog.rs@@ -14,6 +14,118 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-struct HyperLogLog<const N: usize = 16> {+use std::array;++#[derive(Clone)]+pub struct HyperLogLog<const N: usize> {registers: [u8; N],}++impl<const N: usize> Default for HyperLogLog<N> {+ fn default() -> Self {+ Self {+ registers: array::from_fn(|_| 0),+ }+ }+}++impl<const N: usize> HyperLogLog<N> {+ fn hash(item: u64) -> u64 {+ item.wrapping_mul(11400714819323198549)+ }++ fn am(&self) -> f64 {+ let m = self.registers.len();++ if m >= 128 {+ 0.7213 / (1. + 1.079 / (m as f64))+ } else if m >= 64 {+ 0.709+ } else if m >= 32 {+ 0.697+ } else {+ 0.673+ }+ }++ pub fn add(&mut self, item: u64) {+ let b = (N as f64).log2() as usize;+ let hash = Self::hash(item) as usize;++ let index_mask = (1usize << b) - 1;+ let i = hash & index_mask;+ let p = ((hash & !index_mask).leading_zeros() + 1) as u8;++ self.registers[i] = self.registers[i].max(p);+ }++ pub fn size(&self) -> usize {+ let m = self.registers.len();++ let z = 1f64+ / self+ .registers+ .iter()+ .map(|&val| 2_f64.powi(-(i32::from(val))))+ .sum::<f64>();++ (self.am() * (m as f64).powi(2) * z) as usize+ }++ #[cfg(test)]+ pub fn relative_error(&self) -> f64 {+ (3f64 * 2f64.ln() - 1f64).sqrt() / (self.registers.len() as f64).sqrt()+ }++ pub fn merge(&mut self, other: &Self) {+ for i in 0..N {+ self.registers[i] = self.registers[i].max(other.registers[i]);+ }+ }++ pub fn registers(&self) -> &[u8] {+ &self.registers+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn size_estimate_within_bounds() {+ let mut set: HyperLogLog<16> = HyperLogLog::default();++ for item in 0..10_000_000 {+ set.add(item);+ }++ let delta = (set.relative_error() * (set.size() as f64)) as usize;+ let lower_bound = set.size() - delta;+ let upper_bound = set.size() + delta;++ assert!(set.size() > lower_bound && set.size() < upper_bound);+ }++ #[test]+ fn merge() {+ let mut without_merge: HyperLogLog<16> = HyperLogLog::default();++ let mut a: HyperLogLog<16> = HyperLogLog::default();+ let mut b: HyperLogLog<16> = HyperLogLog::default();++ for item in 0..10_000 {+ without_merge.add(item);+ a.add(item);+ }++ for item in 10_001..20_000 {+ without_merge.add(item);+ b.add(item);+ }++ a.merge(&b);++ assert_eq!(a.registers, without_merge.registers);+ }+}diff --git a/src/intmap.rs b/src/intmap.rsindex 4b9cb967..f6944f4b 100644--- a/src/intmap.rs+++ b/src/intmap.rs@@ -24,6 +24,15 @@ pub struct IntMap<V> {len: usize,}+impl<V: Clone> Clone for IntMap<V> {+ fn clone(&self) -> Self {+ Self {+ bins: self.bins.clone(),+ len: self.len,+ }+ }+}+impl<V> IntMap<V> {pub fn new() -> Self {let bins = vec![Vec::new(), Vec::new()];@@ -77,6 +86,26 @@ impl<V> IntMap<V> {.find(|(stored_key, _)| stored_key == key).map(|(_, val)| val)}++ pub fn get_mut(&mut self, key: &Key) -> Option<&mut V> {+ let bin = self.bin_idx(key);+ self.bins[bin]+ .iter_mut()+ .find(|(stored_key, _)| stored_key == key)+ .map(|(_, val)| val)+ }++ pub fn len(&self) -> usize {+ self.len+ }++ pub fn into_iter(self) -> impl Iterator<Item = (Key, V)> {+ self.bins.into_iter().flat_map(|bin| bin.into_iter())+ }++ pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut (Key, V)> {+ self.bins.iter_mut().flat_map(|bin| bin.iter_mut())+ }}impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rsindex 4b1cd609..05fc0c44 100644--- a/src/webgraph/centrality/harmonic.rs+++ b/src/webgraph/centrality/harmonic.rs@@ -16,46 +16,87 @@use std::collections::HashMap;-use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};use tracing::info;-use crate::webgraph::{graph_store::GraphStore, Node, NodeID, Store, Webgraph};+use crate::{+ hyperloglog::HyperLogLog,+ intmap::IntMap,+ webgraph::{graph_store::GraphStore, Node, Store, Webgraph},+};pub struct HarmonicCentrality {pub full: HashMap<Node, f64>,pub host: HashMap<Node, f64>,}-fn calculate_centrality<S, F>(graph: &GraphStore<S>, node_distances: F) -> HashMap<Node, f64>+fn calculate_centrality<S>(graph: &GraphStore<S>) -> HashMap<Node, f64>whereS: Store,- F: Fn(Node) -> HashMap<NodeID, usize>,{let nodes: Vec<_> = graph.nodes().collect();info!("Found {} nodes in the graph", nodes.len());- let pb = ProgressBar::new(nodes.len() as u64);- pb.set_style(- ProgressStyle::default_bar()- .template("{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})")- .progress_chars("#>-"),- );let norm_factor = (nodes.len() - 1) as f64;- nodes++ let mut counters: IntMap<HyperLogLog<16>> = nodes.iter()- .progress_with(pb)- .map(|node_id| {- let node = graph.id2node(node_id).expect("unknown node");- let centrality_values: HashMap<NodeID, f64> = node_distances(node.clone())- .into_iter()- .filter(|(other_id, _)| *other_id != *node_id)- .map(|(other_node, dist)| (other_node, 1f64 / dist as f64))- .collect();-- let centrality = centrality_values.into_values().sum::<f64>() / norm_factor;-- (node, centrality)+ .map(|node| {+ let mut counter = HyperLogLog::default();+ counter.add(*node);++ (*node, counter)})+ .collect();++ let mut counter_changes = counters.len();+ let mut t = 0;+ let mut centralities: IntMap<f64> = nodes.iter().map(|node| (*node, 0.0)).collect();++ loop {+ if counter_changes == 0 {+ break;+ }++ counter_changes = 0;+ let mut new_counters: IntMap<HyperLogLog<16>> = counters.clone();++ for edge in graph.edges() {+ let counter_to = new_counters.get_mut(&edge.to).unwrap();+ let counter_from = counters.get(&edge.from).unwrap();++ if counter_to+ .registers()+ .iter()+ .zip(counter_from.registers().iter())+ .any(|(to, from)| *from > *to)+ {+ counter_to.merge(counter_from);+ counter_changes += 1;+ }+ }++ for (node, score) in centralities.iter_mut() {+ *score += new_counters+ .get(node)+ .map(|counter| counter.size())+ .unwrap_or_default()+ .checked_sub(+ counters+ .get(node)+ .map(|counter| counter.size())+ .unwrap_or_default(),+ )+ .unwrap_or_default() as f64+ / (t + 1) as f64;+ }++ counters = new_counters;+ t += 1;+ }++ centralities+ .into_iter().filter(|(_, centrality)| *centrality > 0.0)+ .map(|(node_id, centrality)| (graph.id2node(&node_id).unwrap(), centrality / norm_factor)).collect()}@@ -63,9 +104,7 @@ fn calculate_full(graph: &Webgraph) -> HashMap<Node, f64> {graph.full.as_ref()- .map(|full_graph| {- calculate_centrality(full_graph, |node| graph.raw_reversed_distances(node))- })+ .map(calculate_centrality).unwrap_or_default()}@@ -73,9 +112,7 @@ fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {graph.host.as_ref()- .map(|host_graph| {- calculate_centrality(host_graph, |node| graph.raw_host_reversed_distances(node))- })+ .map(calculate_centrality).unwrap_or_default()}@@ -163,16 +200,15 @@ mod tests {let centrality = HarmonicCentrality::calculate(&graph);- assert_eq!(centrality.full.get(&Node::from("C")).unwrap(), &1.0);- assert_eq!(centrality.full.get(&Node::from("D")), None);- assert_eq!(- (*centrality.full.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,- 0.67+ assert!(+ centrality.full.get(&Node::from("C")).unwrap()+ > centrality.full.get(&Node::from("A")).unwrap());- assert_eq!(- (*centrality.full.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,- 0.61+ assert!(+ centrality.full.get(&Node::from("A")).unwrap()+ > centrality.full.get(&Node::from("B")).unwrap());+ assert_eq!(centrality.full.get(&Node::from("D")), None);}#[test]@@ -184,11 +220,13 @@ mod tests {graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());+ graph.insert(Node::from("C.com"), Node::from("A.com"), String::new());+ graph.insert(Node::from("C.com"), Node::from("www.A.com"), String::new());graph.flush();let centrality = HarmonicCentrality::calculate(&graph);- assert_eq!(centrality.host.get(&Node::from("A.com")), Some(&1.0));+ assert!(centrality.host.get(&Node::from("A.com")).is_some());assert_eq!(centrality.host.get(&Node::from("www.A.com")), None);}@@ -207,15 +245,14 @@ mod tests {graph.flush();let centrality = HarmonicCentrality::calculate(&graph);- assert_eq!(centrality.full.get(&Node::from("C")).unwrap(), &1.0);- assert_eq!(centrality.full.get(&Node::from("D")), None);- assert_eq!(- (*centrality.full.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,- 0.67+ assert!(+ centrality.full.get(&Node::from("C")).unwrap()+ > centrality.full.get(&Node::from("A")).unwrap());- assert_eq!(- (*centrality.full.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,- 0.61+ assert!(+ centrality.full.get(&Node::from("A")).unwrap()+ > centrality.full.get(&Node::from("B")).unwrap());+ assert_eq!(centrality.full.get(&Node::from("D")), None);}}