Skip to content

web search done right

calculate harmonic centrality using hyperball

Mikkel Denker769890a64bc3
diff --git a/src/hyperloglog.rs b/src/hyperloglog.rs
index 811ff243..5e701120 100644
--- a/src/hyperloglog.rs
+++ b/src/hyperloglog.rs
@@ -14,6 +14,118 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-struct HyperLogLog<const N: usize = 16> {
+use std::array;
+
+#[derive(Clone)]
+pub struct HyperLogLog<const N: usize> {
registers: [u8; N],
}
+
+impl<const N: usize> Default for HyperLogLog<N> {
+ fn default() -> Self {
+ Self {
+ registers: array::from_fn(|_| 0),
+ }
+ }
+}
+
+impl<const N: usize> HyperLogLog<N> {
+ fn hash(item: u64) -> u64 {
+ item.wrapping_mul(11400714819323198549)
+ }
+
+ fn am(&self) -> f64 {
+ let m = self.registers.len();
+
+ if m >= 128 {
+ 0.7213 / (1. + 1.079 / (m as f64))
+ } else if m >= 64 {
+ 0.709
+ } else if m >= 32 {
+ 0.697
+ } else {
+ 0.673
+ }
+ }
+
+ pub fn add(&mut self, item: u64) {
+ let b = (N as f64).log2() as usize;
+ let hash = Self::hash(item) as usize;
+
+ let index_mask = (1usize << b) - 1;
+ let i = hash & index_mask;
+ let p = ((hash & !index_mask).leading_zeros() + 1) as u8;
+
+ self.registers[i] = self.registers[i].max(p);
+ }
+
+ pub fn size(&self) -> usize {
+ let m = self.registers.len();
+
+ let z = 1f64
+ / self
+ .registers
+ .iter()
+ .map(|&val| 2_f64.powi(-(i32::from(val))))
+ .sum::<f64>();
+
+ (self.am() * (m as f64).powi(2) * z) as usize
+ }
+
+ #[cfg(test)]
+ pub fn relative_error(&self) -> f64 {
+ (3f64 * 2f64.ln() - 1f64).sqrt() / (self.registers.len() as f64).sqrt()
+ }
+
+ pub fn merge(&mut self, other: &Self) {
+ for i in 0..N {
+ self.registers[i] = self.registers[i].max(other.registers[i]);
+ }
+ }
+
+ pub fn registers(&self) -> &[u8] {
+ &self.registers
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn size_estimate_within_bounds() {
+ let mut set: HyperLogLog<16> = HyperLogLog::default();
+
+ for item in 0..10_000_000 {
+ set.add(item);
+ }
+
+ let delta = (set.relative_error() * (set.size() as f64)) as usize;
+ let lower_bound = set.size() - delta;
+ let upper_bound = set.size() + delta;
+
+ assert!(set.size() > lower_bound && set.size() < upper_bound);
+ }
+
+ #[test]
+ fn merge() {
+ let mut without_merge: HyperLogLog<16> = HyperLogLog::default();
+
+ let mut a: HyperLogLog<16> = HyperLogLog::default();
+ let mut b: HyperLogLog<16> = HyperLogLog::default();
+
+ for item in 0..10_000 {
+ without_merge.add(item);
+ a.add(item);
+ }
+
+ for item in 10_001..20_000 {
+ without_merge.add(item);
+ b.add(item);
+ }
+
+ a.merge(&b);
+
+ assert_eq!(a.registers, without_merge.registers);
+ }
+}
diff --git a/src/intmap.rs b/src/intmap.rs
index 4b9cb967..f6944f4b 100644
--- a/src/intmap.rs
+++ b/src/intmap.rs
@@ -24,6 +24,15 @@ pub struct IntMap<V> {
len: usize,
}
+impl<V: Clone> Clone for IntMap<V> {
+ fn clone(&self) -> Self {
+ Self {
+ bins: self.bins.clone(),
+ len: self.len,
+ }
+ }
+}
+
impl<V> IntMap<V> {
pub fn new() -> Self {
let bins = vec![Vec::new(), Vec::new()];
@@ -77,6 +86,26 @@ impl<V> IntMap<V> {
.find(|(stored_key, _)| stored_key == key)
.map(|(_, val)| val)
}
+
+ pub fn get_mut(&mut self, key: &Key) -> Option<&mut V> {
+ let bin = self.bin_idx(key);
+ self.bins[bin]
+ .iter_mut()
+ .find(|(stored_key, _)| stored_key == key)
+ .map(|(_, val)| val)
+ }
+
+ pub fn len(&self) -> usize {
+ self.len
+ }
+
+ pub fn into_iter(self) -> impl Iterator<Item = (Key, V)> {
+ self.bins.into_iter().flat_map(|bin| bin.into_iter())
+ }
+
+ pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut (Key, V)> {
+ self.bins.iter_mut().flat_map(|bin| bin.iter_mut())
+ }
}
impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {
diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rs
index 4b1cd609..05fc0c44 100644
--- a/src/webgraph/centrality/harmonic.rs
+++ b/src/webgraph/centrality/harmonic.rs
@@ -16,46 +16,87 @@
use std::collections::HashMap;
-use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
use tracing::info;
-use crate::webgraph::{graph_store::GraphStore, Node, NodeID, Store, Webgraph};
+use crate::{
+ hyperloglog::HyperLogLog,
+ intmap::IntMap,
+ webgraph::{graph_store::GraphStore, Node, Store, Webgraph},
+};
pub struct HarmonicCentrality {
pub full: HashMap<Node, f64>,
pub host: HashMap<Node, f64>,
}
-fn calculate_centrality<S, F>(graph: &GraphStore<S>, node_distances: F) -> HashMap<Node, f64>
+fn calculate_centrality<S>(graph: &GraphStore<S>) -> HashMap<Node, f64>
where
S: Store,
- F: Fn(Node) -> HashMap<NodeID, usize>,
{
let nodes: Vec<_> = graph.nodes().collect();
info!("Found {} nodes in the graph", nodes.len());
- let pb = ProgressBar::new(nodes.len() as u64);
- pb.set_style(
- ProgressStyle::default_bar()
- .template("{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})")
- .progress_chars("#>-"),
- );
let norm_factor = (nodes.len() - 1) as f64;
- nodes
+
+ let mut counters: IntMap<HyperLogLog<16>> = nodes
.iter()
- .progress_with(pb)
- .map(|node_id| {
- let node = graph.id2node(node_id).expect("unknown node");
- let centrality_values: HashMap<NodeID, f64> = node_distances(node.clone())
- .into_iter()
- .filter(|(other_id, _)| *other_id != *node_id)
- .map(|(other_node, dist)| (other_node, 1f64 / dist as f64))
- .collect();
-
- let centrality = centrality_values.into_values().sum::<f64>() / norm_factor;
-
- (node, centrality)
+ .map(|node| {
+ let mut counter = HyperLogLog::default();
+ counter.add(*node);
+
+ (*node, counter)
})
+ .collect();
+
+ let mut counter_changes = counters.len();
+ let mut t = 0;
+ let mut centralities: IntMap<f64> = nodes.iter().map(|node| (*node, 0.0)).collect();
+
+ loop {
+ if counter_changes == 0 {
+ break;
+ }
+
+ counter_changes = 0;
+ let mut new_counters: IntMap<HyperLogLog<16>> = counters.clone();
+
+ for edge in graph.edges() {
+ let counter_to = new_counters.get_mut(&edge.to).unwrap();
+ let counter_from = counters.get(&edge.from).unwrap();
+
+ if counter_to
+ .registers()
+ .iter()
+ .zip(counter_from.registers().iter())
+ .any(|(to, from)| *from > *to)
+ {
+ counter_to.merge(counter_from);
+ counter_changes += 1;
+ }
+ }
+
+ for (node, score) in centralities.iter_mut() {
+ *score += new_counters
+ .get(node)
+ .map(|counter| counter.size())
+ .unwrap_or_default()
+ .checked_sub(
+ counters
+ .get(node)
+ .map(|counter| counter.size())
+ .unwrap_or_default(),
+ )
+ .unwrap_or_default() as f64
+ / (t + 1) as f64;
+ }
+
+ counters = new_counters;
+ t += 1;
+ }
+
+ centralities
+ .into_iter()
.filter(|(_, centrality)| *centrality > 0.0)
+ .map(|(node_id, centrality)| (graph.id2node(&node_id).unwrap(), centrality / norm_factor))
.collect()
}
@@ -63,9 +104,7 @@ fn calculate_full(graph: &Webgraph) -> HashMap<Node, f64> {
graph
.full
.as_ref()
- .map(|full_graph| {
- calculate_centrality(full_graph, |node| graph.raw_reversed_distances(node))
- })
+ .map(calculate_centrality)
.unwrap_or_default()
}
@@ -73,9 +112,7 @@ fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {
graph
.host
.as_ref()
- .map(|host_graph| {
- calculate_centrality(host_graph, |node| graph.raw_host_reversed_distances(node))
- })
+ .map(calculate_centrality)
.unwrap_or_default()
}
@@ -163,16 +200,15 @@ mod tests {
let centrality = HarmonicCentrality::calculate(&graph);
- assert_eq!(centrality.full.get(&Node::from("C")).unwrap(), &1.0);
- assert_eq!(centrality.full.get(&Node::from("D")), None);
- assert_eq!(
- (*centrality.full.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,
- 0.67
+ assert!(
+ centrality.full.get(&Node::from("C")).unwrap()
+ > centrality.full.get(&Node::from("A")).unwrap()
);
- assert_eq!(
- (*centrality.full.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,
- 0.61
+ assert!(
+ centrality.full.get(&Node::from("A")).unwrap()
+ > centrality.full.get(&Node::from("B")).unwrap()
);
+ assert_eq!(centrality.full.get(&Node::from("D")), None);
}
#[test]
@@ -184,11 +220,13 @@ mod tests {
graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());
graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());
+ graph.insert(Node::from("C.com"), Node::from("A.com"), String::new());
+ graph.insert(Node::from("C.com"), Node::from("www.A.com"), String::new());
graph.flush();
let centrality = HarmonicCentrality::calculate(&graph);
- assert_eq!(centrality.host.get(&Node::from("A.com")), Some(&1.0));
+ assert!(centrality.host.get(&Node::from("A.com")).is_some());
assert_eq!(centrality.host.get(&Node::from("www.A.com")), None);
}
@@ -207,15 +245,14 @@ mod tests {
graph.flush();
let centrality = HarmonicCentrality::calculate(&graph);
- assert_eq!(centrality.full.get(&Node::from("C")).unwrap(), &1.0);
- assert_eq!(centrality.full.get(&Node::from("D")), None);
- assert_eq!(
- (*centrality.full.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,
- 0.67
+ assert!(
+ centrality.full.get(&Node::from("C")).unwrap()
+ > centrality.full.get(&Node::from("A")).unwrap()
);
- assert_eq!(
- (*centrality.full.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,
- 0.61
+ assert!(
+ centrality.full.get(&Node::from("A")).unwrap()
+ > centrality.full.get(&Node::from("B")).unwrap()
);
+ assert_eq!(centrality.full.get(&Node::from("D")), None);
}
}