Skip to content

web search done right

harmonic centrality exact counting threshold

Mikkel Denker0ea90aefdd29
diff --git a/core/src/intmap.rs b/core/src/intmap.rs
index e5fae824..02bbdab6 100644
--- a/core/src/intmap.rs
+++ b/core/src/intmap.rs
@@ -150,6 +150,7 @@ impl<V> Default for IntMap<V> {
}
}
+#[derive(Default)]
pub struct IntSet {
map: IntMap<()>,
}
@@ -166,6 +167,10 @@ impl IntSet {
pub fn into_iter(self) -> impl Iterator<Item = Key> {
self.map.into_iter().map(|(key, _)| key)
}
+
+ pub fn len(&self) -> usize {
+ self.map.len()
+ }
}
#[cfg(test)]
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index 9bf34e3a..e8d3376f 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -21,12 +21,13 @@ use tracing::info;
use crate::{
hyperloglog::HyperLogLog,
- intmap::IntMap,
+ intmap::{IntMap, IntSet},
kahan_sum::KahanSum,
webgraph::{Node, NodeID, Webgraph},
};
const HYPERLOGLOG_COUNTERS: usize = 64;
+const EXACT_COUNTING_THRESHOLD: u64 = 1_000_000;
#[derive(Clone)]
struct JankyBloomFilter {
@@ -60,6 +61,22 @@ impl JankyBloomFilter {
let h = Self::hash(item);
self.bit_vec[h % self.num_bits as usize]
}
+
+ pub fn estimate_card(&self) -> u64 {
+ let num_ones = self.bit_vec.count_ones() as u64;
+
+ if num_ones == 0 || self.num_bits == 0 {
+ return 0;
+ }
+
+ if num_ones == self.num_bits {
+ return u64::MAX;
+ }
+
+ (-(self.num_bits as i64) * (1.0 - (num_ones as f64) / (self.num_bits as f64)).ln() as i64)
+ .try_into()
+ .unwrap_or_default()
+ }
}
pub struct HarmonicCentrality {
@@ -81,45 +98,81 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {
})
.collect();
- let mut counter_changes = counters.len() as u64;
+ let mut exact_counting = false;
+ let mut has_changes = true;
let mut t = 0;
let mut centralities: IntMap<KahanSum> = nodes
.iter()
.map(|node| (node.0, KahanSum::default()))
.collect();
+ let mut exact_changed_nodes = IntSet::default();
let mut changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);
for node in &nodes {
changed_nodes.insert(node.0);
}
loop {
- if counter_changes == 0 {
+ if !has_changes {
break;
}
let mut new_counters: IntMap<_> = counters.clone();
- counter_changes = 0;
+ has_changes = false;
let mut new_changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);
- for edge in graph.edges() {
- if !changed_nodes.contains(&edge.from.0) {
- continue;
+ if exact_changed_nodes.len() > 0 {
+ let mut new_exact_changed_nodes = IntSet::default();
+
+ for changed_node in exact_changed_nodes.into_iter() {
+ for edge in graph.raw_outgoing_edges(&NodeID(changed_node)) {
+ if let (Some(counter_to), Some(counter_from)) =
+ (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))
+ {
+ if counter_to
+ .registers()
+ .iter()
+ .zip(counter_from.registers().iter())
+ .any(|(to, from)| *from > *to)
+ {
+ counter_to.merge(counter_from);
+ new_changed_nodes.insert(edge.to.0);
+
+ new_exact_changed_nodes.insert(edge.to.0);
+
+ has_changes = true;
+ }
+ }
+ }
}
- if let (Some(counter_to), Some(counter_from)) =
- (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))
- {
- if counter_to
- .registers()
- .iter()
- .zip(counter_from.registers().iter())
- .any(|(to, from)| *from > *to)
+ exact_changed_nodes = new_exact_changed_nodes;
+ } else {
+ exact_changed_nodes = IntSet::default();
+ for edge in graph.edges() {
+ if !changed_nodes.contains(&edge.from.0) {
+ continue;
+ }
+
+ if let (Some(counter_to), Some(counter_from)) =
+ (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))
{
- counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.0);
- counter_changes += 1;
+ if counter_to
+ .registers()
+ .iter()
+ .zip(counter_from.registers().iter())
+ .any(|(to, from)| *from > *to)
+ {
+ counter_to.merge(counter_from);
+ new_changed_nodes.insert(edge.to.0);
+
+ if exact_counting {
+ exact_changed_nodes.insert(edge.to.0);
+ }
+
+ has_changes = true;
+ }
}
}
}
@@ -142,6 +195,10 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {
counters = new_counters;
changed_nodes = new_changed_nodes;
t += 1;
+
+ if changed_nodes.estimate_card() <= EXACT_COUNTING_THRESHOLD {
+ exact_counting = true;
+ }
}
centralities
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
deleted file mode 100644
index a66f9c5b..00000000
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ /dev/null
@@ -1,586 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-//! Algorithm in broad strokes
-//! We first calculate the betweenness centrality for all nodes an choose top k as proxy nodes.
-//! The distances from every node to every proxy node and reverse is then calculated and stored.
-//!
-//! During search the user can choose a set of liked and disliked nodes. For each user node, we find the best k proxy nodes.
-//! For all proxy nodes we then estimate the harmonic centrality from the liked nodes - disliked nodes by estimating
-//! the shortest distance as the distance through their best proxy node. This distance estimate is very fast since
-//! all distances are pre-computed for the proxy nodes.
-
-use std::cmp;
-use std::collections::{BTreeMap, BinaryHeap, HashMap};
-use std::fs::File;
-use std::io::{BufReader, BufWriter, Read};
-use std::path::Path;
-use std::sync::Arc;
-
-use dashmap::DashMap;
-use serde::{Deserialize, Serialize};
-
-use crate::intmap::IntMap;
-use crate::ranking::centrality_store::HarmonicCentralityStore;
-use crate::webgraph::{Edge, Node};
-use crate::webgraph::{NodeID, Webgraph};
-use crate::Result;
-
-const NUM_PROXY_NODES: usize = 1000;
-const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;
-const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
-const MAX_DIST_PROXY: u8 = 10;
-const MAX_NUM_OUT_EDGES: usize = 1000;
-
-#[derive(Serialize, Deserialize)]
-pub struct ProxyNode {
- pub id: NodeID,
- pub dist_from_node: IntMap<u8>, // from node to proxy node
- pub dist_to_node: IntMap<u8>, // from proxy node to other node
-}
-
-impl ProxyNode {
- fn dist(&self, from: &NodeID, to: &NodeID) -> Option<u8> {
- if self.id == *from || self.id == *to {
- return None;
- }
-
- if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {
- if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {
- return Some(from_node_to_proxy + from_proxy_to_node);
- }
- }
-
- None
- }
-}
-
-struct WeightedProxyNode {
- node: Arc<ProxyNode>,
- dist: u8,
-}
-
-impl PartialOrd for WeightedProxyNode {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- self.dist.partial_cmp(&other.dist)
- }
-}
-impl Ord for WeightedProxyNode {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
- }
-}
-
-impl PartialEq for WeightedProxyNode {
- fn eq(&self, other: &Self) -> bool {
- self.dist == other.dist
- }
-}
-
-impl Eq for WeightedProxyNode {}
-
-pub struct Scorer {
- fixed_scores: HashMap<NodeID, f64>,
- liked_nodes: Vec<UserNode>,
- disliked_nodes: Vec<UserNode>,
- cache: DashMap<NodeID, f64>,
-}
-
-fn create_user_nodes(nodes: &[NodeID], proxy_nodes: &[Arc<ProxyNode>]) -> Vec<UserNode> {
- let mut res = Vec::new();
-
- let nodes = nodes.to_vec();
-
- for id in nodes {
- let user_node = UserNode::new(id, proxy_nodes);
- if res.len() < USER_NODES_LIMIT {
- res.push(user_node);
- } else {
- let best = res
- .iter_mut()
- .min_by_key(|curr| user_node.best_dist(&curr.id).unwrap_or(255))
- .unwrap();
-
- best.weight += 1;
- }
- }
-
- res
-}
-
-impl Scorer {
- fn new(
- proxy_nodes: &[Arc<ProxyNode>],
- liked_nodes: &[NodeID],
- disliked_nodes: &[NodeID],
- fixed_scores: HashMap<NodeID, f64>,
- ) -> Self {
- Self {
- fixed_scores,
- liked_nodes: create_user_nodes(liked_nodes, proxy_nodes),
- disliked_nodes: create_user_nodes(disliked_nodes, proxy_nodes),
- cache: DashMap::new(),
- }
- }
-
- pub fn score(&self, node: NodeID) -> f64 {
- let norm = self.liked_nodes.len() as f64 + self.disliked_nodes.len() as f64;
- if norm == 0.0 {
- return 0.0;
- }
-
- if let Some(score) = self.fixed_scores.get(&node) {
- *score
- } else {
- if let Some(cached) = self.cache.get(&node) {
- return *cached;
- }
- let res = (self.disliked_nodes.len() as f64
- + (self
- .liked_nodes
- .iter()
- .filter_map(|liked_node| {
- liked_node.best_dist(&node).map(|dist| (dist, liked_node))
- })
- .map(|(dist, liked_node)| liked_node.weight as f64 / (dist as f64 + 1.0))
- .sum::<f64>()
- - self
- .disliked_nodes
- .iter()
- .filter_map(|disliked_node| {
- disliked_node
- .best_dist(&node)
- .map(|dist| (dist, disliked_node))
- })
- .map(|(dist, disliked_node)| {
- disliked_node.weight as f64 / (dist as f64 + 1.0)
- })
- .sum::<f64>()))
- .max(0.0)
- / norm;
-
- let res = if res.is_finite() { res } else { 0.0 };
- self.cache.insert(node, res);
-
- res
- }
- }
-}
-
-struct UserNode {
- id: NodeID,
- proxy_nodes: Vec<Arc<ProxyNode>>,
- weight: usize,
-}
-
-impl UserNode {
- fn new(id: NodeID, proxy_nodes: &[Arc<ProxyNode>]) -> Self {
- let mut heap: BinaryHeap<WeightedProxyNode> =
- BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);
-
- for proxy in proxy_nodes {
- if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id.0) {
- let weighted_node = WeightedProxyNode {
- node: Arc::clone(proxy),
- dist: *dist_to_proxy,
- };
-
- if heap.len() >= BEST_PROXY_NODES_PER_USER_NODE {
- if let Some(mut worst) = heap.peek_mut() {
- if worst.dist > weighted_node.dist {
- *worst = weighted_node;
- }
- }
- } else {
- heap.push(weighted_node);
- }
- }
- }
-
- Self {
- id,
- weight: 1,
- proxy_nodes: heap.into_iter().map(|weighted| weighted.node).collect(),
- }
- }
-
- fn best_dist(&self, node: &NodeID) -> Option<u8> {
- let mut best = None;
-
- for proxy in &self.proxy_nodes {
- if let Some(dist) = proxy.dist(&self.id, node) {
- best = match best {
- Some((best_dist, best_proxy)) => {
- if dist < best_dist {
- Some((dist, proxy))
- } else {
- Some((best_dist, best_proxy))
- }
- }
- None => Some((dist, proxy)),
- }
- }
- }
-
- best.map(|(dist, _)| dist)
- }
-}
-
-struct ProxyNodeCandidate {
- node: Node,
- score: f64,
-}
-
-impl PartialOrd for ProxyNodeCandidate {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- self.score.partial_cmp(&other.score)
- }
-}
-
-impl PartialEq for ProxyNodeCandidate {
- fn eq(&self, other: &Self) -> bool {
- self.score == other.score
- }
-}
-
-impl Ord for ProxyNodeCandidate {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
- }
-}
-
-impl Eq for ProxyNodeCandidate {}
-
-fn modified_dijkstra<F1, F2>(
- source: Node,
- node_edges: F1,
- edge_node: F2,
- graph: &Webgraph,
-) -> BTreeMap<NodeID, u8>
-where
- F1: Fn(NodeID) -> Vec<Edge>,
- F2: Fn(&Edge) -> NodeID,
-{
- let source_id = graph.node2id(&source).copied();
-
- if source_id.is_none() {
- return BTreeMap::new();
- }
-
- let source_id = source_id.unwrap();
- let mut distances: BTreeMap<NodeID, u8> = BTreeMap::default();
-
- let mut queue = BinaryHeap::new();
-
- queue.push(cmp::Reverse((0, source_id)));
- distances.insert(source_id, 0);
-
- while let Some(state) = queue.pop() {
- let (cost, v) = state.0;
-
- if cost >= MAX_DIST_PROXY {
- continue;
- }
-
- let current_dist = distances.get(&v).unwrap_or(&u8::MAX);
-
- if cost > *current_dist {
- continue;
- }
-
- let mut out_edges_taken = 0;
- for edge in node_edges(v).into_iter() {
- if out_edges_taken >= MAX_NUM_OUT_EDGES {
- break;
- }
-
- if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {
- let d = cost + 1;
-
- if d > MAX_DIST_PROXY {
- continue;
- }
-
- out_edges_taken += 1;
- let next = cmp::Reverse((d, edge_node(&edge)));
- queue.push(next);
- distances.insert(edge_node(&edge), d);
- }
- }
- }
-
- distances
-}
-
-fn distances(graph: &Webgraph, source: Node) -> BTreeMap<NodeID, u8> {
- modified_dijkstra(
- source,
- |node| graph.raw_outgoing_edges(&node),
- |edge| edge.to,
- graph,
- )
-}
-
-fn reversed_distances(graph: &Webgraph, source: Node) -> BTreeMap<NodeID, u8> {
- modified_dijkstra(
- source,
- |node| graph.raw_ingoing_edges(&node),
- |edge| edge.from,
- graph,
- )
-}
-
-#[derive(Serialize, Deserialize, Default)]
-pub struct OnlineHarmonicCentrality {
- pub proxy_nodes: Vec<Arc<ProxyNode>>,
-}
-
-impl OnlineHarmonicCentrality {
- pub fn new(graph: &Webgraph, centrality: &HarmonicCentralityStore) -> Self {
- Self::new_with_num_proxy(graph, centrality, NUM_PROXY_NODES)
- }
-
- fn new_with_num_proxy(
- graph: &Webgraph,
- centrality: &HarmonicCentralityStore,
- num_proxy_nodes: usize,
- ) -> Self {
- // we should probably choose the proxy nodes based on their
- // betweenness centrality, but I don't know how we can approximate betweenness
- // on a graph that cannot be in memory.
- // For now, we will choose the nodes with highest harmonic centrality
- // as an estimate
-
- let mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();
- for (node, node_id) in graph.node_ids().map(|(node, id)| (node.clone(), *id)) {
- let score = centrality.host.get(&node_id).unwrap_or(0.0);
- let candidate = ProxyNodeCandidate { node, score };
-
- if nodes.len() >= num_proxy_nodes {
- if let Some(mut existing_node) = nodes.peek_mut() {
- if candidate.score > existing_node.score {
- *existing_node = candidate;
- }
- }
- } else {
- nodes.push(candidate);
- }
- }
-
- let proxy_nodes: Vec<_> = nodes
- .into_iter()
- .map(|candidate| candidate.node)
- .take(num_proxy_nodes)
- .map(|node| {
- let dist_to_node = distances(graph, node.clone())
- .into_iter()
- .map(|(n, v)| (n.0, v))
- .collect();
-
- let dist_from_node = reversed_distances(graph, node.clone())
- .into_iter()
- .map(|(n, v)| (n.0, v))
- .collect();
-
- Arc::new(ProxyNode {
- id: *graph.node2id(&node).unwrap(),
- dist_to_node,
- dist_from_node,
- })
- })
- .collect();
-
- Self { proxy_nodes }
- }
-
- pub fn scorer(&self, liked_nodes: &[NodeID], disliked_nodes: &[NodeID]) -> Scorer {
- Scorer::new(
- &self.proxy_nodes,
- liked_nodes,
- disliked_nodes,
- HashMap::new(),
- )
- }
-
- pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
- let mut file = BufWriter::new(
- File::options()
- .create(true)
- .truncate(true)
- .write(true)
- .open(path)?,
- );
-
- bincode::serialize_into(&mut file, &self)?;
-
- Ok(())
- }
-
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let file = File::open(path)?;
- let mut reader = BufReader::new(file);
-
- let mut buf = Vec::new();
- reader.read_to_end(&mut buf)?;
-
- Ok(bincode::deserialize(&buf)?)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::webgraph::{centrality::harmonic::HarmonicCentrality, WebgraphBuilder};
-
- fn test_graph() -> Webgraph {
- /*
- G ◄---- F
- | |
- | ▼
- ------► E -------► H
- ▲ |
- A -- | |
- | | ▼
- B ----► D ◄------- C
- */
-
- let mut graph = WebgraphBuilder::new_memory().open();
-
- graph.insert(Node::from("A"), Node::from("D"), String::new());
- graph.insert(Node::from("B"), Node::from("D"), String::new());
- graph.insert(Node::from("C"), Node::from("D"), String::new());
- graph.insert(Node::from("D"), Node::from("E"), String::new());
- graph.insert(Node::from("E"), Node::from("H"), String::new());
- graph.insert(Node::from("H"), Node::from("C"), String::new());
- graph.insert(Node::from("F"), Node::from("E"), String::new());
- graph.insert(Node::from("F"), Node::from("G"), String::new());
- graph.insert(Node::from("G"), Node::from("E"), String::new());
-
- graph.commit();
-
- graph
- }
-
- #[test]
- fn ordering() {
- let graph = test_graph();
- let harmonic = HarmonicCentrality::calculate(&graph);
-
- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
- for (node, centrality) in harmonic.host {
- harmonic_centrality_store
- .host
- .insert(*graph.node2id(&node).unwrap(), centrality);
- }
- harmonic_centrality_store.host.flush();
-
- let centrality =
- OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);
-
- let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())]
- .into_iter()
- .filter_map(|node| graph.node2id(&node).copied())
- .collect();
-
- let scorer = centrality.scorer(&liked_nodes, &[]);
-
- assert!(
- scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
- );
- assert!(
- scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
- );
-
- assert!(
- scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())
- );
- }
-
- #[test]
- fn disliked_nodes_centrality() {
- let graph = test_graph();
- let harmonic = HarmonicCentrality::calculate(&graph);
-
- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
- for (node, centrality) in harmonic.host {
- harmonic_centrality_store
- .host
- .insert(*graph.node2id(&node).unwrap(), centrality);
- }
- harmonic_centrality_store.host.flush();
-
- let centrality =
- OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);
-
- let disliked_nodes: Vec<_> = vec![Node::from("D".to_string()), Node::from("E".to_string())]
- .into_iter()
- .map(|node| *graph.node2id(&node).unwrap())
- .collect();
-
- let scorer = centrality.scorer(&[], &disliked_nodes);
-
- for node in &disliked_nodes {
- assert!(scorer.score(*node) < disliked_nodes.len() as f64);
- }
- }
-
- #[test]
- fn ordering_with_dislikes() {
- let graph = test_graph();
- let harmonic = HarmonicCentrality::calculate(&graph);
-
- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
- for (node, centrality) in harmonic.host {
- harmonic_centrality_store
- .host
- .insert(*graph.node2id(&node).unwrap(), centrality);
- }
- harmonic_centrality_store.host.flush();
-
- let centrality =
- OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);
-
- let liked_nodes: Vec<_> = vec![Node::from("B".to_string())]
- .into_iter()
- .filter_map(|node| graph.node2id(&node).copied())
- .collect();
-
- let disliked_nodes: Vec<_> = vec![Node::from("F".to_string())]
- .into_iter()
- .filter_map(|node| graph.node2id(&node).copied())
- .collect();
-
- let scorer = centrality.scorer(&liked_nodes, &disliked_nodes);
-
- assert!(
- scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
- );
-
- assert!(
- scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
- );
-
- assert!(
- scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
- );
- }
-}