web search done right
harmonic centrality exact counting threshold
Mikkel Denker0ea90aefdd29
diff --git a/core/src/intmap.rs b/core/src/intmap.rsindex e5fae824..02bbdab6 100644--- a/core/src/intmap.rs+++ b/core/src/intmap.rs@@ -150,6 +150,7 @@ impl<V> Default for IntMap<V> {}}+#[derive(Default)]pub struct IntSet {map: IntMap<()>,}@@ -166,6 +167,10 @@ impl IntSet {pub fn into_iter(self) -> impl Iterator<Item = Key> {self.map.into_iter().map(|(key, _)| key)}++ pub fn len(&self) -> usize {+ self.map.len()+ }}#[cfg(test)]diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 9bf34e3a..e8d3376f 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -21,12 +21,13 @@ use tracing::info;use crate::{hyperloglog::HyperLogLog,- intmap::IntMap,+ intmap::{IntMap, IntSet},kahan_sum::KahanSum,webgraph::{Node, NodeID, Webgraph},};const HYPERLOGLOG_COUNTERS: usize = 64;+const EXACT_COUNTING_THRESHOLD: u64 = 1_000_000;#[derive(Clone)]struct JankyBloomFilter {@@ -60,6 +61,22 @@ impl JankyBloomFilter {let h = Self::hash(item);self.bit_vec[h % self.num_bits as usize]}++ pub fn estimate_card(&self) -> u64 {+ let num_ones = self.bit_vec.count_ones() as u64;++ if num_ones == 0 || self.num_bits == 0 {+ return 0;+ }++ if num_ones == self.num_bits {+ return u64::MAX;+ }++ (-(self.num_bits as i64) * (1.0 - (num_ones as f64) / (self.num_bits as f64)).ln() as i64)+ .try_into()+ .unwrap_or_default()+ }}pub struct HarmonicCentrality {@@ -81,45 +98,81 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {}).collect();- let mut counter_changes = counters.len() as u64;+ let mut exact_counting = false;+ let mut has_changes = true;let mut t = 0;let mut centralities: IntMap<KahanSum> = nodes.iter().map(|node| (node.0, KahanSum::default())).collect();+ let mut exact_changed_nodes = IntSet::default();let mut changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);for node in &nodes {changed_nodes.insert(node.0);}loop {- if counter_changes == 0 {+ if !has_changes {break;}let mut new_counters: IntMap<_> = counters.clone();- counter_changes = 0;+ has_changes = false;let mut new_changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);- for edge in graph.edges() {- if !changed_nodes.contains(&edge.from.0) {- continue;+ if exact_changed_nodes.len() > 0 {+ let mut new_exact_changed_nodes = IntSet::default();++ for changed_node in exact_changed_nodes.into_iter() {+ for edge in graph.raw_outgoing_edges(&NodeID(changed_node)) {+ if let (Some(counter_to), Some(counter_from)) =+ (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))+ {+ if counter_to+ .registers()+ .iter()+ .zip(counter_from.registers().iter())+ .any(|(to, from)| *from > *to)+ {+ counter_to.merge(counter_from);+ new_changed_nodes.insert(edge.to.0);++ new_exact_changed_nodes.insert(edge.to.0);++ has_changes = true;+ }+ }+ }}- if let (Some(counter_to), Some(counter_from)) =- (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))- {- if counter_to- .registers()- .iter()- .zip(counter_from.registers().iter())- .any(|(to, from)| *from > *to)+ exact_changed_nodes = new_exact_changed_nodes;+ } else {+ exact_changed_nodes = IntSet::default();+ for edge in graph.edges() {+ if !changed_nodes.contains(&edge.from.0) {+ continue;+ }++ if let (Some(counter_to), Some(counter_from)) =+ (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0)){- counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.0);- counter_changes += 1;+ if counter_to+ .registers()+ .iter()+ .zip(counter_from.registers().iter())+ .any(|(to, from)| *from > *to)+ {+ counter_to.merge(counter_from);+ new_changed_nodes.insert(edge.to.0);++ if exact_counting {+ exact_changed_nodes.insert(edge.to.0);+ }++ has_changes = true;+ }}}}@@ -142,6 +195,10 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {counters = new_counters;changed_nodes = new_changed_nodes;t += 1;++ if changed_nodes.estimate_card() <= EXACT_COUNTING_THRESHOLD {+ exact_counting = true;+ }}centralitiesdiff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsdeleted file mode 100644index a66f9c5b..00000000--- a/core/src/webgraph/centrality/online_harmonic.rs+++ /dev/null@@ -1,586 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--//! Algorithm in broad strokes-//! We first calculate the betweenness centrality for all nodes an choose top k as proxy nodes.-//! The distances from every node to every proxy node and reverse is then calculated and stored.-//!-//! During search the user can choose a set of liked and disliked nodes. For each user node, we find the best k proxy nodes.-//! For all proxy nodes we then estimate the harmonic centrality from the liked nodes - disliked nodes by estimating-//! the shortest distance as the distance through their best proxy node. This distance estimate is very fast since-//! all distances are pre-computed for the proxy nodes.--use std::cmp;-use std::collections::{BTreeMap, BinaryHeap, HashMap};-use std::fs::File;-use std::io::{BufReader, BufWriter, Read};-use std::path::Path;-use std::sync::Arc;--use dashmap::DashMap;-use serde::{Deserialize, Serialize};--use crate::intmap::IntMap;-use crate::ranking::centrality_store::HarmonicCentralityStore;-use crate::webgraph::{Edge, Node};-use crate::webgraph::{NodeID, Webgraph};-use crate::Result;--const NUM_PROXY_NODES: usize = 1000;-const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;-const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing-const MAX_DIST_PROXY: u8 = 10;-const MAX_NUM_OUT_EDGES: usize = 1000;--#[derive(Serialize, Deserialize)]-pub struct ProxyNode {- pub id: NodeID,- pub dist_from_node: IntMap<u8>, // from node to proxy node- pub dist_to_node: IntMap<u8>, // from proxy node to other node-}--impl ProxyNode {- fn dist(&self, from: &NodeID, to: &NodeID) -> Option<u8> {- if self.id == *from || self.id == *to {- return None;- }-- if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {- if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {- return Some(from_node_to_proxy + from_proxy_to_node);- }- }-- None- }-}--struct WeightedProxyNode {- node: Arc<ProxyNode>,- dist: u8,-}--impl PartialOrd for WeightedProxyNode {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- self.dist.partial_cmp(&other.dist)- }-}-impl Ord for WeightedProxyNode {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)- }-}--impl PartialEq for WeightedProxyNode {- fn eq(&self, other: &Self) -> bool {- self.dist == other.dist- }-}--impl Eq for WeightedProxyNode {}--pub struct Scorer {- fixed_scores: HashMap<NodeID, f64>,- liked_nodes: Vec<UserNode>,- disliked_nodes: Vec<UserNode>,- cache: DashMap<NodeID, f64>,-}--fn create_user_nodes(nodes: &[NodeID], proxy_nodes: &[Arc<ProxyNode>]) -> Vec<UserNode> {- let mut res = Vec::new();-- let nodes = nodes.to_vec();-- for id in nodes {- let user_node = UserNode::new(id, proxy_nodes);- if res.len() < USER_NODES_LIMIT {- res.push(user_node);- } else {- let best = res- .iter_mut()- .min_by_key(|curr| user_node.best_dist(&curr.id).unwrap_or(255))- .unwrap();-- best.weight += 1;- }- }-- res-}--impl Scorer {- fn new(- proxy_nodes: &[Arc<ProxyNode>],- liked_nodes: &[NodeID],- disliked_nodes: &[NodeID],- fixed_scores: HashMap<NodeID, f64>,- ) -> Self {- Self {- fixed_scores,- liked_nodes: create_user_nodes(liked_nodes, proxy_nodes),- disliked_nodes: create_user_nodes(disliked_nodes, proxy_nodes),- cache: DashMap::new(),- }- }-- pub fn score(&self, node: NodeID) -> f64 {- let norm = self.liked_nodes.len() as f64 + self.disliked_nodes.len() as f64;- if norm == 0.0 {- return 0.0;- }-- if let Some(score) = self.fixed_scores.get(&node) {- *score- } else {- if let Some(cached) = self.cache.get(&node) {- return *cached;- }- let res = (self.disliked_nodes.len() as f64- + (self- .liked_nodes- .iter()- .filter_map(|liked_node| {- liked_node.best_dist(&node).map(|dist| (dist, liked_node))- })- .map(|(dist, liked_node)| liked_node.weight as f64 / (dist as f64 + 1.0))- .sum::<f64>()- - self- .disliked_nodes- .iter()- .filter_map(|disliked_node| {- disliked_node- .best_dist(&node)- .map(|dist| (dist, disliked_node))- })- .map(|(dist, disliked_node)| {- disliked_node.weight as f64 / (dist as f64 + 1.0)- })- .sum::<f64>()))- .max(0.0)- / norm;-- let res = if res.is_finite() { res } else { 0.0 };- self.cache.insert(node, res);-- res- }- }-}--struct UserNode {- id: NodeID,- proxy_nodes: Vec<Arc<ProxyNode>>,- weight: usize,-}--impl UserNode {- fn new(id: NodeID, proxy_nodes: &[Arc<ProxyNode>]) -> Self {- let mut heap: BinaryHeap<WeightedProxyNode> =- BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);-- for proxy in proxy_nodes {- if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id.0) {- let weighted_node = WeightedProxyNode {- node: Arc::clone(proxy),- dist: *dist_to_proxy,- };-- if heap.len() >= BEST_PROXY_NODES_PER_USER_NODE {- if let Some(mut worst) = heap.peek_mut() {- if worst.dist > weighted_node.dist {- *worst = weighted_node;- }- }- } else {- heap.push(weighted_node);- }- }- }-- Self {- id,- weight: 1,- proxy_nodes: heap.into_iter().map(|weighted| weighted.node).collect(),- }- }-- fn best_dist(&self, node: &NodeID) -> Option<u8> {- let mut best = None;-- for proxy in &self.proxy_nodes {- if let Some(dist) = proxy.dist(&self.id, node) {- best = match best {- Some((best_dist, best_proxy)) => {- if dist < best_dist {- Some((dist, proxy))- } else {- Some((best_dist, best_proxy))- }- }- None => Some((dist, proxy)),- }- }- }-- best.map(|(dist, _)| dist)- }-}--struct ProxyNodeCandidate {- node: Node,- score: f64,-}--impl PartialOrd for ProxyNodeCandidate {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- self.score.partial_cmp(&other.score)- }-}--impl PartialEq for ProxyNodeCandidate {- fn eq(&self, other: &Self) -> bool {- self.score == other.score- }-}--impl Ord for ProxyNodeCandidate {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)- }-}--impl Eq for ProxyNodeCandidate {}--fn modified_dijkstra<F1, F2>(- source: Node,- node_edges: F1,- edge_node: F2,- graph: &Webgraph,-) -> BTreeMap<NodeID, u8>-where- F1: Fn(NodeID) -> Vec<Edge>,- F2: Fn(&Edge) -> NodeID,-{- let source_id = graph.node2id(&source).copied();-- if source_id.is_none() {- return BTreeMap::new();- }-- let source_id = source_id.unwrap();- let mut distances: BTreeMap<NodeID, u8> = BTreeMap::default();-- let mut queue = BinaryHeap::new();-- queue.push(cmp::Reverse((0, source_id)));- distances.insert(source_id, 0);-- while let Some(state) = queue.pop() {- let (cost, v) = state.0;-- if cost >= MAX_DIST_PROXY {- continue;- }-- let current_dist = distances.get(&v).unwrap_or(&u8::MAX);-- if cost > *current_dist {- continue;- }-- let mut out_edges_taken = 0;- for edge in node_edges(v).into_iter() {- if out_edges_taken >= MAX_NUM_OUT_EDGES {- break;- }-- if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {- let d = cost + 1;-- if d > MAX_DIST_PROXY {- continue;- }-- out_edges_taken += 1;- let next = cmp::Reverse((d, edge_node(&edge)));- queue.push(next);- distances.insert(edge_node(&edge), d);- }- }- }-- distances-}--fn distances(graph: &Webgraph, source: Node) -> BTreeMap<NodeID, u8> {- modified_dijkstra(- source,- |node| graph.raw_outgoing_edges(&node),- |edge| edge.to,- graph,- )-}--fn reversed_distances(graph: &Webgraph, source: Node) -> BTreeMap<NodeID, u8> {- modified_dijkstra(- source,- |node| graph.raw_ingoing_edges(&node),- |edge| edge.from,- graph,- )-}--#[derive(Serialize, Deserialize, Default)]-pub struct OnlineHarmonicCentrality {- pub proxy_nodes: Vec<Arc<ProxyNode>>,-}--impl OnlineHarmonicCentrality {- pub fn new(graph: &Webgraph, centrality: &HarmonicCentralityStore) -> Self {- Self::new_with_num_proxy(graph, centrality, NUM_PROXY_NODES)- }-- fn new_with_num_proxy(- graph: &Webgraph,- centrality: &HarmonicCentralityStore,- num_proxy_nodes: usize,- ) -> Self {- // we should probably choose the proxy nodes based on their- // betweenness centrality, but I don't know how we can approximate betweenness- // on a graph that cannot be in memory.- // For now, we will choose the nodes with highest harmonic centrality- // as an estimate-- let mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();- for (node, node_id) in graph.node_ids().map(|(node, id)| (node.clone(), *id)) {- let score = centrality.host.get(&node_id).unwrap_or(0.0);- let candidate = ProxyNodeCandidate { node, score };-- if nodes.len() >= num_proxy_nodes {- if let Some(mut existing_node) = nodes.peek_mut() {- if candidate.score > existing_node.score {- *existing_node = candidate;- }- }- } else {- nodes.push(candidate);- }- }-- let proxy_nodes: Vec<_> = nodes- .into_iter()- .map(|candidate| candidate.node)- .take(num_proxy_nodes)- .map(|node| {- let dist_to_node = distances(graph, node.clone())- .into_iter()- .map(|(n, v)| (n.0, v))- .collect();-- let dist_from_node = reversed_distances(graph, node.clone())- .into_iter()- .map(|(n, v)| (n.0, v))- .collect();-- Arc::new(ProxyNode {- id: *graph.node2id(&node).unwrap(),- dist_to_node,- dist_from_node,- })- })- .collect();-- Self { proxy_nodes }- }-- pub fn scorer(&self, liked_nodes: &[NodeID], disliked_nodes: &[NodeID]) -> Scorer {- Scorer::new(- &self.proxy_nodes,- liked_nodes,- disliked_nodes,- HashMap::new(),- )- }-- pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {- let mut file = BufWriter::new(- File::options()- .create(true)- .truncate(true)- .write(true)- .open(path)?,- );-- bincode::serialize_into(&mut file, &self)?;-- Ok(())- }-- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let file = File::open(path)?;- let mut reader = BufReader::new(file);-- let mut buf = Vec::new();- reader.read_to_end(&mut buf)?;-- Ok(bincode::deserialize(&buf)?)- }-}--#[cfg(test)]-mod tests {- use super::*;- use crate::webgraph::{centrality::harmonic::HarmonicCentrality, WebgraphBuilder};-- fn test_graph() -> Webgraph {- /*- G ◄---- F- | |- | ▼- ------► E -------► H- ▲ |- A -- | |- | | ▼- B ----► D ◄------- C- */-- let mut graph = WebgraphBuilder::new_memory().open();-- graph.insert(Node::from("A"), Node::from("D"), String::new());- graph.insert(Node::from("B"), Node::from("D"), String::new());- graph.insert(Node::from("C"), Node::from("D"), String::new());- graph.insert(Node::from("D"), Node::from("E"), String::new());- graph.insert(Node::from("E"), Node::from("H"), String::new());- graph.insert(Node::from("H"), Node::from("C"), String::new());- graph.insert(Node::from("F"), Node::from("E"), String::new());- graph.insert(Node::from("F"), Node::from("G"), String::new());- graph.insert(Node::from("G"), Node::from("E"), String::new());-- graph.commit();-- graph- }-- #[test]- fn ordering() {- let graph = test_graph();- let harmonic = HarmonicCentrality::calculate(&graph);-- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());- for (node, centrality) in harmonic.host {- harmonic_centrality_store- .host- .insert(*graph.node2id(&node).unwrap(), centrality);- }- harmonic_centrality_store.host.flush();-- let centrality =- OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);-- let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())]- .into_iter()- .filter_map(|node| graph.node2id(&node).copied())- .collect();-- let scorer = centrality.scorer(&liked_nodes, &[]);-- assert!(- scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())- );- assert!(- scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())- );-- assert!(- scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())- );- }-- #[test]- fn disliked_nodes_centrality() {- let graph = test_graph();- let harmonic = HarmonicCentrality::calculate(&graph);-- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());- for (node, centrality) in harmonic.host {- harmonic_centrality_store- .host- .insert(*graph.node2id(&node).unwrap(), centrality);- }- harmonic_centrality_store.host.flush();-- let centrality =- OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);-- let disliked_nodes: Vec<_> = vec![Node::from("D".to_string()), Node::from("E".to_string())]- .into_iter()- .map(|node| *graph.node2id(&node).unwrap())- .collect();-- let scorer = centrality.scorer(&[], &disliked_nodes);-- for node in &disliked_nodes {- assert!(scorer.score(*node) < disliked_nodes.len() as f64);- }- }-- #[test]- fn ordering_with_dislikes() {- let graph = test_graph();- let harmonic = HarmonicCentrality::calculate(&graph);-- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());- for (node, centrality) in harmonic.host {- harmonic_centrality_store- .host- .insert(*graph.node2id(&node).unwrap(), centrality);- }- harmonic_centrality_store.host.flush();-- let centrality =- OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);-- let liked_nodes: Vec<_> = vec![Node::from("B".to_string())]- .into_iter()- .filter_map(|node| graph.node2id(&node).copied())- .collect();-- let disliked_nodes: Vec<_> = vec![Node::from("F".to_string())]- .into_iter()- .filter_map(|node| graph.node2id(&node).copied())- .collect();-- let scorer = centrality.scorer(&liked_nodes, &disliked_nodes);-- assert!(- scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())- );-- assert!(- scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())- );-- assert!(- scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())- );- }-}