web search done right
harmonic centrality improvements (kahan sum, bloom filter, ability to controll number of hyperloglogcounters)
Mikkel Denkerfe9fa4e94512
diff --git a/Cargo.lock b/Cargo.lockindex aa0a7084..11d36150 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -306,6 +306,18 @@ dependencies = ["crunchy",]+[[package]]+name = "bitvec"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"+dependencies = [+ "funty",+ "radium",+ "tap",+ "wyz",+]+[[package]]name = "block-buffer"version = "0.9.0"@@ -720,6 +732,7 @@ dependencies = ["axum-macros","base64","bincode",+ "bitvec","bzip2","chrono","clap 3.1.18",@@ -1008,6 +1021,12 @@ dependencies = ["utf8-ranges",]+[[package]]+name = "funty"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"+[[package]]name = "futf"version = "0.1.5"@@ -2342,6 +2361,12 @@ dependencies = ["proc-macro2",]+[[package]]+name = "radium"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"+[[package]]name = "rand"version = "0.7.3"@@ -3108,6 +3133,12 @@ dependencies = ["regex",]+[[package]]+name = "tap"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"+[[package]]name = "tempfile"version = "3.3.0"@@ -3844,6 +3875,15 @@ dependencies = ["winapi",]+[[package]]+name = "wyz"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e"+dependencies = [+ "tap",+]+[[package]]name = "xml-rs"version = "0.8.4"diff --git a/Cargo.toml b/Cargo.tomlindex 4e6fae6f..a3ec058b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -64,6 +64,7 @@ axum-macros = "0.2.3"urlencoding = "2.1.2"rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false, optional = true }rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"], optional = true }+bitvec = "1.0.1"[dev-dependencies]criterion = "0.3.6"diff --git a/src/hyperloglog.rs b/src/hyperloglog.rsindex 5e701120..70d3c7a5 100644--- a/src/hyperloglog.rs+++ b/src/hyperloglog.rs@@ -94,7 +94,7 @@ mod tests {#[test]fn size_estimate_within_bounds() {- let mut set: HyperLogLog<16> = HyperLogLog::default();+ let mut set: HyperLogLog<128> = HyperLogLog::default();for item in 0..10_000_000 {set.add(item);@@ -109,10 +109,10 @@ mod tests {#[test]fn merge() {- let mut without_merge: HyperLogLog<16> = HyperLogLog::default();+ let mut without_merge: HyperLogLog<128> = HyperLogLog::default();- let mut a: HyperLogLog<16> = HyperLogLog::default();- let mut b: HyperLogLog<16> = HyperLogLog::default();+ let mut a: HyperLogLog<128> = HyperLogLog::default();+ let mut b: HyperLogLog<128> = HyperLogLog::default();for item in 0..10_000 {without_merge.add(item);diff --git a/src/kahan_sum.rs b/src/kahan_sum.rsnew file mode 100644index 00000000..0e0afe49--- /dev/null+++ b/src/kahan_sum.rs@@ -0,0 +1,76 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::ops::{Add, AddAssign};++#[derive(Default)]+pub struct KahanSum {+ sum: f64,+ err: f64,+}++impl From<KahanSum> for f64 {+ fn from(kahan: KahanSum) -> Self {+ kahan.sum+ }+}++impl From<f64> for KahanSum {+ fn from(val: f64) -> Self {+ Self { sum: val, err: 0.0 }+ }+}++impl AddAssign<f64> for KahanSum {+ fn add_assign(&mut self, rhs: f64) {+ let y = rhs - self.err;+ let t = self.sum + y;+ self.err = (t - self.sum) - y;+ self.sum = t;+ }+}++impl Add<f64> for KahanSum {+ type Output = Self;+ fn add(self, rhs: f64) -> Self::Output {+ let mut k = self;+ k += rhs;+ k+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn it_works() {+ let mut sum = KahanSum::default();+ for elem in [+ 10000.0f64,+ std::f64::consts::PI,+ std::f64::consts::E,+ std::f64::consts::PI,+ std::f64::consts::E,+ std::f64::consts::PI,+ std::f64::consts::E,+ ] {+ sum += elem;+ }++ assert_eq!(10017.579623446147f64, f64::from(sum));+ }+}diff --git a/src/lib.rs b/src/lib.rsindex a61d624f..95fed2db 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -45,6 +45,7 @@ mod image_downloader;mod image_store;pub mod index;mod intmap;+mod kahan_sum;mod kv;pub mod prehashed;mod query;diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rsindex 302eb1e9..ffda6057 100644--- a/src/webgraph/centrality/harmonic.rs+++ b/src/webgraph/centrality/harmonic.rs@@ -22,15 +22,53 @@ use std::{},};+use bitvec::vec::BitVec;use rayon::prelude::*;use tracing::info;use crate::{hyperloglog::HyperLogLog,intmap::IntMap,+ kahan_sum::KahanSum,webgraph::{graph_store::GraphStore, Node, Store, Webgraph},};+const HYPERLOGLOG_COUNTERS: usize = 16;++#[derive(Clone)]+struct JankyBloomFilter {+ bit_vec: BitVec,+ num_bits: u64,+}++impl JankyBloomFilter {+ pub fn new(estimated_items: u64, fp: f64) -> Self {+ let num_bits = Self::num_bits(estimated_items, fp);+ Self {+ bit_vec: BitVec::repeat(false, num_bits as usize),+ num_bits,+ }+ }++ fn num_bits(estimated_items: u64, fp: f64) -> u64 {+ ((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64+ }++ fn hash(item: &u64) -> usize {+ item.wrapping_mul(11400714819323198549) as usize+ }++ pub fn insert(&mut self, item: u64) {+ let h = Self::hash(&item);+ self.bit_vec.set(h % self.num_bits as usize, true);+ }++ pub fn contains(&self, item: &u64) -> bool {+ let h = Self::hash(item);+ self.bit_vec[h % self.num_bits as usize]+ }+}+pub struct HarmonicCentrality {pub full: HashMap<Node, f64>,pub host: HashMap<Node, f64>,@@ -44,7 +82,7 @@ whereinfo!("Found {} nodes in the graph", nodes.len());let norm_factor = (nodes.len() - 1) as f64;- let mut counters: IntMap<HyperLogLog<16>> = nodes+ let mut counters: IntMap<HyperLogLog<HYPERLOGLOG_COUNTERS>> = nodes.iter().map(|node| {let mut counter = HyperLogLog::default();@@ -56,7 +94,15 @@ wherelet counter_changes = AtomicU64::new(counters.len() as u64);let mut t = 0;- let mut centralities: IntMap<f64> = nodes.iter().map(|node| (*node, 0.0)).collect();+ let mut centralities: IntMap<KahanSum> = nodes+ .iter()+ .map(|node| (*node, KahanSum::default()))+ .collect();++ let mut changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);+ for node in &nodes {+ changed_nodes.insert(*node);+ }loop {if counter_changes.load(Ordering::SeqCst) == 0 {@@ -70,8 +116,14 @@ where.collect();counter_changes.store(0, Ordering::SeqCst);+ let new_changed_nodes = Mutex::new(JankyBloomFilter::new(nodes.len() as u64, 0.05));+nodes.par_iter().for_each(|node| {for edge in graph.ingoing_edges(*node) {+ if !changed_nodes.contains(&edge.from) {+ continue;+ }+if let (Some(counter_to), Some(counter_from)) =(new_counters.get(&edge.to), counters.get(&edge.from)){@@ -83,6 +135,7 @@ where.any(|(to, from)| *from > *to){counter_to.merge(counter_from);+ new_changed_nodes.lock().unwrap().insert(edge.to);counter_changes.fetch_add(1, Ordering::SeqCst);}}@@ -108,11 +161,13 @@ where.into_iter().map(|(node, counter)| (node, counter.into_inner().unwrap())).collect();+ changed_nodes = new_changed_nodes.into_inner().unwrap();t += 1;}centralities.into_iter()+ .map(|(node_id, sum)| (node_id, f64::from(sum))).filter(|(_, centrality)| *centrality > 0.0).map(|(node_id, centrality)| (graph.id2node(&node_id).unwrap(), centrality / norm_factor)).collect()@@ -217,6 +272,7 @@ mod tests {let graph = test_graph();let centrality = HarmonicCentrality::calculate(&graph);+ dbg!(¢rality.full);assert!(centrality.full.get(&Node::from("C")).unwrap()