Skip to content

web search done right

harmonic centrality improvements (kahan sum, bloom filter, ability to controll number of hyperloglogcounters)

Mikkel Denkerfe9fa4e94512
diff --git a/Cargo.lock b/Cargo.lock
index aa0a7084..11d36150 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -306,6 +306,18 @@ dependencies = [
"crunchy",
]
+[[package]]
+name = "bitvec"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
[[package]]
name = "block-buffer"
version = "0.9.0"
@@ -720,6 +732,7 @@ dependencies = [
"axum-macros",
"base64",
"bincode",
+ "bitvec",
"bzip2",
"chrono",
"clap 3.1.18",
@@ -1008,6 +1021,12 @@ dependencies = [
"utf8-ranges",
]
+[[package]]
+name = "funty"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
+
[[package]]
name = "futf"
version = "0.1.5"
@@ -2342,6 +2361,12 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "radium"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
+
[[package]]
name = "rand"
version = "0.7.3"
@@ -3108,6 +3133,12 @@ dependencies = [
"regex",
]
+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
[[package]]
name = "tempfile"
version = "3.3.0"
@@ -3844,6 +3875,15 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "wyz"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e"
+dependencies = [
+ "tap",
+]
+
[[package]]
name = "xml-rs"
version = "0.8.4"
diff --git a/Cargo.toml b/Cargo.toml
index 4e6fae6f..a3ec058b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,6 +64,7 @@ axum-macros = "0.2.3"
urlencoding = "2.1.2"
rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false, optional = true }
rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"], optional = true }
+bitvec = "1.0.1"
[dev-dependencies]
criterion = "0.3.6"
diff --git a/src/hyperloglog.rs b/src/hyperloglog.rs
index 5e701120..70d3c7a5 100644
--- a/src/hyperloglog.rs
+++ b/src/hyperloglog.rs
@@ -94,7 +94,7 @@ mod tests {
#[test]
fn size_estimate_within_bounds() {
- let mut set: HyperLogLog<16> = HyperLogLog::default();
+ let mut set: HyperLogLog<128> = HyperLogLog::default();
for item in 0..10_000_000 {
set.add(item);
@@ -109,10 +109,10 @@ mod tests {
#[test]
fn merge() {
- let mut without_merge: HyperLogLog<16> = HyperLogLog::default();
+ let mut without_merge: HyperLogLog<128> = HyperLogLog::default();
- let mut a: HyperLogLog<16> = HyperLogLog::default();
- let mut b: HyperLogLog<16> = HyperLogLog::default();
+ let mut a: HyperLogLog<128> = HyperLogLog::default();
+ let mut b: HyperLogLog<128> = HyperLogLog::default();
for item in 0..10_000 {
without_merge.add(item);
diff --git a/src/kahan_sum.rs b/src/kahan_sum.rs
new file mode 100644
index 00000000..0e0afe49
--- /dev/null
+++ b/src/kahan_sum.rs
@@ -0,0 +1,76 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::ops::{Add, AddAssign};
+
+#[derive(Default)]
+pub struct KahanSum {
+ sum: f64,
+ err: f64,
+}
+
+impl From<KahanSum> for f64 {
+ fn from(kahan: KahanSum) -> Self {
+ kahan.sum
+ }
+}
+
+impl From<f64> for KahanSum {
+ fn from(val: f64) -> Self {
+ Self { sum: val, err: 0.0 }
+ }
+}
+
+impl AddAssign<f64> for KahanSum {
+ fn add_assign(&mut self, rhs: f64) {
+ let y = rhs - self.err;
+ let t = self.sum + y;
+ self.err = (t - self.sum) - y;
+ self.sum = t;
+ }
+}
+
+impl Add<f64> for KahanSum {
+ type Output = Self;
+ fn add(self, rhs: f64) -> Self::Output {
+ let mut k = self;
+ k += rhs;
+ k
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn it_works() {
+ let mut sum = KahanSum::default();
+ for elem in [
+ 10000.0f64,
+ std::f64::consts::PI,
+ std::f64::consts::E,
+ std::f64::consts::PI,
+ std::f64::consts::E,
+ std::f64::consts::PI,
+ std::f64::consts::E,
+ ] {
+ sum += elem;
+ }
+
+ assert_eq!(10017.579623446147f64, f64::from(sum));
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index a61d624f..95fed2db 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -45,6 +45,7 @@ mod image_downloader;
mod image_store;
pub mod index;
mod intmap;
+mod kahan_sum;
mod kv;
pub mod prehashed;
mod query;
diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rs
index 302eb1e9..ffda6057 100644
--- a/src/webgraph/centrality/harmonic.rs
+++ b/src/webgraph/centrality/harmonic.rs
@@ -22,15 +22,53 @@ use std::{
},
};
+use bitvec::vec::BitVec;
use rayon::prelude::*;
use tracing::info;
use crate::{
hyperloglog::HyperLogLog,
intmap::IntMap,
+ kahan_sum::KahanSum,
webgraph::{graph_store::GraphStore, Node, Store, Webgraph},
};
+const HYPERLOGLOG_COUNTERS: usize = 16;
+
+#[derive(Clone)]
+struct JankyBloomFilter {
+ bit_vec: BitVec,
+ num_bits: u64,
+}
+
+impl JankyBloomFilter {
+ pub fn new(estimated_items: u64, fp: f64) -> Self {
+ let num_bits = Self::num_bits(estimated_items, fp);
+ Self {
+ bit_vec: BitVec::repeat(false, num_bits as usize),
+ num_bits,
+ }
+ }
+
+ fn num_bits(estimated_items: u64, fp: f64) -> u64 {
+ ((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64
+ }
+
+ fn hash(item: &u64) -> usize {
+ item.wrapping_mul(11400714819323198549) as usize
+ }
+
+ pub fn insert(&mut self, item: u64) {
+ let h = Self::hash(&item);
+ self.bit_vec.set(h % self.num_bits as usize, true);
+ }
+
+ pub fn contains(&self, item: &u64) -> bool {
+ let h = Self::hash(item);
+ self.bit_vec[h % self.num_bits as usize]
+ }
+}
+
pub struct HarmonicCentrality {
pub full: HashMap<Node, f64>,
pub host: HashMap<Node, f64>,
@@ -44,7 +82,7 @@ where
info!("Found {} nodes in the graph", nodes.len());
let norm_factor = (nodes.len() - 1) as f64;
- let mut counters: IntMap<HyperLogLog<16>> = nodes
+ let mut counters: IntMap<HyperLogLog<HYPERLOGLOG_COUNTERS>> = nodes
.iter()
.map(|node| {
let mut counter = HyperLogLog::default();
@@ -56,7 +94,15 @@ where
let counter_changes = AtomicU64::new(counters.len() as u64);
let mut t = 0;
- let mut centralities: IntMap<f64> = nodes.iter().map(|node| (*node, 0.0)).collect();
+ let mut centralities: IntMap<KahanSum> = nodes
+ .iter()
+ .map(|node| (*node, KahanSum::default()))
+ .collect();
+
+ let mut changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);
+ for node in &nodes {
+ changed_nodes.insert(*node);
+ }
loop {
if counter_changes.load(Ordering::SeqCst) == 0 {
@@ -70,8 +116,14 @@ where
.collect();
counter_changes.store(0, Ordering::SeqCst);
+ let new_changed_nodes = Mutex::new(JankyBloomFilter::new(nodes.len() as u64, 0.05));
+
nodes.par_iter().for_each(|node| {
for edge in graph.ingoing_edges(*node) {
+ if !changed_nodes.contains(&edge.from) {
+ continue;
+ }
+
if let (Some(counter_to), Some(counter_from)) =
(new_counters.get(&edge.to), counters.get(&edge.from))
{
@@ -83,6 +135,7 @@ where
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
+ new_changed_nodes.lock().unwrap().insert(edge.to);
counter_changes.fetch_add(1, Ordering::SeqCst);
}
}
@@ -108,11 +161,13 @@ where
.into_iter()
.map(|(node, counter)| (node, counter.into_inner().unwrap()))
.collect();
+ changed_nodes = new_changed_nodes.into_inner().unwrap();
t += 1;
}
centralities
.into_iter()
+ .map(|(node_id, sum)| (node_id, f64::from(sum)))
.filter(|(_, centrality)| *centrality > 0.0)
.map(|(node_id, centrality)| (graph.id2node(&node_id).unwrap(), centrality / norm_factor))
.collect()
@@ -217,6 +272,7 @@ mod tests {
let graph = test_graph();
let centrality = HarmonicCentrality::calculate(&graph);
+ dbg!(&centrality.full);
assert!(
centrality.full.get(&Node::from("C")).unwrap()