Skip to content

web search done right

each inbound link is now scaled by host centrality in derived_centrality. This should make the centrality metric more robust to linkbuilders

Mikkel Denker32d94b908289
diff --git a/core/src/bloom.rs b/core/src/bloom.rs
new file mode 100644
index 00000000..7a56427e
--- /dev/null
+++ b/core/src/bloom.rs
@@ -0,0 +1,71 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use bitvec::vec::BitVec;
+
+#[derive(Clone)]
+pub struct BloomFilter {
+ bit_vec: BitVec,
+ num_bits: u64,
+}
+
+impl BloomFilter {
+ pub fn new(estimated_items: u64, fp: f64) -> Self {
+ let num_bits = Self::num_bits(estimated_items, fp);
+ Self {
+ bit_vec: BitVec::repeat(false, num_bits as usize),
+ num_bits,
+ }
+ }
+
+ fn num_bits(estimated_items: u64, fp: f64) -> u64 {
+ ((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64
+ }
+
+ fn hash(item: &u64) -> usize {
+ item.wrapping_mul(11400714819323198549) as usize
+ }
+
+ pub fn insert(&mut self, item: u64) {
+ let h = Self::hash(&item);
+ self.bit_vec.set(h % self.num_bits as usize, true);
+ }
+
+ pub fn contains(&self, item: &u64) -> bool {
+ let h = Self::hash(item);
+ self.bit_vec[h % self.num_bits as usize]
+ }
+
+ pub fn estimate_card(&self) -> u64 {
+ let num_ones = self.bit_vec.count_ones() as u64;
+
+ if num_ones == 0 || self.num_bits == 0 {
+ return 0;
+ }
+
+ if num_ones == self.num_bits {
+ return u64::MAX;
+ }
+
+ (-(self.num_bits as i64) * (1.0 - (num_ones as f64) / (self.num_bits as f64)).ln() as i64)
+ .try_into()
+ .unwrap_or_default()
+ }
+
+ pub fn merge(&mut self, other: Self) {
+ self.bit_vec |= other.bit_vec;
+ }
+}
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index 9fab3b48..3e6279bb 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -117,7 +117,7 @@ impl Centrality {
base_output: P,
) -> Result<()> {
tracing::info!("Building derived harmonic centrality");
- let graph = WebgraphBuilder::new(webgraph_path).open();
+ let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();
let host_centrality = RocksDbStore::open(host_centrality_path.as_ref().join("harmonic"));
let derived = DerivedCentrality::build(
diff --git a/core/src/lib.rs b/core/src/lib.rs
index ee8fc991..69bc9cd0 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -36,6 +36,7 @@ pub mod alice;
mod api;
mod autosuggest;
mod bangs;
+mod bloom;
mod collector;
pub mod config;
pub mod crawler;
diff --git a/core/src/webgraph/centrality/derived_harmonic.rs b/core/src/webgraph/centrality/derived_harmonic.rs
index d3d8f0f7..74d8db5d 100644
--- a/core/src/webgraph/centrality/derived_harmonic.rs
+++ b/core/src/webgraph/centrality/derived_harmonic.rs
@@ -19,12 +19,49 @@
//! derive a centrality measure for the page graph.
use anyhow::Result;
-use std::{collections::BTreeMap, path::Path};
+use rayon::prelude::*;
+use std::{collections::BTreeMap, path::Path, sync::Mutex};
use crate::{
+ bloom::BloomFilter,
kv::{rocksdb_store::RocksDbStore, Kv},
webgraph::{NodeID, Webgraph},
};
+
+struct BloomMap {
+ map: Vec<Mutex<BloomFilter>>,
+}
+
+impl BloomMap {
+ fn new(num_blooms: usize, estimated_items: u64, fp: f64) -> Self {
+ let mut map = Vec::new();
+
+ for _ in 0..num_blooms {
+ map.push(Mutex::new(BloomFilter::new(estimated_items, fp)));
+ }
+
+ Self { map }
+ }
+
+ fn insert(&self, item: &NodeID) {
+ let h = item.bit_64();
+ self.map[(h as usize) % self.map.len()]
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .insert(h);
+ }
+
+ fn finalize(mut self) -> BloomFilter {
+ let mut bf = self.map.pop().unwrap().into_inner().unwrap();
+
+ for m in self.map {
+ bf.merge(m.into_inner().unwrap());
+ }
+
+ bf
+ }
+}
+
pub struct DerivedCentrality {
inner: RocksDbStore<NodeID, f64>,
}
@@ -44,38 +81,56 @@ impl DerivedCentrality {
return Err(anyhow::anyhow!("output path already exists"));
}
- let non_normalized = RocksDbStore::open(output.as_ref().join("non_normalized"));
-
- let mut norms: BTreeMap<NodeID, usize> = BTreeMap::new();
+ let num_nodes = page_graph.nodes().count();
- for (node, id) in page_graph.node_ids() {
- let host_node = node.clone().into_host().id();
+ let has_outgoing = BloomMap::new(8, num_nodes as u64, 0.01);
- if let Some(harmonic) = host_harmonic.get(&host_node) {
- let mut ingoing: Vec<_> = page_graph
- .raw_ingoing_edges(&id)
- .into_iter()
- .filter_map(|e| page_graph.id2node(&e.from))
- .map(|n| n.into_host())
- .collect();
- ingoing.sort();
- ingoing.dedup();
+ page_graph.par_edges().for_each(|edge| {
+ has_outgoing.insert(&edge.from);
+ });
- let count = ingoing.len();
- let page_score = harmonic * count as f64;
+ let has_outgoing = has_outgoing.finalize();
- non_normalized.insert(id, page_score);
+ let non_normalized = RocksDbStore::open(output.as_ref().join("non_normalized"));
- let norm = norms.entry(host_node).or_insert(0);
- *norm = (*norm).max(count);
+ let norms: Mutex<BTreeMap<NodeID, f64>> = Mutex::new(BTreeMap::new());
+
+ page_graph.node_ids().par_bridge().for_each(|(node, id)| {
+ if has_outgoing.contains(&id.bit_64()) {
+ let host_node = node.clone().into_host().id();
+
+ if let Some(harmonic) = host_harmonic.get(&host_node) {
+ let mut ingoing: Vec<_> = page_graph
+ .raw_ingoing_edges(&id)
+ .into_iter()
+ .filter_map(|e| page_graph.id2node(&e.from))
+ .map(|n| n.into_host())
+ .collect();
+ ingoing.sort();
+ ingoing.dedup();
+
+ let votes = ingoing
+ .into_iter()
+ .filter_map(|n| host_harmonic.get(&n.id()))
+ .sum::<f64>();
+ let page_score = harmonic * votes;
+
+ non_normalized.insert(id, page_score);
+
+ let mut l = norms.lock().unwrap_or_else(|e| e.into_inner());
+ let norm = l.entry(host_node).or_insert(0.0);
+ *norm = (*norm).max(votes);
+ }
}
- }
+ });
+
+ let norms = norms.into_inner().unwrap();
let db = RocksDbStore::open(output.as_ref());
for (id, score) in non_normalized.iter() {
let node = page_graph.id2node(&id).unwrap().into_host().id();
let norm = norms.get(&node).unwrap();
- let normalized = score / (*norm as f64);
+ let normalized = score / *norm;
db.insert(id, normalized);
}
db.flush();
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index bbedfe07..14eab905 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -21,7 +21,7 @@ use std::{
use std::sync::atomic::Ordering;
-use bitvec::vec::BitVec;
+use crate::bloom::BloomFilter;
use tracing::info;
use crate::{
@@ -32,56 +32,6 @@ use crate::{
const HYPERLOGLOG_COUNTERS: usize = 64;
-#[derive(Clone)]
-struct JankyBloomFilter {
- bit_vec: BitVec,
- num_bits: u64,
-}
-
-impl JankyBloomFilter {
- pub fn new(estimated_items: u64, fp: f64) -> Self {
- let num_bits = Self::num_bits(estimated_items, fp);
- Self {
- bit_vec: BitVec::repeat(false, num_bits as usize),
- num_bits,
- }
- }
-
- fn num_bits(estimated_items: u64, fp: f64) -> u64 {
- ((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64
- }
-
- fn hash(item: &u64) -> usize {
- item.wrapping_mul(11400714819323198549) as usize
- }
-
- pub fn insert(&mut self, item: u64) {
- let h = Self::hash(&item);
- self.bit_vec.set(h % self.num_bits as usize, true);
- }
-
- pub fn contains(&self, item: &u64) -> bool {
- let h = Self::hash(item);
- self.bit_vec[h % self.num_bits as usize]
- }
-
- pub fn estimate_card(&self) -> u64 {
- let num_ones = self.bit_vec.count_ones() as u64;
-
- if num_ones == 0 || self.num_bits == 0 {
- return 0;
- }
-
- if num_ones == self.num_bits {
- return u64::MAX;
- }
-
- (-(self.num_bits as i64) * (1.0 - (num_ones as f64) / (self.num_bits as f64)).ln() as i64)
- .try_into()
- .unwrap_or_default()
- }
-}
-
fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let mut num_nodes = 0;
@@ -99,7 +49,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
num_nodes += 1;
}
- let mut changed_nodes = JankyBloomFilter::new(num_nodes as u64, 0.05);
+ let mut changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);
for node in graph.nodes() {
changed_nodes.insert(node.bit_64());
@@ -123,7 +73,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let mut new_counters = counters.clone();
has_changes.store(false, Ordering::Relaxed);
- let mut new_changed_nodes = JankyBloomFilter::new(num_nodes as u64, 0.05);
+ let mut new_changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);
if !exact_changed_nodes.is_empty()
&& exact_changed_nodes.len() as u64 <= exact_counting_threshold