web search done right
each inbound link is now scaled by host centrality in derived_centrality. This should make the centrality metric more robust to linkbuilders
Mikkel Denker32d94b908289
diff --git a/core/src/bloom.rs b/core/src/bloom.rsnew file mode 100644index 00000000..7a56427e--- /dev/null+++ b/core/src/bloom.rs@@ -0,0 +1,71 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use bitvec::vec::BitVec;++#[derive(Clone)]+pub struct BloomFilter {+ bit_vec: BitVec,+ num_bits: u64,+}++impl BloomFilter {+ pub fn new(estimated_items: u64, fp: f64) -> Self {+ let num_bits = Self::num_bits(estimated_items, fp);+ Self {+ bit_vec: BitVec::repeat(false, num_bits as usize),+ num_bits,+ }+ }++ fn num_bits(estimated_items: u64, fp: f64) -> u64 {+ ((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64+ }++ fn hash(item: &u64) -> usize {+ item.wrapping_mul(11400714819323198549) as usize+ }++ pub fn insert(&mut self, item: u64) {+ let h = Self::hash(&item);+ self.bit_vec.set(h % self.num_bits as usize, true);+ }++ pub fn contains(&self, item: &u64) -> bool {+ let h = Self::hash(item);+ self.bit_vec[h % self.num_bits as usize]+ }++ pub fn estimate_card(&self) -> u64 {+ let num_ones = self.bit_vec.count_ones() as u64;++ if num_ones == 0 || self.num_bits == 0 {+ return 0;+ }++ if num_ones == self.num_bits {+ return u64::MAX;+ }++ (-(self.num_bits as i64) * (1.0 - (num_ones as f64) / (self.num_bits as f64)).ln() as i64)+ .try_into()+ .unwrap_or_default()+ }++ pub fn merge(&mut self, other: Self) {+ self.bit_vec |= other.bit_vec;+ }+}diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex 9fab3b48..3e6279bb 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -117,7 +117,7 @@ impl Centrality {base_output: P,) -> Result<()> {tracing::info!("Building derived harmonic centrality");- let graph = WebgraphBuilder::new(webgraph_path).open();+ let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();let host_centrality = RocksDbStore::open(host_centrality_path.as_ref().join("harmonic"));let derived = DerivedCentrality::build(diff --git a/core/src/lib.rs b/core/src/lib.rsindex ee8fc991..69bc9cd0 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -36,6 +36,7 @@ pub mod alice;mod api;mod autosuggest;mod bangs;+mod bloom;mod collector;pub mod config;pub mod crawler;diff --git a/core/src/webgraph/centrality/derived_harmonic.rs b/core/src/webgraph/centrality/derived_harmonic.rsindex d3d8f0f7..74d8db5d 100644--- a/core/src/webgraph/centrality/derived_harmonic.rs+++ b/core/src/webgraph/centrality/derived_harmonic.rs@@ -19,12 +19,49 @@//! derive a centrality measure for the page graph.use anyhow::Result;-use std::{collections::BTreeMap, path::Path};+use rayon::prelude::*;+use std::{collections::BTreeMap, path::Path, sync::Mutex};use crate::{+ bloom::BloomFilter,kv::{rocksdb_store::RocksDbStore, Kv},webgraph::{NodeID, Webgraph},};++struct BloomMap {+ map: Vec<Mutex<BloomFilter>>,+}++impl BloomMap {+ fn new(num_blooms: usize, estimated_items: u64, fp: f64) -> Self {+ let mut map = Vec::new();++ for _ in 0..num_blooms {+ map.push(Mutex::new(BloomFilter::new(estimated_items, fp)));+ }++ Self { map }+ }++ fn insert(&self, item: &NodeID) {+ let h = item.bit_64();+ self.map[(h as usize) % self.map.len()]+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .insert(h);+ }++ fn finalize(mut self) -> BloomFilter {+ let mut bf = self.map.pop().unwrap().into_inner().unwrap();++ for m in self.map {+ bf.merge(m.into_inner().unwrap());+ }++ bf+ }+}+pub struct DerivedCentrality {inner: RocksDbStore<NodeID, f64>,}@@ -44,38 +81,56 @@ impl DerivedCentrality {return Err(anyhow::anyhow!("output path already exists"));}- let non_normalized = RocksDbStore::open(output.as_ref().join("non_normalized"));-- let mut norms: BTreeMap<NodeID, usize> = BTreeMap::new();+ let num_nodes = page_graph.nodes().count();- for (node, id) in page_graph.node_ids() {- let host_node = node.clone().into_host().id();+ let has_outgoing = BloomMap::new(8, num_nodes as u64, 0.01);- if let Some(harmonic) = host_harmonic.get(&host_node) {- let mut ingoing: Vec<_> = page_graph- .raw_ingoing_edges(&id)- .into_iter()- .filter_map(|e| page_graph.id2node(&e.from))- .map(|n| n.into_host())- .collect();- ingoing.sort();- ingoing.dedup();+ page_graph.par_edges().for_each(|edge| {+ has_outgoing.insert(&edge.from);+ });- let count = ingoing.len();- let page_score = harmonic * count as f64;+ let has_outgoing = has_outgoing.finalize();- non_normalized.insert(id, page_score);+ let non_normalized = RocksDbStore::open(output.as_ref().join("non_normalized"));- let norm = norms.entry(host_node).or_insert(0);- *norm = (*norm).max(count);+ let norms: Mutex<BTreeMap<NodeID, f64>> = Mutex::new(BTreeMap::new());++ page_graph.node_ids().par_bridge().for_each(|(node, id)| {+ if has_outgoing.contains(&id.bit_64()) {+ let host_node = node.clone().into_host().id();++ if let Some(harmonic) = host_harmonic.get(&host_node) {+ let mut ingoing: Vec<_> = page_graph+ .raw_ingoing_edges(&id)+ .into_iter()+ .filter_map(|e| page_graph.id2node(&e.from))+ .map(|n| n.into_host())+ .collect();+ ingoing.sort();+ ingoing.dedup();++ let votes = ingoing+ .into_iter()+ .filter_map(|n| host_harmonic.get(&n.id()))+ .sum::<f64>();+ let page_score = harmonic * votes;++ non_normalized.insert(id, page_score);++ let mut l = norms.lock().unwrap_or_else(|e| e.into_inner());+ let norm = l.entry(host_node).or_insert(0.0);+ *norm = (*norm).max(votes);+ }}- }+ });++ let norms = norms.into_inner().unwrap();let db = RocksDbStore::open(output.as_ref());for (id, score) in non_normalized.iter() {let node = page_graph.id2node(&id).unwrap().into_host().id();let norm = norms.get(&node).unwrap();- let normalized = score / (*norm as f64);+ let normalized = score / *norm;db.insert(id, normalized);}db.flush();diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex bbedfe07..14eab905 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -21,7 +21,7 @@ use std::{use std::sync::atomic::Ordering;-use bitvec::vec::BitVec;+use crate::bloom::BloomFilter;use tracing::info;use crate::{@@ -32,56 +32,6 @@ use crate::{const HYPERLOGLOG_COUNTERS: usize = 64;-#[derive(Clone)]-struct JankyBloomFilter {- bit_vec: BitVec,- num_bits: u64,-}--impl JankyBloomFilter {- pub fn new(estimated_items: u64, fp: f64) -> Self {- let num_bits = Self::num_bits(estimated_items, fp);- Self {- bit_vec: BitVec::repeat(false, num_bits as usize),- num_bits,- }- }-- fn num_bits(estimated_items: u64, fp: f64) -> u64 {- ((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64- }-- fn hash(item: &u64) -> usize {- item.wrapping_mul(11400714819323198549) as usize- }-- pub fn insert(&mut self, item: u64) {- let h = Self::hash(&item);- self.bit_vec.set(h % self.num_bits as usize, true);- }-- pub fn contains(&self, item: &u64) -> bool {- let h = Self::hash(item);- self.bit_vec[h % self.num_bits as usize]- }-- pub fn estimate_card(&self) -> u64 {- let num_ones = self.bit_vec.count_ones() as u64;-- if num_ones == 0 || self.num_bits == 0 {- return 0;- }-- if num_ones == self.num_bits {- return u64::MAX;- }-- (-(self.num_bits as i64) * (1.0 - (num_ones as f64) / (self.num_bits as f64)).ln() as i64)- .try_into()- .unwrap_or_default()- }-}-fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {let mut num_nodes = 0;@@ -99,7 +49,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {num_nodes += 1;}- let mut changed_nodes = JankyBloomFilter::new(num_nodes as u64, 0.05);+ let mut changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);for node in graph.nodes() {changed_nodes.insert(node.bit_64());@@ -123,7 +73,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {let mut new_counters = counters.clone();has_changes.store(false, Ordering::Relaxed);- let mut new_changed_nodes = JankyBloomFilter::new(num_nodes as u64, 0.05);+ let mut new_changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);if !exact_changed_nodes.is_empty()&& exact_changed_nodes.len() as u64 <= exact_counting_threshold