web search done right
try an approximate harmonic centrality calculation for the page graph that randomly samples O(log n / epsilon) nodes and solves SSSP problem from these
Mikkel Denkere1966f0def9b
diff --git a/configs/crawler/planner.toml b/configs/crawler/planner.tomlindex cf2824be..81af0656 100644--- a/configs/crawler/planner.toml+++ b/configs/crawler/planner.toml@@ -2,7 +2,7 @@ host_graph_path = "data/webgraph_host"host_harmonic_path = "data/centrality/harmonic"output_path = "data/crawlplan"page_graph_path = "data/webgraph_page"-page_harmonic_path = "data/centrality_page/derived_harmonic"+page_harmonic_path = "data/centrality_page/approx_harmonic"num_job_queues = 5diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex 1f0a6984..2bc3ae9c 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -21,7 +21,7 @@ use crate::{kv::{rocksdb_store::RocksDbStore, Kv},ranking::inbound_similarity::InboundSimilarity,webgraph::{- centrality::{derived_harmonic::DerivedCentrality, harmonic::HarmonicCentrality},+ centrality::{approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality},Node, WebgraphBuilder,},};@@ -111,24 +111,15 @@ impl Centrality {.unwrap();}- pub fn build_derived_harmonic<P: AsRef<Path>>(- webgraph_path: P,- host_centrality_path: P,- base_output: P,- ) -> Result<()> {- tracing::info!("Building derived harmonic centrality");+ pub fn build_approx_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) -> Result<()> {+ tracing::info!("Building approximated harmonic centrality for page graph");let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();- let host_centrality = RocksDbStore::open(host_centrality_path.as_ref().join("harmonic"));- let derived = DerivedCentrality::build(- &host_centrality,- &graph,- base_output.as_ref().join("derived_harmonic"),- )?;+ let approx = ApproxHarmonic::build(&graph, base_output.as_ref().join("approx_harmonic"));let mut top_nodes = BinaryHeap::new();- for (node_id, centrality) in derived.iter() {+ for (node_id, centrality) in approx.iter() {if top_nodes.len() < 1_000_000 {top_nodes.push((Reverse(SortableFloat(centrality)), node_id));} else {@@ -144,7 +135,7 @@ impl Centrality {.map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0)).collect();- store_csv(derived, base_output.as_ref().join("derived_centrality.csv"));+ store_csv(derived, base_output.as_ref().join("approx_harmonic.csv"));Ok(())}diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 122c2dfc..7e3ed07d 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -144,7 +144,7 @@ fn calculate_centrality() {let out_path_page = Path::new(DATA_PATH).join("centrality_page");if !out_path_page.exists() {- Centrality::build_derived_harmonic(webgraph_page, out_path, out_path_page).unwrap();+ Centrality::build_approx_harmonic(webgraph_page, out_path_page).unwrap();}}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex b2b2f43f..6b6caac6 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -72,7 +72,7 @@ impl IndexingWorker {Path::new(&host_centrality_store_path).join("harmonic"),),page_centrality_store: page_centrality_store_path- .map(|p| RocksDbStore::open(Path::new(&p).join("derived_harmonic"))),+ .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic"))),page_webgraph: page_webgraph_path.map(|path| WebgraphBuilder::new(path).single_threaded().open()),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),diff --git a/core/src/main.rs b/core/src/main.rsindex 74ba767f..815f26e2 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -173,7 +173,6 @@ enum CentralityMode {/// Calculate metrics for the page webgraph.Page {webgraph_path: String,- host_centrality_path: String,output_path: String,},}@@ -269,13 +268,8 @@ fn main() -> Result<()> {}CentralityMode::Page {webgraph_path,- host_centrality_path,output_path,- } => entrypoint::Centrality::build_derived_harmonic(- webgraph_path,- host_centrality_path,- output_path,- )?,+ } => entrypoint::Centrality::build_approx_harmonic(webgraph_path, output_path)?,}tracing::info!("Done");}diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rsnew file mode 100644index 00000000..8252ae36--- /dev/null+++ b/core/src/webgraph/centrality/approx_harmonic.rs@@ -0,0 +1,76 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{path::Path, sync::Mutex};++use indicatif::ParallelProgressIterator;+use rayon::prelude::*;++use crate::{+ kv::{rocksdb_store::RocksDbStore, Kv},+ webgraph::{NodeID, ShortestPaths, Webgraph},+};++const EPSILON: f64 = 0.05;++// Approximate harmonic centrality by sampling O(log n / epsilon^2) nodes and+// computing single-source shortest paths from each of them.+//+// Epsilong is set to 0.05.+pub struct ApproxHarmonic {+ inner: RocksDbStore<NodeID, f64>,+}++impl ApproxHarmonic {+ pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {+ let num_nodes = graph.nodes().count();++ let num_samples = ((num_nodes as f64).log2() / EPSILON.powi(2)).ceil() as usize;+ let sampled = graph.random_nodes(num_samples);++ let res = Mutex::new(Self {+ inner: RocksDbStore::open(output),+ });++ let norm = num_nodes as f64 / (num_samples as f64 * (num_nodes as f64 - 1.0));++ sampled.into_par_iter().progress().for_each(|source| {+ let dists = graph.raw_distances(source);++ let res = res.lock().unwrap();+ for (target, dist) in dists {+ if dist == 0 {+ continue;+ }++ let dist = dist as f64;++ let old = res.inner.get(&target).unwrap_or(0.0);+ res.inner.insert(target, old + ((1.0 / dist) * norm));+ }+ });++ res.into_inner().unwrap()+ }++ pub fn get(&self, node: &NodeID) -> Option<f64> {+ self.inner.get(node)+ }++ pub fn iter(&self) -> impl Iterator<Item = (NodeID, f64)> + '_ {+ self.inner.iter()+ }+}diff --git a/core/src/webgraph/centrality/mod.rs b/core/src/webgraph/centrality/mod.rsindex 122cb320..012a8847 100644--- a/core/src/webgraph/centrality/mod.rs+++ b/core/src/webgraph/centrality/mod.rs@@ -18,6 +18,7 @@ use crate::kv::{rocksdb_store::RocksDbStore, Kv};use super::NodeID;+pub mod approx_harmonic;pub mod betweenness;pub mod derived_harmonic;pub mod harmonic;diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex fdeb80fd..06bb286f 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -22,6 +22,7 @@ use std::path::Path;use std::sync::Arc;use std::{cmp, fs};+use rand::seq::SliceRandom;use rayon::prelude::*;use url::Url;use utoipa::ToSchema;@@ -333,24 +334,29 @@ impl WebgraphBuilder {pub trait ShortestPaths {fn distances(&self, source: Node) -> BTreeMap<Node, u8>;- fn raw_distances(&self, source: Node) -> BTreeMap<NodeID, u8>;- fn raw_reversed_distances(&self, source: Node) -> BTreeMap<NodeID, u8>;+ fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;+ fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;}-fn dijkstra<F1, F2, L>(source: Node, node_edges: F1, edge_node: F2) -> BTreeMap<NodeID, u8>+fn dijkstra_multi<F1, F2, L>(+ sources: &[NodeID],+ node_edges: F1,+ edge_node: F2,+) -> BTreeMap<NodeID, u8>whereL: EdgeLabel,F1: Fn(NodeID) -> Vec<Edge<L>>,F2: Fn(&Edge<L>) -> NodeID,{- let source_id = source.id();let mut distances: BTreeMap<NodeID, u8> = BTreeMap::default();let mut queue = BinaryHeap::new();- queue.push(cmp::Reverse((0, source_id)));- distances.insert(source_id, 0);+ for source_id in sources.iter().copied() {+ queue.push(cmp::Reverse((0, source_id)));+ distances.insert(source_id, 0);+ }while let Some(state) = queue.pop() {let (cost, v) = state.0;@@ -377,30 +383,30 @@ whereimpl ShortestPaths for Webgraph {fn distances(&self, source: Node) -> BTreeMap<Node, u8> {- self.raw_distances(source)+ self.raw_distances(source.id()).into_iter().filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist))).collect()}- fn raw_distances(&self, source: Node) -> BTreeMap<NodeID, u8> {- dijkstra(- source,+ fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {+ dijkstra_multi(+ &[source],|node| self.raw_outgoing_edges(&node),|edge| edge.to,)}- fn raw_reversed_distances(&self, source: Node) -> BTreeMap<NodeID, u8> {- dijkstra(- source,+ fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {+ dijkstra_multi(+ &[source],|node| self.raw_ingoing_edges(&node),|edge| edge.from,)}fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {- self.raw_reversed_distances(source)+ self.raw_reversed_distances(source.id()).into_iter().filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist))).collect()@@ -828,6 +834,13 @@ impl Webgraph {self.id2node.keys()}+ pub fn random_nodes(&self, num: usize) -> Vec<NodeID> {+ let mut rng = rand::thread_rng();+ let mut nodes = self.nodes().collect::<Vec<_>>();+ nodes.shuffle(&mut rng);+ nodes.into_iter().take(num).collect()+ }+pub fn par_nodes(&self) -> impl ParallelIterator<Item = NodeID> + '_ {self.id2node.keys().par_bridge()}