Skip to content

web search done right

try an approximate harmonic centrality calculation for the page graph that randomly samples O(log n / epsilon) nodes and solves SSSP problem from these

Mikkel Denkere1966f0def9b
diff --git a/configs/crawler/planner.toml b/configs/crawler/planner.toml
index cf2824be..81af0656 100644
--- a/configs/crawler/planner.toml
+++ b/configs/crawler/planner.toml
@@ -2,7 +2,7 @@ host_graph_path = "data/webgraph_host"
host_harmonic_path = "data/centrality/harmonic"
output_path = "data/crawlplan"
page_graph_path = "data/webgraph_page"
-page_harmonic_path = "data/centrality_page/derived_harmonic"
+page_harmonic_path = "data/centrality_page/approx_harmonic"
num_job_queues = 5
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index 1f0a6984..2bc3ae9c 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -21,7 +21,7 @@ use crate::{
kv::{rocksdb_store::RocksDbStore, Kv},
ranking::inbound_similarity::InboundSimilarity,
webgraph::{
- centrality::{derived_harmonic::DerivedCentrality, harmonic::HarmonicCentrality},
+ centrality::{approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality},
Node, WebgraphBuilder,
},
};
@@ -111,24 +111,15 @@ impl Centrality {
.unwrap();
}
- pub fn build_derived_harmonic<P: AsRef<Path>>(
- webgraph_path: P,
- host_centrality_path: P,
- base_output: P,
- ) -> Result<()> {
- tracing::info!("Building derived harmonic centrality");
+ pub fn build_approx_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) -> Result<()> {
+ tracing::info!("Building approximated harmonic centrality for page graph");
let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();
- let host_centrality = RocksDbStore::open(host_centrality_path.as_ref().join("harmonic"));
- let derived = DerivedCentrality::build(
- &host_centrality,
- &graph,
- base_output.as_ref().join("derived_harmonic"),
- )?;
+ let approx = ApproxHarmonic::build(&graph, base_output.as_ref().join("approx_harmonic"));
let mut top_nodes = BinaryHeap::new();
- for (node_id, centrality) in derived.iter() {
+ for (node_id, centrality) in approx.iter() {
if top_nodes.len() < 1_000_000 {
top_nodes.push((Reverse(SortableFloat(centrality)), node_id));
} else {
@@ -144,7 +135,7 @@ impl Centrality {
.map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))
.collect();
- store_csv(derived, base_output.as_ref().join("derived_centrality.csv"));
+ store_csv(derived, base_output.as_ref().join("approx_harmonic.csv"));
Ok(())
}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 122c2dfc..7e3ed07d 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -144,7 +144,7 @@ fn calculate_centrality() {
let out_path_page = Path::new(DATA_PATH).join("centrality_page");
if !out_path_page.exists() {
- Centrality::build_derived_harmonic(webgraph_page, out_path, out_path_page).unwrap();
+ Centrality::build_approx_harmonic(webgraph_page, out_path_page).unwrap();
}
}
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index b2b2f43f..6b6caac6 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -72,7 +72,7 @@ impl IndexingWorker {
Path::new(&host_centrality_store_path).join("harmonic"),
),
page_centrality_store: page_centrality_store_path
- .map(|p| RocksDbStore::open(Path::new(&p).join("derived_harmonic"))),
+ .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic"))),
page_webgraph: page_webgraph_path
.map(|path| WebgraphBuilder::new(path).single_threaded().open()),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
diff --git a/core/src/main.rs b/core/src/main.rs
index 74ba767f..815f26e2 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -173,7 +173,6 @@ enum CentralityMode {
/// Calculate metrics for the page webgraph.
Page {
webgraph_path: String,
- host_centrality_path: String,
output_path: String,
},
}
@@ -269,13 +268,8 @@ fn main() -> Result<()> {
}
CentralityMode::Page {
webgraph_path,
- host_centrality_path,
output_path,
- } => entrypoint::Centrality::build_derived_harmonic(
- webgraph_path,
- host_centrality_path,
- output_path,
- )?,
+ } => entrypoint::Centrality::build_approx_harmonic(webgraph_path, output_path)?,
}
tracing::info!("Done");
}
diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rs
new file mode 100644
index 00000000..8252ae36
--- /dev/null
+++ b/core/src/webgraph/centrality/approx_harmonic.rs
@@ -0,0 +1,76 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{path::Path, sync::Mutex};
+
+use indicatif::ParallelProgressIterator;
+use rayon::prelude::*;
+
+use crate::{
+ kv::{rocksdb_store::RocksDbStore, Kv},
+ webgraph::{NodeID, ShortestPaths, Webgraph},
+};
+
+const EPSILON: f64 = 0.05;
+
+// Approximate harmonic centrality by sampling O(log n / epsilon^2) nodes and
+// computing single-source shortest paths from each of them.
+//
+// Epsilong is set to 0.05.
+pub struct ApproxHarmonic {
+ inner: RocksDbStore<NodeID, f64>,
+}
+
+impl ApproxHarmonic {
+ pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {
+ let num_nodes = graph.nodes().count();
+
+ let num_samples = ((num_nodes as f64).log2() / EPSILON.powi(2)).ceil() as usize;
+ let sampled = graph.random_nodes(num_samples);
+
+ let res = Mutex::new(Self {
+ inner: RocksDbStore::open(output),
+ });
+
+ let norm = num_nodes as f64 / (num_samples as f64 * (num_nodes as f64 - 1.0));
+
+ sampled.into_par_iter().progress().for_each(|source| {
+ let dists = graph.raw_distances(source);
+
+ let res = res.lock().unwrap();
+ for (target, dist) in dists {
+ if dist == 0 {
+ continue;
+ }
+
+ let dist = dist as f64;
+
+ let old = res.inner.get(&target).unwrap_or(0.0);
+ res.inner.insert(target, old + ((1.0 / dist) * norm));
+ }
+ });
+
+ res.into_inner().unwrap()
+ }
+
+ pub fn get(&self, node: &NodeID) -> Option<f64> {
+ self.inner.get(node)
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = (NodeID, f64)> + '_ {
+ self.inner.iter()
+ }
+}
diff --git a/core/src/webgraph/centrality/mod.rs b/core/src/webgraph/centrality/mod.rs
index 122cb320..012a8847 100644
--- a/core/src/webgraph/centrality/mod.rs
+++ b/core/src/webgraph/centrality/mod.rs
@@ -18,6 +18,7 @@ use crate::kv::{rocksdb_store::RocksDbStore, Kv};
use super::NodeID;
+pub mod approx_harmonic;
pub mod betweenness;
pub mod derived_harmonic;
pub mod harmonic;
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index fdeb80fd..06bb286f 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -22,6 +22,7 @@ use std::path::Path;
use std::sync::Arc;
use std::{cmp, fs};
+use rand::seq::SliceRandom;
use rayon::prelude::*;
use url::Url;
use utoipa::ToSchema;
@@ -333,24 +334,29 @@ impl WebgraphBuilder {
pub trait ShortestPaths {
fn distances(&self, source: Node) -> BTreeMap<Node, u8>;
- fn raw_distances(&self, source: Node) -> BTreeMap<NodeID, u8>;
- fn raw_reversed_distances(&self, source: Node) -> BTreeMap<NodeID, u8>;
+ fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
+ fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;
}
-fn dijkstra<F1, F2, L>(source: Node, node_edges: F1, edge_node: F2) -> BTreeMap<NodeID, u8>
+fn dijkstra_multi<F1, F2, L>(
+ sources: &[NodeID],
+ node_edges: F1,
+ edge_node: F2,
+) -> BTreeMap<NodeID, u8>
where
L: EdgeLabel,
F1: Fn(NodeID) -> Vec<Edge<L>>,
F2: Fn(&Edge<L>) -> NodeID,
{
- let source_id = source.id();
let mut distances: BTreeMap<NodeID, u8> = BTreeMap::default();
let mut queue = BinaryHeap::new();
- queue.push(cmp::Reverse((0, source_id)));
- distances.insert(source_id, 0);
+ for source_id in sources.iter().copied() {
+ queue.push(cmp::Reverse((0, source_id)));
+ distances.insert(source_id, 0);
+ }
while let Some(state) = queue.pop() {
let (cost, v) = state.0;
@@ -377,30 +383,30 @@ where
impl ShortestPaths for Webgraph {
fn distances(&self, source: Node) -> BTreeMap<Node, u8> {
- self.raw_distances(source)
+ self.raw_distances(source.id())
.into_iter()
.filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist)))
.collect()
}
- fn raw_distances(&self, source: Node) -> BTreeMap<NodeID, u8> {
- dijkstra(
- source,
+ fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {
+ dijkstra_multi(
+ &[source],
|node| self.raw_outgoing_edges(&node),
|edge| edge.to,
)
}
- fn raw_reversed_distances(&self, source: Node) -> BTreeMap<NodeID, u8> {
- dijkstra(
- source,
+ fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {
+ dijkstra_multi(
+ &[source],
|node| self.raw_ingoing_edges(&node),
|edge| edge.from,
)
}
fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {
- self.raw_reversed_distances(source)
+ self.raw_reversed_distances(source.id())
.into_iter()
.filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist)))
.collect()
@@ -828,6 +834,13 @@ impl Webgraph {
self.id2node.keys()
}
+ pub fn random_nodes(&self, num: usize) -> Vec<NodeID> {
+ let mut rng = rand::thread_rng();
+ let mut nodes = self.nodes().collect::<Vec<_>>();
+ nodes.shuffle(&mut rng);
+ nodes.into_iter().take(num).collect()
+ }
+
pub fn par_nodes(&self) -> impl ParallelIterator<Item = NodeID> + '_ {
self.id2node.keys().par_bridge()
}