Skip to content

web search done right

[webgraph] speedup similar_hosts by estimating the best candidate nodes based on how many of the backlink nodes they have a link from

Mikkel Denker5639813d890a
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index 6c791d1b..e761a420 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -16,6 +16,7 @@
use std::{cmp::Reverse, sync::Arc};
+use fnv::FnvHashMap;
use hashbrown::HashSet;
use itertools::Itertools;
use url::Url;
@@ -31,6 +32,11 @@ use crate::{
SortableFloat,
};
+const NUM_BACKLINK_APPROXIMATION_THRESHOLD: usize = 32;
+const NUM_BACKLINK_APPROXIMATION_FRACTION: f64 = 0.25;
+const APPROXIMATION_CANDIDATES: usize = 256;
+const CANDIDATES_LIMIT: usize = 1024;
+
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]
pub struct ScoredNode {
pub node: Node,
@@ -93,20 +99,40 @@ impl SimilarHostsFinder {
.unique()
.collect::<Vec<_>>();
+ let num_backlink_nodes = backlink_nodes.len();
+
let outgoing_edges = self
.batch_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(512))
.await;
- outgoing_edges
+ let mut counts = FnvHashMap::default();
+
+ for e in outgoing_edges
.iter()
.flatten()
.filter(|e| !e.rel_flags.contains(RelFlags::NOFOLLOW))
- .map(|e| (e.to, 1))
- .into_group_map()
+ {
+ *counts.entry(e.to).or_insert(0) += 1;
+ }
+
+ let apply_filter = num_backlink_nodes > NUM_BACKLINK_APPROXIMATION_THRESHOLD;
+ let num_candidates = if apply_filter {
+ APPROXIMATION_CANDIDATES
+ } else {
+ CANDIDATES_LIMIT
+ };
+
+ counts
.into_iter()
- .sorted_by_key(|(_, c)| Reverse(c.len()))
+ .filter(|(_, c)| {
+ !apply_filter
+ || (*c as usize)
+ <= (num_backlink_nodes as f64 * NUM_BACKLINK_APPROXIMATION_FRACTION).ceil()
+ as usize
+ })
+ .sorted_by_key(|(_, c)| Reverse(*c))
.map(|(n, _)| n)
- .take(1024)
+ .take(num_candidates)
.filter(|n| !nodes.contains(n))
.collect()
}