Skip to content

web search done right

[crawler] prioritise urls during the wandering phase by the sum of the page weights that links to them. this prevents us from having to calculate the full harmonic centralities and therefore store the entire site graph in memory, which caused oom for some of the very big sites.

Mikkel Denkerf76034648304
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 00e74954..bf0be611 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -31,7 +31,7 @@ pub mod router;
pub use router::Router;
mod file_queue;
pub mod planner;
-mod site_graph;
+mod wander_prirotiser;
mod warc_writer;
mod worker;
@@ -119,12 +119,18 @@ impl Domain {
}
}
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
+pub struct WeightedUrl {
+ pub url: Url,
+ pub weight: f64,
+}
+
/// All urls in a job must be from the same domain and only one job per domain.
/// at a time. This ensures that we stay polite when crawling.
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
pub struct Job {
pub domain: Domain,
- pub urls: VecDeque<Url>,
+ pub urls: VecDeque<WeightedUrl>,
pub wandering_urls: u64,
}
@@ -177,13 +183,22 @@ pub struct DomainCrawled {
}
pub struct RetrieableUrl {
- url: Url,
+ weighted_url: WeightedUrl,
retries: u8,
}
-impl From<Url> for RetrieableUrl {
- fn from(url: Url) -> Self {
- Self { url, retries: 0 }
+impl RetrieableUrl {
+ pub fn url(&self) -> &Url {
+ &self.weighted_url.url
+ }
+}
+
+impl From<WeightedUrl> for RetrieableUrl {
+ fn from(weighted_url: WeightedUrl) -> Self {
+ Self {
+ weighted_url,
+ retries: 0,
+ }
}
}
diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rs
index 6a6d22f5..ebddb907 100644
--- a/core/src/crawler/planner.rs
+++ b/core/src/crawler/planner.rs
@@ -27,6 +27,7 @@ use std::{
};
use url::Url;
+use crate::crawler::WeightedUrl;
use crate::webgraph::centrality::{top_hosts, TopHosts};
use crate::{
config::CrawlPlannerConfig,
@@ -256,10 +257,12 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
urls.extend(
pages
.into_iter()
- .map(|(id, _)| id)
- .filter_map(|id| page_graph.id2node(&id))
- .map(|n| n.name)
- .filter_map(|n| Url::parse(&format!("http://{n}")).ok())
+ .filter_map(|(id, score)| page_graph.id2node(&id).map(|n| (n, score)))
+ .map(|(n, score)| (n.name, score))
+ .filter_map(|(n, score)| {
+ Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))
+ })
+ .map(|(url, score)| WeightedUrl { url, weight: score })
.take(schedule_budget as usize),
);
diff --git a/core/src/crawler/site_graph.rs b/core/src/crawler/site_graph.rs
deleted file mode 100644
index 56aef931..00000000
--- a/core/src/crawler/site_graph.rs
+++ /dev/null
@@ -1,159 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-//! In-memory graph that the worker constructs for the site during crawl.
-
-use std::collections::{BTreeMap, BTreeSet};
-
-use url::Url;
-
-use crate::{hyperloglog::HyperLogLog, kahan_sum::KahanSum};
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct Node {
- url: Url,
-}
-
-impl From<Url> for Node {
- fn from(url: Url) -> Self {
- Self { url }
- }
-}
-
-impl From<Node> for Url {
- fn from(node: Node) -> Self {
- node.url
- }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct NodeRef(usize);
-
-#[derive(Default)]
-pub struct SiteGraph {
- nodes: Vec<Node>,
- node_refs: BTreeMap<Node, NodeRef>,
- edges: Vec<(NodeRef, NodeRef)>,
-}
-
-impl SiteGraph {
- pub fn new() -> Self {
- Self::default()
- }
-
- pub fn add_node(&mut self, node: Node) -> NodeRef {
- if let Some(node_ref) = self.node_refs.get(&node) {
- return *node_ref;
- }
-
- let node_ref = NodeRef(self.nodes.len());
- self.nodes.push(node.clone());
- self.node_refs.insert(node, node_ref);
-
- node_ref
- }
-
- pub fn add_edge(&mut self, from: NodeRef, to: NodeRef) {
- self.edges.push((from, to));
- }
-
- pub fn get_node(&self, node_ref: NodeRef) -> Option<&Node> {
- self.nodes.get(node_ref.0)
- }
-
- pub fn compute_centralities(&self) -> BTreeMap<NodeRef, f64> {
- let mut counters: BTreeMap<_, _> = self
- .node_refs
- .values()
- .map(|node_ref| {
- let mut counter: HyperLogLog<64> = HyperLogLog::default();
- counter.add(node_ref.0 as u64);
- (*node_ref, counter)
- })
- .collect();
-
- let mut centralities: BTreeMap<_, _> = self
- .node_refs
- .values()
- .map(|node_ref| {
- let kahan = KahanSum::default();
-
- (*node_ref, kahan)
- })
- .collect();
-
- let mut changed_nodes: BTreeSet<_> = self.node_refs.values().copied().collect();
- let mut t = 0;
-
- loop {
- if changed_nodes.is_empty() {
- break;
- }
-
- let mut new_counters = counters.clone();
- let mut new_changed_nodes = BTreeSet::new();
-
- for (from, to) in &self.edges {
- if changed_nodes.contains(from) {
- let counter_to = new_counters.get_mut(to).unwrap();
- let counter_from = &counters[from];
-
- if counter_to
- .registers()
- .iter()
- .zip(counter_from.registers().iter())
- .any(|(to, from)| *from > *to)
- {
- counter_to.merge(counter_from);
- new_changed_nodes.insert(*to);
- }
- }
- }
-
- centralities.iter_mut().for_each(|(node, score)| {
- *score += new_counters
- .get(node)
- .map(|counter| counter.size())
- .unwrap_or_default()
- .checked_sub(
- counters
- .get(node)
- .map(|counter| counter.size())
- .unwrap_or_default(),
- )
- .unwrap_or_default() as f64
- / (t + 1) as f64;
- });
-
- counters = new_counters;
- changed_nodes = new_changed_nodes;
- t += 1;
- }
-
- centralities
- .into_iter()
- .map(|(node_ref, kahan)| (node_ref, f64::from(kahan)))
- .filter(|(_, centrality)| *centrality > 0.0)
- .map(|(node_ref, centrality)| {
- if !centrality.is_finite() {
- (node_ref, 0.0)
- } else {
- (node_ref, centrality)
- }
- })
- .collect()
- }
-}
diff --git a/core/src/crawler/wander_prirotiser.rs b/core/src/crawler/wander_prirotiser.rs
new file mode 100644
index 00000000..30ee425f
--- /dev/null
+++ b/core/src/crawler/wander_prirotiser.rs
@@ -0,0 +1,59 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! In-memory graph that the worker constructs for the site during crawl.
+
+use std::collections::BTreeMap;
+
+use url::Url;
+
+#[derive(Default)]
+pub struct WanderPrioritiser {
+ url_weights: BTreeMap<Url, f64>,
+}
+
+impl WanderPrioritiser {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn inc(&mut self, url: Url, weight: f64) {
+ self.url_weights
+ .entry(url)
+ .and_modify(|w| *w += weight)
+ .or_insert(weight);
+ }
+
+ pub fn top_and_clear(&mut self, top_n: usize) -> Vec<(Url, f64)> {
+ let mut urls: Vec<_> = self
+ .url_weights
+ .iter()
+ .map(|(url, weight)| (weight, url))
+ .collect();
+
+ urls.sort_by(|(w1, _), (w2, _)| w2.total_cmp(w1));
+
+ let res = urls
+ .into_iter()
+ .take(top_n)
+ .map(|(w, url)| (url.clone(), *w))
+ .collect();
+
+ self.url_weights = BTreeMap::new();
+
+ res
+ }
+}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index a42f070b..412464a9 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -40,8 +40,9 @@ use crate::{
};
use super::{
- reqwest_client, robots_txt::RobotsTxtManager, site_graph::SiteGraph, CrawlDatum, DatumStream,
- Domain, Error, Result, RetrieableUrl, Site, UrlResponse, WarcWriter, WorkerJob,
+ reqwest_client, robots_txt::RobotsTxtManager, wander_prirotiser::WanderPrioritiser, CrawlDatum,
+ DatumStream, Domain, Error, Result, RetrieableUrl, Site, UrlResponse, WarcWriter, WeightedUrl,
+ WorkerJob,
};
const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
@@ -125,7 +126,7 @@ pub struct JobExecutor<S: DatumStream> {
crawled_sitemaps: HashSet<Site>,
sitemap_urls: HashSet<Url>,
config: Arc<CrawlerConfig>,
- site_graph: SiteGraph,
+ wander_prioritiser: WanderPrioritiser,
job: WorkerJob,
}
@@ -148,7 +149,7 @@ impl<S: DatumStream> JobExecutor<S> {
crawled_sitemaps: HashSet::new(),
sitemap_urls: HashSet::new(),
config,
- site_graph: SiteGraph::new(),
+ wander_prioritiser: WanderPrioritiser::new(),
job,
}
}
@@ -170,15 +171,9 @@ impl<S: DatumStream> JobExecutor<S> {
async fn wander(&mut self) {
let mut urls: Vec<(Url, f64)> = self
- .site_graph
- .compute_centralities()
+ .wander_prioritiser
+ .top_and_clear(self.job.wandering_urls as usize)
.into_iter()
- .map(|(node_ref, score)| {
- (
- Url::from(self.site_graph.get_node(node_ref).unwrap().clone()),
- score,
- )
- })
.chain(self.sitemap_urls.drain().map(|url| (url.clone(), 0.0)))
.filter(|(url, _)| !self.crawled_urls.contains(url))
.filter(|(url, _)| self.job.domain == Domain::from(url))
@@ -186,7 +181,7 @@ impl<S: DatumStream> JobExecutor<S> {
.collect();
urls.sort_by(|(a, _), (b, _)| a.cmp(b));
- urls.dedup();
+ urls.dedup_by(|(a, _), (b, _)| a == b);
urls.sort_by(|(_, a), (_, b)| b.total_cmp(a));
@@ -194,6 +189,7 @@ impl<S: DatumStream> JobExecutor<S> {
.into_iter()
.take(self.job.wandering_urls as usize)
.map(|(url, _)| url)
+ .map(|url| WeightedUrl { url, weight: 0.0 })
.map(RetrieableUrl::from)
.collect();
@@ -202,11 +198,11 @@ impl<S: DatumStream> JobExecutor<S> {
async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {
while let Some(retryable_url) = urls.pop_front() {
- if Domain::from(&retryable_url.url) != self.job.domain {
+ if Domain::from(retryable_url.url()) != self.job.domain {
continue;
}
- if self.crawled_urls.contains(&retryable_url.url) {
+ if self.crawled_urls.contains(retryable_url.url()) {
continue;
}
@@ -214,8 +210,8 @@ impl<S: DatumStream> JobExecutor<S> {
continue;
}
- if retryable_url.url.host_str().is_none()
- || !matches!(retryable_url.url.scheme(), "http" | "https")
+ if retryable_url.url().host_str().is_none()
+ || !matches!(retryable_url.url().scheme(), "http" | "https")
{
continue;
}
@@ -223,41 +219,44 @@ impl<S: DatumStream> JobExecutor<S> {
if !self.config.dry_run
&& !self
.robotstxt
- .is_allowed(&retryable_url.url, &self.config.user_agent.token)
+ .is_allowed(retryable_url.url(), &self.config.user_agent.token)
.await
{
continue;
}
- let site = Site(retryable_url.url.host_str().unwrap_or_default().to_string());
+ let site = Site(
+ retryable_url
+ .url()
+ .host_str()
+ .unwrap_or_default()
+ .to_string(),
+ );
if !self.config.dry_run && fetch_sitemap && !self.crawled_sitemaps.contains(&site) {
self.crawled_sitemaps.insert(site.clone());
- if let Some(sitemap) = self.robotstxt.sitemap(&retryable_url.url).await {
+ if let Some(sitemap) = self.robotstxt.sitemap(retryable_url.url()).await {
self.sitemap_urls
.extend(self.urls_from_sitemap(sitemap, 0, 5).await);
}
}
- let res = self.process_url(retryable_url.url.clone()).await;
+ let res = self.process_url(retryable_url.url().clone()).await;
match res.response {
- UrlResponse::Success { url } => {
- self.crawled_urls.insert(url.clone());
-
- let from_node = self.site_graph.add_node(url.clone().into());
+ UrlResponse::Success { url: _ } => {
+ let weight = retryable_url.weighted_url.weight;
for new_url in res.new_urls {
if new_url.host_str().is_none() {
continue;
}
- if new_url.root_domain() != retryable_url.url.root_domain() {
+ if new_url.root_domain() != retryable_url.url().root_domain() {
continue;
}
- let to_node = self.site_graph.add_node(new_url.clone().into());
- self.site_graph.add_edge(from_node, to_node);
+ self.wander_prioritiser.inc(new_url, weight);
}
}
UrlResponse::Failed {
diff --git a/core/src/live_index.rs b/core/src/live_index.rs
index f69a28c5..f0822a24 100644
--- a/core/src/live_index.rs
+++ b/core/src/live_index.rs
@@ -26,7 +26,9 @@ use url::Url;
use crate::{
config::{CrawlerConfig, LiveIndexConfig},
- crawler::{reqwest_client, CrawlDatum, DatumStream, JobExecutor, RetrieableUrl, WorkerJob},
+ crawler::{
+ reqwest_client, CrawlDatum, DatumStream, JobExecutor, RetrieableUrl, WeightedUrl, WorkerJob,
+ },
entrypoint::indexer::IndexingWorker,
feed::{
self,
@@ -147,7 +149,11 @@ impl Crawler {
let domain = urls.first().unwrap().into();
let job = WorkerJob {
domain,
- urls: urls.clone().into_iter().map(RetrieableUrl::from).collect(),
+ urls: urls
+ .clone()
+ .into_iter()
+ .map(|url| RetrieableUrl::from(WeightedUrl { url, weight: 1.0 }))
+ .collect(),
wandering_urls: 0,
};