Skip to content

web search done right

Better prioritization for which domains and urls to crawl. Each domain now starts with a score of 1.0 and is added with the score of all the incoming links for that domain. A domains score is distributed amongst all the outgoing links for that domain when it is sampled.

Mikkel Denkerceb4c83c7fb8
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
index 306bd6e2..54b924ea 100644
--- a/core/benches/crawl-coordinator.rs
+++ b/core/benches/crawl-coordinator.rs
@@ -52,6 +52,7 @@ fn random_responses(num: usize) -> Vec<JobResponse> {
domain: domain.clone(),
url_responses,
discovered_urls,
+ weight_budget: 1.0,
});
}
diff --git a/core/examples/crawl_coordinator_memory_stress.rs b/core/examples/crawl_coordinator_memory_stress.rs
index edeb71e1..919bfcd6 100644
--- a/core/examples/crawl_coordinator_memory_stress.rs
+++ b/core/examples/crawl_coordinator_memory_stress.rs
@@ -59,6 +59,7 @@ fn random_responses(num: usize) -> Vec<JobResponse> {
domain: domain.clone(),
url_responses,
discovered_urls,
+ weight_budget: 1.0,
});
}
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index d9b57ca1..d871eb21 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -20,16 +20,16 @@ use rand::Rng;
use rayon::prelude::*;
use std::hash::Hash;
use std::path::PathBuf;
-use std::{
- cmp::Ordering,
- collections::{BinaryHeap, VecDeque},
- path::Path,
-};
+use std::{collections::BinaryHeap, path::Path};
use url::Url;
use super::{Domain, Job, JobResponse, Result, UrlResponse};
-const MAX_URL_DB_SIZE_BYTES: u64 = 10 * 1024 * 1024 * 1024; // 10GB
+const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB
+
+/// TLDs that are known to be spammy and should be ignored.
+/// Whenever a TLD is free to register, it is likely to be abused by spammers.
+const BLOCKED_TLD: [&str; 1] = ["tk"];
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum UrlStatus {
@@ -71,7 +71,7 @@ impl<T> Ord for SampledItem<T> {
}
}
-fn weighted_sample<T>(items: impl Iterator<Item = (T, f64)>, num_items: usize) -> Vec<T> {
+fn weighted_sample<T>(items: impl Iterator<Item = (T, f64)>, num_items: usize) -> Vec<(T, f64)> {
let mut sampled_items: BinaryHeap<SampledItem<T>> = BinaryHeap::with_capacity(num_items);
let mut rng = rand::thread_rng();
@@ -90,7 +90,10 @@ fn weighted_sample<T>(items: impl Iterator<Item = (T, f64)>, num_items: usize) -
}
}
- sampled_items.into_iter().map(|s| s.item).collect()
+ sampled_items
+ .into_iter()
+ .map(|s| (s.item, s.priority))
+ .collect()
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
@@ -112,15 +115,13 @@ impl Default for UrlState {
struct DomainState {
weight: f64,
status: DomainStatus,
- total_urls: u64,
}
impl Default for DomainState {
fn default() -> Self {
Self {
- weight: 0.0,
+ weight: 1.0,
status: DomainStatus::Pending,
- total_urls: 0,
}
}
}
@@ -222,7 +223,7 @@ impl<T> From<T> for CachedValue<T> {
struct UrlToInsert {
url: Url,
- different_domain: bool,
+ weight: f64,
}
struct UrlStateDbShard {
@@ -433,7 +434,7 @@ impl UrlStateDb {
pub fn get(&self, domain: &Domain, url: &UrlString) -> Result<Option<UrlState>> {
// we iterate in reverse order so that we get the most recent state
- // since we insert new states at the last shard.
+ // since new states are inserted into the last shard.
for shard in self.shards.iter().rev() {
if let Some(state) = shard.get(domain, url)? {
return Ok(Some(state));
@@ -581,21 +582,7 @@ impl CrawlDb {
for url in urls {
let domain = Domain::from(url);
- match self.domain_state.get(&domain)? {
- Some(mut state) => {
- state.total_urls += 1;
- self.domain_state.put(&domain, &state)?;
- }
- None => self.domain_state.put(
- &domain,
- &DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- total_urls: 1,
- },
- )?,
- }
-
+ self.domain_state.put(&domain, &DomainState::default())?;
self.urls
.put_batch(&domain, &[(UrlString::from(url), UrlState::default())])?;
}
@@ -607,16 +594,62 @@ impl CrawlDb {
let domains: DashMap<Domain, Vec<UrlToInsert>> = DashMap::new();
responses.par_iter().for_each(|res| {
+ let urls: Vec<(Domain, Url)> = res
+ .discovered_urls
+ .iter()
+ .filter_map(|url| {
+ let domain = Domain::from(url);
+
+ for tld in BLOCKED_TLD.iter() {
+ if domain.as_str().ends_with(tld) {
+ return None;
+ }
+ }
+
+ Some((domain, url.clone()))
+ })
+ .collect();
+
+ let diff_domains = urls
+ .iter()
+ .filter(|(domain, _)| res.domain != *domain)
+ .count() as f64;
+ let mut used_budget = 0.0;
+
for url in &res.discovered_urls {
let domain = Domain::from(url);
+
+ for tld in BLOCKED_TLD.iter() {
+ if domain.as_str().ends_with(tld) {
+ continue;
+ }
+ }
+
let different_domain = res.domain != domain;
+ let weight = if different_domain {
+ (res.weight_budget / diff_domains).min(1.0)
+ } else {
+ 0.0
+ };
+
+ used_budget += weight;
+
domains.entry(domain).or_default().push(UrlToInsert {
url: url.clone(),
- different_domain,
+ weight,
});
}
+ let mut domain_state = self
+ .domain_state
+ .get(&res.domain)
+ .unwrap()
+ .unwrap_or_default();
+
+ domain_state.weight = (domain_state.weight - used_budget).max(0.0);
+ self.domain_state.put(&res.domain, &domain_state).unwrap();
+
for url_res in &res.url_responses {
if let UrlResponse::Redirected { url, new_url } = url_res {
self.redirects.put(url, new_url).ok();
@@ -627,19 +660,7 @@ impl CrawlDb {
let mut nonempty_domains = HashSet::new();
for (domain, urls) in domains.into_iter() {
- let mut domain_state = match self.domain_state.get(&domain)? {
- Some(state) => state,
- None => {
- let state = DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- total_urls: 0,
- };
- self.domain_state.put(&domain, &state)?;
-
- state
- }
- };
+ let mut domain_state = self.domain_state.get(&domain)?.unwrap_or_default();
if !urls.is_empty() {
nonempty_domains.insert(domain.clone());
@@ -650,19 +671,11 @@ impl CrawlDb {
for url in urls {
let mut url_state = match self.urls.get(&domain, &UrlString::from(&url.url))? {
Some(state) => state,
- None => {
- domain_state.total_urls += 1;
- UrlState::default()
- }
+ None => UrlState::default(),
};
- if url.different_domain {
- url_state.weight += 1.0;
- }
-
- if url_state.weight > domain_state.weight {
- domain_state.weight = url_state.weight;
- }
+ url_state.weight += url.weight;
+ domain_state.weight += url_state.weight;
url_states.push((UrlString::from(&url.url), url_state));
}
@@ -697,13 +710,13 @@ impl CrawlDb {
num_jobs,
);
- for domain in sampled.iter() {
+ for (domain, _) in sampled.iter() {
let mut state = self.domain_state.get(domain)?.unwrap_or_default();
state.status = DomainStatus::CrawlInProgress;
self.domain_state.put(domain, &state)?;
}
- Ok(sampled)
+ Ok(sampled.into_iter().map(|(d, _)| d).collect())
}
pub fn prepare_jobs(&mut self, domains: &[Domain], urls_per_job: usize) -> Result<Vec<Job>> {
@@ -729,7 +742,7 @@ impl CrawlDb {
let mut new_url_states = Vec::new();
- for url in &sampled {
+ for (url, _) in &sampled {
let mut state = self.urls.get(domain, url)?.unwrap_or_default();
state.status = UrlStatus::Crawling;
@@ -738,32 +751,15 @@ impl CrawlDb {
self.urls.put_batch(domain, &new_url_states)?;
- let mut domain_state = self.domain_state.get(domain)?.unwrap_or_default();
+ let domain_state = self.domain_state.get(domain)?.unwrap_or_default();
- domain_state.weight = urls
- .iter()
- .filter_map(|(_, state)| {
- if state.status == UrlStatus::Pending {
- Some(state.weight)
- } else {
- None
- }
- })
- .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal))
- .unwrap_or(0.0);
-
- self.domain_state.put(domain, &domain_state)?;
-
- let mut job = Job {
+ let job = Job {
domain: domain.clone(),
fetch_sitemap: false, // todo: fetch for new sites
- urls: VecDeque::with_capacity(urls_per_job),
+ urls: sampled.iter().map(|(url, _)| (*url).into()).collect(),
+ weight_budget: domain_state.weight,
};
- for url in sampled {
- job.urls.push_back(url.into());
- }
-
jobs.push(job);
}
@@ -794,7 +790,7 @@ mod tests {
let items: Vec<(usize, f64)> = vec![(0, 1000000000.0), (1, 2.0)];
let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 1);
assert_eq!(sampled.len(), 1);
- assert_eq!(*sampled[0], 0);
+ assert_eq!(*sampled[0].0, 0);
}
#[test]
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 5cd96947..9da3b382 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -111,6 +111,7 @@ pub struct Job {
pub domain: Domain,
pub fetch_sitemap: bool,
pub urls: VecDeque<Url>,
+ pub weight_budget: f64,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
@@ -125,6 +126,7 @@ pub struct JobResponse {
pub domain: Domain,
pub url_responses: Vec<UrlResponse>,
pub discovered_urls: Vec<Url>,
+ pub weight_budget: f64,
}
struct RetrieableUrl {
@@ -142,6 +144,7 @@ struct WorkerJob {
pub domain: Domain,
pub fetch_sitemap: bool,
pub urls: VecDeque<RetrieableUrl>,
+ pub weight_budget: f64,
}
impl From<Job> for WorkerJob {
@@ -150,6 +153,7 @@ impl From<Job> for WorkerJob {
domain: value.domain,
fetch_sitemap: value.fetch_sitemap,
urls: value.urls.into_iter().map(RetrieableUrl::from).collect(),
+ weight_budget: value.weight_budget,
}
}
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index a9f4a6ac..560aac1a 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -272,6 +272,7 @@ impl WorkerThread {
.into_iter()
.filter(|url| url.as_str().len() < MAX_URL_LENGTH)
.collect(),
+ weight_budget: job.weight_budget,
};
self.results.lock().await.push(job_response);
diff --git a/docs/src/crawler.md b/docs/src/crawler.md
index e9b56c20..28ce3ead 100644
--- a/docs/src/crawler.md
+++ b/docs/src/crawler.md
@@ -10,10 +10,10 @@ The coordinator starts with a list of seed urls, schedules these to the availabl
You can imagine that the url frontier can grow very large, very quickly. This begs the question: How does the coordinator determine which urls to crawl next? We could just crawl the urls in the order they were discovered, but this might not lead to the most interesting results.
-Instead, the coordinator assigns a score to each url and performs a weighted random selection of the next url to crawl. The score is determined by the number of incoming links to the url from other urls on different domains. The more incoming links a url has from other domains, the higher its score and the more likely it is to be selected for crawling.
-This prioritizes urls that are more likely to be interesting to the user. After a url has been chosen, we again sample from the url frontier but this time only choosing urls from the same domain as the chosen url. This ensures that we get a fairly good coverage of the domain before moving on to the next one.
+Instead, the coordinator assigns a score to each url and performs a weighted random selection of the next url to crawl.
+Each domain starts with a score of 1.0 and is summed with the weight of all the ingoing links. Whenever a domain is sampled, it spreads its score to all urls that are linked to from the domain. Thus if a domain has many outgoing links, the weight for each of those links will be relatively low compared to if the domain only has a few outgoing links.
-The sampled urls are then scheduled to the available workers and the process repeats.
+This prioritizes urls and domains that has many high-valued incoming links and are therefore more likely to be interesting for the user. The sampled urls are then scheduled to the available workers and the process repeats.
### Respectfullness
It is of utmost importance that we are respectful of the websites we crawl. We do not want to overload a website with requests and we do not want to crawl pages from the website that the website owner does not want us to crawl.