Skip to content

web search done right

distribute crawl coordinator by introducing a crawl router middleman and use hash(domain) % coordinators.len() as the responsible coordinator for that domain

Mikkel Denkerfe5966b01454
diff --git a/configs/crawler/coordinator.toml b/configs/crawler/coordinator.toml
index 43018773..5dfc8bc4 100644
--- a/configs/crawler/coordinator.toml
+++ b/configs/crawler/coordinator.toml
@@ -1,4 +1,3 @@
crawldb_folder = "data/crawldb"
host = "0.0.0.0:8080"
-num_urls_to_crawl = 3
-seed_urls = ["https://weld.app"]
\ No newline at end of file
+seed_urls = ["https://en.wikipedia.org"]
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.toml
index 577019f1..c65bc35e 100644
--- a/configs/crawler/crawler.toml
+++ b/configs/crawler/crawler.toml
@@ -1,16 +1,16 @@
-num_worker_threads = 10
-num_jobs_per_fetch = 10
+num_jobs_per_fetch = 10
+num_worker_threads = 1
politeness_factor = 1.0
+router_hosts = ["0.0.0.0:8181"]
timeout_seconds = 30
-coordinator_host = "0.0.0.0:8080"
[user_agent]
-token = "StractBot"
full = "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)"
+token = "StractBot"
[s3]
+access_key = "<access_key>"
bucket = "crawl"
+endpoint = "http://s3.trystract.com"
folder = "test"
-access_key = "<access_key>"
secret_key = "<secret_key>"
-endpoint = "http://s3.trystract.com"
\ No newline at end of file
diff --git a/configs/crawler/router.toml b/configs/crawler/router.toml
new file mode 100644
index 00000000..fca9e23a
--- /dev/null
+++ b/configs/crawler/router.toml
@@ -0,0 +1,2 @@
+coordinator_addrs = ["0.0.0.0:8080"]
+host = "0.0.0.0:8181"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 147537ef..554e6da7 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -141,10 +141,6 @@ name = "build-similarity"
harness = false
name = "spell-correction"
-[[bench]]
-harness = false
-name = "crawl-coordinator"
-
[[bench]]
harness = false
name = "similar-sites"
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
deleted file mode 100644
index 54b924ea..00000000
--- a/core/benches/crawl-coordinator.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-// rust-analyzer: disable
-use criterion::{criterion_group, criterion_main, Criterion};
-use rand::Rng;
-use stract::crawler::{CrawlCoordinator, Domain, JobResponse, UrlResponse};
-use url::Url;
-
-fn rand_domain() -> Domain {
- // generate random domain name by taking from a-z
- // and appending .com
- let mut rng = rand::thread_rng();
- let mut domain = String::new();
- for _ in 0..rng.gen_range(1..10) {
- domain.push(rng.gen_range(b'a'..=b'z') as char);
- }
- domain.push_str(".com");
- domain.into()
-}
-
-fn rand_url(domain: &Domain) -> Url {
- let mut path = String::new();
-
- let mut rng = rand::thread_rng();
- for _ in 0..rng.gen_range(1..100) {
- path.push(rng.gen_range(b'a'..=b'z') as char);
- }
-
- Url::parse(&format!("https://{}/{}", domain.as_str(), path)).unwrap()
-}
-
-fn random_responses(num: usize) -> Vec<JobResponse> {
- let mut responses = Vec::with_capacity(num);
- for _ in 0..num {
- let domain = rand_domain();
-
- let url_responses = vec![
- UrlResponse::Success {
- url: rand_url(&domain)
- };
- 100
- ];
-
- let mut discovered_urls = Vec::new();
- for _ in 0..3 {
- let domain = rand_domain();
-
- for _ in 0..10 {
- discovered_urls.push(rand_url(&domain));
- }
- }
-
- responses.push(JobResponse {
- domain: domain.clone(),
- url_responses,
- discovered_urls,
- weight_budget: 1.0,
- });
- }
-
- responses
-}
-
-pub fn criterion_benchmark(c: &mut Criterion) {
- let coordinator = CrawlCoordinator::new("data/crawldb", 100, vec![]).unwrap();
-
- for _ in 0..1_000 {
- c.bench_function("Add response to coordinator", |b| {
- b.iter(|| {
- let responses = random_responses(1024);
- coordinator.add_responses(&responses).unwrap();
- })
- });
-
- c.bench_function("Sample jobs", |b| {
- b.iter(|| {
- coordinator.sample_jobs(256).unwrap();
- })
- });
- }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/core/examples/crawl_coordinator_memory_stress.rs b/core/examples/crawl_coordinator_memory_stress.rs
deleted file mode 100644
index 919bfcd6..00000000
--- a/core/examples/crawl_coordinator_memory_stress.rs
+++ /dev/null
@@ -1,95 +0,0 @@
-use anyhow::Result;
-use indicatif::{ProgressBar, ProgressStyle};
-use rand::Rng;
-use stract::crawler::{CrawlCoordinator, Domain, JobResponse, UrlResponse};
-use url::Url;
-
-#[cfg(not(target_env = "msvc"))]
-use tikv_jemallocator::Jemalloc;
-
-#[cfg(not(target_env = "msvc"))]
-#[global_allocator]
-static GLOBAL: Jemalloc = Jemalloc;
-
-fn rand_domain() -> Domain {
- // generate random domain name by taking from a-z
- // and appending .com
- let mut rng = rand::thread_rng();
- let mut domain = String::new();
- for _ in 0..rng.gen_range(1..3) {
- domain.push(rng.gen_range(b'a'..=b'z') as char);
- }
- domain.push_str(".com");
- domain.into()
-}
-
-fn rand_url(domain: &Domain) -> Url {
- let mut path = String::new();
-
- let mut rng = rand::thread_rng();
- for _ in 0..rng.gen_range(1..100) {
- path.push(rng.gen_range(b'a'..=b'z') as char);
- }
-
- Url::parse(&format!("https://{}/{}", domain.as_str(), path)).unwrap()
-}
-
-fn random_responses(num: usize) -> Vec<JobResponse> {
- let mut responses = Vec::with_capacity(num);
- for _ in 0..num {
- let domain = rand_domain();
-
- let url_responses = vec![
- UrlResponse::Success {
- url: rand_url(&domain)
- };
- 100
- ];
-
- let mut discovered_urls = Vec::new();
- for _ in 0..3 {
- let domain = rand_domain();
-
- for _ in 0..10 {
- discovered_urls.push(rand_url(&domain));
- }
- }
-
- responses.push(JobResponse {
- domain: domain.clone(),
- url_responses,
- discovered_urls,
- weight_budget: 1.0,
- });
- }
-
- responses
-}
-
-fn main() -> Result<()> {
- let test_size = 1_000_000_000;
- if std::path::Path::new("data/crawldb").exists() {
- std::fs::remove_dir_all("data/crawldb")?;
- }
-
- let coordinator = CrawlCoordinator::new("data/crawldb", test_size, vec![])?;
-
- let pb = ProgressBar::new(test_size);
- pb.set_style(
- ProgressStyle::default_bar()
- .template(
- "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({per_sec})",
- )
- .progress_chars("#>-"),
- );
-
- for _ in 0..test_size {
- pb.inc(1);
- let responses = random_responses(1024);
- coordinator.add_responses(&responses)?;
-
- coordinator.sample_jobs(1024)?;
- }
-
- Ok(())
-}
diff --git a/core/src/call_counter.rs b/core/src/call_counter.rs
deleted file mode 100644
index d051ffb6..00000000
--- a/core/src/call_counter.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::collections::VecDeque;
-use std::time::{Duration, Instant};
-
-#[derive(Debug, Clone, Copy)]
-struct WeightedInstant {
- weight: usize,
- instant: Instant,
-}
-
-pub struct CallCounter {
- call_times: VecDeque<WeightedInstant>,
- window: Duration,
- cur_weights: usize,
-}
-
-impl CallCounter {
- // Constructs a new `CallCounter` with a specified window duration for the rolling average.
- pub fn new(window: Duration) -> CallCounter {
- CallCounter {
- call_times: VecDeque::new(),
- window,
- cur_weights: 0,
- }
- }
-
- pub fn count_with_weight(&mut self, weight: usize) {
- let now = Instant::now();
- self.call_times.push_back(WeightedInstant {
- weight,
- instant: now,
- });
- self.cur_weights += weight;
- }
-
- fn prune(&mut self) {
- let now = Instant::now();
-
- while let Some(&old) = self.call_times.front() {
- if now.duration_since(old.instant) < self.window {
- break;
- }
-
- self.cur_weights -= old.weight;
- self.call_times.pop_front();
- }
- }
-
- // Calculate the average calls per second.
- pub fn avg_per_second(&mut self) -> f64 {
- self.prune();
- self.cur_weights as f64 / self.window.as_secs_f64()
- }
-}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 92f7785e..96ff736a 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -44,7 +44,6 @@ pub struct WebgraphConstructConfig {
pub host_graph_base_path: String,
pub page_graph_base_path: String,
pub warc_source: WarcSource,
- pub redirect_db_path: Option<String>,
pub limit_warc_files: Option<usize>,
pub batch_size: Option<usize>,
}
@@ -264,7 +263,6 @@ pub struct SearchServerConfig {
pub struct CrawlCoordinatorConfig {
pub crawldb_folder: String,
pub host: SocketAddr,
- pub num_urls_to_crawl: u64,
pub seed_urls: Vec<String>,
}
@@ -303,7 +301,13 @@ pub struct CrawlerConfig {
pub timeout_seconds: u64,
pub s3: S3Config,
- pub coordinator_host: String,
+ pub router_hosts: Vec<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CrawlRouterConfig {
+ pub host: SocketAddr,
+ pub coordinator_addrs: Vec<SocketAddr>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 27fa24cb..afe47ce8 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -14,38 +14,20 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use hashbrown::HashMap;
use url::Url;
-use crate::call_counter::CallCounter;
-
-use super::{
- crawl_db::{CrawlDb, DomainStatus},
- Job, JobResponse, Result,
-};
-use std::{
- path::Path,
- sync::{
- atomic::{AtomicU64, Ordering},
- Mutex,
- },
- time::{Duration, Instant},
-};
+use super::{crawl_db::CrawlDb, Domain, DomainCrawled, Job, Result, UrlToInsert};
+use std::{path::Path, sync::Mutex, time::Instant};
const DEFAULT_JOB_URLS: usize = 200;
pub struct CrawlCoordinator {
db: Mutex<CrawlDb>,
- num_crawled_urls: AtomicU64,
- num_urls_to_crawl: u64,
- call_counter: Mutex<CallCounter>,
}
impl CrawlCoordinator {
- pub fn new<P: AsRef<Path>>(
- crawldb_folder: P,
- num_urls_to_crawl: u64,
- seed_urls: Vec<String>,
- ) -> Result<Self> {
+ pub fn new<P: AsRef<Path>>(crawldb_folder: P, seed_urls: Vec<String>) -> Result<Self> {
let mut db = CrawlDb::open(crawldb_folder)?;
let mut parsed_seed_urls = Vec::new();
@@ -55,53 +37,31 @@ impl CrawlCoordinator {
db.insert_seed_urls(&parsed_seed_urls)?;
- Ok(Self {
- db: Mutex::new(db),
- num_urls_to_crawl,
- num_crawled_urls: AtomicU64::new(0),
- call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),
- })
+ Ok(Self { db: Mutex::new(db) })
}
- fn log_crawls_per_second(&self, num_urls: usize) {
- if self.is_done() {
- return;
- }
+ pub fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {
+ let mut db = self.db.lock().unwrap();
+ let start = Instant::now();
- let mut call_counter = self.call_counter.lock().unwrap();
+ db.insert_urls(urls)?;
- call_counter.count_with_weight(num_urls);
- tracing::info!("avg crawls per second: {}", call_counter.avg_per_second());
- }
-
- pub fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {
- let num_crawled_urls = responses.iter().map(|res| res.url_responses.len()).sum();
+ tracing::info!("inserted responses in {:?}", start.elapsed());
- self.log_crawls_per_second(num_crawled_urls);
- self.num_crawled_urls
- .fetch_add(num_crawled_urls as u64, Ordering::SeqCst);
+ Ok(())
+ }
+ pub fn mark_jobs_complete(&self, domains: &[DomainCrawled]) -> Result<()> {
let mut db = self.db.lock().unwrap();
let start = Instant::now();
- db.insert_urls(responses)?;
-
- let response_domains = responses
- .iter()
- .map(|resp| resp.domain.clone())
- .collect::<Vec<_>>();
-
- db.set_domain_status(&response_domains, DomainStatus::Pending)?;
+ db.mark_jobs_complete(domains)?;
- tracing::info!("inserted responses in {:?}", start.elapsed());
+ tracing::info!("marked jobs complete in {:?}", start.elapsed());
Ok(())
}
- pub fn is_done(&self) -> bool {
- self.num_crawled_urls.load(Ordering::SeqCst) >= self.num_urls_to_crawl
- }
-
pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {
let mut db = self.db.lock().unwrap();
let start = Instant::now();
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 07c92542..cb627517 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -21,7 +21,7 @@ use std::path::PathBuf;
use std::{collections::BinaryHeap, path::Path};
use url::Url;
-use super::{Domain, Job, JobResponse, Result, UrlResponse};
+use super::{Domain, DomainCrawled, Job, Result, UrlToInsert};
const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB
@@ -119,51 +119,6 @@ impl Default for DomainState {
}
}
-pub struct RedirectDb {
- inner: rocksdb::DB,
-}
-
-impl RedirectDb {
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mut options = rocksdb::Options::default();
-
- options.create_if_missing(true);
-
- let mut block_options = rocksdb::BlockBasedOptions::default();
- block_options.set_format_version(5);
-
- options.set_block_based_table_factory(&block_options);
-
- let inner = rocksdb::DB::open(&options, path.as_ref())?;
-
- Ok(Self { inner })
- }
-
- pub fn put(&self, from: &Url, to: &Url) -> Result<()> {
- let url_bytes = bincode::serialize(from)?;
- let redirect_bytes = bincode::serialize(to)?;
-
- let mut write_options = rocksdb::WriteOptions::default();
- write_options.disable_wal(true);
- self.inner
- .put_opt(url_bytes, redirect_bytes, &write_options)?;
-
- Ok(())
- }
-
- pub fn get(&self, from: &Url) -> Result<Option<Url>> {
- let url_bytes = bincode::serialize(from)?;
- let redirect_bytes = self.inner.get(url_bytes)?;
-
- if let Some(redirect_bytes) = redirect_bytes {
- let redirect: Url = bincode::deserialize(&redirect_bytes)?;
- return Ok(Some(redirect));
- }
-
- Ok(None)
- }
-}
-
struct MetaDb {
db: rocksdb::DB,
}
@@ -214,11 +169,6 @@ impl<T> From<T> for CachedValue<T> {
}
}
-struct UrlToInsert {
- url: Url,
- weight: f64,
-}
-
struct UrlStateDbShard {
db: rocksdb::DB,
meta: MetaDb,
@@ -603,13 +553,11 @@ impl From<&UrlString> for Url {
pub struct CrawlDb {
domain_state: DomainStateDb,
urls: UrlStateDb,
- redirects: RedirectDb,
}
impl CrawlDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
Ok(Self {
- redirects: RedirectDb::open(path.as_ref().join("redirects"))?,
domain_state: DomainStateDb::open(path.as_ref().join("domains"))?,
urls: UrlStateDb::open(path.as_ref().join("urls"))?,
})
@@ -627,73 +575,13 @@ impl CrawlDb {
Ok(())
}
- pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<()> {
- let mut domain_urls: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();
- let mut domain_batches = Vec::new();
-
- let response_domains: Vec<_> = responses.iter().map(|res| res.domain.clone()).collect();
- let response_domains = self.domain_state.multi_get(&response_domains)?;
-
- for res in responses {
- let mut urls: Vec<(Domain, Url)> = res
- .discovered_urls
- .iter()
- .map(|url| {
- let domain = Domain::from(url);
- (domain, url.clone())
- })
- .collect();
-
- let diff_domains = urls
- .iter()
- .filter(|(domain, _)| res.domain != *domain)
- .count() as f64;
-
- urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));
- urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());
-
- let mut used_budget = 0.0;
-
- for (domain, url) in urls {
- let different_domain = res.domain != domain;
-
- let weight = if different_domain {
- (res.weight_budget / diff_domains).min(1.0)
- } else {
- 0.0
- };
-
- used_budget += weight;
-
- domain_urls
- .entry(domain)
- .or_default()
- .push(UrlToInsert { url, weight });
- }
-
- let mut domain_state = response_domains
- .get(&res.domain)
- .cloned()
- .unwrap_or_default();
-
- domain_state.weight = (domain_state.weight - used_budget).max(0.0);
- domain_batches.push((res.domain.clone(), domain_state));
-
- for url_res in &res.url_responses {
- if let UrlResponse::Redirected { url, new_url } = url_res {
- self.redirects.put(url, new_url).ok();
- }
- }
- }
-
- self.domain_state.put_batch(&domain_batches)?;
-
+ pub fn insert_urls(&mut self, domain_urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {
let mut url_batches = Vec::new();
let mut domain_batches = Vec::new();
let domains = domain_urls.keys().cloned().collect::<Vec<_>>();
let domain_states = self.domain_state.multi_get(&domains)?;
- let url_states = self.urls.multi_get(
+ let mut url_states = self.urls.multi_get(
domain_urls
.iter()
.flat_map(|(domain, urls)| {
@@ -710,17 +598,22 @@ impl CrawlDb {
let mut updated_url_states = Vec::new();
for url in urls {
- let mut url_state = match url_states.get(&UrlString::from(&url.url)) {
- Some(state) => state.clone(),
- None => UrlState::default(),
- };
-
- url_state.weight += url.weight;
- domain_state.weight += url_state.weight;
+ match url_states.get_mut(&UrlString::from(&url.url)) {
+ Some(state) => {
+ state.weight += url.weight;
+ domain_state.weight += url.weight;
+ if url.weight > 0.0 {
+ updated_url_states.push((UrlString::from(&url.url), state.clone()));
+ }
+ }
+ None => {
+ let mut state = UrlState::default();
+ state.weight += url.weight;
+ domain_state.weight += url.weight;
- if url.weight > 0.0 {
- updated_url_states.push((UrlString::from(&url.url), url_state));
- }
+ updated_url_states.push((UrlString::from(&url.url), state));
+ }
+ };
}
url_batches.push((domain.clone(), updated_url_states));
@@ -733,7 +626,7 @@ impl CrawlDb {
Ok(())
}
- pub fn set_domain_status(&mut self, domains: &[Domain], status: DomainStatus) -> Result<()> {
+ fn set_domain_status(&mut self, domains: &[Domain], status: DomainStatus) -> Result<()> {
let domain_states = self.domain_state.multi_get(domains)?;
let mut batches = Vec::new();
@@ -750,6 +643,33 @@ impl CrawlDb {
Ok(())
}
+ pub fn mark_jobs_complete(&mut self, domains: &[DomainCrawled]) -> Result<()> {
+ let domain_states = self.domain_state.multi_get(
+ domains
+ .iter()
+ .map(|d| d.domain.clone())
+ .collect::<Vec<_>>()
+ .as_slice(),
+ )?;
+
+ let mut batches = Vec::new();
+
+ for domain in domains {
+ let mut domain_state = domain_states
+ .get(&domain.domain)
+ .cloned()
+ .unwrap_or_default();
+ domain_state.status = DomainStatus::Pending;
+ domain_state.weight -= domain.budget_used;
+
+ batches.push((domain.domain.clone(), domain_state));
+ }
+
+ self.domain_state.put_batch(&batches)?;
+
+ Ok(())
+ }
+
pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<Domain>> {
let sampled = weighted_sample(
self.domain_state.iter().filter_map(|(domain, state)| {
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 9da3b382..e1dede69 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -30,6 +30,8 @@ use self::{warc_writer::WarcWriter, worker::WorkerThread};
pub mod coordinator;
pub mod crawl_db;
mod robots_txt;
+pub mod router;
+pub use router::Router;
mod warc_writer;
mod worker;
@@ -129,6 +131,23 @@ pub struct JobResponse {
pub weight_budget: f64,
}
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct UrlToInsert {
+ pub url: Url,
+ pub weight: f64,
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct DiscoveredUrls {
+ pub urls: HashMap<Domain, Vec<UrlToInsert>>,
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct DomainCrawled {
+ pub domain: Domain,
+ pub budget_used: f64,
+}
+
struct RetrieableUrl {
url: Url,
retries: u8,
@@ -184,7 +203,11 @@ impl Crawler {
let writer = Arc::new(WarcWriter::new(config.s3.clone()));
let timeout = Duration::from_secs(config.timeout_seconds);
let mut handles = Vec::new();
- let coordinator_host = config.coordinator_host.parse()?;
+ let mut router_hosts = Vec::new();
+
+ for host in &config.router_hosts {
+ router_hosts.push(host.parse()?);
+ }
for _ in 0..config.num_worker_threads {
let worker = WorkerThread::new(
@@ -193,7 +216,7 @@ impl Crawler {
Arc::clone(&results),
config.clone(),
timeout,
- coordinator_host,
+ router_hosts.clone(),
)?;
handles.push(tokio::spawn(async move {
@@ -204,7 +227,7 @@ impl Crawler {
Ok(Self { writer, handles })
}
- pub async fn wait(self) {
+ pub async fn run(self) {
for handle in self.handles {
handle.await.ok();
}
diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rs
new file mode 100644
index 00000000..ad0dee4b
--- /dev/null
+++ b/core/src/crawler/router.rs
@@ -0,0 +1,177 @@
+use anyhow::Result;
+use hashbrown::HashMap;
+use rand::seq::SliceRandom;
+use std::{net::SocketAddr, time::Duration};
+use url::Url;
+
+use crate::{
+ crawler::{DomainCrawled, UrlToInsert},
+ distributed::{retry_strategy::ExponentialBackoff, sonic},
+ entrypoint::crawler::coordinator::{CoordinatorService, GetJobs, InsertUrls, MarkJobsComplete},
+};
+
+use super::{Domain, Job, JobResponse};
+
+struct RemoteCoordinator {
+ addr: SocketAddr,
+}
+
+impl RemoteCoordinator {
+ async fn conn(&self) -> Result<sonic::service::ResilientConnection<CoordinatorService>> {
+ let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));
+
+ Ok(sonic::service::ResilientConnection::create_with_timeout(
+ self.addr,
+ Duration::from_secs(3600),
+ retry,
+ )
+ .await?)
+ }
+
+ async fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {
+ let conn = self.conn().await?;
+
+ let response = conn
+ .send_with_timeout(&GetJobs { num_jobs }, Duration::from_secs(60 * 60))
+ .await?;
+
+ Ok(response)
+ }
+
+ async fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {
+ let conn = self.conn().await?;
+
+ conn.send_with_timeout(&InsertUrls { urls }, Duration::from_secs(60 * 60))
+ .await?;
+
+ Ok(())
+ }
+
+ async fn mark_jobs_complete(&self, domains: Vec<DomainCrawled>) -> Result<()> {
+ let conn = self.conn().await?;
+
+ conn.send_with_timeout(&MarkJobsComplete { domains }, Duration::from_secs(60 * 60))
+ .await?;
+
+ Ok(())
+ }
+}
+
+pub struct Router {
+ coordinators: Vec<RemoteCoordinator>,
+}
+
+impl Router {
+ pub fn new(coordinator_addrs: Vec<SocketAddr>) -> Self {
+ Self {
+ coordinators: coordinator_addrs
+ .into_iter()
+ .map(|addr| RemoteCoordinator { addr })
+ .collect(),
+ }
+ }
+
+ fn coordinator_index(&self, domain: &Domain) -> usize {
+ let hash = md5::compute(domain.0.as_bytes());
+ let hash = u128::from_le_bytes(hash.0) as usize;
+ hash % self.coordinators.len()
+ }
+
+ pub async fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {
+ let random_coordinator = self.coordinators.choose(&mut rand::thread_rng()).unwrap();
+
+ random_coordinator.sample_jobs(num_jobs).await
+ }
+
+ pub async fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {
+ let mut domain_urls: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();
+ let mut domain_budgets: Vec<DomainCrawled> = Vec::new();
+
+ for res in responses {
+ let mut urls: Vec<(Domain, Url)> = res
+ .discovered_urls
+ .iter()
+ .map(|url| {
+ let domain = Domain::from(url);
+ (domain, url.clone())
+ })
+ .collect();
+
+ let diff_domains = urls
+ .iter()
+ .filter(|(domain, _)| res.domain != *domain)
+ .count() as f64;
+
+ urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));
+ urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());
+
+ let mut used_budget = 0.0;
+
+ for (domain, url) in urls {
+ let different_domain = res.domain != domain;
+
+ let weight = if different_domain {
+ (res.weight_budget / diff_domains).min(1.0)
+ } else {
+ 0.0
+ };
+
+ used_budget += weight;
+
+ domain_urls
+ .entry(domain)
+ .or_default()
+ .push(UrlToInsert { url, weight });
+ }
+
+ domain_budgets.push(DomainCrawled {
+ domain: res.domain.clone(),
+ budget_used: used_budget,
+ });
+ }
+
+ let mut coordinator_urls: HashMap<usize, HashMap<Domain, Vec<UrlToInsert>>> =
+ HashMap::new();
+
+ for (domain, urls) in domain_urls {
+ let coordinator_index = self.coordinator_index(&domain);
+
+ coordinator_urls
+ .entry(coordinator_index)
+ .or_default()
+ .insert(domain, urls);
+ }
+
+ let mut futures = Vec::new();
+
+ for (coordinator_index, urls) in coordinator_urls {
+ let coordinator = &self.coordinators[coordinator_index];
+
+ futures.push(coordinator.insert_urls(urls));
+ }
+
+ futures::future::join_all(futures).await;
+
+ let mut futures = Vec::new();
+ let mut coordinator_budgets: HashMap<usize, Vec<DomainCrawled>> = HashMap::new();
+
+ for domain in domain_budgets {
+ let coordinator_index = self.coordinator_index(&domain.domain);
+
+ coordinator_budgets
+ .entry(coordinator_index)
+ .or_default()
+ .push(domain);
+ }
+
+ for (coordinator_index, budgets) in coordinator_budgets {
+ let coordinator = &self.coordinators[coordinator_index];
+
+ futures.push(coordinator.mark_jobs_complete(budgets));
+ }
+
+ futures::future::join_all(futures).await;
+
+ Ok(())
+ }
+}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 560aac1a..1bb2184a 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -17,6 +17,7 @@ use encoding_rs::{Encoding, UTF_8};
use futures::{future::BoxFuture, FutureExt};
use mime::Mime;
use quick_xml::events::Event;
+use rand::seq::SliceRandom;
use tokio_stream::StreamExt;
use std::{
@@ -33,7 +34,7 @@ use crate::{
config::CrawlerConfig,
crawler::{JobResponse, Response},
distributed::{retry_strategy::ExponentialBackoff, sonic},
- entrypoint::crawler::{CoordinatorService, NewJobs},
+ entrypoint::crawler::router::{NewJobs, RouterService},
webpage::Html,
};
@@ -54,7 +55,7 @@ pub struct WorkerThread {
client: reqwest::Client,
config: CrawlerConfig,
politeness_factor: f32,
- coordinator_host: SocketAddr,
+ router_hosts: Vec<SocketAddr>,
num_jobs_per_fetch: usize,
robotstxt: RobotsTxtManager,
}
@@ -66,7 +67,7 @@ impl WorkerThread {
results: Arc<Mutex<Vec<JobResponse>>>,
config: CrawlerConfig,
timeout: Duration,
- coordinator_host: SocketAddr,
+ router_hosts: Vec<SocketAddr>,
) -> Result<Self> {
if config.politeness_factor < config.min_politeness_factor {
return Err(Error::InvalidPolitenessFactor.into());
@@ -104,19 +105,19 @@ impl WorkerThread {
num_jobs_per_fetch: config.num_jobs_per_fetch,
politeness_factor: config.politeness_factor,
config,
- coordinator_host,
+ router_hosts,
robotstxt,
})
}
- async fn coordinator_conn(
- &self,
- ) -> Result<sonic::service::ResilientConnection<CoordinatorService>> {
+ async fn router_conn(&self) -> Result<sonic::service::ResilientConnection<RouterService>> {
let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));
+ let router = *self.router_hosts.choose(&mut rand::thread_rng()).unwrap();
+
Ok(sonic::service::ResilientConnection::create_with_timeout(
- self.coordinator_host,
- Duration::from_secs(30),
+ router,
+ Duration::from_secs(3600),
retry,
)
.await?)
@@ -145,7 +146,7 @@ impl WorkerThread {
}
}
} else {
- let conn = self.coordinator_conn().await.unwrap();
+ let conn = self.router_conn().await.unwrap();
let results = self.results.lock().await.drain(..).collect::<Vec<_>>();
let res = conn
@@ -154,7 +155,7 @@ impl WorkerThread {
responses: results,
num_jobs: self.num_jobs_per_fetch,
},
- Duration::from_secs(10 * 60),
+ Duration::from_secs(60 * 60),
)
.await;
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index ae3dd8f9..96947c29 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -84,7 +84,6 @@ fn create_webgraph() -> Result<()> {
};
let mut worker = webgraph::WebgraphWorker {
- redirect: None,
host_graph: webgraph::open_host_graph_writer(&out_path_host),
page_graph: webgraph::open_page_graph_writer(&out_path_page),
};
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index f008227e..d2e5598e 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -16,11 +16,12 @@
use std::{net::SocketAddr, sync::Arc};
+use hashbrown::HashMap;
use serde::{Deserialize, Serialize};
use crate::{
config,
- crawler::{CrawlCoordinator, Crawler, JobResponse},
+ crawler::{self, CrawlCoordinator, Crawler, Domain, DomainCrawled, JobResponse, UrlToInsert},
distributed::sonic::{self, service::Message},
sonic_service, Result,
};
@@ -28,53 +29,123 @@ use crate::{
pub async fn worker(config: config::CrawlerConfig) -> Result<()> {
let crawler = Crawler::new(config).await?;
- crawler.wait().await;
+ crawler.run().await;
Ok(())
}
-pub struct CoordinatorService {
- coordinator: Arc<CrawlCoordinator>,
+pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {
+ let coordinator = Arc::new(CrawlCoordinator::new(
+ config.crawldb_folder,
+ config.seed_urls,
+ )?);
+
+ let addr: SocketAddr = config.host;
+ let server = coordinator::CoordinatorService { coordinator }
+ .bind(addr)
+ .await
+ .unwrap();
+
+ tracing::info!("Crawl coordinator listening on {}", addr);
+
+ loop {
+ let _ = server.accept().await;
+ }
}
-sonic_service!(CoordinatorService, [NewJobs]);
+pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {
+ let router = crawler::Router::new(config.coordinator_addrs.clone());
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NewJobs {
- pub responses: Vec<JobResponse>,
- pub num_jobs: usize,
+ let addr: SocketAddr = config.host;
+
+ let server = router::RouterService { router }.bind(addr).await.unwrap();
+
+ tracing::info!("Crawl router listening on {}", addr);
+
+ loop {
+ let _ = server.accept().await;
+ }
}
-#[async_trait::async_trait]
-impl Message<CoordinatorService> for NewJobs {
- type Response = crate::crawler::Response;
- async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {
- server.coordinator.add_responses(&self.responses)?;
+pub mod router {
+ use super::*;
+ pub struct RouterService {
+ pub router: crawler::Router,
+ }
- if server.coordinator.is_done() {
- tracing::info!("Crawl is done. Waiting for workers to finish.");
- Ok(crate::crawler::Response::Done)
- } else {
- let jobs = server.coordinator.sample_jobs(self.num_jobs)?;
+ sonic_service!(RouterService, [NewJobs]);
+
+ #[derive(Debug, Clone, Serialize, Deserialize)]
+ pub struct NewJobs {
+ pub responses: Vec<JobResponse>,
+ pub num_jobs: usize,
+ }
+ #[async_trait::async_trait]
+ impl Message<RouterService> for NewJobs {
+ type Response = crate::crawler::Response;
+
+ async fn handle(self, server: &RouterService) -> sonic::Result<Self::Response> {
+ server.router.add_responses(&self.responses).await?;
+ let jobs = server.router.sample_jobs(self.num_jobs).await?;
Ok(crate::crawler::Response::NewJobs { jobs })
}
}
}
-pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {
- let coordinator = Arc::new(CrawlCoordinator::new(
- config.crawldb_folder,
- config.num_urls_to_crawl,
- config.seed_urls,
- )?);
+pub mod coordinator {
+ use crate::crawler::Job;
- let addr: SocketAddr = config.host;
- let server = CoordinatorService { coordinator }.bind(addr).await.unwrap();
+ use super::*;
- tracing::info!("Crawl coordinator listening on {}", addr);
+ pub struct CoordinatorService {
+ pub coordinator: Arc<CrawlCoordinator>,
+ }
- loop {
- let _ = server.accept().await;
+ sonic_service!(CoordinatorService, [InsertUrls, GetJobs, MarkJobsComplete]);
+
+ #[derive(Debug, Clone, Serialize, Deserialize)]
+ pub struct InsertUrls {
+ pub urls: HashMap<Domain, Vec<UrlToInsert>>,
+ }
+
+ #[derive(Debug, Clone, Serialize, Deserialize)]
+ pub struct GetJobs {
+ pub num_jobs: usize,
+ }
+
+ #[derive(Debug, Clone, Serialize, Deserialize)]
+ pub struct MarkJobsComplete {
+ pub domains: Vec<DomainCrawled>,
+ }
+
+ #[async_trait::async_trait]
+ impl Message<CoordinatorService> for InsertUrls {
+ type Response = ();
+
+ async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {
+ server.coordinator.insert_urls(self.urls)?;
+ Ok(())
+ }
+ }
+
+ #[async_trait::async_trait]
+ impl Message<CoordinatorService> for GetJobs {
+ type Response = Vec<Job>;
+
+ async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {
+ let jobs = server.coordinator.sample_jobs(self.num_jobs)?;
+ Ok(jobs)
+ }
+ }
+
+ #[async_trait::async_trait]
+ impl Message<CoordinatorService> for MarkJobsComplete {
+ type Response = ();
+
+ async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {
+ server.coordinator.mark_jobs_complete(&self.domains)?;
+ Ok(())
+ }
}
}
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index aff872dd..72a9fbc2 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -16,7 +16,6 @@
use crate::{
config::WarcSource,
config::{self, WebgraphConstructConfig},
- crawler::crawl_db::RedirectDb,
entrypoint::download_all_warc_files,
mapreduce::Worker,
webgraph::{self, Node, WebgraphWriter},
@@ -25,7 +24,7 @@ use crate::{
};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
-use std::{fs, path::Path, sync::Arc};
+use std::{fs, path::Path};
use tokio::pin;
use tracing::{info, trace};
@@ -86,7 +85,6 @@ pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWrit
pub struct WebgraphWorker {
pub host_graph: webgraph::WebgraphWriter,
pub page_graph: webgraph::WebgraphWriter,
- pub redirect: Option<Arc<RedirectDb>>,
}
impl WebgraphWorker {
@@ -122,14 +120,7 @@ impl WebgraphWorker {
})
{
let source = link.source.clone();
- let mut destination = link.destination.clone();
-
- if let Some(redirect) = &self.redirect {
- if let Some(new_destination) = redirect.get(&destination).unwrap() {
- trace!("redirecting {:?} to {:?}", destination, new_destination);
- destination = new_destination;
- }
- }
+ let destination = link.destination.clone();
if source.root_domain() == destination.root_domain() {
continue;
@@ -168,11 +159,6 @@ impl Webgraph {
let job_config = JobConfig::from(config.warc_source.clone());
- let redirect = match &config.redirect_db_path {
- Some(path) => Some(Arc::new(RedirectDb::open(path)?)),
- None => None,
- };
-
let jobs: Vec<_> = warc_paths
.into_iter()
.take(config.limit_warc_files.unwrap_or(usize::MAX))
@@ -200,7 +186,6 @@ impl Webgraph {
let page_path = page_path.join(format!("worker_{i}"));
let mut worker = WebgraphWorker {
- redirect: redirect.clone(),
host_graph: open_host_graph_writer(host_path),
page_graph: open_page_graph_writer(page_path),
};
diff --git a/core/src/lib.rs b/core/src/lib.rs
index b2e94909..681f0f59 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -35,7 +35,6 @@ pub mod alice;
mod api;
mod autosuggest;
mod bangs;
-mod call_counter;
mod collector;
pub mod config;
pub mod crawler;
diff --git a/core/src/main.rs b/core/src/main.rs
index 2d979fc3..1a20e327 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -94,13 +94,11 @@ enum Commands {
output_dir: String,
},
- /// Deploy the crawl coordinator. The crawl coordinator is responsible for
- /// distributing crawl jobs to the crawles and deciding which urls to crawl next.
- CrawlCoordinator { config_path: String },
-
- /// Deploy the crawler. The crawler is responsible for downloading webpages, saving them to S3,
- /// and sending newly discovered urls back to the crawl coordinator.
- Crawler { config_path: String },
+ /// Deploy the crawler.
+ Crawler {
+ #[clap(subcommand)]
+ options: Crawler,
+ },
/// Train or run inference on the classifier that predicts if a webpage is NSFW or SFW.
SafetyClassifier {
@@ -119,6 +117,21 @@ enum Commands {
},
}
+#[derive(Subcommand)]
+enum Crawler {
+ /// Deploy the crawl worker. The worker is responsible for downloading webpages, saving them to S3,
+ /// and sending newly discovered urls back to the crawl coordinator.
+ Worker { config_path: String },
+
+ /// Deploy the crawl coordinator. The crawl coordinator is responsible for
+ /// distributing crawl jobs to the crawles and deciding which urls to crawl next.
+ Coordinator { config_path: String },
+
+ /// Deploy the crawl router. The crawl router is responsible for routing job responses and requests
+ /// from the workers to the correct crawl coordinators.
+ Router { config_path: String },
+}
+
/// Commands to deploy Alice.
#[derive(Subcommand)]
enum AliceOptions {
@@ -311,22 +324,32 @@ fn main() -> Result<()> {
}
AliceOptions::GenerateKey => entrypoint::alice::generate_key(),
},
- Commands::CrawlCoordinator { config_path } => {
- let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);
+ Commands::Crawler { options } => match options {
+ Crawler::Worker { config_path } => {
+ let config: config::CrawlerConfig = load_toml_config(config_path);
- tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()?
- .block_on(entrypoint::crawler::coordinator(config))?
- }
- Commands::Crawler { config_path } => {
- let config: config::CrawlerConfig = load_toml_config(config_path);
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(entrypoint::crawler::worker(config))?
+ }
+ Crawler::Coordinator { config_path } => {
+ let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);
- tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()?
- .block_on(entrypoint::crawler::worker(config))?
- }
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(entrypoint::crawler::coordinator(config))?
+ }
+ Crawler::Router { config_path } => {
+ let config: config::CrawlRouterConfig = load_toml_config(config_path);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(entrypoint::crawler::router(config))?
+ }
+ },
Commands::SafetyClassifier { options } => match options {
SafetyClassifierOptions::Train {
dataset_path,