web search done right
distribute crawl coordinator by introducing a crawl router middleman and use hash(domain) % coordinators.len() as the responsible coordinator for that domain
Mikkel Denkerfe5966b01454
diff --git a/configs/crawler/coordinator.toml b/configs/crawler/coordinator.tomlindex 43018773..5dfc8bc4 100644--- a/configs/crawler/coordinator.toml+++ b/configs/crawler/coordinator.toml@@ -1,4 +1,3 @@crawldb_folder = "data/crawldb"host = "0.0.0.0:8080"-num_urls_to_crawl = 3-seed_urls = ["https://weld.app"]\ No newline at end of file+seed_urls = ["https://en.wikipedia.org"]diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.tomlindex 577019f1..c65bc35e 100644--- a/configs/crawler/crawler.toml+++ b/configs/crawler/crawler.toml@@ -1,16 +1,16 @@-num_worker_threads = 10-num_jobs_per_fetch = 10+num_jobs_per_fetch = 10+num_worker_threads = 1politeness_factor = 1.0+router_hosts = ["0.0.0.0:8181"]timeout_seconds = 30-coordinator_host = "0.0.0.0:8080"[user_agent]-token = "StractBot"full = "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)"+token = "StractBot"[s3]+access_key = "<access_key>"bucket = "crawl"+endpoint = "http://s3.trystract.com"folder = "test"-access_key = "<access_key>"secret_key = "<secret_key>"-endpoint = "http://s3.trystract.com"\ No newline at end of filediff --git a/configs/crawler/router.toml b/configs/crawler/router.tomlnew file mode 100644index 00000000..fca9e23a--- /dev/null+++ b/configs/crawler/router.toml@@ -0,0 +1,2 @@+coordinator_addrs = ["0.0.0.0:8080"]+host = "0.0.0.0:8181"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 147537ef..554e6da7 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -141,10 +141,6 @@ name = "build-similarity"harness = falsename = "spell-correction"-[[bench]]-harness = false-name = "crawl-coordinator"-[[bench]]harness = falsename = "similar-sites"diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rsdeleted file mode 100644index 54b924ea..00000000--- a/core/benches/crawl-coordinator.rs+++ /dev/null@@ -1,82 +0,0 @@-// rust-analyzer: disable-use criterion::{criterion_group, criterion_main, Criterion};-use rand::Rng;-use stract::crawler::{CrawlCoordinator, Domain, JobResponse, UrlResponse};-use url::Url;--fn rand_domain() -> Domain {- // generate random domain name by taking from a-z- // and appending .com- let mut rng = rand::thread_rng();- let mut domain = String::new();- for _ in 0..rng.gen_range(1..10) {- domain.push(rng.gen_range(b'a'..=b'z') as char);- }- domain.push_str(".com");- domain.into()-}--fn rand_url(domain: &Domain) -> Url {- let mut path = String::new();-- let mut rng = rand::thread_rng();- for _ in 0..rng.gen_range(1..100) {- path.push(rng.gen_range(b'a'..=b'z') as char);- }-- Url::parse(&format!("https://{}/{}", domain.as_str(), path)).unwrap()-}--fn random_responses(num: usize) -> Vec<JobResponse> {- let mut responses = Vec::with_capacity(num);- for _ in 0..num {- let domain = rand_domain();-- let url_responses = vec![- UrlResponse::Success {- url: rand_url(&domain)- };- 100- ];-- let mut discovered_urls = Vec::new();- for _ in 0..3 {- let domain = rand_domain();-- for _ in 0..10 {- discovered_urls.push(rand_url(&domain));- }- }-- responses.push(JobResponse {- domain: domain.clone(),- url_responses,- discovered_urls,- weight_budget: 1.0,- });- }-- responses-}--pub fn criterion_benchmark(c: &mut Criterion) {- let coordinator = CrawlCoordinator::new("data/crawldb", 100, vec![]).unwrap();-- for _ in 0..1_000 {- c.bench_function("Add response to coordinator", |b| {- b.iter(|| {- let responses = random_responses(1024);- coordinator.add_responses(&responses).unwrap();- })- });-- c.bench_function("Sample jobs", |b| {- b.iter(|| {- coordinator.sample_jobs(256).unwrap();- })- });- }-}--criterion_group!(benches, criterion_benchmark);-criterion_main!(benches);diff --git a/core/examples/crawl_coordinator_memory_stress.rs b/core/examples/crawl_coordinator_memory_stress.rsdeleted file mode 100644index 919bfcd6..00000000--- a/core/examples/crawl_coordinator_memory_stress.rs+++ /dev/null@@ -1,95 +0,0 @@-use anyhow::Result;-use indicatif::{ProgressBar, ProgressStyle};-use rand::Rng;-use stract::crawler::{CrawlCoordinator, Domain, JobResponse, UrlResponse};-use url::Url;--#[cfg(not(target_env = "msvc"))]-use tikv_jemallocator::Jemalloc;--#[cfg(not(target_env = "msvc"))]-#[global_allocator]-static GLOBAL: Jemalloc = Jemalloc;--fn rand_domain() -> Domain {- // generate random domain name by taking from a-z- // and appending .com- let mut rng = rand::thread_rng();- let mut domain = String::new();- for _ in 0..rng.gen_range(1..3) {- domain.push(rng.gen_range(b'a'..=b'z') as char);- }- domain.push_str(".com");- domain.into()-}--fn rand_url(domain: &Domain) -> Url {- let mut path = String::new();-- let mut rng = rand::thread_rng();- for _ in 0..rng.gen_range(1..100) {- path.push(rng.gen_range(b'a'..=b'z') as char);- }-- Url::parse(&format!("https://{}/{}", domain.as_str(), path)).unwrap()-}--fn random_responses(num: usize) -> Vec<JobResponse> {- let mut responses = Vec::with_capacity(num);- for _ in 0..num {- let domain = rand_domain();-- let url_responses = vec![- UrlResponse::Success {- url: rand_url(&domain)- };- 100- ];-- let mut discovered_urls = Vec::new();- for _ in 0..3 {- let domain = rand_domain();-- for _ in 0..10 {- discovered_urls.push(rand_url(&domain));- }- }-- responses.push(JobResponse {- domain: domain.clone(),- url_responses,- discovered_urls,- weight_budget: 1.0,- });- }-- responses-}--fn main() -> Result<()> {- let test_size = 1_000_000_000;- if std::path::Path::new("data/crawldb").exists() {- std::fs::remove_dir_all("data/crawldb")?;- }-- let coordinator = CrawlCoordinator::new("data/crawldb", test_size, vec![])?;-- let pb = ProgressBar::new(test_size);- pb.set_style(- ProgressStyle::default_bar()- .template(- "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({per_sec})",- )- .progress_chars("#>-"),- );-- for _ in 0..test_size {- pb.inc(1);- let responses = random_responses(1024);- coordinator.add_responses(&responses)?;-- coordinator.sample_jobs(1024)?;- }-- Ok(())-}diff --git a/core/src/call_counter.rs b/core/src/call_counter.rsdeleted file mode 100644index d051ffb6..00000000--- a/core/src/call_counter.rs+++ /dev/null@@ -1,68 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::collections::VecDeque;-use std::time::{Duration, Instant};--#[derive(Debug, Clone, Copy)]-struct WeightedInstant {- weight: usize,- instant: Instant,-}--pub struct CallCounter {- call_times: VecDeque<WeightedInstant>,- window: Duration,- cur_weights: usize,-}--impl CallCounter {- // Constructs a new `CallCounter` with a specified window duration for the rolling average.- pub fn new(window: Duration) -> CallCounter {- CallCounter {- call_times: VecDeque::new(),- window,- cur_weights: 0,- }- }-- pub fn count_with_weight(&mut self, weight: usize) {- let now = Instant::now();- self.call_times.push_back(WeightedInstant {- weight,- instant: now,- });- self.cur_weights += weight;- }-- fn prune(&mut self) {- let now = Instant::now();-- while let Some(&old) = self.call_times.front() {- if now.duration_since(old.instant) < self.window {- break;- }-- self.cur_weights -= old.weight;- self.call_times.pop_front();- }- }-- // Calculate the average calls per second.- pub fn avg_per_second(&mut self) -> f64 {- self.prune();- self.cur_weights as f64 / self.window.as_secs_f64()- }-}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 92f7785e..96ff736a 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -44,7 +44,6 @@ pub struct WebgraphConstructConfig {pub host_graph_base_path: String,pub page_graph_base_path: String,pub warc_source: WarcSource,- pub redirect_db_path: Option<String>,pub limit_warc_files: Option<usize>,pub batch_size: Option<usize>,}@@ -264,7 +263,6 @@ pub struct SearchServerConfig {pub struct CrawlCoordinatorConfig {pub crawldb_folder: String,pub host: SocketAddr,- pub num_urls_to_crawl: u64,pub seed_urls: Vec<String>,}@@ -303,7 +301,13 @@ pub struct CrawlerConfig {pub timeout_seconds: u64,pub s3: S3Config,- pub coordinator_host: String,+ pub router_hosts: Vec<String>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct CrawlRouterConfig {+ pub host: SocketAddr,+ pub coordinator_addrs: Vec<SocketAddr>,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 27fa24cb..afe47ce8 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -14,38 +14,20 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use hashbrown::HashMap;use url::Url;-use crate::call_counter::CallCounter;--use super::{- crawl_db::{CrawlDb, DomainStatus},- Job, JobResponse, Result,-};-use std::{- path::Path,- sync::{- atomic::{AtomicU64, Ordering},- Mutex,- },- time::{Duration, Instant},-};+use super::{crawl_db::CrawlDb, Domain, DomainCrawled, Job, Result, UrlToInsert};+use std::{path::Path, sync::Mutex, time::Instant};const DEFAULT_JOB_URLS: usize = 200;pub struct CrawlCoordinator {db: Mutex<CrawlDb>,- num_crawled_urls: AtomicU64,- num_urls_to_crawl: u64,- call_counter: Mutex<CallCounter>,}impl CrawlCoordinator {- pub fn new<P: AsRef<Path>>(- crawldb_folder: P,- num_urls_to_crawl: u64,- seed_urls: Vec<String>,- ) -> Result<Self> {+ pub fn new<P: AsRef<Path>>(crawldb_folder: P, seed_urls: Vec<String>) -> Result<Self> {let mut db = CrawlDb::open(crawldb_folder)?;let mut parsed_seed_urls = Vec::new();@@ -55,53 +37,31 @@ impl CrawlCoordinator {db.insert_seed_urls(&parsed_seed_urls)?;- Ok(Self {- db: Mutex::new(db),- num_urls_to_crawl,- num_crawled_urls: AtomicU64::new(0),- call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),- })+ Ok(Self { db: Mutex::new(db) })}- fn log_crawls_per_second(&self, num_urls: usize) {- if self.is_done() {- return;- }+ pub fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {+ let mut db = self.db.lock().unwrap();+ let start = Instant::now();- let mut call_counter = self.call_counter.lock().unwrap();+ db.insert_urls(urls)?;- call_counter.count_with_weight(num_urls);- tracing::info!("avg crawls per second: {}", call_counter.avg_per_second());- }-- pub fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {- let num_crawled_urls = responses.iter().map(|res| res.url_responses.len()).sum();+ tracing::info!("inserted responses in {:?}", start.elapsed());- self.log_crawls_per_second(num_crawled_urls);- self.num_crawled_urls- .fetch_add(num_crawled_urls as u64, Ordering::SeqCst);+ Ok(())+ }+ pub fn mark_jobs_complete(&self, domains: &[DomainCrawled]) -> Result<()> {let mut db = self.db.lock().unwrap();let start = Instant::now();- db.insert_urls(responses)?;-- let response_domains = responses- .iter()- .map(|resp| resp.domain.clone())- .collect::<Vec<_>>();-- db.set_domain_status(&response_domains, DomainStatus::Pending)?;+ db.mark_jobs_complete(domains)?;- tracing::info!("inserted responses in {:?}", start.elapsed());+ tracing::info!("marked jobs complete in {:?}", start.elapsed());Ok(())}- pub fn is_done(&self) -> bool {- self.num_crawled_urls.load(Ordering::SeqCst) >= self.num_urls_to_crawl- }-pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {let mut db = self.db.lock().unwrap();let start = Instant::now();diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 07c92542..cb627517 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -21,7 +21,7 @@ use std::path::PathBuf;use std::{collections::BinaryHeap, path::Path};use url::Url;-use super::{Domain, Job, JobResponse, Result, UrlResponse};+use super::{Domain, DomainCrawled, Job, Result, UrlToInsert};const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB@@ -119,51 +119,6 @@ impl Default for DomainState {}}-pub struct RedirectDb {- inner: rocksdb::DB,-}--impl RedirectDb {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut options = rocksdb::Options::default();-- options.create_if_missing(true);-- let mut block_options = rocksdb::BlockBasedOptions::default();- block_options.set_format_version(5);-- options.set_block_based_table_factory(&block_options);-- let inner = rocksdb::DB::open(&options, path.as_ref())?;-- Ok(Self { inner })- }-- pub fn put(&self, from: &Url, to: &Url) -> Result<()> {- let url_bytes = bincode::serialize(from)?;- let redirect_bytes = bincode::serialize(to)?;-- let mut write_options = rocksdb::WriteOptions::default();- write_options.disable_wal(true);- self.inner- .put_opt(url_bytes, redirect_bytes, &write_options)?;-- Ok(())- }-- pub fn get(&self, from: &Url) -> Result<Option<Url>> {- let url_bytes = bincode::serialize(from)?;- let redirect_bytes = self.inner.get(url_bytes)?;-- if let Some(redirect_bytes) = redirect_bytes {- let redirect: Url = bincode::deserialize(&redirect_bytes)?;- return Ok(Some(redirect));- }-- Ok(None)- }-}-struct MetaDb {db: rocksdb::DB,}@@ -214,11 +169,6 @@ impl<T> From<T> for CachedValue<T> {}}-struct UrlToInsert {- url: Url,- weight: f64,-}-struct UrlStateDbShard {db: rocksdb::DB,meta: MetaDb,@@ -603,13 +553,11 @@ impl From<&UrlString> for Url {pub struct CrawlDb {domain_state: DomainStateDb,urls: UrlStateDb,- redirects: RedirectDb,}impl CrawlDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {Ok(Self {- redirects: RedirectDb::open(path.as_ref().join("redirects"))?,domain_state: DomainStateDb::open(path.as_ref().join("domains"))?,urls: UrlStateDb::open(path.as_ref().join("urls"))?,})@@ -627,73 +575,13 @@ impl CrawlDb {Ok(())}- pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<()> {- let mut domain_urls: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();- let mut domain_batches = Vec::new();-- let response_domains: Vec<_> = responses.iter().map(|res| res.domain.clone()).collect();- let response_domains = self.domain_state.multi_get(&response_domains)?;-- for res in responses {- let mut urls: Vec<(Domain, Url)> = res- .discovered_urls- .iter()- .map(|url| {- let domain = Domain::from(url);- (domain, url.clone())- })- .collect();-- let diff_domains = urls- .iter()- .filter(|(domain, _)| res.domain != *domain)- .count() as f64;-- urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));- urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());-- let mut used_budget = 0.0;-- for (domain, url) in urls {- let different_domain = res.domain != domain;-- let weight = if different_domain {- (res.weight_budget / diff_domains).min(1.0)- } else {- 0.0- };-- used_budget += weight;-- domain_urls- .entry(domain)- .or_default()- .push(UrlToInsert { url, weight });- }-- let mut domain_state = response_domains- .get(&res.domain)- .cloned()- .unwrap_or_default();-- domain_state.weight = (domain_state.weight - used_budget).max(0.0);- domain_batches.push((res.domain.clone(), domain_state));-- for url_res in &res.url_responses {- if let UrlResponse::Redirected { url, new_url } = url_res {- self.redirects.put(url, new_url).ok();- }- }- }-- self.domain_state.put_batch(&domain_batches)?;-+ pub fn insert_urls(&mut self, domain_urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {let mut url_batches = Vec::new();let mut domain_batches = Vec::new();let domains = domain_urls.keys().cloned().collect::<Vec<_>>();let domain_states = self.domain_state.multi_get(&domains)?;- let url_states = self.urls.multi_get(+ let mut url_states = self.urls.multi_get(domain_urls.iter().flat_map(|(domain, urls)| {@@ -710,17 +598,22 @@ impl CrawlDb {let mut updated_url_states = Vec::new();for url in urls {- let mut url_state = match url_states.get(&UrlString::from(&url.url)) {- Some(state) => state.clone(),- None => UrlState::default(),- };-- url_state.weight += url.weight;- domain_state.weight += url_state.weight;+ match url_states.get_mut(&UrlString::from(&url.url)) {+ Some(state) => {+ state.weight += url.weight;+ domain_state.weight += url.weight;+ if url.weight > 0.0 {+ updated_url_states.push((UrlString::from(&url.url), state.clone()));+ }+ }+ None => {+ let mut state = UrlState::default();+ state.weight += url.weight;+ domain_state.weight += url.weight;- if url.weight > 0.0 {- updated_url_states.push((UrlString::from(&url.url), url_state));- }+ updated_url_states.push((UrlString::from(&url.url), state));+ }+ };}url_batches.push((domain.clone(), updated_url_states));@@ -733,7 +626,7 @@ impl CrawlDb {Ok(())}- pub fn set_domain_status(&mut self, domains: &[Domain], status: DomainStatus) -> Result<()> {+ fn set_domain_status(&mut self, domains: &[Domain], status: DomainStatus) -> Result<()> {let domain_states = self.domain_state.multi_get(domains)?;let mut batches = Vec::new();@@ -750,6 +643,33 @@ impl CrawlDb {Ok(())}+ pub fn mark_jobs_complete(&mut self, domains: &[DomainCrawled]) -> Result<()> {+ let domain_states = self.domain_state.multi_get(+ domains+ .iter()+ .map(|d| d.domain.clone())+ .collect::<Vec<_>>()+ .as_slice(),+ )?;++ let mut batches = Vec::new();++ for domain in domains {+ let mut domain_state = domain_states+ .get(&domain.domain)+ .cloned()+ .unwrap_or_default();+ domain_state.status = DomainStatus::Pending;+ domain_state.weight -= domain.budget_used;++ batches.push((domain.domain.clone(), domain_state));+ }++ self.domain_state.put_batch(&batches)?;++ Ok(())+ }+pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<Domain>> {let sampled = weighted_sample(self.domain_state.iter().filter_map(|(domain, state)| {diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 9da3b382..e1dede69 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -30,6 +30,8 @@ use self::{warc_writer::WarcWriter, worker::WorkerThread};pub mod coordinator;pub mod crawl_db;mod robots_txt;+pub mod router;+pub use router::Router;mod warc_writer;mod worker;@@ -129,6 +131,23 @@ pub struct JobResponse {pub weight_budget: f64,}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+pub struct UrlToInsert {+ pub url: Url,+ pub weight: f64,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+pub struct DiscoveredUrls {+ pub urls: HashMap<Domain, Vec<UrlToInsert>>,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+pub struct DomainCrawled {+ pub domain: Domain,+ pub budget_used: f64,+}+struct RetrieableUrl {url: Url,retries: u8,@@ -184,7 +203,11 @@ impl Crawler {let writer = Arc::new(WarcWriter::new(config.s3.clone()));let timeout = Duration::from_secs(config.timeout_seconds);let mut handles = Vec::new();- let coordinator_host = config.coordinator_host.parse()?;+ let mut router_hosts = Vec::new();++ for host in &config.router_hosts {+ router_hosts.push(host.parse()?);+ }for _ in 0..config.num_worker_threads {let worker = WorkerThread::new(@@ -193,7 +216,7 @@ impl Crawler {Arc::clone(&results),config.clone(),timeout,- coordinator_host,+ router_hosts.clone(),)?;handles.push(tokio::spawn(async move {@@ -204,7 +227,7 @@ impl Crawler {Ok(Self { writer, handles })}- pub async fn wait(self) {+ pub async fn run(self) {for handle in self.handles {handle.await.ok();}diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rsnew file mode 100644index 00000000..ad0dee4b--- /dev/null+++ b/core/src/crawler/router.rs@@ -0,0 +1,177 @@+use anyhow::Result;+use hashbrown::HashMap;+use rand::seq::SliceRandom;+use std::{net::SocketAddr, time::Duration};+use url::Url;++use crate::{+ crawler::{DomainCrawled, UrlToInsert},+ distributed::{retry_strategy::ExponentialBackoff, sonic},+ entrypoint::crawler::coordinator::{CoordinatorService, GetJobs, InsertUrls, MarkJobsComplete},+};++use super::{Domain, Job, JobResponse};++struct RemoteCoordinator {+ addr: SocketAddr,+}++impl RemoteCoordinator {+ async fn conn(&self) -> Result<sonic::service::ResilientConnection<CoordinatorService>> {+ let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));++ Ok(sonic::service::ResilientConnection::create_with_timeout(+ self.addr,+ Duration::from_secs(3600),+ retry,+ )+ .await?)+ }++ async fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {+ let conn = self.conn().await?;++ let response = conn+ .send_with_timeout(&GetJobs { num_jobs }, Duration::from_secs(60 * 60))+ .await?;++ Ok(response)+ }++ async fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {+ let conn = self.conn().await?;++ conn.send_with_timeout(&InsertUrls { urls }, Duration::from_secs(60 * 60))+ .await?;++ Ok(())+ }++ async fn mark_jobs_complete(&self, domains: Vec<DomainCrawled>) -> Result<()> {+ let conn = self.conn().await?;++ conn.send_with_timeout(&MarkJobsComplete { domains }, Duration::from_secs(60 * 60))+ .await?;++ Ok(())+ }+}++pub struct Router {+ coordinators: Vec<RemoteCoordinator>,+}++impl Router {+ pub fn new(coordinator_addrs: Vec<SocketAddr>) -> Self {+ Self {+ coordinators: coordinator_addrs+ .into_iter()+ .map(|addr| RemoteCoordinator { addr })+ .collect(),+ }+ }++ fn coordinator_index(&self, domain: &Domain) -> usize {+ let hash = md5::compute(domain.0.as_bytes());+ let hash = u128::from_le_bytes(hash.0) as usize;+ hash % self.coordinators.len()+ }++ pub async fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {+ let random_coordinator = self.coordinators.choose(&mut rand::thread_rng()).unwrap();++ random_coordinator.sample_jobs(num_jobs).await+ }++ pub async fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {+ let mut domain_urls: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();+ let mut domain_budgets: Vec<DomainCrawled> = Vec::new();++ for res in responses {+ let mut urls: Vec<(Domain, Url)> = res+ .discovered_urls+ .iter()+ .map(|url| {+ let domain = Domain::from(url);+ (domain, url.clone())+ })+ .collect();++ let diff_domains = urls+ .iter()+ .filter(|(domain, _)| res.domain != *domain)+ .count() as f64;++ urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));+ urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());++ let mut used_budget = 0.0;++ for (domain, url) in urls {+ let different_domain = res.domain != domain;++ let weight = if different_domain {+ (res.weight_budget / diff_domains).min(1.0)+ } else {+ 0.0+ };++ used_budget += weight;++ domain_urls+ .entry(domain)+ .or_default()+ .push(UrlToInsert { url, weight });+ }++ domain_budgets.push(DomainCrawled {+ domain: res.domain.clone(),+ budget_used: used_budget,+ });+ }++ let mut coordinator_urls: HashMap<usize, HashMap<Domain, Vec<UrlToInsert>>> =+ HashMap::new();++ for (domain, urls) in domain_urls {+ let coordinator_index = self.coordinator_index(&domain);++ coordinator_urls+ .entry(coordinator_index)+ .or_default()+ .insert(domain, urls);+ }++ let mut futures = Vec::new();++ for (coordinator_index, urls) in coordinator_urls {+ let coordinator = &self.coordinators[coordinator_index];++ futures.push(coordinator.insert_urls(urls));+ }++ futures::future::join_all(futures).await;++ let mut futures = Vec::new();+ let mut coordinator_budgets: HashMap<usize, Vec<DomainCrawled>> = HashMap::new();++ for domain in domain_budgets {+ let coordinator_index = self.coordinator_index(&domain.domain);++ coordinator_budgets+ .entry(coordinator_index)+ .or_default()+ .push(domain);+ }++ for (coordinator_index, budgets) in coordinator_budgets {+ let coordinator = &self.coordinators[coordinator_index];++ futures.push(coordinator.mark_jobs_complete(budgets));+ }++ futures::future::join_all(futures).await;++ Ok(())+ }+}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 560aac1a..1bb2184a 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -17,6 +17,7 @@ use encoding_rs::{Encoding, UTF_8};use futures::{future::BoxFuture, FutureExt};use mime::Mime;use quick_xml::events::Event;+use rand::seq::SliceRandom;use tokio_stream::StreamExt;use std::{@@ -33,7 +34,7 @@ use crate::{config::CrawlerConfig,crawler::{JobResponse, Response},distributed::{retry_strategy::ExponentialBackoff, sonic},- entrypoint::crawler::{CoordinatorService, NewJobs},+ entrypoint::crawler::router::{NewJobs, RouterService},webpage::Html,};@@ -54,7 +55,7 @@ pub struct WorkerThread {client: reqwest::Client,config: CrawlerConfig,politeness_factor: f32,- coordinator_host: SocketAddr,+ router_hosts: Vec<SocketAddr>,num_jobs_per_fetch: usize,robotstxt: RobotsTxtManager,}@@ -66,7 +67,7 @@ impl WorkerThread {results: Arc<Mutex<Vec<JobResponse>>>,config: CrawlerConfig,timeout: Duration,- coordinator_host: SocketAddr,+ router_hosts: Vec<SocketAddr>,) -> Result<Self> {if config.politeness_factor < config.min_politeness_factor {return Err(Error::InvalidPolitenessFactor.into());@@ -104,19 +105,19 @@ impl WorkerThread {num_jobs_per_fetch: config.num_jobs_per_fetch,politeness_factor: config.politeness_factor,config,- coordinator_host,+ router_hosts,robotstxt,})}- async fn coordinator_conn(- &self,- ) -> Result<sonic::service::ResilientConnection<CoordinatorService>> {+ async fn router_conn(&self) -> Result<sonic::service::ResilientConnection<RouterService>> {let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));+ let router = *self.router_hosts.choose(&mut rand::thread_rng()).unwrap();+Ok(sonic::service::ResilientConnection::create_with_timeout(- self.coordinator_host,- Duration::from_secs(30),+ router,+ Duration::from_secs(3600),retry,).await?)@@ -145,7 +146,7 @@ impl WorkerThread {}}} else {- let conn = self.coordinator_conn().await.unwrap();+ let conn = self.router_conn().await.unwrap();let results = self.results.lock().await.drain(..).collect::<Vec<_>>();let res = conn@@ -154,7 +155,7 @@ impl WorkerThread {responses: results,num_jobs: self.num_jobs_per_fetch,},- Duration::from_secs(10 * 60),+ Duration::from_secs(60 * 60),).await;diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex ae3dd8f9..96947c29 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -84,7 +84,6 @@ fn create_webgraph() -> Result<()> {};let mut worker = webgraph::WebgraphWorker {- redirect: None,host_graph: webgraph::open_host_graph_writer(&out_path_host),page_graph: webgraph::open_page_graph_writer(&out_path_page),};diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex f008227e..d2e5598e 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -16,11 +16,12 @@use std::{net::SocketAddr, sync::Arc};+use hashbrown::HashMap;use serde::{Deserialize, Serialize};use crate::{config,- crawler::{CrawlCoordinator, Crawler, JobResponse},+ crawler::{self, CrawlCoordinator, Crawler, Domain, DomainCrawled, JobResponse, UrlToInsert},distributed::sonic::{self, service::Message},sonic_service, Result,};@@ -28,53 +29,123 @@ use crate::{pub async fn worker(config: config::CrawlerConfig) -> Result<()> {let crawler = Crawler::new(config).await?;- crawler.wait().await;+ crawler.run().await;Ok(())}-pub struct CoordinatorService {- coordinator: Arc<CrawlCoordinator>,+pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {+ let coordinator = Arc::new(CrawlCoordinator::new(+ config.crawldb_folder,+ config.seed_urls,+ )?);++ let addr: SocketAddr = config.host;+ let server = coordinator::CoordinatorService { coordinator }+ .bind(addr)+ .await+ .unwrap();++ tracing::info!("Crawl coordinator listening on {}", addr);++ loop {+ let _ = server.accept().await;+ }}-sonic_service!(CoordinatorService, [NewJobs]);+pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {+ let router = crawler::Router::new(config.coordinator_addrs.clone());-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct NewJobs {- pub responses: Vec<JobResponse>,- pub num_jobs: usize,+ let addr: SocketAddr = config.host;++ let server = router::RouterService { router }.bind(addr).await.unwrap();++ tracing::info!("Crawl router listening on {}", addr);++ loop {+ let _ = server.accept().await;+ }}-#[async_trait::async_trait]-impl Message<CoordinatorService> for NewJobs {- type Response = crate::crawler::Response;- async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {- server.coordinator.add_responses(&self.responses)?;+pub mod router {+ use super::*;+ pub struct RouterService {+ pub router: crawler::Router,+ }- if server.coordinator.is_done() {- tracing::info!("Crawl is done. Waiting for workers to finish.");- Ok(crate::crawler::Response::Done)- } else {- let jobs = server.coordinator.sample_jobs(self.num_jobs)?;+ sonic_service!(RouterService, [NewJobs]);++ #[derive(Debug, Clone, Serialize, Deserialize)]+ pub struct NewJobs {+ pub responses: Vec<JobResponse>,+ pub num_jobs: usize,+ }+ #[async_trait::async_trait]+ impl Message<RouterService> for NewJobs {+ type Response = crate::crawler::Response;++ async fn handle(self, server: &RouterService) -> sonic::Result<Self::Response> {+ server.router.add_responses(&self.responses).await?;+ let jobs = server.router.sample_jobs(self.num_jobs).await?;Ok(crate::crawler::Response::NewJobs { jobs })}}}-pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {- let coordinator = Arc::new(CrawlCoordinator::new(- config.crawldb_folder,- config.num_urls_to_crawl,- config.seed_urls,- )?);+pub mod coordinator {+ use crate::crawler::Job;- let addr: SocketAddr = config.host;- let server = CoordinatorService { coordinator }.bind(addr).await.unwrap();+ use super::*;- tracing::info!("Crawl coordinator listening on {}", addr);+ pub struct CoordinatorService {+ pub coordinator: Arc<CrawlCoordinator>,+ }- loop {- let _ = server.accept().await;+ sonic_service!(CoordinatorService, [InsertUrls, GetJobs, MarkJobsComplete]);++ #[derive(Debug, Clone, Serialize, Deserialize)]+ pub struct InsertUrls {+ pub urls: HashMap<Domain, Vec<UrlToInsert>>,+ }++ #[derive(Debug, Clone, Serialize, Deserialize)]+ pub struct GetJobs {+ pub num_jobs: usize,+ }++ #[derive(Debug, Clone, Serialize, Deserialize)]+ pub struct MarkJobsComplete {+ pub domains: Vec<DomainCrawled>,+ }++ #[async_trait::async_trait]+ impl Message<CoordinatorService> for InsertUrls {+ type Response = ();++ async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {+ server.coordinator.insert_urls(self.urls)?;+ Ok(())+ }+ }++ #[async_trait::async_trait]+ impl Message<CoordinatorService> for GetJobs {+ type Response = Vec<Job>;++ async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {+ let jobs = server.coordinator.sample_jobs(self.num_jobs)?;+ Ok(jobs)+ }+ }++ #[async_trait::async_trait]+ impl Message<CoordinatorService> for MarkJobsComplete {+ type Response = ();++ async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {+ server.coordinator.mark_jobs_complete(&self.domains)?;+ Ok(())+ }}}diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex aff872dd..72a9fbc2 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -16,7 +16,6 @@use crate::{config::WarcSource,config::{self, WebgraphConstructConfig},- crawler::crawl_db::RedirectDb,entrypoint::download_all_warc_files,mapreduce::Worker,webgraph::{self, Node, WebgraphWriter},@@ -25,7 +24,7 @@ use crate::{};use itertools::Itertools;use serde::{Deserialize, Serialize};-use std::{fs, path::Path, sync::Arc};+use std::{fs, path::Path};use tokio::pin;use tracing::{info, trace};@@ -86,7 +85,6 @@ pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWritpub struct WebgraphWorker {pub host_graph: webgraph::WebgraphWriter,pub page_graph: webgraph::WebgraphWriter,- pub redirect: Option<Arc<RedirectDb>>,}impl WebgraphWorker {@@ -122,14 +120,7 @@ impl WebgraphWorker {}){let source = link.source.clone();- let mut destination = link.destination.clone();-- if let Some(redirect) = &self.redirect {- if let Some(new_destination) = redirect.get(&destination).unwrap() {- trace!("redirecting {:?} to {:?}", destination, new_destination);- destination = new_destination;- }- }+ let destination = link.destination.clone();if source.root_domain() == destination.root_domain() {continue;@@ -168,11 +159,6 @@ impl Webgraph {let job_config = JobConfig::from(config.warc_source.clone());- let redirect = match &config.redirect_db_path {- Some(path) => Some(Arc::new(RedirectDb::open(path)?)),- None => None,- };-let jobs: Vec<_> = warc_paths.into_iter().take(config.limit_warc_files.unwrap_or(usize::MAX))@@ -200,7 +186,6 @@ impl Webgraph {let page_path = page_path.join(format!("worker_{i}"));let mut worker = WebgraphWorker {- redirect: redirect.clone(),host_graph: open_host_graph_writer(host_path),page_graph: open_page_graph_writer(page_path),};diff --git a/core/src/lib.rs b/core/src/lib.rsindex b2e94909..681f0f59 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -35,7 +35,6 @@ pub mod alice;mod api;mod autosuggest;mod bangs;-mod call_counter;mod collector;pub mod config;pub mod crawler;diff --git a/core/src/main.rs b/core/src/main.rsindex 2d979fc3..1a20e327 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -94,13 +94,11 @@ enum Commands {output_dir: String,},- /// Deploy the crawl coordinator. The crawl coordinator is responsible for- /// distributing crawl jobs to the crawles and deciding which urls to crawl next.- CrawlCoordinator { config_path: String },-- /// Deploy the crawler. The crawler is responsible for downloading webpages, saving them to S3,- /// and sending newly discovered urls back to the crawl coordinator.- Crawler { config_path: String },+ /// Deploy the crawler.+ Crawler {+ #[clap(subcommand)]+ options: Crawler,+ },/// Train or run inference on the classifier that predicts if a webpage is NSFW or SFW.SafetyClassifier {@@ -119,6 +117,21 @@ enum Commands {},}+#[derive(Subcommand)]+enum Crawler {+ /// Deploy the crawl worker. The worker is responsible for downloading webpages, saving them to S3,+ /// and sending newly discovered urls back to the crawl coordinator.+ Worker { config_path: String },++ /// Deploy the crawl coordinator. The crawl coordinator is responsible for+ /// distributing crawl jobs to the crawles and deciding which urls to crawl next.+ Coordinator { config_path: String },++ /// Deploy the crawl router. The crawl router is responsible for routing job responses and requests+ /// from the workers to the correct crawl coordinators.+ Router { config_path: String },+}+/// Commands to deploy Alice.#[derive(Subcommand)]enum AliceOptions {@@ -311,22 +324,32 @@ fn main() -> Result<()> {}AliceOptions::GenerateKey => entrypoint::alice::generate_key(),},- Commands::CrawlCoordinator { config_path } => {- let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);+ Commands::Crawler { options } => match options {+ Crawler::Worker { config_path } => {+ let config: config::CrawlerConfig = load_toml_config(config_path);- tokio::runtime::Builder::new_multi_thread()- .enable_all()- .build()?- .block_on(entrypoint::crawler::coordinator(config))?- }- Commands::Crawler { config_path } => {- let config: config::CrawlerConfig = load_toml_config(config_path);+ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::crawler::worker(config))?+ }+ Crawler::Coordinator { config_path } => {+ let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);- tokio::runtime::Builder::new_multi_thread()- .enable_all()- .build()?- .block_on(entrypoint::crawler::worker(config))?- }+ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::crawler::coordinator(config))?+ }+ Crawler::Router { config_path } => {+ let config: config::CrawlRouterConfig = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::crawler::router(config))?+ }+ },Commands::SafetyClassifier { options } => match options {SafetyClassifierOptions::Train {dataset_path,