web search done right
Moved crawler seed urls to router. This ensures the correct coordinators gets the correct urls based on hash(domain) % coordinators.len()
Mikkel Denkerb044f25228b7
diff --git a/configs/crawler/coordinator.toml b/configs/crawler/coordinator.tomlindex 5dfc8bc4..c5c38269 100644--- a/configs/crawler/coordinator.toml+++ b/configs/crawler/coordinator.toml@@ -1,3 +1,2 @@crawldb_folder = "data/crawldb"host = "0.0.0.0:8080"-seed_urls = ["https://en.wikipedia.org"]diff --git a/configs/crawler/router.toml b/configs/crawler/router.tomlindex fca9e23a..dcc3fa08 100644--- a/configs/crawler/router.toml+++ b/configs/crawler/router.toml@@ -1,2 +1,3 @@coordinator_addrs = ["0.0.0.0:8080"]host = "0.0.0.0:8181"+seed_urls = ["https://en.wikipedia.org"]diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 96ff736a..a653b67c 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -263,7 +263,6 @@ pub struct SearchServerConfig {pub struct CrawlCoordinatorConfig {pub crawldb_folder: String,pub host: SocketAddr,- pub seed_urls: Vec<String>,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -308,6 +307,7 @@ pub struct CrawlerConfig {pub struct CrawlRouterConfig {pub host: SocketAddr,pub coordinator_addrs: Vec<SocketAddr>,+ pub seed_urls: Vec<String>,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex afe47ce8..8e70a577 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use hashbrown::HashMap;-use url::Url;use super::{crawl_db::CrawlDb, Domain, DomainCrawled, Job, Result, UrlToInsert};use std::{path::Path, sync::Mutex, time::Instant};@@ -27,15 +26,8 @@ pub struct CrawlCoordinator {}impl CrawlCoordinator {- pub fn new<P: AsRef<Path>>(crawldb_folder: P, seed_urls: Vec<String>) -> Result<Self> {- let mut db = CrawlDb::open(crawldb_folder)?;- let mut parsed_seed_urls = Vec::new();-- for url in seed_urls {- parsed_seed_urls.push(Url::parse(&url)?);- }-- db.insert_seed_urls(&parsed_seed_urls)?;+ pub fn new<P: AsRef<Path>>(crawldb_folder: P) -> Result<Self> {+ let db = CrawlDb::open(crawldb_folder)?;Ok(Self { db: Mutex::new(db) })}diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex cb627517..c21d634c 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -503,17 +503,6 @@ impl DomainStateDb {Ok(())}- fn put(&self, domain: &Domain, state: &DomainState) -> Result<()> {- let domain_bytes = bincode::serialize(domain)?;- let state_bytes = bincode::serialize(state)?;-- let mut write_options = rocksdb::WriteOptions::default();- write_options.disable_wal(true);- self.db.put_opt(domain_bytes, state_bytes, &write_options)?;-- Ok(())- }-fn iter(&self) -> impl Iterator<Item = (Domain, DomainState)> + '_ {let iter = self.db.iterator(rocksdb::IteratorMode::Start);@@ -563,18 +552,6 @@ impl CrawlDb {})}- pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {- for url in urls {- let domain = Domain::from(url);-- self.domain_state.put(&domain, &DomainState::default())?;- self.urls- .put_batch(&[(domain, vec![(UrlString::from(url), UrlState::default())])])?;- }-- Ok(())- }-pub fn insert_urls(&mut self, domain_urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {let mut url_batches = Vec::new();let mut domain_batches = Vec::new();@@ -749,6 +726,25 @@ mod tests {use super::*;+ fn insert_seed_urls(urls: &[Url], db: &mut CrawlDb) {+ let mut domain_urls = HashMap::new();++ for url in urls {+ let domain = Domain::from(url);+ let url_to_insert = UrlToInsert {+ url: url.clone(),+ weight: 0.0,+ };++ domain_urls+ .entry(domain)+ .or_insert_with(Vec::new)+ .push(url_to_insert);+ }++ db.insert_urls(domain_urls).unwrap();+ }+#[test]fn sampling() {let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];@@ -773,8 +769,8 @@ mod tests {fn simple_politeness() {let mut db = CrawlDb::open(gen_temp_path()).unwrap();- db.insert_seed_urls(&[Url::parse("https://example.com").unwrap()])- .unwrap();+ let urls = vec![Url::parse("https://example.com").unwrap()];+ insert_seed_urls(&urls, &mut db);let domain = Domain::from(&Url::parse("https://example.com").unwrap());@@ -795,11 +791,11 @@ mod tests {fn get_all_urls() {let mut db = CrawlDb::open(gen_temp_path()).unwrap();- db.insert_seed_urls(&[+ let urls = vec![Url::parse("https://a.com").unwrap(),Url::parse("https://b.com").unwrap(),- ])- .unwrap();+ ];+ insert_seed_urls(&urls, &mut db);let domain = Domain::from(&Url::parse("https://a.com").unwrap());diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rsindex ad0dee4b..10707588 100644--- a/core/src/crawler/router.rs+++ b/core/src/crawler/router.rs@@ -62,13 +62,41 @@ pub struct Router {}impl Router {- pub fn new(coordinator_addrs: Vec<SocketAddr>) -> Self {- Self {+ pub async fn new(coordinator_addrs: Vec<SocketAddr>, seed_urls: Vec<String>) -> Result<Self> {+ let s = Self {coordinators: coordinator_addrs.into_iter().map(|addr| RemoteCoordinator { addr }).collect(),+ };++ let mut coordinator_urls: HashMap<usize, HashMap<Domain, Vec<UrlToInsert>>> =+ HashMap::new();++ for url in seed_urls {+ let url = Url::parse(&url)?;+ let domain = Domain::from(&url);++ let coordinator_index = s.coordinator_index(&domain);++ coordinator_urls+ .entry(coordinator_index)+ .or_default()+ .entry(domain)+ .or_default()+ .push(UrlToInsert { url, weight: 0.0 });}++ let mut futures = Vec::new();+ for (coordinator_index, urls) in coordinator_urls {+ let coordinator = &s.coordinators[coordinator_index];++ futures.push(coordinator.insert_urls(urls));+ }++ futures::future::join_all(futures).await;++ Ok(s)}fn coordinator_index(&self, domain: &Domain) -> usize {diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex d2e5598e..63e57b4e 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -35,10 +35,7 @@ pub async fn worker(config: config::CrawlerConfig) -> Result<()> {}pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {- let coordinator = Arc::new(CrawlCoordinator::new(- config.crawldb_folder,- config.seed_urls,- )?);+ let coordinator = Arc::new(CrawlCoordinator::new(config.crawldb_folder)?);let addr: SocketAddr = config.host;let server = coordinator::CoordinatorService { coordinator }@@ -54,7 +51,7 @@ pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {}pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {- let router = crawler::Router::new(config.coordinator_addrs.clone());+ let router = crawler::Router::new(config.coordinator_addrs.clone(), config.seed_urls).await?;let addr: SocketAddr = config.host;