Skip to content

web search done right

Moved crawler seed urls to router. This ensures the correct coordinators gets the correct urls based on hash(domain) % coordinators.len()

Mikkel Denkerb044f25228b7
diff --git a/configs/crawler/coordinator.toml b/configs/crawler/coordinator.toml
index 5dfc8bc4..c5c38269 100644
--- a/configs/crawler/coordinator.toml
+++ b/configs/crawler/coordinator.toml
@@ -1,3 +1,2 @@
crawldb_folder = "data/crawldb"
host = "0.0.0.0:8080"
-seed_urls = ["https://en.wikipedia.org"]
diff --git a/configs/crawler/router.toml b/configs/crawler/router.toml
index fca9e23a..dcc3fa08 100644
--- a/configs/crawler/router.toml
+++ b/configs/crawler/router.toml
@@ -1,2 +1,3 @@
coordinator_addrs = ["0.0.0.0:8080"]
host = "0.0.0.0:8181"
+seed_urls = ["https://en.wikipedia.org"]
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 96ff736a..a653b67c 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -263,7 +263,6 @@ pub struct SearchServerConfig {
pub struct CrawlCoordinatorConfig {
pub crawldb_folder: String,
pub host: SocketAddr,
- pub seed_urls: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -308,6 +307,7 @@ pub struct CrawlerConfig {
pub struct CrawlRouterConfig {
pub host: SocketAddr,
pub coordinator_addrs: Vec<SocketAddr>,
+ pub seed_urls: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index afe47ce8..8e70a577 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use hashbrown::HashMap;
-use url::Url;
use super::{crawl_db::CrawlDb, Domain, DomainCrawled, Job, Result, UrlToInsert};
use std::{path::Path, sync::Mutex, time::Instant};
@@ -27,15 +26,8 @@ pub struct CrawlCoordinator {
}
impl CrawlCoordinator {
- pub fn new<P: AsRef<Path>>(crawldb_folder: P, seed_urls: Vec<String>) -> Result<Self> {
- let mut db = CrawlDb::open(crawldb_folder)?;
- let mut parsed_seed_urls = Vec::new();
-
- for url in seed_urls {
- parsed_seed_urls.push(Url::parse(&url)?);
- }
-
- db.insert_seed_urls(&parsed_seed_urls)?;
+ pub fn new<P: AsRef<Path>>(crawldb_folder: P) -> Result<Self> {
+ let db = CrawlDb::open(crawldb_folder)?;
Ok(Self { db: Mutex::new(db) })
}
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index cb627517..c21d634c 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -503,17 +503,6 @@ impl DomainStateDb {
Ok(())
}
- fn put(&self, domain: &Domain, state: &DomainState) -> Result<()> {
- let domain_bytes = bincode::serialize(domain)?;
- let state_bytes = bincode::serialize(state)?;
-
- let mut write_options = rocksdb::WriteOptions::default();
- write_options.disable_wal(true);
- self.db.put_opt(domain_bytes, state_bytes, &write_options)?;
-
- Ok(())
- }
-
fn iter(&self) -> impl Iterator<Item = (Domain, DomainState)> + '_ {
let iter = self.db.iterator(rocksdb::IteratorMode::Start);
@@ -563,18 +552,6 @@ impl CrawlDb {
})
}
- pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {
- for url in urls {
- let domain = Domain::from(url);
-
- self.domain_state.put(&domain, &DomainState::default())?;
- self.urls
- .put_batch(&[(domain, vec![(UrlString::from(url), UrlState::default())])])?;
- }
-
- Ok(())
- }
-
pub fn insert_urls(&mut self, domain_urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {
let mut url_batches = Vec::new();
let mut domain_batches = Vec::new();
@@ -749,6 +726,25 @@ mod tests {
use super::*;
+ fn insert_seed_urls(urls: &[Url], db: &mut CrawlDb) {
+ let mut domain_urls = HashMap::new();
+
+ for url in urls {
+ let domain = Domain::from(url);
+ let url_to_insert = UrlToInsert {
+ url: url.clone(),
+ weight: 0.0,
+ };
+
+ domain_urls
+ .entry(domain)
+ .or_insert_with(Vec::new)
+ .push(url_to_insert);
+ }
+
+ db.insert_urls(domain_urls).unwrap();
+ }
+
#[test]
fn sampling() {
let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];
@@ -773,8 +769,8 @@ mod tests {
fn simple_politeness() {
let mut db = CrawlDb::open(gen_temp_path()).unwrap();
- db.insert_seed_urls(&[Url::parse("https://example.com").unwrap()])
- .unwrap();
+ let urls = vec![Url::parse("https://example.com").unwrap()];
+ insert_seed_urls(&urls, &mut db);
let domain = Domain::from(&Url::parse("https://example.com").unwrap());
@@ -795,11 +791,11 @@ mod tests {
fn get_all_urls() {
let mut db = CrawlDb::open(gen_temp_path()).unwrap();
- db.insert_seed_urls(&[
+ let urls = vec![
Url::parse("https://a.com").unwrap(),
Url::parse("https://b.com").unwrap(),
- ])
- .unwrap();
+ ];
+ insert_seed_urls(&urls, &mut db);
let domain = Domain::from(&Url::parse("https://a.com").unwrap());
diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rs
index ad0dee4b..10707588 100644
--- a/core/src/crawler/router.rs
+++ b/core/src/crawler/router.rs
@@ -62,13 +62,41 @@ pub struct Router {
}
impl Router {
- pub fn new(coordinator_addrs: Vec<SocketAddr>) -> Self {
- Self {
+ pub async fn new(coordinator_addrs: Vec<SocketAddr>, seed_urls: Vec<String>) -> Result<Self> {
+ let s = Self {
coordinators: coordinator_addrs
.into_iter()
.map(|addr| RemoteCoordinator { addr })
.collect(),
+ };
+
+ let mut coordinator_urls: HashMap<usize, HashMap<Domain, Vec<UrlToInsert>>> =
+ HashMap::new();
+
+ for url in seed_urls {
+ let url = Url::parse(&url)?;
+ let domain = Domain::from(&url);
+
+ let coordinator_index = s.coordinator_index(&domain);
+
+ coordinator_urls
+ .entry(coordinator_index)
+ .or_default()
+ .entry(domain)
+ .or_default()
+ .push(UrlToInsert { url, weight: 0.0 });
}
+
+ let mut futures = Vec::new();
+ for (coordinator_index, urls) in coordinator_urls {
+ let coordinator = &s.coordinators[coordinator_index];
+
+ futures.push(coordinator.insert_urls(urls));
+ }
+
+ futures::future::join_all(futures).await;
+
+ Ok(s)
}
fn coordinator_index(&self, domain: &Domain) -> usize {
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index d2e5598e..63e57b4e 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -35,10 +35,7 @@ pub async fn worker(config: config::CrawlerConfig) -> Result<()> {
}
pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {
- let coordinator = Arc::new(CrawlCoordinator::new(
- config.crawldb_folder,
- config.seed_urls,
- )?);
+ let coordinator = Arc::new(CrawlCoordinator::new(config.crawldb_folder)?);
let addr: SocketAddr = config.host;
let server = coordinator::CoordinatorService { coordinator }
@@ -54,7 +51,7 @@ pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {
}
pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {
- let router = crawler::Router::new(config.coordinator_addrs.clone());
+ let router = crawler::Router::new(config.coordinator_addrs.clone(), config.seed_urls).await?;
let addr: SocketAddr = config.host;