web search done right
simplify crawler states
Mikkel Denker6d26f1f94208
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex f19b214e..ce9d4014 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -84,16 +84,10 @@ impl CrawlCoordinator {let mut db = self.db.lock().unwrap();let start = Instant::now();- let nonempty_domains = db.insert_urls(responses)?;+ db.insert_urls(responses)?;- for domain in &nonempty_domains {- db.set_domain_status(domain, DomainStatus::Pending)?;- }-- for response in responses.iter() {- if !nonempty_domains.contains(&response.domain) {- db.set_domain_status(&response.domain, DomainStatus::NoUncrawledUrls)?;- }+ for resp in responses {+ db.set_domain_status(&resp.domain, DomainStatus::Pending)?;}tracing::info!("inserted responses in {:?}", start.elapsed());diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 5f6c180d..9c974f0d 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use dashmap::DashMap;-use hashbrown::{HashMap, HashSet};+use hashbrown::HashMap;use rand::Rng;use rayon::prelude::*;use std::hash::Hash;@@ -38,7 +38,6 @@ pub enum UrlStatus {#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]pub enum DomainStatus {Pending,- NoUncrawledUrls,CrawlInProgress,}@@ -586,7 +585,7 @@ impl CrawlDb {Ok(())}- pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<HashSet<Domain>> {+ pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<()> {let domains: DashMap<Domain, Vec<UrlToInsert>> = DashMap::new();responses.par_iter().for_each(|res| {@@ -642,15 +641,9 @@ impl CrawlDb {}});- let mut nonempty_domains = HashSet::new();-for (domain, urls) in domains.into_iter() {let mut domain_state = self.domain_state.get(&domain)?.unwrap_or_default();- if !urls.is_empty() {- nonempty_domains.insert(domain.clone());- }-let mut url_states = Vec::new();for url in urls {@@ -670,7 +663,7 @@ impl CrawlDb {self.domain_state.put(&domain, &domain_state)?;}- Ok(nonempty_domains)+ Ok(())}pub fn set_domain_status(&mut self, domain: &Domain, status: DomainStatus) -> Result<()> {