Skip to content

web search done right

simplify crawler states

Mikkel Denker6d26f1f94208
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index f19b214e..ce9d4014 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -84,16 +84,10 @@ impl CrawlCoordinator {
let mut db = self.db.lock().unwrap();
let start = Instant::now();
- let nonempty_domains = db.insert_urls(responses)?;
+ db.insert_urls(responses)?;
- for domain in &nonempty_domains {
- db.set_domain_status(domain, DomainStatus::Pending)?;
- }
-
- for response in responses.iter() {
- if !nonempty_domains.contains(&response.domain) {
- db.set_domain_status(&response.domain, DomainStatus::NoUncrawledUrls)?;
- }
+ for resp in responses {
+ db.set_domain_status(&resp.domain, DomainStatus::Pending)?;
}
tracing::info!("inserted responses in {:?}", start.elapsed());
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 5f6c180d..9c974f0d 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use dashmap::DashMap;
-use hashbrown::{HashMap, HashSet};
+use hashbrown::HashMap;
use rand::Rng;
use rayon::prelude::*;
use std::hash::Hash;
@@ -38,7 +38,6 @@ pub enum UrlStatus {
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum DomainStatus {
Pending,
- NoUncrawledUrls,
CrawlInProgress,
}
@@ -586,7 +585,7 @@ impl CrawlDb {
Ok(())
}
- pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<HashSet<Domain>> {
+ pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<()> {
let domains: DashMap<Domain, Vec<UrlToInsert>> = DashMap::new();
responses.par_iter().for_each(|res| {
@@ -642,15 +641,9 @@ impl CrawlDb {
}
});
- let mut nonempty_domains = HashSet::new();
-
for (domain, urls) in domains.into_iter() {
let mut domain_state = self.domain_state.get(&domain)?.unwrap_or_default();
- if !urls.is_empty() {
- nonempty_domains.insert(domain.clone());
- }
-
let mut url_states = Vec::new();
for url in urls {
@@ -670,7 +663,7 @@ impl CrawlDb {
self.domain_state.put(&domain, &domain_state)?;
}
- Ok(nonempty_domains)
+ Ok(())
}
pub fn set_domain_status(&mut self, domain: &Domain, status: DomainStatus) -> Result<()> {