Skip to content

web search done right

simple test for crawldb politeness

Mikkel Denkerd9bcc5f92573
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index c1207480..8483465b 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -44,7 +44,7 @@ pub enum UrlStatus {
Done,
}
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DomainStatus {
Pending,
CrawlInProgress,
@@ -877,4 +877,26 @@ mod tests {
assert!(db.contains(&321));
assert_eq!(db.get(&321).unwrap(), "world");
}
+
+ #[test]
+ fn simple_politeness() {
+ let mut db = CrawlDb::open(gen_temp_path()).unwrap();
+
+ db.insert_seed_urls(&[Url::parse("https://example.com").unwrap()])
+ .unwrap();
+
+ let domain = Domain::from(&Url::parse("https://example.com").unwrap());
+ let domain_id = db.domain_ids.id(&domain).unwrap().into();
+ let sample = db.sample_domains(128).unwrap();
+
+ assert_eq!(sample.len(), 1);
+ assert_eq!(&sample[0], &domain_id);
+ assert_eq!(
+ db.domain_state.get(&domain_id).unwrap().status,
+ DomainStatus::CrawlInProgress
+ );
+
+ let new_sample = db.sample_domains(128).unwrap();
+ assert_eq!(new_sample.len(), 0);
+ }
}