web search done right
Some domain states got set to "Pending" even though a worker was already crawling them. This primarily happened for popular sites that has many incoming links from many different sites.
Mikkel Denker69e06bb57469
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex a8d8ba9b..429c1da7 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -70,11 +70,11 @@ impl Crawler {}pub fn max_crawl_delay_ms() -> u64 {- 30_000+ 60_000}pub fn max_politeness_factor() -> f32 {- 1024.0+ 2048.0}pub fn min_politeness_factor() -> f32 {diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex ba06e265..34379369 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -54,7 +54,7 @@ impl CrawlCoordinator {db: Mutex::new(db),num_urls_to_crawl,num_crawled_urls: AtomicU64::new(0),- call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),+ call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),})}@@ -74,8 +74,8 @@ impl CrawlCoordinator {self.log_crawls_per_second(responses.iter().map(|res| res.url_responses.len()).sum());let mut db = self.db.lock().unwrap();- db.insert_urls(&responses).unwrap();- db.update_url_status(&responses).unwrap();+ db.insert_urls(responses).unwrap();+ db.update_url_status(responses).unwrap();for response in responses.iter() {db.set_domain_status(&response.domain, DomainStatus::Pending)diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 6e87ae5d..4ab4208d 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -100,6 +100,8 @@ wherelet mut batch_id2t = rocksdb::WriteBatch::default();let mut batch_t2id = rocksdb::WriteBatch::default();+ let mut batch_assignments = HashMap::new();+for item in items {// check cacheif let Some(id) = self.t2id_cache.get(&item) {@@ -121,12 +123,22 @@ where}// insert item- let id = self.next_id;- self.next_id += 1;+ let assigned_id = batch_assignments.get(&item).copied();++ let id = assigned_id.unwrap_or_else(|| {+ let id = self.next_id;+ self.next_id += 1;+ id+ });+let id_bytes = bincode::serialize(&id)?;batch_t2id.put(&item_bytes, &id_bytes);batch_id2t.put(&id_bytes, &item_bytes);+ if assigned_id.is_none() {+ batch_assignments.insert(item.clone(), id);+ }+// update cacheself.t2id_cache.put(item, id);@@ -420,8 +432,6 @@ impl CrawlDb {for url in urls {let url_id: UrlId = self.url_ids.id(url.url.clone())?.into();- domain_state.status = DomainStatus::Pending;-let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {weight: 0.0,status: UrlStatus::Pending,@@ -495,14 +505,13 @@ impl CrawlDb {for (domain, responses) in url_responses {let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();- let domain_state = self- .domain_state++ self.domain_state.entry(domain_id).or_insert_with(|| DomainState {weight: 0.0,status: DomainStatus::Pending,});- domain_state.status = DomainStatus::Pending;let url_states = self.urls.entry(domain_id).or_default();@@ -545,7 +554,7 @@ impl CrawlDb {.entry(domain_id).or_insert_with(|| DomainState {weight: 0.0,- status: DomainStatus::Pending,+ status: status.clone(),});domain_state.status = status;diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rsindex 2386340d..2430f247 100644--- a/core/src/search_prettifier/entity.rs+++ b/core/src/search_prettifier/entity.rs@@ -167,7 +167,7 @@ fn entity_link_to_html(span: Span, trunace_to: usize) -> String {.collect();res += format!(- "<a href=\"https://en.wikipedia.org/wiki/{}\">",+ "<a class=\"hover:underline\" href=\"https://en.wikipedia.org/wiki/{}\">",link.target.replace(' ', "_")).as_str();@@ -212,7 +212,7 @@ mod tests {},10000),- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"+ "some <a class=\"hover:underline\" href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link".to_string());}@@ -231,7 +231,7 @@ mod tests {},7),- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()+ "some <a class=\"hover:underline\" href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string());}diff --git a/frontend/src/components/Entity.astro b/frontend/src/components/Entity.astroindex 487a2a64..0be4f40e 100644--- a/frontend/src/components/Entity.astro+++ b/frontend/src/components/Entity.astro@@ -24,7 +24,7 @@ export interface Props {))}<div class="mb-5 text-xl">- <a href="https://en.wikipedia.org/wiki/{{ entity.title }}"+ <a class="hover:underline" href="https://en.wikipedia.org/wiki/{{ entity.title }}">{askama`entity.title $ {{name.fullName}}`}</a></div>@@ -32,7 +32,7 @@ export interface Props {{askama`entity.small_abstract|safe $ {{lorem.lines}}`}<span class="italic">source: <a- class="text-blue-600"+ class="hover:underline text-blue-600"href="https://en.wikipedia.org/wiki/{{ entity.title|urlencode }}">wikipedia</a>@@ -85,4 +85,4 @@ export interface Props {))}</div>-</div>+</div>\ No newline at end of filediff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdxindex 2bfe7aa3..f53b9ec8 100644--- a/frontend/src/pages/crawler.mdx+++ b/frontend/src/pages/crawler.mdx@@ -10,8 +10,8 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; +https:## PolitenessStractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 30 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 30 seconds.-This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 1024).+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 60 seconds.+This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.