Skip to content

web search done right

Some domain states got set to "Pending" even though a worker was already crawling them. This primarily happened for popular sites that has many incoming links from many different sites.

Mikkel Denker69e06bb57469
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index a8d8ba9b..429c1da7 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -70,11 +70,11 @@ impl Crawler {
}
pub fn max_crawl_delay_ms() -> u64 {
- 30_000
+ 60_000
}
pub fn max_politeness_factor() -> f32 {
- 1024.0
+ 2048.0
}
pub fn min_politeness_factor() -> f32 {
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index ba06e265..34379369 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -54,7 +54,7 @@ impl CrawlCoordinator {
db: Mutex::new(db),
num_urls_to_crawl,
num_crawled_urls: AtomicU64::new(0),
- call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),
+ call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),
})
}
@@ -74,8 +74,8 @@ impl CrawlCoordinator {
self.log_crawls_per_second(responses.iter().map(|res| res.url_responses.len()).sum());
let mut db = self.db.lock().unwrap();
- db.insert_urls(&responses).unwrap();
- db.update_url_status(&responses).unwrap();
+ db.insert_urls(responses).unwrap();
+ db.update_url_status(responses).unwrap();
for response in responses.iter() {
db.set_domain_status(&response.domain, DomainStatus::Pending)
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 6e87ae5d..4ab4208d 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -100,6 +100,8 @@ where
let mut batch_id2t = rocksdb::WriteBatch::default();
let mut batch_t2id = rocksdb::WriteBatch::default();
+ let mut batch_assignments = HashMap::new();
+
for item in items {
// check cache
if let Some(id) = self.t2id_cache.get(&item) {
@@ -121,12 +123,22 @@ where
}
// insert item
- let id = self.next_id;
- self.next_id += 1;
+ let assigned_id = batch_assignments.get(&item).copied();
+
+ let id = assigned_id.unwrap_or_else(|| {
+ let id = self.next_id;
+ self.next_id += 1;
+ id
+ });
+
let id_bytes = bincode::serialize(&id)?;
batch_t2id.put(&item_bytes, &id_bytes);
batch_id2t.put(&id_bytes, &item_bytes);
+ if assigned_id.is_none() {
+ batch_assignments.insert(item.clone(), id);
+ }
+
// update cache
self.t2id_cache.put(item, id);
@@ -420,8 +432,6 @@ impl CrawlDb {
for url in urls {
let url_id: UrlId = self.url_ids.id(url.url.clone())?.into();
- domain_state.status = DomainStatus::Pending;
-
let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
weight: 0.0,
status: UrlStatus::Pending,
@@ -495,14 +505,13 @@ impl CrawlDb {
for (domain, responses) in url_responses {
let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
- let domain_state = self
- .domain_state
+
+ self.domain_state
.entry(domain_id)
.or_insert_with(|| DomainState {
weight: 0.0,
status: DomainStatus::Pending,
});
- domain_state.status = DomainStatus::Pending;
let url_states = self.urls.entry(domain_id).or_default();
@@ -545,7 +554,7 @@ impl CrawlDb {
.entry(domain_id)
.or_insert_with(|| DomainState {
weight: 0.0,
- status: DomainStatus::Pending,
+ status: status.clone(),
});
domain_state.status = status;
diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rs
index 2386340d..2430f247 100644
--- a/core/src/search_prettifier/entity.rs
+++ b/core/src/search_prettifier/entity.rs
@@ -167,7 +167,7 @@ fn entity_link_to_html(span: Span, trunace_to: usize) -> String {
.collect();
res += format!(
- "<a href=\"https://en.wikipedia.org/wiki/{}\">",
+ "<a class=\"hover:underline\" href=\"https://en.wikipedia.org/wiki/{}\">",
link.target.replace(' ', "_")
)
.as_str();
@@ -212,7 +212,7 @@ mod tests {
},
10000
),
- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"
+ "some <a class=\"hover:underline\" href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"
.to_string()
);
}
@@ -231,7 +231,7 @@ mod tests {
},
7
),
- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()
+ "some <a class=\"hover:underline\" href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()
);
}
diff --git a/frontend/src/components/Entity.astro b/frontend/src/components/Entity.astro
index 487a2a64..0be4f40e 100644
--- a/frontend/src/components/Entity.astro
+++ b/frontend/src/components/Entity.astro
@@ -24,7 +24,7 @@ export interface Props {
))
}
<div class="mb-5 text-xl">
- <a href="https://en.wikipedia.org/wiki/{{ entity.title }}"
+ <a class="hover:underline" href="https://en.wikipedia.org/wiki/{{ entity.title }}"
>{askama`entity.title $ {{name.fullName}}`}
</a>
</div>
@@ -32,7 +32,7 @@ export interface Props {
{askama`entity.small_abstract|safe $ {{lorem.lines}}`}
<span class="italic">
source: <a
- class="text-blue-600"
+ class="hover:underline text-blue-600"
href="https://en.wikipedia.org/wiki/{{ entity.title|urlencode }}"
>wikipedia</a
>
@@ -85,4 +85,4 @@ export interface Props {
))
}
</div>
-</div>
+</div>
\ No newline at end of file
diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdx
index 2bfe7aa3..f53b9ec8 100644
--- a/frontend/src/pages/crawler.mdx
+++ b/frontend/src/pages/crawler.mdx
@@ -10,8 +10,8 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; +https:
## Politeness
StractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.
-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 30 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 30 seconds.
-This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 1024).
+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 60 seconds.
+This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).
The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.