Skip to content

web search done right

remove all url point lookups in crawldb

Mikkel Denker41da2a2d852a
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 9d27c9a3..dc5b33e0 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -286,23 +286,6 @@ impl UrlStateDbShard {
})
}
- pub fn get(&self, domain: &Domain, url: &UrlString) -> Result<Option<UrlState>> {
- let url_bytes = bincode::serialize(url)?;
- let domain_bytes = bincode::serialize(domain)?;
-
- let key_bytes = [domain_bytes.as_slice(), &[b'/'], url_bytes.as_slice()].concat();
-
- let state_bytes = self.db.get(key_bytes)?;
-
- match state_bytes {
- Some(state_bytes) => {
- let state = bincode::deserialize(&state_bytes).unwrap();
- Ok(Some(state))
- }
- None => Ok(None),
- }
- }
-
pub fn put_batch(&mut self, domain: &Domain, urls: &[(UrlString, UrlState)]) -> Result<()> {
let domain_bytes = bincode::serialize(domain)?;
@@ -427,18 +410,6 @@ impl UrlStateDb {
}
}
- pub fn get(&self, domain: &Domain, url: &UrlString) -> Result<Option<UrlState>> {
- // we iterate in reverse order so that we get the most recent state
- // since new states are inserted into the last shard.
- for shard in self.shards.iter().rev() {
- if let Some(state) = shard.get(domain, url)? {
- return Ok(Some(state));
- }
- }
-
- Ok(None)
- }
-
pub fn put_batch(&mut self, domain: &Domain, urls: &[(UrlString, UrlState)]) -> Result<()> {
let last_shard = self.shards.last_mut().unwrap();
@@ -459,7 +430,7 @@ impl UrlStateDb {
Ok(())
}
- pub fn get_all_urls(&self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {
+ pub fn get_all_urls(&self, domain: &Domain) -> Result<HashMap<UrlString, UrlState>> {
let mut res = HashMap::new();
for shard in &self.shards {
@@ -468,7 +439,7 @@ impl UrlStateDb {
}
}
- Ok(res.into_iter().collect())
+ Ok(res)
}
}
@@ -646,7 +617,7 @@ impl CrawlDb {
for (domain, urls) in domains.into_iter() {
let mut domain_state = self.domain_state.get(&domain)?.unwrap_or_default();
- let all_urls: HashMap<_, _> = self.urls.get_all_urls(&domain)?.into_iter().collect();
+ let all_urls: HashMap<_, _> = self.urls.get_all_urls(&domain)?;
let mut url_states = Vec::new();
@@ -725,7 +696,7 @@ impl CrawlDb {
let mut new_url_states = Vec::new();
for (url, _) in &sampled {
- let mut state = self.urls.get(domain, url)?.unwrap_or_default();
+ let mut state = urls.get(*url).cloned().unwrap_or_default();
state.status = UrlStatus::Crawling;
new_url_states.push(((*url).clone(), state));
@@ -809,7 +780,7 @@ mod tests {
let domain = Domain::from(&Url::parse("https://a.com").unwrap());
- let urls = db.urls.get_all_urls(&domain).unwrap();
+ let urls: Vec<_> = db.urls.get_all_urls(&domain).unwrap().into_iter().collect();
assert_eq!(urls.len(), 1);
assert_eq!(