web search done right
less clones when inserting results into crawl coordinator
Mikkel Denker0eec60152021
diff --git a/Cargo.lock b/Cargo.lockindex 6c13afcf..786d041c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3269,6 +3269,7 @@ version = "2.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457"dependencies = [+ "hashbrown 0.12.3","idna","psl-types",]diff --git a/Cargo.toml b/Cargo.tomlindex 6b5d5f23..c1699707 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -86,7 +86,7 @@ torch-sys = "0.13.0"libc = "0.2.142"lz-str = "0.2.1"safetensors = "0.3.1"-publicsuffix = "2"+publicsuffix = { version = "2", features = ["hashbrown"] }robotstxt-with-cache = "0.4.0"rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rsindex 721902a6..f2755d74 100644--- a/core/benches/crawl-coordinator.rs+++ b/core/benches/crawl-coordinator.rs@@ -1772,7 +1772,7 @@ Url::from("https://www.apple.com/legal/internet-services/itunes/us/terms.html".t];let mut rng = rand::thread_rng();- for _ in 0..10 {+ for _ in 0..1_000 {c.bench_function("Add response to coordinator", |b| {b.iter(|| {let response = responses.choose(&mut rng).unwrap();diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 4ab4208d..f0c6f690 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -94,7 +94,7 @@ where})}- pub fn bulk_ids(&mut self, items: impl Iterator<Item = T>) -> Result<Vec<u64>> {+ pub fn bulk_ids<'a>(&'a mut self, items: impl Iterator<Item = &'a T>) -> Result<Vec<u64>> {let mut ids = Vec::new();let mut batch_id2t = rocksdb::WriteBatch::default();@@ -104,26 +104,26 @@ wherefor item in items {// check cache- if let Some(id) = self.t2id_cache.get(&item) {+ if let Some(id) = self.t2id_cache.get(item) {ids.push(*id);continue;}// check if item exists- let item_bytes = bincode::serialize(&item)?;+ let item_bytes = bincode::serialize(item)?;let id = self.t2id.get(&item_bytes)?;if let Some(id) = id {let id = bincode::deserialize(&id)?;// update cache- self.t2id_cache.put(item, id);+ self.t2id_cache.put(item.clone(), id);ids.push(id);continue;}// insert item- let assigned_id = batch_assignments.get(&item).copied();+ let assigned_id = batch_assignments.get(item).copied();let id = assigned_id.unwrap_or_else(|| {let id = self.next_id;@@ -140,7 +140,7 @@ where}// update cache- self.t2id_cache.put(item, id);+ self.t2id_cache.put(item.clone(), id);ids.push(id);}@@ -410,13 +410,13 @@ impl CrawlDb {let domain_ids: Vec<DomainId> = self.domain_ids- .bulk_ids(domains.keys().cloned())?+ .bulk_ids(domains.keys())?.into_iter().map(DomainId::from).collect();self.url_ids- .bulk_ids(domains.values().flatten().map(|u| u.url.clone()))?;+ .bulk_ids(domains.values().flatten().map(|u| &u.url))?;for (domain_id, urls) in domain_ids.into_iter().zip_eq(domains.values()) {let domain_state = self@@ -485,23 +485,18 @@ impl CrawlDb {}// bulk register urls- self.url_ids.bulk_ids(- url_responses- .values()- .flatten()- .flat_map(|res| match res {- UrlResponse::Success { url } => vec![url].into_iter(),- UrlResponse::Failed {- url,- status_code: _,- } => vec![url].into_iter(),- UrlResponse::Redirected { url, new_url } => vec![url, new_url].into_iter(),- })- .cloned(),- )?;+ self.url_ids+ .bulk_ids(url_responses.values().flatten().flat_map(|res| match res {+ UrlResponse::Success { url } => vec![url].into_iter(),+ UrlResponse::Failed {+ url,+ status_code: _,+ } => vec![url].into_iter(),+ UrlResponse::Redirected { url, new_url } => vec![url, new_url].into_iter(),+ }))?;// bulk register domains- self.domain_ids.bulk_ids(url_responses.keys().cloned())?;+ self.domain_ids.bulk_ids(url_responses.keys())?;for (domain, responses) in url_responses {let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rsindex 09f4b146..e08fb475 100644--- a/core/src/webpage/url.rs+++ b/core/src/webpage/url.rs@@ -132,14 +132,16 @@ impl Url {pub fn strip_protocol(&self) -> &str {let mut start_host = 0;let url = &self.0;- if url.starts_with("http://") || url.starts_with("https://") || url.starts_with("//") {- start_host = url- .find('/')- .expect("It was checked that url starts with protocol");- start_host += 2; // skip the two '/'++ if url.starts_with("http://") {+ start_host = 7;+ } else if url.starts_with("https://") {+ start_host = 8;+ } else if url.starts_with("//") {+ start_host = 2;}- let start_host = ceil_char_boundary(url, start_host);+ let start_host = floor_char_boundary(url, std::cmp::min(start_host, url.len()));&url[start_host..]}