Skip to content

web search done right

less clones when inserting results into crawl coordinator

Mikkel Denker0eec60152021
diff --git a/Cargo.lock b/Cargo.lock
index 6c13afcf..786d041c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3269,6 +3269,7 @@ version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457"
dependencies = [
+ "hashbrown 0.12.3",
"idna",
"psl-types",
]
diff --git a/Cargo.toml b/Cargo.toml
index 6b5d5f23..c1699707 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -86,7 +86,7 @@ torch-sys = "0.13.0"
libc = "0.2.142"
lz-str = "0.2.1"
safetensors = "0.3.1"
-publicsuffix = "2"
+publicsuffix = { version = "2", features = ["hashbrown"] }
robotstxt-with-cache = "0.4.0"
rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }
rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
index 721902a6..f2755d74 100644
--- a/core/benches/crawl-coordinator.rs
+++ b/core/benches/crawl-coordinator.rs
@@ -1772,7 +1772,7 @@ Url::from("https://www.apple.com/legal/internet-services/itunes/us/terms.html".t
];
let mut rng = rand::thread_rng();
- for _ in 0..10 {
+ for _ in 0..1_000 {
c.bench_function("Add response to coordinator", |b| {
b.iter(|| {
let response = responses.choose(&mut rng).unwrap();
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 4ab4208d..f0c6f690 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -94,7 +94,7 @@ where
})
}
- pub fn bulk_ids(&mut self, items: impl Iterator<Item = T>) -> Result<Vec<u64>> {
+ pub fn bulk_ids<'a>(&'a mut self, items: impl Iterator<Item = &'a T>) -> Result<Vec<u64>> {
let mut ids = Vec::new();
let mut batch_id2t = rocksdb::WriteBatch::default();
@@ -104,26 +104,26 @@ where
for item in items {
// check cache
- if let Some(id) = self.t2id_cache.get(&item) {
+ if let Some(id) = self.t2id_cache.get(item) {
ids.push(*id);
continue;
}
// check if item exists
- let item_bytes = bincode::serialize(&item)?;
+ let item_bytes = bincode::serialize(item)?;
let id = self.t2id.get(&item_bytes)?;
if let Some(id) = id {
let id = bincode::deserialize(&id)?;
// update cache
- self.t2id_cache.put(item, id);
+ self.t2id_cache.put(item.clone(), id);
ids.push(id);
continue;
}
// insert item
- let assigned_id = batch_assignments.get(&item).copied();
+ let assigned_id = batch_assignments.get(item).copied();
let id = assigned_id.unwrap_or_else(|| {
let id = self.next_id;
@@ -140,7 +140,7 @@ where
}
// update cache
- self.t2id_cache.put(item, id);
+ self.t2id_cache.put(item.clone(), id);
ids.push(id);
}
@@ -410,13 +410,13 @@ impl CrawlDb {
let domain_ids: Vec<DomainId> = self
.domain_ids
- .bulk_ids(domains.keys().cloned())?
+ .bulk_ids(domains.keys())?
.into_iter()
.map(DomainId::from)
.collect();
self.url_ids
- .bulk_ids(domains.values().flatten().map(|u| u.url.clone()))?;
+ .bulk_ids(domains.values().flatten().map(|u| &u.url))?;
for (domain_id, urls) in domain_ids.into_iter().zip_eq(domains.values()) {
let domain_state = self
@@ -485,23 +485,18 @@ impl CrawlDb {
}
// bulk register urls
- self.url_ids.bulk_ids(
- url_responses
- .values()
- .flatten()
- .flat_map(|res| match res {
- UrlResponse::Success { url } => vec![url].into_iter(),
- UrlResponse::Failed {
- url,
- status_code: _,
- } => vec![url].into_iter(),
- UrlResponse::Redirected { url, new_url } => vec![url, new_url].into_iter(),
- })
- .cloned(),
- )?;
+ self.url_ids
+ .bulk_ids(url_responses.values().flatten().flat_map(|res| match res {
+ UrlResponse::Success { url } => vec![url].into_iter(),
+ UrlResponse::Failed {
+ url,
+ status_code: _,
+ } => vec![url].into_iter(),
+ UrlResponse::Redirected { url, new_url } => vec![url, new_url].into_iter(),
+ }))?;
// bulk register domains
- self.domain_ids.bulk_ids(url_responses.keys().cloned())?;
+ self.domain_ids.bulk_ids(url_responses.keys())?;
for (domain, responses) in url_responses {
let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index 09f4b146..e08fb475 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -132,14 +132,16 @@ impl Url {
pub fn strip_protocol(&self) -> &str {
let mut start_host = 0;
let url = &self.0;
- if url.starts_with("http://") || url.starts_with("https://") || url.starts_with("//") {
- start_host = url
- .find('/')
- .expect("It was checked that url starts with protocol");
- start_host += 2; // skip the two '/'
+
+ if url.starts_with("http://") {
+ start_host = 7;
+ } else if url.starts_with("https://") {
+ start_host = 8;
+ } else if url.starts_with("//") {
+ start_host = 2;
}
- let start_host = ceil_char_boundary(url, start_host);
+ let start_host = floor_char_boundary(url, std::cmp::min(start_host, url.len()));
&url[start_host..]
}