web search done right
sharded url state db to deal with domains that has a lot of urls
Mikkel Denkerdc0390baac6c
diff --git a/Cargo.lock b/Cargo.lockindex a199c67e..760315ae 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -63,6 +63,17 @@ dependencies = ["version_check",]+[[package]]+name = "ahash"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"+dependencies = [+ "cfg-if",+ "once_cell",+ "version_check",+]+[[package]]name = "aho-corasick"version = "0.7.20"@@ -81,6 +92,12 @@ dependencies = ["memchr",]+[[package]]+name = "allocator-api2"+version = "0.2.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"+[[package]]name = "android-tzdata"version = "0.1.1"@@ -1752,7 +1769,7 @@ version = "0.12.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"dependencies = [- "ahash",+ "ahash 0.7.6",][[package]]@@ -1760,6 +1777,11 @@ name = "hashbrown"version = "0.14.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"+dependencies = [+ "ahash 0.8.3",+ "allocator-api2",+ "rkyv",+][[package]]name = "heck"@@ -4231,6 +4253,7 @@ dependencies = ["fst","futures","half 2.3.1",+ "hashbrown 0.14.0","html-escape","http","image",diff --git a/Cargo.toml b/Cargo.tomlindex 9852fc51..be2136c2 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -97,5 +97,6 @@ utoipa = { version = "3.4.4", features = ["axum_extras"] }utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }tikv-jemallocator = "0.5"publicsuffix = "2.2.3"+hashbrown = {version = "0.14.0", features = ["rkyv"] }rkyv = { version = "0.7.42", default_features = false, features = ["std", "validation", "size_64"] }diff --git a/core/Cargo.toml b/core/Cargo.tomlindex ca2c2721..ad2fbb34 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -95,6 +95,7 @@ utoipa = { workspace = true }utoipa-swagger-ui = { workspace = true }publicsuffix = { workspace = true }rkyv = { workspace = true }+hashbrown = { workspace = true }[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator = { workspace = true }diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rsindex 306bd6e2..50d4b416 100644--- a/core/benches/crawl-coordinator.rs+++ b/core/benches/crawl-coordinator.rs@@ -9,7 +9,7 @@ fn rand_domain() -> Domain {// and appending .comlet mut rng = rand::thread_rng();let mut domain = String::new();- for _ in 0..rng.gen_range(1..10) {+ for _ in 0..rng.gen_range(1..3) {domain.push(rng.gen_range(b'a'..=b'z') as char);}domain.push_str(".com");diff --git a/core/examples/crawl_coordinator_memory_stress.rs b/core/examples/crawl_coordinator_memory_stress.rsindex 4c2803ce..edeb71e1 100644--- a/core/examples/crawl_coordinator_memory_stress.rs+++ b/core/examples/crawl_coordinator_memory_stress.rs@@ -86,6 +86,8 @@ fn main() -> Result<()> {pb.inc(1);let responses = random_responses(1024);coordinator.add_responses(&responses)?;++ coordinator.sample_jobs(1024)?;}Ok(())diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 370f5a14..2a55f130 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -14,9 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use hashbrown::HashMap;use rand::Rng;use rkyv::{Archive, Deserialize, Serialize};-use std::collections::{BTreeMap, HashMap};+use std::hash::{Hash, Hasher};use std::{cmp::Ordering,collections::{BinaryHeap, VecDeque},@@ -26,7 +27,7 @@ use url::Url;use super::{Domain, Job, JobResponse, Result, UrlResponse};-const URLS_PER_SHARD: usize = 5_000;+const URLS_PER_SHARD: usize = 10_000;#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Archive)]#[archive(check_bytes)]@@ -105,7 +106,8 @@ struct UrlState {struct DomainState {weight: f64,status: DomainStatus,- max_shard_id: u64,+ total_urls: u64,+ num_shards: u64,}pub struct RedirectDb {@@ -152,7 +154,7 @@ impl RedirectDb {}}-#[derive(Clone, Serialize, Deserialize, Archive)]+#[derive(Clone, Serialize, Deserialize, Archive, Hash, PartialEq, Eq)]#[archive(check_bytes)]struct DomainShard {domain: Domain,@@ -164,6 +166,11 @@ struct UrlToInsert {different_domain: bool,}+/// The UrlStateDb is a key-value store that maps a domain shard to a map of urls to their state.+/// The domain shard is a combination of a domain and a shard id. The shard id is used to split the+/// domain into multiple shards, so that we can store more urls per domain. Some domains have a lot (!)+/// of urls, and it would be slow to store all of them in a single key-value pair as we would then need to+/// read gb's of data from disk to update a single url state.struct UrlStateDb {db: rocksdb::DB,}@@ -193,14 +200,14 @@ impl UrlStateDb {Ok(Self { db })}- pub fn get(&mut self, key: &DomainShard) -> Result<Option<BTreeMap<UrlString, UrlState>>> {- let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;+ pub fn get(&mut self, key: &DomainShard) -> Result<Option<HashMap<UrlString, UrlState>>> {+ let key_bytes = rkyv::to_bytes::<_, 4096>(key)?;let value_bytes = self.db.get(key_bytes)?;match value_bytes {Some(value_bytes) => {let archived =- rkyv::check_archived_root::<BTreeMap<UrlString, UrlState>>(&value_bytes[..])+ rkyv::check_archived_root::<HashMap<UrlString, UrlState>>(&value_bytes[..]).unwrap();let value = archived.deserialize(&mut rkyv::Infallible).unwrap();Ok(Some(value))@@ -209,9 +216,9 @@ impl UrlStateDb {}}- pub fn put(&mut self, key: &DomainShard, value: &BTreeMap<UrlString, UrlState>) -> Result<()> {- let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;- let value_bytes = rkyv::to_bytes::<_, 1024>(value)?;+ pub fn put(&mut self, key: &DomainShard, value: &HashMap<UrlString, UrlState>) -> Result<()> {+ let key_bytes = rkyv::to_bytes::<_, 4096>(key)?;+ let value_bytes = rkyv::to_bytes::<_, 4096>(value)?;let mut write_options = rocksdb::WriteOptions::default();write_options.disable_wal(true);@@ -219,6 +226,51 @@ impl UrlStateDb {Ok(())}++ fn rebuild_shards(&mut self, domain: &Domain, domain_state: &mut DomainState) -> Result<()> {+ let mut urls = HashMap::new();++ for shard_id in 0..domain_state.num_shards {+ let shard = DomainShard {+ domain: domain.clone(),+ shard_id,+ };++ let shard_urls = self.get(&shard)?.unwrap_or_default();++ urls.extend(shard_urls);+ }++ let new_num_shards = domain_state.num_shards * 2;++ let mut new_shards: HashMap<DomainShard, HashMap<UrlString, UrlState>> = HashMap::new();++ for (url, state) in urls {+ let mut hasher = std::collections::hash_map::DefaultHasher::new();+ url.hash(&mut hasher);+ let hash = hasher.finish();++ let shard_id = hash % new_num_shards;++ let shard = DomainShard {+ domain: domain.clone(),+ shard_id,+ };++ new_shards+ .entry(shard)+ .or_default()+ .insert(url, state.clone());+ }++ for (shard, urls) in new_shards {+ self.put(&shard, &urls)?;+ }++ domain_state.num_shards = new_num_shards;++ Ok(())+ }}struct DomainStateDb {@@ -335,14 +387,21 @@ impl CrawlDb {for url in urls {let domain = Domain::from(url);- self.domain_state.put(- &domain,- &DomainState {- weight: 0.0,- status: DomainStatus::Pending,- max_shard_id: 0,- },- )?;+ match self.domain_state.get(&domain)? {+ Some(mut state) => {+ state.total_urls += 1;+ self.domain_state.put(&domain, &state)?;+ }+ None => self.domain_state.put(+ &domain,+ &DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ total_urls: 1,+ num_shards: 1,+ },+ )?,+ }let sharded_domain = DomainShard {domain,@@ -393,7 +452,8 @@ impl CrawlDb {let state = DomainState {weight: 0.0,status: DomainStatus::Pending,- max_shard_id: 0,+ num_shards: 1,+ total_urls: 0,};self.domain_state.put(&domain, &state)?;@@ -401,44 +461,54 @@ impl CrawlDb {}};- let mut sharded_domain = DomainShard {- domain,- shard_id: domain_state.max_shard_id,- };+ if domain_state.total_urls / domain_state.num_shards > URLS_PER_SHARD as u64 {+ self.urls.rebuild_shards(&domain, &mut domain_state)?;+ }+ let mut shards: HashMap<DomainShard, Vec<UrlToInsert>> = HashMap::new();- let mut url_states = self.urls.get(&sharded_domain)?.unwrap_or_default();+ for url in urls {+ let mut hasher = std::collections::hash_map::DefaultHasher::new();+ let url_string = UrlString::from(&url.url);+ url_string.hash(&mut hasher);+ let hash = hasher.finish();++ let shard_id = hash % domain_state.num_shards;+ let shard = DomainShard {+ domain: domain.clone(),+ shard_id,+ };++ shards.entry(shard).or_default().push(url);+ }- if url_states.len() >= URLS_PER_SHARD {- domain_state.max_shard_id += 1;- sharded_domain.shard_id = domain_state.max_shard_id;+ for (sharded_domain, urls) in shards {+ let mut url_states = self.urls.get(&sharded_domain)?.unwrap_or_default();- url_states = BTreeMap::new();- }+ for url in urls {+ let mut url_state = url_states+ .get(&UrlString::from(&url.url))+ .cloned()+ .unwrap_or(UrlState {+ weight: 0.0,+ status: UrlStatus::Pending,+ });- for url in urls {- let mut url_state = url_states- .get(&UrlString::from(&url.url))- .cloned()- .unwrap_or(UrlState {- weight: 0.0,- status: UrlStatus::Pending,- });+ if url.different_domain {+ url_state.weight += 1.0;+ }- if url.different_domain {- url_state.weight += 1.0;- }+ if url_state.weight > domain_state.weight {+ domain_state.weight = url_state.weight;+ }- if url_state.weight > domain_state.weight {- domain_state.weight = url_state.weight;+ if url_states.insert(url.url.into(), url_state).is_none() {+ domain_state.total_urls += 1;+ }}-- url_states.insert(url.url.into(), url_state);+ self.urls.put(&sharded_domain, &url_states)?;}- self.domain_state- .put(&sharded_domain.domain, &domain_state)?;-- self.urls.put(&sharded_domain, &url_states)?;+ self.domain_state.put(&domain, &domain_state)?;}Ok(())@@ -448,7 +518,8 @@ impl CrawlDb {let mut domain_state = self.domain_state.get(domain)?.unwrap_or(DomainState {weight: 0.0,status,- max_shard_id: 0,+ num_shards: 1,+ total_urls: 0,});domain_state.status = status;@@ -484,7 +555,7 @@ impl CrawlDb {for domain in domains {let state = self.domain_state.get(domain)?.unwrap();- let shard_id = rand::thread_rng().gen_range(0..=state.max_shard_id);+ let shard_id = rand::thread_rng().gen_range(0..state.num_shards);let shard = DomainShard {domain: domain.clone(),@@ -509,7 +580,7 @@ impl CrawlDb {);for url in &sampled {- let mut state = urls.get(url).unwrap().clone();+ let mut state = urls.get(*url).unwrap().clone();state.status = UrlStatus::Crawling;urls.insert((*url).clone(), state);