Skip to content

web search done right

sharded url state db to deal with domains that has a lot of urls

Mikkel Denkerdc0390baac6c
diff --git a/Cargo.lock b/Cargo.lock
index a199c67e..760315ae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -63,6 +63,17 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "ahash"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "version_check",
+]
+
[[package]]
name = "aho-corasick"
version = "0.7.20"
@@ -81,6 +92,12 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "allocator-api2"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
+
[[package]]
name = "android-tzdata"
version = "0.1.1"
@@ -1752,7 +1769,7 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
- "ahash",
+ "ahash 0.7.6",
]
[[package]]
@@ -1760,6 +1777,11 @@ name = "hashbrown"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+dependencies = [
+ "ahash 0.8.3",
+ "allocator-api2",
+ "rkyv",
+]
[[package]]
name = "heck"
@@ -4231,6 +4253,7 @@ dependencies = [
"fst",
"futures",
"half 2.3.1",
+ "hashbrown 0.14.0",
"html-escape",
"http",
"image",
diff --git a/Cargo.toml b/Cargo.toml
index 9852fc51..be2136c2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -97,5 +97,6 @@ utoipa = { version = "3.4.4", features = ["axum_extras"] }
utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }
tikv-jemallocator = "0.5"
publicsuffix = "2.2.3"
+hashbrown = {version = "0.14.0", features = ["rkyv"] }
rkyv = { version = "0.7.42", default_features = false, features = ["std", "validation", "size_64"] }
diff --git a/core/Cargo.toml b/core/Cargo.toml
index ca2c2721..ad2fbb34 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -95,6 +95,7 @@ utoipa = { workspace = true }
utoipa-swagger-ui = { workspace = true }
publicsuffix = { workspace = true }
rkyv = { workspace = true }
+hashbrown = { workspace = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { workspace = true }
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
index 306bd6e2..50d4b416 100644
--- a/core/benches/crawl-coordinator.rs
+++ b/core/benches/crawl-coordinator.rs
@@ -9,7 +9,7 @@ fn rand_domain() -> Domain {
// and appending .com
let mut rng = rand::thread_rng();
let mut domain = String::new();
- for _ in 0..rng.gen_range(1..10) {
+ for _ in 0..rng.gen_range(1..3) {
domain.push(rng.gen_range(b'a'..=b'z') as char);
}
domain.push_str(".com");
diff --git a/core/examples/crawl_coordinator_memory_stress.rs b/core/examples/crawl_coordinator_memory_stress.rs
index 4c2803ce..edeb71e1 100644
--- a/core/examples/crawl_coordinator_memory_stress.rs
+++ b/core/examples/crawl_coordinator_memory_stress.rs
@@ -86,6 +86,8 @@ fn main() -> Result<()> {
pb.inc(1);
let responses = random_responses(1024);
coordinator.add_responses(&responses)?;
+
+ coordinator.sample_jobs(1024)?;
}
Ok(())
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 370f5a14..2a55f130 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -14,9 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use hashbrown::HashMap;
use rand::Rng;
use rkyv::{Archive, Deserialize, Serialize};
-use std::collections::{BTreeMap, HashMap};
+use std::hash::{Hash, Hasher};
use std::{
cmp::Ordering,
collections::{BinaryHeap, VecDeque},
@@ -26,7 +27,7 @@ use url::Url;
use super::{Domain, Job, JobResponse, Result, UrlResponse};
-const URLS_PER_SHARD: usize = 5_000;
+const URLS_PER_SHARD: usize = 10_000;
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Archive)]
#[archive(check_bytes)]
@@ -105,7 +106,8 @@ struct UrlState {
struct DomainState {
weight: f64,
status: DomainStatus,
- max_shard_id: u64,
+ total_urls: u64,
+ num_shards: u64,
}
pub struct RedirectDb {
@@ -152,7 +154,7 @@ impl RedirectDb {
}
}
-#[derive(Clone, Serialize, Deserialize, Archive)]
+#[derive(Clone, Serialize, Deserialize, Archive, Hash, PartialEq, Eq)]
#[archive(check_bytes)]
struct DomainShard {
domain: Domain,
@@ -164,6 +166,11 @@ struct UrlToInsert {
different_domain: bool,
}
+/// The UrlStateDb is a key-value store that maps a domain shard to a map of urls to their state.
+/// The domain shard is a combination of a domain and a shard id. The shard id is used to split the
+/// domain into multiple shards, so that we can store more urls per domain. Some domains have a lot (!)
+/// of urls, and it would be slow to store all of them in a single key-value pair as we would then need to
+/// read gb's of data from disk to update a single url state.
struct UrlStateDb {
db: rocksdb::DB,
}
@@ -193,14 +200,14 @@ impl UrlStateDb {
Ok(Self { db })
}
- pub fn get(&mut self, key: &DomainShard) -> Result<Option<BTreeMap<UrlString, UrlState>>> {
- let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;
+ pub fn get(&mut self, key: &DomainShard) -> Result<Option<HashMap<UrlString, UrlState>>> {
+ let key_bytes = rkyv::to_bytes::<_, 4096>(key)?;
let value_bytes = self.db.get(key_bytes)?;
match value_bytes {
Some(value_bytes) => {
let archived =
- rkyv::check_archived_root::<BTreeMap<UrlString, UrlState>>(&value_bytes[..])
+ rkyv::check_archived_root::<HashMap<UrlString, UrlState>>(&value_bytes[..])
.unwrap();
let value = archived.deserialize(&mut rkyv::Infallible).unwrap();
Ok(Some(value))
@@ -209,9 +216,9 @@ impl UrlStateDb {
}
}
- pub fn put(&mut self, key: &DomainShard, value: &BTreeMap<UrlString, UrlState>) -> Result<()> {
- let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;
- let value_bytes = rkyv::to_bytes::<_, 1024>(value)?;
+ pub fn put(&mut self, key: &DomainShard, value: &HashMap<UrlString, UrlState>) -> Result<()> {
+ let key_bytes = rkyv::to_bytes::<_, 4096>(key)?;
+ let value_bytes = rkyv::to_bytes::<_, 4096>(value)?;
let mut write_options = rocksdb::WriteOptions::default();
write_options.disable_wal(true);
@@ -219,6 +226,51 @@ impl UrlStateDb {
Ok(())
}
+
+ fn rebuild_shards(&mut self, domain: &Domain, domain_state: &mut DomainState) -> Result<()> {
+ let mut urls = HashMap::new();
+
+ for shard_id in 0..domain_state.num_shards {
+ let shard = DomainShard {
+ domain: domain.clone(),
+ shard_id,
+ };
+
+ let shard_urls = self.get(&shard)?.unwrap_or_default();
+
+ urls.extend(shard_urls);
+ }
+
+ let new_num_shards = domain_state.num_shards * 2;
+
+ let mut new_shards: HashMap<DomainShard, HashMap<UrlString, UrlState>> = HashMap::new();
+
+ for (url, state) in urls {
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ url.hash(&mut hasher);
+ let hash = hasher.finish();
+
+ let shard_id = hash % new_num_shards;
+
+ let shard = DomainShard {
+ domain: domain.clone(),
+ shard_id,
+ };
+
+ new_shards
+ .entry(shard)
+ .or_default()
+ .insert(url, state.clone());
+ }
+
+ for (shard, urls) in new_shards {
+ self.put(&shard, &urls)?;
+ }
+
+ domain_state.num_shards = new_num_shards;
+
+ Ok(())
+ }
}
struct DomainStateDb {
@@ -335,14 +387,21 @@ impl CrawlDb {
for url in urls {
let domain = Domain::from(url);
- self.domain_state.put(
- &domain,
- &DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- max_shard_id: 0,
- },
- )?;
+ match self.domain_state.get(&domain)? {
+ Some(mut state) => {
+ state.total_urls += 1;
+ self.domain_state.put(&domain, &state)?;
+ }
+ None => self.domain_state.put(
+ &domain,
+ &DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ total_urls: 1,
+ num_shards: 1,
+ },
+ )?,
+ }
let sharded_domain = DomainShard {
domain,
@@ -393,7 +452,8 @@ impl CrawlDb {
let state = DomainState {
weight: 0.0,
status: DomainStatus::Pending,
- max_shard_id: 0,
+ num_shards: 1,
+ total_urls: 0,
};
self.domain_state.put(&domain, &state)?;
@@ -401,44 +461,54 @@ impl CrawlDb {
}
};
- let mut sharded_domain = DomainShard {
- domain,
- shard_id: domain_state.max_shard_id,
- };
+ if domain_state.total_urls / domain_state.num_shards > URLS_PER_SHARD as u64 {
+ self.urls.rebuild_shards(&domain, &mut domain_state)?;
+ }
+ let mut shards: HashMap<DomainShard, Vec<UrlToInsert>> = HashMap::new();
- let mut url_states = self.urls.get(&sharded_domain)?.unwrap_or_default();
+ for url in urls {
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ let url_string = UrlString::from(&url.url);
+ url_string.hash(&mut hasher);
+ let hash = hasher.finish();
+
+ let shard_id = hash % domain_state.num_shards;
+ let shard = DomainShard {
+ domain: domain.clone(),
+ shard_id,
+ };
+
+ shards.entry(shard).or_default().push(url);
+ }
- if url_states.len() >= URLS_PER_SHARD {
- domain_state.max_shard_id += 1;
- sharded_domain.shard_id = domain_state.max_shard_id;
+ for (sharded_domain, urls) in shards {
+ let mut url_states = self.urls.get(&sharded_domain)?.unwrap_or_default();
- url_states = BTreeMap::new();
- }
+ for url in urls {
+ let mut url_state = url_states
+ .get(&UrlString::from(&url.url))
+ .cloned()
+ .unwrap_or(UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
- for url in urls {
- let mut url_state = url_states
- .get(&UrlString::from(&url.url))
- .cloned()
- .unwrap_or(UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
+ if url.different_domain {
+ url_state.weight += 1.0;
+ }
- if url.different_domain {
- url_state.weight += 1.0;
- }
+ if url_state.weight > domain_state.weight {
+ domain_state.weight = url_state.weight;
+ }
- if url_state.weight > domain_state.weight {
- domain_state.weight = url_state.weight;
+ if url_states.insert(url.url.into(), url_state).is_none() {
+ domain_state.total_urls += 1;
+ }
}
-
- url_states.insert(url.url.into(), url_state);
+ self.urls.put(&sharded_domain, &url_states)?;
}
- self.domain_state
- .put(&sharded_domain.domain, &domain_state)?;
-
- self.urls.put(&sharded_domain, &url_states)?;
+ self.domain_state.put(&domain, &domain_state)?;
}
Ok(())
@@ -448,7 +518,8 @@ impl CrawlDb {
let mut domain_state = self.domain_state.get(domain)?.unwrap_or(DomainState {
weight: 0.0,
status,
- max_shard_id: 0,
+ num_shards: 1,
+ total_urls: 0,
});
domain_state.status = status;
@@ -484,7 +555,7 @@ impl CrawlDb {
for domain in domains {
let state = self.domain_state.get(domain)?.unwrap();
- let shard_id = rand::thread_rng().gen_range(0..=state.max_shard_id);
+ let shard_id = rand::thread_rng().gen_range(0..state.num_shards);
let shard = DomainShard {
domain: domain.clone(),
@@ -509,7 +580,7 @@ impl CrawlDb {
);
for url in &sampled {
- let mut state = urls.get(url).unwrap().clone();
+ let mut state = urls.get(*url).unwrap().clone();
state.status = UrlStatus::Crawling;
urls.insert((*url).clone(), state);