Skip to content

web search done right

Significantly decrease memory usage by storing url states on disk. Hopefully this will not hurt performance too much. It would allow us to make much larger crawls

Mikkel Denker864fe3a7c00c
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 98e9c490..8637f496 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -30,7 +30,7 @@ use crate::webpage::Url;
use super::{Domain, Job, Result, UrlResponse};
-#[derive(Clone, PartialEq, Eq)]
+#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq)]
pub enum UrlStatus {
Pending,
Crawling,
@@ -255,6 +255,7 @@ fn weighted_sample<'a, T: 'a>(
sampled_items.into_iter().map(|s| s.0.item).collect()
}
+#[derive(serde::Serialize, serde::Deserialize)]
struct UrlState {
weight: f64,
status: UrlStatus,
@@ -264,7 +265,7 @@ struct DomainState {
status: DomainStatus,
}
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct DomainId(u64);
impl From<u64> for DomainId {
@@ -273,7 +274,7 @@ impl From<u64> for DomainId {
}
}
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct UrlId(u64);
impl From<u64> for UrlId {
@@ -288,6 +289,8 @@ struct RedirectDb {
impl RedirectDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ rocksdb::DB::destroy(&rocksdb::Options::default(), path.as_ref())?;
+
let inner = rocksdb::DB::open_default(path.as_ref())?;
Ok(Self { inner })
@@ -307,6 +310,59 @@ impl RedirectDb {
}
}
+struct UrlMap {
+ inner: rocksdb::DB, // DomainId -> HashMap<UrlId, UrlState>
+}
+
+impl UrlMap {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ rocksdb::DB::destroy(&rocksdb::Options::default(), path.as_ref())?;
+
+ let mut options = rocksdb::Options::default();
+ options.create_if_missing(true);
+ options.increase_parallelism(8);
+ options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable
+ options.set_max_write_buffer_number(8);
+
+ let mut block_options = BlockBasedOptions::default();
+ block_options.set_bloom_filter(64.0, true);
+
+ options.set_block_based_table_factory(&block_options);
+
+ let inner = rocksdb::DB::open(&options, path.as_ref())?;
+
+ Ok(Self { inner })
+ }
+
+ pub fn put(&self, domain_id: DomainId, map: HashMap<UrlId, UrlState>) -> Result<()> {
+ let domain_bytes = bincode::serialize(&domain_id)?;
+ let map_bytes = bincode::serialize(&map)?;
+
+ let mut write_options = rocksdb::WriteOptions::default();
+ write_options.set_sync(false);
+ write_options.disable_wal(true);
+
+ self.inner
+ .put_opt(domain_bytes, map_bytes, &write_options)?;
+
+ Ok(())
+ }
+
+ pub fn get(&self, domain_id: DomainId) -> Result<Option<HashMap<UrlId, UrlState>>> {
+ let domain_bytes = bincode::serialize(&domain_id)?;
+
+ let map_bytes = self.inner.get(domain_bytes)?;
+
+ match map_bytes {
+ Some(bytes) => {
+ let map = bincode::deserialize(&bytes)?;
+ Ok(Some(map))
+ }
+ None => Ok(None),
+ }
+ }
+}
+
pub struct CrawlDb {
url_ids: IdTable<Url>,
domain_ids: IdTable<Domain>,
@@ -315,13 +371,14 @@ pub struct CrawlDb {
domain_state: HashMap<DomainId, DomainState>,
- urls: HashMap<DomainId, HashMap<UrlId, UrlState>>,
+ urls: UrlMap,
}
impl CrawlDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let url_ids = IdTable::open(path.as_ref().join("urls"))?;
- let domain_ids = IdTable::open(path.as_ref().join("domains"))?;
+ let url_ids = IdTable::open(path.as_ref().join("url_ids"))?;
+ let domain_ids = IdTable::open(path.as_ref().join("domain_ids"))?;
+ let url_map = UrlMap::open(path.as_ref().join("url_map"))?;
let redirects = RedirectDb::open(path.as_ref().join("redirects"))?;
Ok(Self {
@@ -329,29 +386,44 @@ impl CrawlDb {
domain_ids,
redirects,
domain_state: HashMap::new(),
- urls: HashMap::new(),
+ urls: url_map,
})
}
pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {
+ let mut bundled: HashMap<Domain, Vec<Url>> = HashMap::new();
+
for url in urls {
- let domain_id = self.domain_ids.id(url.into())?.into();
- let url_id = self.url_ids.id(url.clone())?.into();
+ bundled
+ .entry(url.domain().to_string().into())
+ .or_default()
+ .push(url.clone());
+ }
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
+ for (domain, urls) in bundled {
+ let domain_id = self.domain_ids.id(domain)?.into();
+ let mut map = self.urls.get(domain_id)?.unwrap_or_default();
+ for url in urls {
+ let domain_id = self.domain_ids.id(url.clone().into())?.into();
+ let url_id = self.url_ids.id(url.clone())?.into();
- self.urls.entry(domain_id).or_default().insert(
- url_id,
- UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- },
- );
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+
+ map.insert(
+ url_id,
+ UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ },
+ );
+ }
+
+ self.urls.put(domain_id, map)?;
}
Ok(())
@@ -376,6 +448,8 @@ impl CrawlDb {
self.url_ids.bulk_ids(domains.values().flatten().cloned())?;
for (domain_id, (domain, urls)) in domain_ids.into_iter().zip(domains.into_iter()) {
+ let mut map = self.urls.get(domain_id)?.unwrap_or_default();
+
for url in urls {
let url_id: UrlId = self.url_ids.id(url.clone())?.into();
@@ -389,15 +463,10 @@ impl CrawlDb {
domain_state.status = DomainStatus::Pending;
- let url_state = self
- .urls
- .entry(domain_id)
- .or_default()
- .entry(url_id)
- .or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
+ let url_state = map.entry(url_id).or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
if &domain != crawled_domain {
url_state.weight += 1.0;
@@ -407,12 +476,39 @@ impl CrawlDb {
domain_state.weight = url_state.weight;
}
}
+
+ self.urls.put(domain_id, map)?;
}
Ok(())
}
pub fn update_url_status(&mut self, url_responses: &[UrlResponse]) -> Result<()> {
+ let mut bundled: HashMap<Domain, Vec<UrlResponse>> = HashMap::new();
+
+ for res in url_responses {
+ match res {
+ UrlResponse::Success { url } => {
+ let domain: Domain = url.domain().to_string().into();
+ bundled.entry(domain).or_default().push(res.clone());
+ }
+ UrlResponse::Failed {
+ url,
+ status_code: _,
+ } => {
+ let domain: Domain = url.domain().to_string().into();
+ bundled.entry(domain).or_default().push(res.clone());
+ }
+ UrlResponse::Redirected { url, new_url: _ } => {
+ let domain: Domain = url.domain().to_string().into();
+ bundled.entry(domain).or_default().push(res.clone());
+ }
+ }
+ }
+
+ // bulk register domains
+ self.domain_ids.bulk_ids(bundled.keys().cloned())?;
+
// bulk register urls
self.url_ids.bulk_ids(
url_responses
@@ -428,66 +524,61 @@ impl CrawlDb {
.cloned(),
)?;
- for response in url_responses {
- match response {
- UrlResponse::Success { url } => {
- let domain: Domain = url.domain().to_string().into();
- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
- let url_id: UrlId = self.url_ids.id(url.clone())?.into();
-
- let domain_state =
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
- domain_state.status = DomainStatus::Pending;
-
- let url_state = self
- .urls
- .entry(domain_id)
- .or_default()
- .entry(url_id)
- .or_insert_with(|| UrlState {
+ for (domain, url_responses) in bundled {
+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
+ let mut map = self.urls.get(domain_id)?.unwrap_or_default();
+
+ for response in url_responses {
+ match response {
+ UrlResponse::Success { url } => {
+ let domain: Domain = url.domain().to_string().into();
+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+
+ let domain_state =
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+ domain_state.status = DomainStatus::Pending;
+
+ let url_state = map.entry(url_id).or_insert_with(|| UrlState {
weight: 0.0,
status: UrlStatus::Pending,
});
- url_state.status = UrlStatus::Done;
- }
- UrlResponse::Failed { url, status_code } => {
- let domain: Domain = url.domain().to_string().into();
- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
- let url_id: UrlId = self.url_ids.id(url.clone())?.into();
-
- let domain_state =
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
- domain_state.status = DomainStatus::Pending;
-
- let url_state = self
- .urls
- .entry(domain_id)
- .or_default()
- .entry(url_id)
- .or_insert_with(|| UrlState {
+ url_state.status = UrlStatus::Done;
+ }
+ UrlResponse::Failed { url, status_code } => {
+ let domain: Domain = url.domain().to_string().into();
+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+
+ let domain_state =
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+ domain_state.status = DomainStatus::Pending;
+
+ let url_state = map.entry(url_id).or_insert_with(|| UrlState {
weight: 0.0,
status: UrlStatus::Pending,
});
- url_state.status = UrlStatus::Failed {
- status_code: *status_code,
- };
- }
- UrlResponse::Redirected { url, new_url } => {
- self.redirects.put(url, new_url)?;
+ url_state.status = UrlStatus::Failed { status_code };
+ }
+ UrlResponse::Redirected { url, new_url } => {
+ self.redirects.put(&url, &new_url)?;
+ }
}
}
+
+ self.urls.put(domain_id, map)?;
}
Ok(())
}
@@ -534,7 +625,7 @@ impl CrawlDb {
pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {
let mut jobs = Vec::with_capacity(domains.len());
for domain_id in domains {
- let urls = self.urls.entry(*domain_id).or_default();
+ let mut urls = self.urls.get(*domain_id)?.unwrap_or_default();
let sampled: Vec<_> = weighted_sample(
urls.iter_mut().filter_map(|(id, state)| {
@@ -566,6 +657,8 @@ impl CrawlDb {
job.urls.push_back(url);
}
+ self.urls.put(*domain_id, urls)?;
+
jobs.push(job);
}
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index a6715f79..c31e88d8 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -92,6 +92,12 @@ impl From<&Url> for Domain {
}
}
+impl From<Url> for Domain {
+ fn from(url: Url) -> Self {
+ Self::from(&url)
+ }
+}
+
impl From<String> for Domain {
fn from(s: String) -> Self {
Self(s)