web search done right
Url states to disk. The idea is to reduce memory usage significantly. Don't know if it will be fast enough for big crawls.
Mikkel Denker9d4c01efc929
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 3e229aac..c1207480 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -22,20 +22,21 @@ use std::{num::NonZeroUsize,ops::Range,path::Path,- sync::Mutex,+ sync::{Arc, Mutex},};-use dashmap::DashMap;use itertools::Itertools;use lru::LruCache;use rand::Rng;+use rayon::prelude::IntoParallelRefIterator;use rayon::prelude::*;use rocksdb::BlockBasedOptions;+use serde::{Deserialize, Serialize};use url::Url;use super::{Domain, Job, JobResponse, Result, UrlResponse};-#[derive(Clone, PartialEq, Eq)]+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]pub enum UrlStatus {Pending,Crawling,@@ -171,7 +172,8 @@ impl AsId for Url {struct IdTable<T> {db: Mutex<MemmapDb<Id, T>>,- cache: Mutex<LruCache<Id, T>>,+ value_cache: Mutex<LruCache<Id, T>>,+ id_cache: Mutex<LruCache<T, Id>>,}impl<T> IdTable<T>@@ -186,30 +188,34 @@ whereOk(Self {db,- cache: Mutex::new(LruCache::new(NonZeroUsize::new(500_000).unwrap())),+ value_cache: Mutex::new(LruCache::new(NonZeroUsize::new(500_000).unwrap())),+ id_cache: Mutex::new(LruCache::new(NonZeroUsize::new(500_000).unwrap())),})}- pub fn bulk_ids<'a>(&'a self, items: impl Iterator<Item = &'a T>) -> Result<Vec<Id>> {- let mut ids = Vec::new();-+ pub fn bulk_insert_ids(&self, items: &[(T, Id)]) -> Result<()> {let mut db = self.db.lock().unwrap();- for item in items {- let id = item.as_id();- if !db.contains(&id) {- db.put(&id, item)?;+ let mut id_cache = self.id_cache.lock().unwrap();++ for (item, id) in items {+ if !db.contains(id) {+ db.put(id, item)?;}- ids.push(id);+ id_cache.put(item.clone(), *id);}db.commit()?;- Ok(ids)+ Ok(())}pub fn id(&self, item: &T) -> Result<Id> {+ if let Some(id) = self.id_cache.lock().unwrap().get(item) {+ return Ok(*id);+ }+let id = item.as_id();let mut db = self.db.lock().unwrap();@@ -223,7 +229,7 @@ where}pub fn value(&mut self, id: Id) -> Result<Option<T>> {- let mut cache = self.cache.lock().unwrap();+ let mut cache = self.value_cache.lock().unwrap();// check cacheif let Some(value) = cache.get(&id) {@@ -293,6 +299,7 @@ fn weighted_sample<'a, T: 'a>(sampled_items.into_iter().map(|s| s.item).collect()}+#[derive(Clone, Serialize, Deserialize)]struct UrlState {weight: f64,status: UrlStatus,@@ -302,7 +309,7 @@ struct DomainState {status: DomainStatus,}-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]pub struct DomainId(u128);impl From<u128> for DomainId {@@ -311,7 +318,7 @@ impl From<u128> for DomainId {}}-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]struct UrlId(u128);impl From<u128> for UrlId {@@ -369,15 +376,101 @@ struct UrlToInsert {different_domain: bool,}+struct UrlStateDb {+ cache: LruCache<DomainId, Arc<Mutex<BTreeMap<UrlId, UrlState>>>>,+ cache_size: usize,+ db: rocksdb::DB,+}++impl UrlStateDb {+ pub fn open<P: AsRef<Path>>(path: P, cache_size: usize) -> Result<Self> {+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);++ let mut block_options = BlockBasedOptions::default();+ block_options.set_ribbon_filter(10.0);+ options.set_block_based_table_factory(&block_options);+ options.set_optimize_filters_for_hits(true);+ options.optimize_for_point_lookup(512 * 1024 * 1024); // 512 MB++ let db = rocksdb::DB::open(&options, path.as_ref())?;++ Ok(Self {+ cache: LruCache::new(NonZeroUsize::new(cache_size + 1).unwrap()),+ cache_size,+ db,+ })+ }++ pub fn get(&mut self, domain_id: DomainId) -> Result<Arc<Mutex<BTreeMap<UrlId, UrlState>>>> {+ // check cache+ if let Some(value) = self.cache.get(&domain_id) {+ return Ok(value.clone());+ }++ // check db+ let domain_id_bytes = bincode::serialize(&domain_id)?;+ let value_bytes = self.db.get(domain_id_bytes)?;++ if let Some(value_bytes) = &value_bytes {+ let value: Arc<Mutex<BTreeMap<UrlId, UrlState>>> = bincode::deserialize(value_bytes)?;+ self.cache.put(domain_id, value.clone());++ self.maybe_prune_cache()?;++ return Ok(value);+ }++ Ok(Arc::new(Mutex::new(BTreeMap::new())))+ }++ fn maybe_prune_cache(&mut self) -> Result<()> {+ // if cache is full, write half of it to disk+ if self.cache.len() >= self.cache_size {+ let mut batch = rocksdb::WriteBatch::default();++ for _ in 0..self.cache_size / 2 {+ if let Some((key, value)) = self.cache.pop_lru() {+ let domain_id_bytes = bincode::serialize(&key)?;+ let value_bytes = bincode::serialize(&value)?;++ batch.put(domain_id_bytes, value_bytes);+ } else {+ break;+ }+ }++ let mut write_options = rocksdb::WriteOptions::default();+ write_options.set_sync(false);+ write_options.disable_wal(true);++ self.db.write_opt(batch, &write_options)?;+ }++ Ok(())+ }++ pub fn put(+ &mut self,+ domain_id: DomainId,+ url_states: Arc<Mutex<BTreeMap<UrlId, UrlState>>>,+ ) -> Result<()> {+ self.cache.put(domain_id, url_states.clone());+ self.maybe_prune_cache()?;++ Ok(())+ }+}+pub struct CrawlDb {url_ids: IdTable<Url>,domain_ids: IdTable<Domain>,redirects: RedirectDb,- domain_state: DashMap<DomainId, DomainState>,+ domain_state: BTreeMap<DomainId, DomainState>,- urls: DashMap<DomainId, BTreeMap<UrlId, UrlState>>,+ urls: UrlStateDb,}impl CrawlDb {@@ -390,8 +483,8 @@ impl CrawlDb {url_ids,domain_ids,redirects,- domain_state: DashMap::new(),- urls: DashMap::new(),+ domain_state: BTreeMap::new(),+ urls: UrlStateDb::open(path.as_ref().join("urls").join("states"), 100_000)?,})}@@ -407,13 +500,17 @@ impl CrawlDb {status: DomainStatus::Pending,});- self.urls.entry(domain_id).or_default().insert(+ let urls = self.urls.get(domain_id)?;++ urls.lock().unwrap().insert(url_id,UrlState {weight: 0.0,status: UrlStatus::Pending,},);++ self.urls.put(domain_id, urls)?;}Ok(())@@ -434,31 +531,41 @@ impl CrawlDb {}});- let domain_ids: Vec<DomainId> = self- .domain_ids- .bulk_ids(domains.keys())?- .into_iter()- .map(DomainId::from)+ let domain_ids: Vec<_> = domains+ .par_iter()+ .map(|(domain, _)| (domain.clone(), domain.as_id())).collect();+ self.domain_ids.bulk_insert_ids(&domain_ids)?;++ let url_ids: Vec<_> = domains+ .par_iter()+ .flat_map(|(_, urls)| {+ urls.iter()+ .map(|url| (url.url.clone(), url.url.as_id()))+ .collect_vec()+ })+ .collect();+ self.url_ids.bulk_insert_ids(&url_ids)?;- self.url_ids- .bulk_ids(domains.values().flatten().map(|u| &u.url))?;-- domain_ids+ for (domain_id, urls) in domain_ids.into_iter()+ .map(|(_, id)| id).zip_eq(domains.values())- .par_bridge()- .for_each(|(domain_id, urls)| {- let mut domain_state =- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });-- let mut url_states = self.urls.entry(domain_id).or_default();+ {+ let domain_id = domain_id.into();++ let domain_state = self+ .domain_state+ .entry(domain_id)+ .or_insert_with(|| DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ });++ let url_states = self.urls.get(domain_id)?;+ {+ let mut url_states = url_states.lock().unwrap();for url in urls {let id = self.url_ids.id(&url.url);@@ -483,7 +590,10 @@ impl CrawlDb {domain_state.weight = url_state.weight;}}- });+ }++ self.urls.put(domain_id, url_states)?;+ }Ok(())}@@ -523,35 +633,47 @@ impl CrawlDb {}// bulk register urls- self.url_ids- .bulk_ids(url_responses.values().flatten().flat_map(|res| match res {- UrlResponse::Success { url } => vec![url].into_iter(),- UrlResponse::Failed {- url,- status_code: _,- } => vec![url].into_iter(),- UrlResponse::Redirected { url, new_url } => vec![url, new_url].into_iter(),- }))?;+ let url_ids: Vec<_> = url_responses+ .par_iter()+ .flat_map(|(_, responses)| {+ responses+ .iter()+ .flat_map(|res| match res {+ UrlResponse::Success { url } => vec![url],+ UrlResponse::Failed {+ url,+ status_code: _,+ } => vec![url],+ UrlResponse::Redirected { url, new_url } => vec![url, new_url],+ })+ .map(|url| (url.clone(), url.as_id()))+ .collect_vec()+ })+ .collect();- // bulk register domains- self.domain_ids.bulk_ids(url_responses.keys())?;+ self.url_ids.bulk_insert_ids(&url_ids)?;- url_responses- .into_par_iter()- .filter_map(|(domain, responses)| {- let domain_id: DomainId = self.domain_ids.id(&domain).ok()?.into();- Some((domain_id, responses))- })- .for_each(|(domain_id, responses)| {- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });+ // bulk register domains+ let domain_ids: Vec<_> = url_responses+ .par_iter()+ .map(|(domain, _)| (domain.clone(), domain.as_id()))+ .collect();+ self.domain_ids.bulk_insert_ids(&domain_ids)?;- let mut url_states = self.urls.entry(domain_id).or_default();+ for (domain_id, responses) in url_responses.into_iter().filter_map(|(domain, responses)| {+ let domain_id: DomainId = self.domain_ids.id(&domain).ok()?.into();+ Some((domain_id, responses))+ }) {+ self.domain_state+ .entry(domain_id)+ .or_insert_with(|| DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ });+ let url_states = self.urls.get(domain_id)?;+ {+ let mut url_states = url_states.lock().unwrap();for response in responses {match response {UrlResponse::Success { url } => {@@ -591,7 +713,10 @@ impl CrawlDb {}}}- });+ }++ self.urls.put(domain_id, url_states)?;+ }Ok(())}@@ -599,7 +724,7 @@ impl CrawlDb {pub fn set_domain_status(&mut self, domain: &Domain, status: DomainStatus) -> Result<()> {let domain_id: DomainId = self.domain_ids.id(domain)?.into();- let mut domain_state = self+ let domain_state = self.domain_state.entry(domain_id).or_insert_with(|| DomainState {@@ -616,8 +741,7 @@ impl CrawlDb {let available_domains: Vec<_> = self.domain_state.iter()- .filter_map(|r| {- let (id, state) = r.pair();+ .filter_map(|(id, state)| {if state.status == DomainStatus::Pending {Some((*id, state.weight))} else {@@ -635,7 +759,7 @@ impl CrawlDb {.collect();for id in &sampled {- let mut state = self.domain_state.get_mut(id).unwrap();+ let state = self.domain_state.get_mut(id).unwrap();state.status = DomainStatus::CrawlInProgress;}@@ -645,53 +769,59 @@ impl CrawlDb {pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {let mut jobs = Vec::with_capacity(domains.len());for domain_id in domains {- let mut urls = self.urls.entry(*domain_id).or_default();-- let sampled: Vec<_> = weighted_sample(- urls.iter_mut().filter_map(|(id, state)| {- if state.status == UrlStatus::Pending {- Some((id, state.weight))- } else {- None- }- }),- urls_per_job,- )- .into_iter()- .copied()- .collect();+ let urls = self.urls.get(*domain_id)?;- for id in &sampled {- let state = urls.get_mut(id).unwrap();- state.status = UrlStatus::Crawling;- }+ {+ let mut urls = urls.lock().unwrap();- let mut domain_state = self.domain_state.get_mut(domain_id).unwrap();+ let sampled: Vec<_> = weighted_sample(+ urls.iter().filter_map(|(id, state)| {+ if state.status == UrlStatus::Pending {+ Some((id, state.weight))+ } else {+ None+ }+ }),+ urls_per_job,+ )+ .into_iter()+ .copied()+ .collect();++ for id in &sampled {+ let state = urls.get_mut(id).unwrap();+ state.status = UrlStatus::Crawling;+ }- domain_state.weight = urls- .iter()- .filter_map(|(_, state)| {- if state.status == UrlStatus::Pending {- Some(state.weight)- } else {- None- }- })- .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal))- .unwrap_or(0.0);-- let mut job = Job {- domain: self.domain_ids.value(domain_id.0)?.unwrap(),- fetch_sitemap: false, // todo: fetch for new sites- urls: VecDeque::with_capacity(urls_per_job),- };-- for url_id in sampled {- let url = self.url_ids.value(url_id.0)?.unwrap();- job.urls.push_back(url);+ let domain_state = self.domain_state.get_mut(domain_id).unwrap();++ domain_state.weight = urls+ .iter()+ .filter_map(|(_, state)| {+ if state.status == UrlStatus::Pending {+ Some(state.weight)+ } else {+ None+ }+ })+ .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal))+ .unwrap_or(0.0);++ let mut job = Job {+ domain: self.domain_ids.value(domain_id.0)?.unwrap(),+ fetch_sitemap: false, // todo: fetch for new sites+ urls: VecDeque::with_capacity(urls_per_job),+ };++ for url_id in sampled {+ let url = self.url_ids.value(url_id.0)?.unwrap();+ job.urls.push_back(url);+ }++ jobs.push(job);}- jobs.push(job);+ self.urls.put(*domain_id, urls)?;}Ok(jobs)diff --git a/core/src/intmap.rs b/core/src/intmap.rsindex 4cdc1fda..9946259a 100644--- a/core/src/intmap.rs+++ b/core/src/intmap.rs@@ -190,10 +190,6 @@ impl<K: Key> IntSet<K> {self.map.insert(item, ());}- pub fn into_iter(self) -> impl Iterator<Item = K> {- self.map.into_iter().map(|(key, _)| key)- }-pub fn contains(&self, item: &K) -> bool {self.map.contains_key(item)}