web search done right
crawler robustness
Mikkel Denker3c12b9a9e66b
diff --git a/Cargo.lock b/Cargo.lockindex d758183b..ef422264 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4303,6 +4303,7 @@ dependencies = ["num_cpus","once_cell","optics",+ "parking_lot","parse_wiki_text","proptest","proptest-derive",diff --git a/Cargo.toml b/Cargo.tomlindex 4c14270f..4ae57dfb 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -58,6 +58,7 @@ mime = "0.3.17"min-max-heap = "1.3.0"num_cpus = "1.15.0"once_cell = "1.13.1"+parking_lot = "0.12.1"parse_wiki_text = "0.1.5"proptest = "1.2.0"proptest-derive = "0.3.0"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 554e6da7..1dc0e1e4 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -66,6 +66,7 @@ min-max-heap = {workspace = true}num_cpus = {workspace = true}once_cell = {workspace = true}optics = {path = "../optics"}+parking_lot = {workspace = true}parse_wiki_text = {workspace = true}publicsuffix = {workspace = true}quick-xml = {workspace = true}diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 8e70a577..c6b8ac24 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -15,9 +15,10 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use hashbrown::HashMap;+use parking_lot::Mutex;use super::{crawl_db::CrawlDb, Domain, DomainCrawled, Job, Result, UrlToInsert};-use std::{path::Path, sync::Mutex, time::Instant};+use std::{path::Path, time::Instant};const DEFAULT_JOB_URLS: usize = 200;@@ -33,7 +34,7 @@ impl CrawlCoordinator {}pub fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {- let mut db = self.db.lock().unwrap();+ let mut db = self.db.lock();let start = Instant::now();db.insert_urls(urls)?;@@ -44,7 +45,7 @@ impl CrawlCoordinator {}pub fn mark_jobs_complete(&self, domains: &[DomainCrawled]) -> Result<()> {- let mut db = self.db.lock().unwrap();+ let mut db = self.db.lock();let start = Instant::now();db.mark_jobs_complete(domains)?;@@ -55,7 +56,7 @@ impl CrawlCoordinator {}pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {- let mut db = self.db.lock().unwrap();+ let mut db = self.db.lock();let start = Instant::now();let domains = db.sample_domains(num_jobs)?;diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex c21d634c..618f141b 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -21,7 +21,7 @@ use std::path::PathBuf;use std::{collections::BinaryHeap, path::Path};use url::Url;-use super::{Domain, DomainCrawled, Job, Result, UrlToInsert};+use super::{Domain, DomainCrawled, Job, Result, UrlToInsert, MAX_URL_LEN_BYTES};const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB@@ -118,43 +118,6 @@ impl Default for DomainState {}}}--struct MetaDb {- db: rocksdb::DB,-}--impl MetaDb {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut options = rocksdb::Options::default();- options.create_if_missing(true);-- let db = rocksdb::DB::open(&options, path.as_ref())?;-- Ok(Self { db })- }-- fn set_longest_url_len(&self, len: usize) -> Result<()> {- let mut write_options = rocksdb::WriteOptions::default();- write_options.disable_wal(true);-- self.db- .put_opt(b"longest_url_len", len.to_be_bytes(), &write_options)?;-- Ok(())- }-- fn get_longest_url_len(&self) -> Result<usize> {- let len_bytes = self.db.get(b"longest_url_len")?;-- if let Some(len_bytes) = len_bytes {- let len: usize = usize::from_be_bytes(len_bytes.try_into().unwrap());- return Ok(len);- }-- Ok(0)- }-}-struct CachedValue<T> {value: T,last_updated: std::time::Instant,@@ -171,11 +134,9 @@ impl<T> From<T> for CachedValue<T> {struct UrlStateDbShard {db: rocksdb::DB,- meta: MetaDb,/// from rocksdb docs: "Cache must outlive DB instance which uses it."_cache: rocksdb::Cache,approx_size_bytes: CachedValue<u64>,- longest_known_url: CachedValue<usize>,}impl UrlStateDbShard {@@ -218,19 +179,13 @@ impl UrlStateDbShard {let db = rocksdb::DB::open(&options, path.as_ref().join("urls"))?;let approx_size_bytes = db.property_int_value(rocksdb::properties::TOTAL_SST_FILES_SIZE)?- .unwrap()+ .unwrap_or_default().into();- let meta = MetaDb::open(path.as_ref().join("meta"))?;-- let longest_known_url = meta.get_longest_url_len()?.into();-Ok(Self {db,approx_size_bytes,_cache: cache,- longest_known_url,- meta,})}@@ -242,14 +197,11 @@ impl UrlStateDbShard {for (url, state) in urls {let url_bytes = bincode::serialize(url)?;-- if url_bytes.len() > self.longest_known_url.value {- self.longest_known_url = url_bytes.len().into();- self.meta- .set_longest_url_len(self.longest_known_url.value)?;+ if url_bytes.len() > MAX_URL_LEN_BYTES {+ continue;}- let key_bytes = [domain_bytes.as_slice(), &[b'/'], url_bytes.as_slice()].concat();+ let key_bytes = [domain_bytes.as_slice(), url_bytes.as_slice()].concat();let state_bytes = bincode::serialize(state)?;@@ -268,12 +220,7 @@ impl UrlStateDbShard {pub fn get_all_urls(&self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {let domain_bytes = bincode::serialize(domain)?;- let start = [- domain_bytes.as_slice(),- &[b'/'],- &[0].repeat(self.longest_known_url.value),- ]- .concat();+ let start = [domain_bytes.as_slice(), &[0].repeat(MAX_URL_LEN_BYTES)].concat();let iter = self.db.iterator(rocksdb::IteratorMode::From(&start,@@ -283,7 +230,11 @@ impl UrlStateDbShard {Ok(iter.take_while(|r| {if let Ok((key, _)) = r.as_ref() {- key[..domain_bytes.len()] == domain_bytes[..]+ if domain_bytes.len() >= key.len() {+ return false;+ }++ key[..domain_bytes.len()] == domain_bytes} else {false}@@ -291,10 +242,13 @@ impl UrlStateDbShard {.filter_map(|r| {let (key, value) = r.ok()?;- let url = bincode::deserialize(&key[domain_bytes.len() + 1..]) // +1 for '/'- .ok()?;+ if domain_bytes.len() >= key.len() {+ return None;+ }++ let url = bincode::deserialize(&key[domain_bytes.len()..]).ok()?;- let state = bincode::deserialize(&value[..]).ok()?;+ let state = bincode::deserialize(&value).ok()?;Some((url, state))})@@ -322,7 +276,7 @@ impl UrlStateDbShard {let domain_bytes = bincode::serialize(domain)?;let url_bytes = bincode::serialize(url)?;- let key_bytes = [domain_bytes.as_slice(), &[b'/'], url_bytes.as_slice()].concat();+ let key_bytes = [domain_bytes.as_slice(), url_bytes.as_slice()].concat();keys.push(key_bytes);}@@ -334,7 +288,7 @@ impl UrlStateDbShard {.zip(urls.iter()){if let Some(val) = val? {- let state: UrlState = bincode::deserialize(&val[..])?;+ let state: UrlState = bincode::deserialize(&val)?;res.insert(url.clone(), state);}}@@ -477,7 +431,7 @@ impl DomainStateDb {.zip(domains.iter()){if let Some(val) = val? {- let domain_state: DomainState = bincode::deserialize(&val[..])?;+ let domain_state: DomainState = bincode::deserialize(&val)?;res.insert(domain.clone(), domain_state);}}@@ -508,8 +462,8 @@ impl DomainStateDb {iter.filter_map(|r| {let (key, value) = r.ok()?;- let domain = bincode::deserialize(&key[..]).ok()?;- let state = bincode::deserialize(&value[..]).ok()?;+ let domain = bincode::deserialize(&key).ok()?;+ let state = bincode::deserialize(&value).ok()?;Some((domain, state))})@@ -575,6 +529,10 @@ impl CrawlDb {let mut updated_url_states = Vec::new();for url in urls {+ if url.url.as_str().len() > MAX_URL_LEN_BYTES {+ continue;+ }+match url_states.get_mut(&UrlString::from(&url.url)) {Some(state) => {state.weight += url.weight;diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex e1dede69..6d705a5d 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -37,6 +37,10 @@ mod worker;pub use coordinator::CrawlCoordinator;+pub const MAX_URL_LEN_BYTES: usize = 8192;+pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 200;+pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 100;+#[derive(thiserror::Error, Debug)]pub enum Error {#[error("invalid content type: {0}")]diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rsindex 10707588..75005ee4 100644--- a/core/src/crawler/router.rs+++ b/core/src/crawler/router.rs@@ -10,7 +10,7 @@ use crate::{entrypoint::crawler::coordinator::{CoordinatorService, GetJobs, InsertUrls, MarkJobsComplete},};-use super::{Domain, Job, JobResponse};+use super::{Domain, Job, JobResponse, MAX_URLS_FOR_DOMAIN_PER_INSERT, MAX_URL_LEN_BYTES};struct RemoteCoordinator {addr: SocketAddr,@@ -136,6 +136,10 @@ impl Router {let mut used_budget = 0.0;for (domain, url) in urls {+ if url.as_str().len() > MAX_URL_LEN_BYTES {+ continue;+ }+let different_domain = res.domain != domain;let weight = if different_domain {@@ -146,10 +150,13 @@ impl Router {used_budget += weight;- domain_urls- .entry(domain)- .or_default()- .push(UrlToInsert { url, weight });+ let urls = domain_urls.entry(domain).or_default();++ if urls.len() >= MAX_URLS_FOR_DOMAIN_PER_INSERT {+ continue;+ }++ urls.push(UrlToInsert { url, weight });}domain_budgets.push(DomainCrawled {diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 1bb2184a..da6f235a 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -32,7 +32,7 @@ use url::Url;use crate::{config::CrawlerConfig,- crawler::{JobResponse, Response},+ crawler::{JobResponse, Response, MAX_OUTGOING_URLS_PER_PAGE, MAX_URL_LEN_BYTES},distributed::{retry_strategy::ExponentialBackoff, sonic},entrypoint::crawler::router::{NewJobs, RouterService},webpage::Html,@@ -45,8 +45,6 @@ use super::{const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB-const MAX_URL_LENGTH: usize = 8192;-pub struct WorkerThread {current_job: Option<WorkerJob>,pending_commands: Arc<Mutex<VecDeque<Command>>>,@@ -226,7 +224,7 @@ impl WorkerThread {discovered_urls.extend(sitemap_urls.into_iter()- .filter(|url| url.as_str().len() < MAX_URL_LENGTH),+ .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES),);}}@@ -261,7 +259,8 @@ impl WorkerThread {discovered_urls.extend(res.new_urls.into_iter()- .filter(|url| url.as_str().len() < MAX_URL_LENGTH),+ .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES)+ .take(MAX_OUTGOING_URLS_PER_PAGE),);url_responses.push(res.response);}@@ -271,7 +270,7 @@ impl WorkerThread {url_responses,discovered_urls: discovered_urls.into_iter()- .filter(|url| url.as_str().len() < MAX_URL_LENGTH)+ .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES).collect(),weight_budget: job.weight_budget,};