Skip to content

web search done right

crawler robustness

Mikkel Denker3c12b9a9e66b
diff --git a/Cargo.lock b/Cargo.lock
index d758183b..ef422264 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4303,6 +4303,7 @@ dependencies = [
"num_cpus",
"once_cell",
"optics",
+ "parking_lot",
"parse_wiki_text",
"proptest",
"proptest-derive",
diff --git a/Cargo.toml b/Cargo.toml
index 4c14270f..4ae57dfb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -58,6 +58,7 @@ mime = "0.3.17"
min-max-heap = "1.3.0"
num_cpus = "1.15.0"
once_cell = "1.13.1"
+parking_lot = "0.12.1"
parse_wiki_text = "0.1.5"
proptest = "1.2.0"
proptest-derive = "0.3.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 554e6da7..1dc0e1e4 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -66,6 +66,7 @@ min-max-heap = {workspace = true}
num_cpus = {workspace = true}
once_cell = {workspace = true}
optics = {path = "../optics"}
+parking_lot = {workspace = true}
parse_wiki_text = {workspace = true}
publicsuffix = {workspace = true}
quick-xml = {workspace = true}
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 8e70a577..c6b8ac24 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -15,9 +15,10 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use hashbrown::HashMap;
+use parking_lot::Mutex;
use super::{crawl_db::CrawlDb, Domain, DomainCrawled, Job, Result, UrlToInsert};
-use std::{path::Path, sync::Mutex, time::Instant};
+use std::{path::Path, time::Instant};
const DEFAULT_JOB_URLS: usize = 200;
@@ -33,7 +34,7 @@ impl CrawlCoordinator {
}
pub fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {
- let mut db = self.db.lock().unwrap();
+ let mut db = self.db.lock();
let start = Instant::now();
db.insert_urls(urls)?;
@@ -44,7 +45,7 @@ impl CrawlCoordinator {
}
pub fn mark_jobs_complete(&self, domains: &[DomainCrawled]) -> Result<()> {
- let mut db = self.db.lock().unwrap();
+ let mut db = self.db.lock();
let start = Instant::now();
db.mark_jobs_complete(domains)?;
@@ -55,7 +56,7 @@ impl CrawlCoordinator {
}
pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {
- let mut db = self.db.lock().unwrap();
+ let mut db = self.db.lock();
let start = Instant::now();
let domains = db.sample_domains(num_jobs)?;
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index c21d634c..618f141b 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -21,7 +21,7 @@ use std::path::PathBuf;
use std::{collections::BinaryHeap, path::Path};
use url::Url;
-use super::{Domain, DomainCrawled, Job, Result, UrlToInsert};
+use super::{Domain, DomainCrawled, Job, Result, UrlToInsert, MAX_URL_LEN_BYTES};
const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB
@@ -118,43 +118,6 @@ impl Default for DomainState {
}
}
}
-
-struct MetaDb {
- db: rocksdb::DB,
-}
-
-impl MetaDb {
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mut options = rocksdb::Options::default();
- options.create_if_missing(true);
-
- let db = rocksdb::DB::open(&options, path.as_ref())?;
-
- Ok(Self { db })
- }
-
- fn set_longest_url_len(&self, len: usize) -> Result<()> {
- let mut write_options = rocksdb::WriteOptions::default();
- write_options.disable_wal(true);
-
- self.db
- .put_opt(b"longest_url_len", len.to_be_bytes(), &write_options)?;
-
- Ok(())
- }
-
- fn get_longest_url_len(&self) -> Result<usize> {
- let len_bytes = self.db.get(b"longest_url_len")?;
-
- if let Some(len_bytes) = len_bytes {
- let len: usize = usize::from_be_bytes(len_bytes.try_into().unwrap());
- return Ok(len);
- }
-
- Ok(0)
- }
-}
-
struct CachedValue<T> {
value: T,
last_updated: std::time::Instant,
@@ -171,11 +134,9 @@ impl<T> From<T> for CachedValue<T> {
struct UrlStateDbShard {
db: rocksdb::DB,
- meta: MetaDb,
/// from rocksdb docs: "Cache must outlive DB instance which uses it."
_cache: rocksdb::Cache,
approx_size_bytes: CachedValue<u64>,
- longest_known_url: CachedValue<usize>,
}
impl UrlStateDbShard {
@@ -218,19 +179,13 @@ impl UrlStateDbShard {
let db = rocksdb::DB::open(&options, path.as_ref().join("urls"))?;
let approx_size_bytes = db
.property_int_value(rocksdb::properties::TOTAL_SST_FILES_SIZE)?
- .unwrap()
+ .unwrap_or_default()
.into();
- let meta = MetaDb::open(path.as_ref().join("meta"))?;
-
- let longest_known_url = meta.get_longest_url_len()?.into();
-
Ok(Self {
db,
approx_size_bytes,
_cache: cache,
- longest_known_url,
- meta,
})
}
@@ -242,14 +197,11 @@ impl UrlStateDbShard {
for (url, state) in urls {
let url_bytes = bincode::serialize(url)?;
-
- if url_bytes.len() > self.longest_known_url.value {
- self.longest_known_url = url_bytes.len().into();
- self.meta
- .set_longest_url_len(self.longest_known_url.value)?;
+ if url_bytes.len() > MAX_URL_LEN_BYTES {
+ continue;
}
- let key_bytes = [domain_bytes.as_slice(), &[b'/'], url_bytes.as_slice()].concat();
+ let key_bytes = [domain_bytes.as_slice(), url_bytes.as_slice()].concat();
let state_bytes = bincode::serialize(state)?;
@@ -268,12 +220,7 @@ impl UrlStateDbShard {
pub fn get_all_urls(&self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {
let domain_bytes = bincode::serialize(domain)?;
- let start = [
- domain_bytes.as_slice(),
- &[b'/'],
- &[0].repeat(self.longest_known_url.value),
- ]
- .concat();
+ let start = [domain_bytes.as_slice(), &[0].repeat(MAX_URL_LEN_BYTES)].concat();
let iter = self.db.iterator(rocksdb::IteratorMode::From(
&start,
@@ -283,7 +230,11 @@ impl UrlStateDbShard {
Ok(iter
.take_while(|r| {
if let Ok((key, _)) = r.as_ref() {
- key[..domain_bytes.len()] == domain_bytes[..]
+ if domain_bytes.len() >= key.len() {
+ return false;
+ }
+
+ key[..domain_bytes.len()] == domain_bytes
} else {
false
}
@@ -291,10 +242,13 @@ impl UrlStateDbShard {
.filter_map(|r| {
let (key, value) = r.ok()?;
- let url = bincode::deserialize(&key[domain_bytes.len() + 1..]) // +1 for '/'
- .ok()?;
+ if domain_bytes.len() >= key.len() {
+ return None;
+ }
+
+ let url = bincode::deserialize(&key[domain_bytes.len()..]).ok()?;
- let state = bincode::deserialize(&value[..]).ok()?;
+ let state = bincode::deserialize(&value).ok()?;
Some((url, state))
})
@@ -322,7 +276,7 @@ impl UrlStateDbShard {
let domain_bytes = bincode::serialize(domain)?;
let url_bytes = bincode::serialize(url)?;
- let key_bytes = [domain_bytes.as_slice(), &[b'/'], url_bytes.as_slice()].concat();
+ let key_bytes = [domain_bytes.as_slice(), url_bytes.as_slice()].concat();
keys.push(key_bytes);
}
@@ -334,7 +288,7 @@ impl UrlStateDbShard {
.zip(urls.iter())
{
if let Some(val) = val? {
- let state: UrlState = bincode::deserialize(&val[..])?;
+ let state: UrlState = bincode::deserialize(&val)?;
res.insert(url.clone(), state);
}
}
@@ -477,7 +431,7 @@ impl DomainStateDb {
.zip(domains.iter())
{
if let Some(val) = val? {
- let domain_state: DomainState = bincode::deserialize(&val[..])?;
+ let domain_state: DomainState = bincode::deserialize(&val)?;
res.insert(domain.clone(), domain_state);
}
}
@@ -508,8 +462,8 @@ impl DomainStateDb {
iter.filter_map(|r| {
let (key, value) = r.ok()?;
- let domain = bincode::deserialize(&key[..]).ok()?;
- let state = bincode::deserialize(&value[..]).ok()?;
+ let domain = bincode::deserialize(&key).ok()?;
+ let state = bincode::deserialize(&value).ok()?;
Some((domain, state))
})
@@ -575,6 +529,10 @@ impl CrawlDb {
let mut updated_url_states = Vec::new();
for url in urls {
+ if url.url.as_str().len() > MAX_URL_LEN_BYTES {
+ continue;
+ }
+
match url_states.get_mut(&UrlString::from(&url.url)) {
Some(state) => {
state.weight += url.weight;
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index e1dede69..6d705a5d 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -37,6 +37,10 @@ mod worker;
pub use coordinator::CrawlCoordinator;
+pub const MAX_URL_LEN_BYTES: usize = 8192;
+pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 200;
+pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 100;
+
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("invalid content type: {0}")]
diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rs
index 10707588..75005ee4 100644
--- a/core/src/crawler/router.rs
+++ b/core/src/crawler/router.rs
@@ -10,7 +10,7 @@ use crate::{
entrypoint::crawler::coordinator::{CoordinatorService, GetJobs, InsertUrls, MarkJobsComplete},
};
-use super::{Domain, Job, JobResponse};
+use super::{Domain, Job, JobResponse, MAX_URLS_FOR_DOMAIN_PER_INSERT, MAX_URL_LEN_BYTES};
struct RemoteCoordinator {
addr: SocketAddr,
@@ -136,6 +136,10 @@ impl Router {
let mut used_budget = 0.0;
for (domain, url) in urls {
+ if url.as_str().len() > MAX_URL_LEN_BYTES {
+ continue;
+ }
+
let different_domain = res.domain != domain;
let weight = if different_domain {
@@ -146,10 +150,13 @@ impl Router {
used_budget += weight;
- domain_urls
- .entry(domain)
- .or_default()
- .push(UrlToInsert { url, weight });
+ let urls = domain_urls.entry(domain).or_default();
+
+ if urls.len() >= MAX_URLS_FOR_DOMAIN_PER_INSERT {
+ continue;
+ }
+
+ urls.push(UrlToInsert { url, weight });
}
domain_budgets.push(DomainCrawled {
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 1bb2184a..da6f235a 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -32,7 +32,7 @@ use url::Url;
use crate::{
config::CrawlerConfig,
- crawler::{JobResponse, Response},
+ crawler::{JobResponse, Response, MAX_OUTGOING_URLS_PER_PAGE, MAX_URL_LEN_BYTES},
distributed::{retry_strategy::ExponentialBackoff, sonic},
entrypoint::crawler::router::{NewJobs, RouterService},
webpage::Html,
@@ -45,8 +45,6 @@ use super::{
const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
-const MAX_URL_LENGTH: usize = 8192;
-
pub struct WorkerThread {
current_job: Option<WorkerJob>,
pending_commands: Arc<Mutex<VecDeque<Command>>>,
@@ -226,7 +224,7 @@ impl WorkerThread {
discovered_urls.extend(
sitemap_urls
.into_iter()
- .filter(|url| url.as_str().len() < MAX_URL_LENGTH),
+ .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES),
);
}
}
@@ -261,7 +259,8 @@ impl WorkerThread {
discovered_urls.extend(
res.new_urls
.into_iter()
- .filter(|url| url.as_str().len() < MAX_URL_LENGTH),
+ .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES)
+ .take(MAX_OUTGOING_URLS_PER_PAGE),
);
url_responses.push(res.response);
}
@@ -271,7 +270,7 @@ impl WorkerThread {
url_responses,
discovered_urls: discovered_urls
.into_iter()
- .filter(|url| url.as_str().len() < MAX_URL_LENGTH)
+ .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES)
.collect(),
weight_budget: job.weight_budget,
};