web search done right
Fix crawler hang/deadlock. It turns out, that a worker had crashed after establishing a socket to the coordinator. This caused the coordinator to hang since it tried to send a response on the worker that had crashed which caused it to hang indefinitely. Killing the socket on the server got the remaining workers to continue the crawl. This should now be fixed by having a timeout in sonic on all the socket communications. I think the worker crashed due to oom, so we now prune the robotstxt map to remove stale robotstxt from memory and have a hard limit on the number of html bytes that the worker will fetch from the site its crawling.
Mikkel Denker442e87f04f27
diff --git a/Cargo.lock b/Cargo.lockindex 0edd293f..9c325c8a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4214,6 +4214,7 @@ dependencies = ["md5","memmap","memmap2",+ "mime","min-max-heap","num_cpus","once_cell",diff --git a/Cargo.toml b/Cargo.tomlindex 0ad10394..d4f7af65 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -102,3 +102,4 @@ rkyv = { version = "0.7.42", default_features = false, features = ["validation","size_64",] }+mime = "0.3.17"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex 85f57e66..c953a5be 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -99,6 +99,7 @@ utoipa-swagger-ui = { workspace = true }publicsuffix = { workspace = true }rkyv = { workspace = true }hashbrown = { workspace = true }+mime = { workspace = true }[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator = { workspace = true }diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 3045ac9d..e86b9396 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -14,8 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use dashmap::DashMap;use hashbrown::{HashMap, HashSet};use rand::Rng;+use rayon::prelude::*;use rkyv::Deserialize;use std::hash::Hash;use std::ops::Range;@@ -174,13 +176,17 @@ impl RedirectDb {struct RangesDb {db: rocksdb::DB, // domain -> range<vec<u8>>+ /// from rocksdb docs: "Cache must outlive DB instance which uses it."+ _cache: rocksdb::Cache,}impl RangesDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let mut options = rocksdb::Options::default();+ let cache = rocksdb::Cache::new_lru_cache(100 * 1024 * 1024)?; // 100MBoptions.create_if_missing(true);+ options.set_row_cache(&cache);let mut block_options = rocksdb::BlockBasedOptions::default();block_options.set_ribbon_filter(10.0);@@ -193,12 +199,12 @@ impl RangesDb {options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);options.set_max_subcompactions(8);- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);+ options.optimize_for_point_lookup(512);options.set_compression_type(rocksdb::DBCompressionType::None);let db = rocksdb::DB::open(&options, path.as_ref())?;- Ok(Self { db })+ Ok(Self { db, _cache: cache })}pub fn get(&self, domain: &Domain) -> Result<Option<Range<Vec<u8>>>> {@@ -247,6 +253,8 @@ struct UrlToInsert {struct UrlStateDbShard {db: rocksdb::DB,ranges: RangesDb,+ /// from rocksdb docs: "Cache must outlive DB instance which uses it."+ _cache: rocksdb::Cache,approx_size_bytes: CachedValue<u64>,}@@ -282,6 +290,7 @@ impl UrlStateDbShard {Ok(Self {db,approx_size_bytes,+ _cache: cache,ranges: RangesDb::open(path.as_ref().join("ranges"))?,})}@@ -303,38 +312,52 @@ impl UrlStateDbShard {}}- pub fn put(&mut self, domain: &Domain, url: &UrlString, value: &UrlState) -> Result<()> {- let url_bytes = bincode::serialize(url)?;+ pub fn put_batch(&mut self, domain: &Domain, urls: &[(UrlString, UrlState)]) -> Result<()> {+ let mut range = self.ranges.get(domain)?;+let domain_bytes = bincode::serialize(domain)?;- let key_bytes = [domain_bytes.as_slice(), &[b'/'], url_bytes.as_slice()].concat();+ let mut batch = rocksdb::WriteBatch::default();- // update ranges- let mut range = self.ranges.get(domain)?.unwrap_or_else(|| Range {- start: key_bytes.clone(),- end: key_bytes.clone(),- });+ for (url, state) in urls {+ let url_bytes = bincode::serialize(url)?;- if key_bytes < range.start {- range.start = key_bytes.clone();- }+ let key_bytes = [domain_bytes.as_slice(), &[b'/'], url_bytes.as_slice()].concat();- if key_bytes > range.end {- range.end = key_bytes.clone();- }+ // update ranges- self.ranges.put(domain, &range)?;+ if range.is_none() {+ range = Some(Range {+ start: key_bytes.clone(),+ end: key_bytes.clone(),+ });+ }++ let range = range.as_mut().unwrap();++ if key_bytes < range.start {+ range.start = key_bytes.clone();+ }++ if key_bytes > range.end {+ range.end = key_bytes.clone();+ }+ let state_bytes = bincode::serialize(state)?;- let value_bytes = bincode::serialize(value)?;+ batch.put(key_bytes, state_bytes);+ }let mut write_options = rocksdb::WriteOptions::default();write_options.disable_wal(true);- self.db.put_opt(key_bytes, value_bytes, &write_options)?;++ self.db.write_opt(batch, &write_options)?;++ self.ranges.put(domain, range.as_ref().unwrap())?;Ok(())}- pub fn get_all_urls(&mut self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {+ pub fn get_all_urls(&self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {match self.ranges.get(domain)? {Some(range) => {let domain_bytes = bincode::serialize(domain)?;@@ -388,24 +411,31 @@ struct UrlStateDb {impl UrlStateDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {if path.as_ref().exists() {- let mut shards = Vec::new();-+ let mut shard_names = Vec::new();for entry in std::fs::read_dir(path.as_ref())? {let entry = entry?;let path = entry.path();if path.is_dir() {- let shard = UrlStateDbShard::open(&path)?;- shards.push(shard);+ shard_names.push(path.to_str().unwrap().to_string());}}+ shard_names.sort();++ let mut shards = Vec::new();++ for shard_name in shard_names {+ shards.push(UrlStateDbShard::open(shard_name)?);+ }+Ok(Self {shards,path: path.as_ref().to_path_buf(),})} else {- let shard_id = uuid::Uuid::new_v4().to_string();+ let shard_id =+ chrono::Utc::now().to_rfc3339() + "_" + uuid::Uuid::new_v4().to_string().as_str();let shard_path = path.as_ref().join(shard_id);std::fs::create_dir_all(&shard_path)?;@@ -420,6 +450,8 @@ impl UrlStateDb {}pub fn get(&self, domain: &Domain, url: &UrlString) -> Result<Option<UrlState>> {+ // we iterate in reverse order so that we get the most recent state+ // since we insert new states at the last shard.for shard in self.shards.iter().rev() {if let Some(state) = shard.get(domain, url)? {return Ok(Some(state));@@ -429,11 +461,12 @@ impl UrlStateDb {Ok(None)}- pub fn put(&mut self, domain: &Domain, url: &UrlString, value: &UrlState) -> Result<()> {+ pub fn put_batch(&mut self, domain: &Domain, urls: &[(UrlString, UrlState)]) -> Result<()> {let last_shard = self.shards.last_mut().unwrap();if last_shard.approximate_size_bytes()? > MAX_URL_DB_SIZE_BYTES {- let shard_id = uuid::Uuid::new_v4().to_string();+ let shard_id =+ chrono::Utc::now().to_rfc3339() + "_" + uuid::Uuid::new_v4().to_string().as_str();let shard_path = self.path.as_path().join(shard_id);std::fs::create_dir_all(&shard_path)?;@@ -443,15 +476,15 @@ impl UrlStateDb {self.shards.push(shard);}- self.shards.last_mut().unwrap().put(domain, url, value)?;+ self.shards.last_mut().unwrap().put_batch(domain, urls)?;Ok(())}- pub fn get_all_urls(&mut self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {+ pub fn get_all_urls(&self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {let mut res = HashMap::new();- for shard in self.shards.iter_mut() {+ for shard in &self.shards {for (url, state) in shard.get_all_urls(domain)? {res.insert(url, state);}@@ -587,17 +620,16 @@ impl CrawlDb {}self.urls- .put(&domain, &UrlString::from(url), &UrlState::default())?;+ .put_batch(&domain, &[(UrlString::from(url), UrlState::default())])?;}Ok(())}pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<HashSet<Domain>> {- let mut domains: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();- let mut nonempty_domains = HashSet::new();+ let domains: DashMap<Domain, Vec<UrlToInsert>> = DashMap::new();- responses.iter().for_each(|res| {+ responses.par_iter().for_each(|res| {for url in &res.discovered_urls {let domain = Domain::from(url);let different_domain = res.domain != domain;@@ -615,6 +647,8 @@ impl CrawlDb {}});+ let mut nonempty_domains = HashSet::new();+for (domain, urls) in domains.into_iter() {let mut domain_state = match self.domain_state.get(&domain)? {Some(state) => state,@@ -634,6 +668,8 @@ impl CrawlDb {nonempty_domains.insert(domain.clone());}+ let mut url_states = Vec::new();+for url in urls {let mut url_state = match self.urls.get(&domain, &UrlString::from(&url.url))? {Some(state) => state,@@ -651,10 +687,11 @@ impl CrawlDb {domain_state.weight = url_state.weight;}- self.urls- .put(&domain, &UrlString::from(&url.url), &url_state)?;+ url_states.push((UrlString::from(&url.url), url_state));}+ self.urls.put_batch(&domain, &url_states)?;+self.domain_state.put(&domain, &domain_state)?;}@@ -713,13 +750,17 @@ impl CrawlDb {urls_per_job,);+ let mut new_url_states = Vec::new();+for url in &sampled {let mut state = self.urls.get(domain, url)?.unwrap();state.status = UrlStatus::Crawling;- self.urls.put(domain, url, &state)?;+ new_url_states.push(((*url).clone(), state));}+ self.urls.put_batch(domain, &new_url_states)?;+let mut domain_state = self.domain_state.get(domain)?.unwrap();domain_state.weight = urlsdiff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsindex 3c42dc7a..f76dd4eb 100644--- a/core/src/crawler/robots_txt.rs+++ b/core/src/crawler/robots_txt.rs@@ -31,6 +31,7 @@ enum Lookup<T> {pub struct RobotsTxtManager {cache: HashMap<Site, Lookup<RobotsTxt>>,+ last_prune: std::time::Instant,client: reqwest::Client,cache_expiration: Duration,}@@ -40,6 +41,7 @@ impl RobotsTxtManager {Self {client,cache_expiration,+ last_prune: std::time::Instant::now(),cache: HashMap::new(),}}@@ -84,7 +86,21 @@ impl RobotsTxtManager {}}+ fn maybe_prune(&mut self) {+ if self.last_prune.elapsed() < Duration::from_secs(60) {+ return;+ }++ self.cache.retain(|_, v| match v {+ Lookup::Found(robots_txt) => !robots_txt.is_expired(&self.cache_expiration),+ _ => true,+ });++ self.last_prune = std::time::Instant::now();+ }+async fn get_mut(&mut self, url: &Url) -> &mut Lookup<RobotsTxt> {+ self.maybe_prune();let site = Site(url.host_str().unwrap_or_default().to_string());let cache_should_update = match self.cache.get_mut(&site) {diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex eaa41111..58c1550f 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -1,3 +1,4 @@+use encoding_rs::{Encoding, UTF_8};// Stract is an open source web search engine.// Copyright (C) 2023 Stract ApS//@@ -14,7 +15,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use futures::{future::BoxFuture, FutureExt};+use mime::Mime;use quick_xml::events::Event;+use tokio_stream::StreamExt;use std::{collections::{HashMap, HashSet, VecDeque},@@ -39,6 +42,8 @@ use super::{WarcWriter, WorkerJob,};+const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB+pub struct WorkerThread {current_job: Option<WorkerJob>,pending_commands: Arc<Mutex<VecDeque<Command>>>,@@ -419,7 +424,7 @@ impl WorkerThread {// check if content length is too largeif let Some(content_length) = headers.get("content-length") {- if content_length.parse::<usize>().unwrap_or(0) > 10_000_000 {+ if content_length.parse::<usize>().unwrap_or(0) > MAX_CONTENT_LENGTH {return Err(Error::ContentTooLarge.into());}}@@ -455,7 +460,37 @@ impl WorkerThread {}let res_url = res.url().clone();- let body = res.text().await?;++ let content_type = res+ .headers()+ .get(reqwest::header::CONTENT_TYPE)+ .and_then(|value| value.to_str().ok())+ .and_then(|value| value.parse::<Mime>().ok());+ let encoding_name = content_type+ .as_ref()+ .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))+ .unwrap_or("utf-8");+ let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8);++ let mut bytes = Vec::new();++ let mut stream = res.bytes_stream();+ while let Some(b) = stream.next().await {+ if b.is_err() {+ return Err(Error::ContentTooLarge.into());+ }++ let b = b.unwrap();++ bytes.extend_from_slice(&b);++ if bytes.len() > MAX_CONTENT_LENGTH {+ return Err(Error::ContentTooLarge.into());+ }+ }++ let (text, _, _) = encoding.decode(&bytes);+ let body = text.to_string();Ok(CrawlDatum {url: res_url,diff --git a/core/src/distributed/sonic/mod.rs b/core/src/distributed/sonic/mod.rsindex 3e1b9017..7b7a72d3 100644--- a/core/src/distributed/sonic/mod.rs+++ b/core/src/distributed/sonic/mod.rs@@ -77,7 +77,7 @@ where}}- pub async fn send_without_timeout(mut self, request: &Req) -> Result<Res> {+ async fn send_without_timeout(mut self, request: &Req) -> Result<Res> {let bytes = bincode::serialize(&request).unwrap();// disable linger to avoid TIME_WAIT.@@ -141,10 +141,7 @@ where})}- pub async fn accept(&self) -> Result<Request<Req, Res>> {- let (mut stream, client) = self.listener.accept().await?;- tracing::debug!(?client, "accepted connection");-+ async fn parse_incoming_stream(&self, mut stream: TcpStream) -> Result<Request<Req, Res>> {let mut header_buf = vec![0; std::mem::size_of::<Header>()];stream.read_exact(&mut header_buf).await?;let header: Header = *bytemuck::from_bytes(&header_buf);@@ -161,6 +158,15 @@ wheremarker: PhantomData,})}++ pub async fn accept(&self) -> Result<Request<Req, Res>> {+ let (stream, client) = self.listener.accept().await?;+ tracing::debug!(?client, "accepted connection");++ tokio::time::timeout(Duration::from_secs(60), self.parse_incoming_stream(stream))+ .await+ .map_err(|_| Error::ConnectionTimeout)?+ }}pub struct Request<Req, Res> {@@ -173,7 +179,7 @@ impl<Req, Res> Request<Req, Res>whereRes: Serialize,{- pub async fn respond(mut self, response: Res) -> Result<()> {+ async fn respond_without_timeout(mut self, response: Res) -> Result<()> {let bytes = bincode::serialize(&response).unwrap();let header = Header {body_size: bytes.len(),@@ -192,6 +198,15 @@ whereOk(())}+ pub async fn respond(self, response: Res) -> Result<()> {+ tokio::time::timeout(+ Duration::from_secs(90),+ self.respond_without_timeout(response),+ )+ .await+ .map_err(|_| Error::RequestTimeout)?+ }+pub fn body(&self) -> &Req {self.body.as_ref().unwrap()}