web search done right
crawler robustness
Mikkel Denkere1ccaf9251c8
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rsindex 82db135c..ab7c2af0 100644--- a/crates/core/src/api/search.rs+++ b/crates/core/src/api/search.rs@@ -292,8 +292,8 @@ pub async fn entity_image(let bytes = result.as_raw_bytes();Ok((- ([(axum::http::header::CONTENT_TYPE, "image/png")]),- axum::response::AppendHeaders([(axum::http::header::CONTENT_TYPE, "image/png")]),+ ([(axum::http::header::CONTENT_TYPE, "image/webp")]),+ axum::response::AppendHeaders([(axum::http::header::CONTENT_TYPE, "image/webp")]),bytes,))}diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex fb1853e7..1aae5dfa 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -121,8 +121,8 @@ impl Crawler {5}- pub fn dry_run() -> bool {- false+ pub fn timeout_seconds() -> u64 {+ 60}}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex f92e80d4..f1ba4798 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -472,6 +472,7 @@ pub struct LiveIndexConfig {pub max_url_slowdown_retry: u8,#[serde(default = "defaults::Crawler::max_redirects")]pub max_redirects: usize,+ #[serde(default = "defaults::Crawler::timeout_seconds")]pub timeout_seconds: u64,// indexerdiff --git a/crates/core/src/crawler/coordinator.rs b/crates/core/src/crawler/coordinator.rsindex b2b21049..13234b15 100644--- a/crates/core/src/crawler/coordinator.rs+++ b/crates/core/src/crawler/coordinator.rs@@ -29,6 +29,10 @@ impl CrawlCoordinator {}pub fn sample_job(&self) -> Result<Option<Job>> {- self.jobs.lock().unwrap_or_else(|e| e.into_inner()).pop()+ self.jobs+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .pop()+ .map_err(Into::into)}}diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex 412a8d1b..ee31da67 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -14,10 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration};+use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc, time::Duration};type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;+use anyhow::anyhow;use url::Url;use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};@@ -49,8 +50,11 @@ pub enum Error {#[error("invalid content type: {0}")]InvalidContentType(String),- #[error("fetch failed: {0}")]- FetchFailed(reqwest::StatusCode),+ #[error("fetch failed: {status_code}")]+ FetchFailed {+ status_code: reqwest::StatusCode,+ headers: reqwest::header::HeaderMap,+ },#[error("content too large")]ContentTooLarge,@@ -60,9 +64,15 @@ pub enum Error {#[error("invalid redirect")]InvalidRedirect,++ #[error("couldn't parse html")]+ InvalidHtml,++ #[error("an error occurred: {0}")]+ Anyhow(#[from] anyhow::Error),}-type Result<T, E = anyhow::Error> = std::result::Result<T, E>;+type Result<T, E = Error> = std::result::Result<T, E>;#[derive(Clone,@@ -148,25 +158,6 @@ pub struct Job {pub wandering_urls: u64,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub enum UrlResponse {- Success {- #[bincode(with_serde)]- url: Url,- },- Failed {- #[bincode(with_serde)]- url: Url,- status_code: Option<u16>,- },- Redirected {- #[bincode(with_serde)]- url: Url,- #[bincode(with_serde)]- new_url: Url,- },-}-#[derive(Debug,Clone,@@ -257,7 +248,6 @@ impl From<Job> for WorkerJob {#[derive(Debug, Clone)]pub struct CrawlDatum {pub url: Url,- pub status_code: u16,pub payload_type: warc::PayloadType,pub body: String,pub fetch_time_ms: u64,@@ -275,7 +265,10 @@ impl Crawler {let mut router_hosts = Vec::new();for host in &config.router_hosts {- router_hosts.push(host.parse()?);+ router_hosts.push(+ host.parse::<SocketAddr>()+ .map_err(|e| Error::from(anyhow!(e)))?,+ );}for _ in 0..config.num_worker_threads {@@ -317,12 +310,13 @@ pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),);- Ok(reqwest::Client::builder()+ reqwest::Client::builder().timeout(timeout).connect_timeout(timeout).http2_keep_alive_interval(None).default_headers(headers).redirect(reqwest::redirect::Policy::limited(config.max_redirects)).user_agent(&config.user_agent.full)- .build()?)+ .build()+ .map_err(|e| Error::from(anyhow!(e)))}diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex 3862b8e9..40730b4c 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -18,7 +18,9 @@ use std::{collections::BTreeMap, panic, time::Duration};use url::Url;-use super::{Error, Result, Site};+use crate::{config::CrawlerConfig, crawler};++use super::{Result, Site};enum Lookup<T> {Found(T),@@ -34,16 +36,26 @@ pub struct RobotsTxtManager {client: reqwest::Client,cache_expiration: Duration,user_agent: String,+ min_crawl_delay: Duration,+ max_crawl_delay: Duration,}impl RobotsTxtManager {- pub fn new(client: reqwest::Client, cache_expiration: Duration, user_agent: &str) -> Self {+ pub fn new(config: &CrawlerConfig) -> Self {+ let client = crawler::reqwest_client(config).unwrap();+ let cache_expiration = Duration::from_secs(config.robots_txt_cache_sec);+ let user_agent = config.user_agent.token.clone();+ let min_crawl_delay = Duration::from_millis(config.min_crawl_delay_ms);+ let max_crawl_delay = Duration::from_millis(config.max_crawl_delay_ms);+Self {client,cache_expiration,last_prune: std::time::Instant::now(),cache: BTreeMap::new(),user_agent: user_agent.to_string(),+ min_crawl_delay,+ max_crawl_delay,}}@@ -55,59 +67,89 @@ impl RobotsTxtManager {}}- async fn fetch_robots_txt_from_url(&self, url: &str) -> Result<RobotsTxt> {- let res = self+ pub async fn crawl_delay(&mut self, url: &Url) -> Option<Duration> {+ match self.get_mut(url).await {+ Lookup::Found(robots_txt) => robots_txt.robots.crawl_delay(),+ Lookup::Unavailable | Lookup::Unreachable => None,+ }+ }++ async fn fetch_robots_txt_from_url(&self, url: &str) -> Lookup<RobotsTxt> {+ let res = match self.client.get(url).timeout(Duration::from_secs(60)).send()- .await;-- let res = res?;+ .await+ {+ Ok(res) => {+ if res.status() != reqwest::StatusCode::OK {+ match res.status() {+ reqwest::StatusCode::NOT_FOUND => return Lookup::Unavailable,+ _ => return Lookup::Unreachable,+ }+ }- if res.status() != reqwest::StatusCode::OK {- return Err(Error::FetchFailed(res.status()).into());- }+ let body = match res.text().await {+ Ok(body) => body,+ Err(_) => return Lookup::Unreachable,+ };- if !res- .headers()- .get(reqwest::header::CONTENT_TYPE)- .map(|h| h.to_str().unwrap_or_default().starts_with("text/plain"))- .unwrap_or(false)- {- return Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into());- }+ let self_user_agent = self.user_agent.clone();+ match panic::catch_unwind(|| RobotsTxt::new(&self_user_agent, body)) {+ Ok(Ok(r)) => Lookup::Found(r),+ _ => Lookup::Unreachable,+ }+ }+ Err(_) => Lookup::Unreachable,+ };- let body = res.text().await?;+ tokio::time::sleep(self.min_crawl_delay).await;- let self_user_agent = self.user_agent.clone();- match panic::catch_unwind(|| RobotsTxt::new(&self_user_agent, body)) {- Ok(Ok(r)) => Ok(r),- _ => Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into()),- }+ res}- async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {- if let Ok(robots_txt) = self+ async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<RobotsTxt> {+ match self.fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0)).await{- return Ok(robots_txt);+ Lookup::Unavailable => {+ match self+ .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))+ .await+ {+ Lookup::Found(robots_txt) => Lookup::Found(robots_txt),+ Lookup::Unreachable => Lookup::Unreachable,+ Lookup::Unavailable+ if !site.0.starts_with("www.")+ && site.0.chars().filter(|&c| c == '.').count() == 1 =>+ {+ self.fetch_robots_txt_from_url(&format!(+ "https://www.{}/robots.txt",+ &site.0+ ))+ .await+ }+ Lookup::Unavailable => Lookup::Unavailable,+ }+ }+ res => res,}+ }- match self- .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))- .await- {- Ok(robots_txt) => Ok(robots_txt),- _ if !site.0.starts_with("www.")- && site.0.chars().filter(|&c| c == '.').count() == 1 =>- {- self.fetch_robots_txt_from_url(&format!("https://www.{}/robots.txt", &site.0))- .await+ async fn fetch_robots_txt(&self, site: &Site) -> Lookup<RobotsTxt> {+ for _ in 0..3 {+ match self.fetch_robots_txt_without_retry(site).await {+ Lookup::Found(robots_txt) => return Lookup::Found(robots_txt),+ Lookup::Unavailable => return Lookup::Unavailable,+ Lookup::Unreachable => {}}- Err(err) => Err(err),++ tokio::time::sleep(self.max_crawl_delay).await;}++ Lookup::Unreachable}fn maybe_prune(&mut self) {@@ -134,22 +176,8 @@ impl RobotsTxtManager {};if cache_should_update {- match self.fetch_robots_txt(&site).await {- Ok(robots_txt) => {- self.cache.insert(site.clone(), Lookup::Found(robots_txt));- }- Err(err) => match err.downcast_ref() {- Some(Error::FetchFailed(status))- if *status == reqwest::StatusCode::NOT_FOUND =>- {- self.cache.insert(site.clone(), Lookup::Unavailable);- }- _ => {- self.cache.insert(site.clone(), Lookup::Unreachable);- tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);- }- },- }+ self.cache+ .insert(site.clone(), self.fetch_robots_txt(&site).await);}self.cache.get_mut(&site).unwrap()diff --git a/crates/core/src/crawler/warc_writer.rs b/crates/core/src/crawler/warc_writer.rsindex a16a2d7b..7b0a7733 100644--- a/crates/core/src/crawler/warc_writer.rs+++ b/crates/core/src/crawler/warc_writer.rs@@ -21,7 +21,8 @@ use crate::{warc,};-use super::{CrawlDatum, DatumStream, Result};+use super::{CrawlDatum, DatumStream, Error, Result};+use anyhow::anyhow;/// The WarcWriter is responsible for storing the crawl datums/// as WARC files on S3.@@ -31,13 +32,19 @@ pub struct WarcWriter {impl DatumStream for WarcWriter {async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {- self.tx.send(WarcWriterMessage::Crawl(crawl_datum)).await?;+ self.tx+ .send(WarcWriterMessage::Crawl(crawl_datum))+ .await+ .map_err(|e| Error::from(anyhow!(e)))?;Ok(())}async fn finish(&self) -> Result<()> {- self.tx.send(WarcWriterMessage::Finish).await?;+ self.tx+ .send(WarcWriterMessage::Finish)+ .await+ .map_err(|e| Error::from(anyhow!(e)))?;self.tx.closed().await;Ok(())diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex ce2b1daf..31a61e4b 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -13,7 +13,7 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use anyhow::{anyhow, bail};+use anyhow::anyhow;use encoding_rs::{Encoding, UTF_8};use hashbrown::HashSet;use mime::Mime;@@ -41,8 +41,7 @@ use crate::{use super::{reqwest_client, robots_txt::RobotsTxtManager, wander_prirotiser::WanderPrioritiser, CrawlDatum,- DatumStream, Domain, Error, Result, RetrieableUrl, Site, UrlResponse, WarcWriter, WeightedUrl,- WorkerJob,+ DatumStream, Domain, Error, Result, RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob,};const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB@@ -53,9 +52,13 @@ const IGNORED_EXTENSIONS: [&str; 27] = [".flac", ".aac", ".ogg", ".m4a", ".m4v",];+enum UrlVisit {+ Skip,+ CanCrawl,+}+struct ProcessedUrl {new_urls: Vec<Url>,- response: UrlResponse,}pub struct WorkerThread {@@ -86,12 +89,13 @@ impl WorkerThread {let router = *self.router_hosts.choose(&mut rand::thread_rng()).unwrap();- Ok(sonic::service::Connection::create_with_timeout_retry(+ sonic::service::Connection::create_with_timeout_retry(router,Duration::from_secs(90),retry,)- .await?)+ .await+ .map_err(|e| Error::from(anyhow!(e)))}pub async fn run(self) {@@ -131,7 +135,10 @@ pub struct JobExecutor<S: DatumStream> {crawled_urls: HashSet<Url>,crawled_sitemaps: HashSet<Site>,sitemap_urls: HashSet<Url>,- config: Arc<CrawlerConfig>,+ min_crawl_delay: Duration,+ max_crawl_delay: Duration,+ max_url_slowdown_retry: u8,+ max_politeness_factor: f32,wander_prioritiser: WanderPrioritiser,wandered_urls: u64,job: WorkerJob,@@ -147,17 +154,16 @@ impl<S: DatumStream> JobExecutor<S> {Self {writer,politeness_factor: config.politeness_factor,- robotstxt: RobotsTxtManager::new(- client.clone(),- Duration::from_secs(config.robots_txt_cache_sec),- &config.user_agent.token,- ),+ robotstxt: RobotsTxtManager::new(&config),client,crawled_urls: HashSet::new(),crawled_sitemaps: HashSet::new(),wandered_urls: 0,sitemap_urls: HashSet::new(),- config,+ min_crawl_delay: Duration::from_millis(config.min_crawl_delay_ms),+ max_crawl_delay: Duration::from_millis(config.max_crawl_delay_ms),+ max_url_slowdown_retry: config.max_url_slowdown_retry,+ max_politeness_factor: config.max_politeness_factor,wander_prioritiser: WanderPrioritiser::new(),job,}@@ -165,7 +171,6 @@ impl<S: DatumStream> JobExecutor<S> {pub async fn run(mut self) {tracing::info!("Processing job: {:?}", self.job.domain);-self.scheduled_urls().await;while self.wandered_urls < self.job.wandering_urls@@ -218,28 +223,48 @@ impl<S: DatumStream> JobExecutor<S> {self.process_urls(urls, false).await;}- async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {- while let Some(retryable_url) = urls.pop_front() {- if Domain::from(retryable_url.url()) != self.job.domain {- continue;- }+ async fn verify_url(&mut self, retryable_url: &RetrieableUrl) -> UrlVisit {+ if Domain::from(retryable_url.url()) != self.job.domain {+ return UrlVisit::Skip;+ }- if self.crawled_urls.contains(retryable_url.url()) {- continue;- }+ if self.crawled_urls.contains(retryable_url.url()) {+ return UrlVisit::Skip;+ }- if retryable_url.retries > self.config.max_url_slowdown_retry {- continue;+ if retryable_url.retries > self.max_url_slowdown_retry {+ return UrlVisit::Skip;+ }++ if retryable_url.url().host_str().is_none()+ || !matches!(retryable_url.url().scheme(), "http" | "https")+ {+ return UrlVisit::Skip;+ }++ if !self.robotstxt.is_allowed(retryable_url.url()).await {+ return UrlVisit::Skip;+ }++ if let Some(port) = retryable_url.url().port() {+ if port != 80 && port != 443 {+ return UrlVisit::Skip;}+ }++ UrlVisit::CanCrawl+ }- if retryable_url.url().host_str().is_none()- || !matches!(retryable_url.url().scheme(), "http" | "https")- {+ async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {+ while let Some(retryable_url) = urls.pop_front() {+ if let UrlVisit::Skip = self.verify_url(&retryable_url).await {continue;}- if !self.robotstxt.is_allowed(retryable_url.url()).await {- continue;+ if let Some(delay) = self.robotstxt.crawl_delay(retryable_url.url()).await {+ if delay > self.min_crawl_delay {+ self.min_crawl_delay = delay.min(self.max_crawl_delay);+ }}let site = Site(@@ -262,8 +287,8 @@ impl<S: DatumStream> JobExecutor<S> {let res = self.process_url(retryable_url.url().clone()).await;- match res.response {- UrlResponse::Success { url: _ } => {+ match res {+ Ok(res) => {let weight = retryable_url.weighted_url.weight;for new_url in res.new_urls {@@ -278,18 +303,32 @@ impl<S: DatumStream> JobExecutor<S> {self.wander_prioritiser.inc(new_url, weight);}}- UrlResponse::Failed {- url: _,+ Err(Error::FetchFailed {status_code,- } => {- if matches!(status_code, Some(429)) {- let mut retryable_url = retryable_url;- retryable_url.retries += 1;- urls.push_back(retryable_url);- continue;+ headers,+ }) if status_code == 429 => {+ if headers.contains_key("retry-after") {+ let retry_after = headers["retry-after"]+ .to_str()+ .ok()+ .and_then(|s| s.parse::<u64>().ok())+ .map(Duration::from_secs);++ if let Some(retry_after) = retry_after {+ if retry_after > self.max_crawl_delay {+ return; // don't crawl anymore from this site+ }++ tokio::time::sleep(retry_after).await;+ }}++ self.increase_politeness();+ let mut retryable_url = retryable_url;+ retryable_url.retries += 1;+ urls.push_back(retryable_url);}- UrlResponse::Redirected { url: _, new_url: _ } => {}+ Err(_) => {}}}}@@ -297,8 +336,8 @@ impl<S: DatumStream> JobExecutor<S> {fn increase_politeness(&mut self) {self.politeness_factor *= 2.0;- if self.politeness_factor > self.config.max_politeness_factor {- self.politeness_factor = self.config.max_politeness_factor;+ if self.politeness_factor > self.max_politeness_factor {+ self.politeness_factor = self.max_politeness_factor;}tracing::warn!("politeness factor increased to {}", self.politeness_factor);@@ -322,82 +361,25 @@ impl<S: DatumStream> JobExecutor<S> {.collect()}- async fn process_url(&mut self, url: Url) -> ProcessedUrl {- let fetch = self.crawl_url(url.clone()).await;-- match fetch {- Ok(datum) => match datum.status_code {- 200 => {- self.save_datum(datum.clone()).await;-- if datum.url != url {- // todo: we might want to save the redirected url- }-- match Html::parse(&datum.body, datum.url.as_str()) {- Ok(html) => {- let new_urls = self.new_urls(&html);- let url_res = UrlResponse::Success { url: datum.url };- ProcessedUrl {- new_urls,- response: url_res,- }- }- Err(_) => ProcessedUrl {- new_urls: Vec::new(),- response: UrlResponse::Failed {- url,- status_code: None,- },- },- }- }- 301 | 302 => {- let url_res = UrlResponse::Redirected {- url,- new_url: datum.url,- };-- ProcessedUrl {- new_urls: Vec::new(),- response: url_res,- }- }-- _ => {- if datum.status_code == 429 {- self.increase_politeness();- }-- tracing::debug!("failed to fetch url ({}): {}", &url, datum.status_code);- ProcessedUrl {- new_urls: Vec::new(),- response: UrlResponse::Failed {- url,- status_code: Some(datum.status_code),- },- }- }- },- Err(err) => {- tracing::debug!("failed to fetch url ({}): {}", &url, err);-- ProcessedUrl {- new_urls: Vec::new(),- response: UrlResponse::Failed {- url,- status_code: None,- },- }+ async fn process_url(&mut self, url: Url) -> Result<ProcessedUrl> {+ let datum = self.crawl_url(url.clone()).await?;+ self.save_datum(datum.clone()).await;++ match Html::parse(&datum.body, datum.url.as_str()) {+ Ok(html) => {+ let root_domain = datum.url.root_domain();+ let new_urls = self+ .new_urls(&html)+ .into_iter()+ .filter(|new_url| new_url.root_domain() == root_domain)+ .collect();+ Ok(ProcessedUrl { new_urls })}+ Err(_) => Err(Error::InvalidHtml),}}async fn save_datum(&self, datum: CrawlDatum) {- if datum.status_code != 200 {- return;- }-self.writer.write(datum).await.ok();}@@ -406,7 +388,7 @@ impl<S: DatumStream> JobExecutor<S> {.get(url.to_string()).send().await- .map_err(|e| e.into())+ .map_err(|e| Error::from(anyhow!(e)))}async fn fetch_with_https_priority(&self, url: Url) -> Result<reqwest::Response> {@@ -419,7 +401,7 @@ impl<S: DatumStream> JobExecutor<S> {match self.fetch(https).await {Ok(res) => Ok(res),Err(_) => {- tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms)).await;+ tokio::time::sleep(self.delay_duration()).await;self.fetch(url.clone()).await}}@@ -428,17 +410,28 @@ impl<S: DatumStream> JobExecutor<S> {}}+ fn delay_duration(&self) -> Duration {+ let mut delay = self.min_crawl_delay;+ delay = delay.mul_f32(self.politeness_factor);++ if delay > self.max_crawl_delay {+ delay = self.max_crawl_delay;+ }++ delay+ }+async fn politeness_delay(&self, fetch_time: Duration) {let mut delay = fetch_time;- if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {- delay = Duration::from_millis(self.config.min_crawl_delay_ms);+ if delay < self.min_crawl_delay {+ delay = self.min_crawl_delay;}delay = delay.mul_f32(self.politeness_factor);- if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {- delay = Duration::from_millis(self.config.max_crawl_delay_ms);+ if delay > self.max_crawl_delay {+ delay = self.max_crawl_delay;}tokio::time::sleep(delay).await;@@ -453,7 +446,7 @@ impl<S: DatumStream> JobExecutor<S> {.and_then(|value| value.parse::<usize>().ok()){if content_length > MAX_CONTENT_LENGTH {- return Err(Error::ContentTooLarge.into());+ return Err(Error::ContentTooLarge);}}@@ -466,7 +459,7 @@ impl<S: DatumStream> JobExecutor<S> {Some(ct) if ct.contains("text/html") => Ok(warc::PayloadType::Html),Some(ct) if ct.contains("application/rss") => Ok(warc::PayloadType::Rss),Some(ct) if ct.contains("application/atom") => Ok(warc::PayloadType::Atom),- ct => Err(Error::InvalidContentType(format!("{ct:?}")).into()),+ ct => Err(Error::InvalidContentType(format!("{ct:?}"))),}}@@ -493,7 +486,6 @@ impl<S: DatumStream> JobExecutor<S> {Ok(Some(CrawlDatum {url,- status_code,payload_type,body: String::new(),fetch_time_ms: fetch_time.as_millis() as u64,@@ -520,7 +512,7 @@ impl<S: DatumStream> JobExecutor<S> {let mut stream = res.bytes_stream();while let Some(b) = stream.next().await {if b.is_err() {- return Err(Error::ContentTooLarge.into());+ return Err(Error::ContentTooLarge);}let b = b.unwrap();@@ -528,7 +520,7 @@ impl<S: DatumStream> JobExecutor<S> {bytes.extend_from_slice(&b);if bytes.len() > MAX_CONTENT_LENGTH {- return Err(Error::ContentTooLarge.into());+ return Err(Error::ContentTooLarge);}}@@ -541,7 +533,7 @@ impl<S: DatumStream> JobExecutor<S> {url.normalize();if self.crawled_urls.contains(&url) {- bail!("url already crawled: {}", url);+ return Err(Error::from(anyhow!("url already crawled: {}", url)));}let start = Instant::now();@@ -559,7 +551,14 @@ impl<S: DatumStream> JobExecutor<S> {return Ok(datum);}- let status_code = res.status().as_u16();+ let status_code = res.status();++ if status_code != reqwest::StatusCode::OK {+ return Err(Error::FetchFailed {+ status_code,+ headers: res.headers().clone(),+ });+ }let mut res_url = res.url().clone();res_url.normalize();@@ -570,7 +569,6 @@ impl<S: DatumStream> JobExecutor<S> {Ok(CrawlDatum {url: res_url,- status_code,body,payload_type,fetch_time_ms: fetch_time.as_millis() as u64,@@ -587,7 +585,7 @@ impl<S: DatumStream> JobExecutor<S> {}let res = self.fetch(url).await;- tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms)).await;+ tokio::time::sleep(self.delay_duration()).await;if res.is_err() {continue;diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rsindex 7365c1b5..9e790d40 100644--- a/crates/core/src/entity_index/mod.rs+++ b/crates/core/src/entity_index/mod.rs@@ -430,9 +430,8 @@ impl EntityIndex {self.image_store.insert(name, image);}- pub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {+ pub fn merge_all_segments(&mut self) -> Result<()> {self.image_store.merge_all_segments();- self.prepare_writer();let base_path = Path::new(&self.path);let segments: Vec<_> = self.tv_index.load_metas()?.segments.into_iter().collect();@@ -440,7 +439,7 @@ impl EntityIndex {self.writer.as_mut().expect("writer has not been prepared"),segments,base_path,- max_num_segments,+ 1,)?;Ok(())diff --git a/crates/core/src/entrypoint/entity.rs b/crates/core/src/entrypoint/entity.rsindex 48a7105a..05b7f454 100644--- a/crates/core/src/entrypoint/entity.rs+++ b/crates/core/src/entrypoint/entity.rs@@ -196,12 +196,18 @@ impl EntityIndexer {let mut index = EntityIndex::open(output_path)?;index.prepare_writer();+ let mut image_bloom = bloom::BytesBloomFilter::new(1_000_000_000, 0.05);+let mut inserts = 0;for entity in EntityIterator::new(&zim)?.filter(|e| !e.is_disambiguation).filter(|e| !e.article_url.starts_with("Portal:")){+ if let Some(image) = entity.image.as_ref() {+ image_bloom.insert(image);+ }+index.insert(entity);inserts += 1;@@ -216,6 +222,10 @@ impl EntityIndexer {for image in zim.images()? {if let Ok(decoded_image) = Image::from_bytes(image.bytes()) {+ if !image_bloom.contains(&image.url) {+ continue;+ }+index.insert_image(image.url, decoded_image);inserts += 1;@@ -228,7 +238,7 @@ impl EntityIndexer {}index.commit();- index.merge_into_max_segments(1)?;+ index.merge_all_segments()?;Ok(())}diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rsindex 9ef48f52..b64861b5 100644--- a/crates/core/src/image_store.rs+++ b/crates/core/src/image_store.rs@@ -150,13 +150,7 @@ impl EntityImageStore {self.store.prepare_writer();}pub fn open<P: AsRef<Path>>(path: P) -> Self {- let store = BaseImageStore::open_with_filters(- path,- vec![Box::new(MaxSizeFilter {- width: 400,- height: 400,- })],- );+ let store = BaseImageStore::open_with_filters(path, vec![]);Self { store }}@@ -206,8 +200,10 @@ impl Image {}pub(crate) fn as_raw_bytes(&self) -> Vec<u8> {+ let img = self.0.clone().into_rgba8();+let mut wrt = BufWriter::new(Cursor::new(Vec::new()));- self.0.write_to(&mut wrt, image::ImageFormat::Png).unwrap();+ img.write_to(&mut wrt, image::ImageFormat::WebP).unwrap();wrt.flush().unwrap();let mut cursor = wrt.into_inner().unwrap();@@ -239,7 +235,12 @@ mod tests {#[test]fn serialize_deserialize_image() {let image = Image(- ImageBuffer::from_pixel(2, 2, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX])).into(),+ ImageBuffer::from_pixel(+ 2,+ 2,+ image::Rgba::<u8>([u8::MAX, u8::MAX, u8::MAX, u8::MAX]),+ )+ .into(),);let bytes = bincode::encode_to_vec(&image, bincode::config::standard()).unwrap();@@ -252,7 +253,12 @@ mod tests {#[test]fn store_and_load_image() {let image = Image(- ImageBuffer::from_pixel(2, 2, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX])).into(),+ ImageBuffer::from_pixel(+ 2,+ 2,+ image::Rgba::<u8>([u8::MAX, u8::MAX, u8::MAX, u8::MAX]),+ )+ .into(),);let key = "test".to_string();let mut store = BaseImageStore::open(crate::gen_temp_path());@@ -267,8 +273,12 @@ mod tests {#[test]fn resize_filter() {let image = Image(- ImageBuffer::from_pixel(32, 32, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]))- .into(),+ ImageBuffer::from_pixel(+ 32,+ 32,+ image::Rgba::<u8>([u8::MAX, u8::MAX, u8::MAX, u8::MAX]),+ )+ .into(),);assert_eq!(image.0.width(), 32);assert_eq!(image.0.height(), 32);diff --git a/crates/core/src/live_index/indexer.rs b/crates/core/src/live_index/indexer.rsindex 6ba6fc15..77b00e06 100644--- a/crates/core/src/live_index/indexer.rs+++ b/crates/core/src/live_index/indexer.rs@@ -18,7 +18,7 @@ use std::sync::{Arc, Mutex, RwLock};use crate::{config::LiveIndexConfig,- crawler::{CrawlDatum, DatumStream},+ crawler::{self, CrawlDatum, DatumStream},entrypoint::indexer::IndexingWorker,Result,};@@ -72,7 +72,7 @@ impl Indexer {}impl DatumStream for Indexer {- async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<(), crawler::Error> {self.write_batch.lock().unwrap_or_else(|e| e.into_inner())@@ -83,7 +83,7 @@ impl DatumStream for Indexer {Ok(())}- async fn finish(&self) -> Result<()> {+ async fn finish(&self) -> Result<(), crawler::Error> {self.write_batch_to_index();self.search_index