Skip to content

web search done right

crawler robustness

Mikkel Denkere1ccaf9251c8
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rs
index 82db135c..ab7c2af0 100644
--- a/crates/core/src/api/search.rs
+++ b/crates/core/src/api/search.rs
@@ -292,8 +292,8 @@ pub async fn entity_image(
let bytes = result.as_raw_bytes();
Ok((
- ([(axum::http::header::CONTENT_TYPE, "image/png")]),
- axum::response::AppendHeaders([(axum::http::header::CONTENT_TYPE, "image/png")]),
+ ([(axum::http::header::CONTENT_TYPE, "image/webp")]),
+ axum::response::AppendHeaders([(axum::http::header::CONTENT_TYPE, "image/webp")]),
bytes,
))
}
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index fb1853e7..1aae5dfa 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -121,8 +121,8 @@ impl Crawler {
5
}
- pub fn dry_run() -> bool {
- false
+ pub fn timeout_seconds() -> u64 {
+ 60
}
}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index f92e80d4..f1ba4798 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -472,6 +472,7 @@ pub struct LiveIndexConfig {
pub max_url_slowdown_retry: u8,
#[serde(default = "defaults::Crawler::max_redirects")]
pub max_redirects: usize,
+ #[serde(default = "defaults::Crawler::timeout_seconds")]
pub timeout_seconds: u64,
// indexer
diff --git a/crates/core/src/crawler/coordinator.rs b/crates/core/src/crawler/coordinator.rs
index b2b21049..13234b15 100644
--- a/crates/core/src/crawler/coordinator.rs
+++ b/crates/core/src/crawler/coordinator.rs
@@ -29,6 +29,10 @@ impl CrawlCoordinator {
}
pub fn sample_job(&self) -> Result<Option<Job>> {
- self.jobs.lock().unwrap_or_else(|e| e.into_inner()).pop()
+ self.jobs
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .pop()
+ .map_err(Into::into)
}
}
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rs
index 412a8d1b..ee31da67 100644
--- a/crates/core/src/crawler/mod.rs
+++ b/crates/core/src/crawler/mod.rs
@@ -14,10 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration};
+use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc, time::Duration};
type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;
+use anyhow::anyhow;
use url::Url;
use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};
@@ -49,8 +50,11 @@ pub enum Error {
#[error("invalid content type: {0}")]
InvalidContentType(String),
- #[error("fetch failed: {0}")]
- FetchFailed(reqwest::StatusCode),
+ #[error("fetch failed: {status_code}")]
+ FetchFailed {
+ status_code: reqwest::StatusCode,
+ headers: reqwest::header::HeaderMap,
+ },
#[error("content too large")]
ContentTooLarge,
@@ -60,9 +64,15 @@ pub enum Error {
#[error("invalid redirect")]
InvalidRedirect,
+
+ #[error("couldn't parse html")]
+ InvalidHtml,
+
+ #[error("an error occurred: {0}")]
+ Anyhow(#[from] anyhow::Error),
}
-type Result<T, E = anyhow::Error> = std::result::Result<T, E>;
+type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(
Clone,
@@ -148,25 +158,6 @@ pub struct Job {
pub wandering_urls: u64,
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub enum UrlResponse {
- Success {
- #[bincode(with_serde)]
- url: Url,
- },
- Failed {
- #[bincode(with_serde)]
- url: Url,
- status_code: Option<u16>,
- },
- Redirected {
- #[bincode(with_serde)]
- url: Url,
- #[bincode(with_serde)]
- new_url: Url,
- },
-}
-
#[derive(
Debug,
Clone,
@@ -257,7 +248,6 @@ impl From<Job> for WorkerJob {
#[derive(Debug, Clone)]
pub struct CrawlDatum {
pub url: Url,
- pub status_code: u16,
pub payload_type: warc::PayloadType,
pub body: String,
pub fetch_time_ms: u64,
@@ -275,7 +265,10 @@ impl Crawler {
let mut router_hosts = Vec::new();
for host in &config.router_hosts {
- router_hosts.push(host.parse()?);
+ router_hosts.push(
+ host.parse::<SocketAddr>()
+ .map_err(|e| Error::from(anyhow!(e)))?,
+ );
}
for _ in 0..config.num_worker_threads {
@@ -317,12 +310,13 @@ pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {
reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),
);
- Ok(reqwest::Client::builder()
+ reqwest::Client::builder()
.timeout(timeout)
.connect_timeout(timeout)
.http2_keep_alive_interval(None)
.default_headers(headers)
.redirect(reqwest::redirect::Policy::limited(config.max_redirects))
.user_agent(&config.user_agent.full)
- .build()?)
+ .build()
+ .map_err(|e| Error::from(anyhow!(e)))
}
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rs
index 3862b8e9..40730b4c 100644
--- a/crates/core/src/crawler/robots_txt.rs
+++ b/crates/core/src/crawler/robots_txt.rs
@@ -18,7 +18,9 @@ use std::{collections::BTreeMap, panic, time::Duration};
use url::Url;
-use super::{Error, Result, Site};
+use crate::{config::CrawlerConfig, crawler};
+
+use super::{Result, Site};
enum Lookup<T> {
Found(T),
@@ -34,16 +36,26 @@ pub struct RobotsTxtManager {
client: reqwest::Client,
cache_expiration: Duration,
user_agent: String,
+ min_crawl_delay: Duration,
+ max_crawl_delay: Duration,
}
impl RobotsTxtManager {
- pub fn new(client: reqwest::Client, cache_expiration: Duration, user_agent: &str) -> Self {
+ pub fn new(config: &CrawlerConfig) -> Self {
+ let client = crawler::reqwest_client(config).unwrap();
+ let cache_expiration = Duration::from_secs(config.robots_txt_cache_sec);
+ let user_agent = config.user_agent.token.clone();
+ let min_crawl_delay = Duration::from_millis(config.min_crawl_delay_ms);
+ let max_crawl_delay = Duration::from_millis(config.max_crawl_delay_ms);
+
Self {
client,
cache_expiration,
last_prune: std::time::Instant::now(),
cache: BTreeMap::new(),
user_agent: user_agent.to_string(),
+ min_crawl_delay,
+ max_crawl_delay,
}
}
@@ -55,59 +67,89 @@ impl RobotsTxtManager {
}
}
- async fn fetch_robots_txt_from_url(&self, url: &str) -> Result<RobotsTxt> {
- let res = self
+ pub async fn crawl_delay(&mut self, url: &Url) -> Option<Duration> {
+ match self.get_mut(url).await {
+ Lookup::Found(robots_txt) => robots_txt.robots.crawl_delay(),
+ Lookup::Unavailable | Lookup::Unreachable => None,
+ }
+ }
+
+ async fn fetch_robots_txt_from_url(&self, url: &str) -> Lookup<RobotsTxt> {
+ let res = match self
.client
.get(url)
.timeout(Duration::from_secs(60))
.send()
- .await;
-
- let res = res?;
+ .await
+ {
+ Ok(res) => {
+ if res.status() != reqwest::StatusCode::OK {
+ match res.status() {
+ reqwest::StatusCode::NOT_FOUND => return Lookup::Unavailable,
+ _ => return Lookup::Unreachable,
+ }
+ }
- if res.status() != reqwest::StatusCode::OK {
- return Err(Error::FetchFailed(res.status()).into());
- }
+ let body = match res.text().await {
+ Ok(body) => body,
+ Err(_) => return Lookup::Unreachable,
+ };
- if !res
- .headers()
- .get(reqwest::header::CONTENT_TYPE)
- .map(|h| h.to_str().unwrap_or_default().starts_with("text/plain"))
- .unwrap_or(false)
- {
- return Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into());
- }
+ let self_user_agent = self.user_agent.clone();
+ match panic::catch_unwind(|| RobotsTxt::new(&self_user_agent, body)) {
+ Ok(Ok(r)) => Lookup::Found(r),
+ _ => Lookup::Unreachable,
+ }
+ }
+ Err(_) => Lookup::Unreachable,
+ };
- let body = res.text().await?;
+ tokio::time::sleep(self.min_crawl_delay).await;
- let self_user_agent = self.user_agent.clone();
- match panic::catch_unwind(|| RobotsTxt::new(&self_user_agent, body)) {
- Ok(Ok(r)) => Ok(r),
- _ => Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into()),
- }
+ res
}
- async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {
- if let Ok(robots_txt) = self
+ async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<RobotsTxt> {
+ match self
.fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))
.await
{
- return Ok(robots_txt);
+ Lookup::Unavailable => {
+ match self
+ .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))
+ .await
+ {
+ Lookup::Found(robots_txt) => Lookup::Found(robots_txt),
+ Lookup::Unreachable => Lookup::Unreachable,
+ Lookup::Unavailable
+ if !site.0.starts_with("www.")
+ && site.0.chars().filter(|&c| c == '.').count() == 1 =>
+ {
+ self.fetch_robots_txt_from_url(&format!(
+ "https://www.{}/robots.txt",
+ &site.0
+ ))
+ .await
+ }
+ Lookup::Unavailable => Lookup::Unavailable,
+ }
+ }
+ res => res,
}
+ }
- match self
- .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))
- .await
- {
- Ok(robots_txt) => Ok(robots_txt),
- _ if !site.0.starts_with("www.")
- && site.0.chars().filter(|&c| c == '.').count() == 1 =>
- {
- self.fetch_robots_txt_from_url(&format!("https://www.{}/robots.txt", &site.0))
- .await
+ async fn fetch_robots_txt(&self, site: &Site) -> Lookup<RobotsTxt> {
+ for _ in 0..3 {
+ match self.fetch_robots_txt_without_retry(site).await {
+ Lookup::Found(robots_txt) => return Lookup::Found(robots_txt),
+ Lookup::Unavailable => return Lookup::Unavailable,
+ Lookup::Unreachable => {}
}
- Err(err) => Err(err),
+
+ tokio::time::sleep(self.max_crawl_delay).await;
}
+
+ Lookup::Unreachable
}
fn maybe_prune(&mut self) {
@@ -134,22 +176,8 @@ impl RobotsTxtManager {
};
if cache_should_update {
- match self.fetch_robots_txt(&site).await {
- Ok(robots_txt) => {
- self.cache.insert(site.clone(), Lookup::Found(robots_txt));
- }
- Err(err) => match err.downcast_ref() {
- Some(Error::FetchFailed(status))
- if *status == reqwest::StatusCode::NOT_FOUND =>
- {
- self.cache.insert(site.clone(), Lookup::Unavailable);
- }
- _ => {
- self.cache.insert(site.clone(), Lookup::Unreachable);
- tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);
- }
- },
- }
+ self.cache
+ .insert(site.clone(), self.fetch_robots_txt(&site).await);
}
self.cache.get_mut(&site).unwrap()
diff --git a/crates/core/src/crawler/warc_writer.rs b/crates/core/src/crawler/warc_writer.rs
index a16a2d7b..7b0a7733 100644
--- a/crates/core/src/crawler/warc_writer.rs
+++ b/crates/core/src/crawler/warc_writer.rs
@@ -21,7 +21,8 @@ use crate::{
warc,
};
-use super::{CrawlDatum, DatumStream, Result};
+use super::{CrawlDatum, DatumStream, Error, Result};
+use anyhow::anyhow;
/// The WarcWriter is responsible for storing the crawl datums
/// as WARC files on S3.
@@ -31,13 +32,19 @@ pub struct WarcWriter {
impl DatumStream for WarcWriter {
async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
- self.tx.send(WarcWriterMessage::Crawl(crawl_datum)).await?;
+ self.tx
+ .send(WarcWriterMessage::Crawl(crawl_datum))
+ .await
+ .map_err(|e| Error::from(anyhow!(e)))?;
Ok(())
}
async fn finish(&self) -> Result<()> {
- self.tx.send(WarcWriterMessage::Finish).await?;
+ self.tx
+ .send(WarcWriterMessage::Finish)
+ .await
+ .map_err(|e| Error::from(anyhow!(e)))?;
self.tx.closed().await;
Ok(())
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index ce2b1daf..31a61e4b 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -13,7 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use anyhow::{anyhow, bail};
+use anyhow::anyhow;
use encoding_rs::{Encoding, UTF_8};
use hashbrown::HashSet;
use mime::Mime;
@@ -41,8 +41,7 @@ use crate::{
use super::{
reqwest_client, robots_txt::RobotsTxtManager, wander_prirotiser::WanderPrioritiser, CrawlDatum,
- DatumStream, Domain, Error, Result, RetrieableUrl, Site, UrlResponse, WarcWriter, WeightedUrl,
- WorkerJob,
+ DatumStream, Domain, Error, Result, RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob,
};
const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
@@ -53,9 +52,13 @@ const IGNORED_EXTENSIONS: [&str; 27] = [
".flac", ".aac", ".ogg", ".m4a", ".m4v",
];
+enum UrlVisit {
+ Skip,
+ CanCrawl,
+}
+
struct ProcessedUrl {
new_urls: Vec<Url>,
- response: UrlResponse,
}
pub struct WorkerThread {
@@ -86,12 +89,13 @@ impl WorkerThread {
let router = *self.router_hosts.choose(&mut rand::thread_rng()).unwrap();
- Ok(sonic::service::Connection::create_with_timeout_retry(
+ sonic::service::Connection::create_with_timeout_retry(
router,
Duration::from_secs(90),
retry,
)
- .await?)
+ .await
+ .map_err(|e| Error::from(anyhow!(e)))
}
pub async fn run(self) {
@@ -131,7 +135,10 @@ pub struct JobExecutor<S: DatumStream> {
crawled_urls: HashSet<Url>,
crawled_sitemaps: HashSet<Site>,
sitemap_urls: HashSet<Url>,
- config: Arc<CrawlerConfig>,
+ min_crawl_delay: Duration,
+ max_crawl_delay: Duration,
+ max_url_slowdown_retry: u8,
+ max_politeness_factor: f32,
wander_prioritiser: WanderPrioritiser,
wandered_urls: u64,
job: WorkerJob,
@@ -147,17 +154,16 @@ impl<S: DatumStream> JobExecutor<S> {
Self {
writer,
politeness_factor: config.politeness_factor,
- robotstxt: RobotsTxtManager::new(
- client.clone(),
- Duration::from_secs(config.robots_txt_cache_sec),
- &config.user_agent.token,
- ),
+ robotstxt: RobotsTxtManager::new(&config),
client,
crawled_urls: HashSet::new(),
crawled_sitemaps: HashSet::new(),
wandered_urls: 0,
sitemap_urls: HashSet::new(),
- config,
+ min_crawl_delay: Duration::from_millis(config.min_crawl_delay_ms),
+ max_crawl_delay: Duration::from_millis(config.max_crawl_delay_ms),
+ max_url_slowdown_retry: config.max_url_slowdown_retry,
+ max_politeness_factor: config.max_politeness_factor,
wander_prioritiser: WanderPrioritiser::new(),
job,
}
@@ -165,7 +171,6 @@ impl<S: DatumStream> JobExecutor<S> {
pub async fn run(mut self) {
tracing::info!("Processing job: {:?}", self.job.domain);
-
self.scheduled_urls().await;
while self.wandered_urls < self.job.wandering_urls
@@ -218,28 +223,48 @@ impl<S: DatumStream> JobExecutor<S> {
self.process_urls(urls, false).await;
}
- async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {
- while let Some(retryable_url) = urls.pop_front() {
- if Domain::from(retryable_url.url()) != self.job.domain {
- continue;
- }
+ async fn verify_url(&mut self, retryable_url: &RetrieableUrl) -> UrlVisit {
+ if Domain::from(retryable_url.url()) != self.job.domain {
+ return UrlVisit::Skip;
+ }
- if self.crawled_urls.contains(retryable_url.url()) {
- continue;
- }
+ if self.crawled_urls.contains(retryable_url.url()) {
+ return UrlVisit::Skip;
+ }
- if retryable_url.retries > self.config.max_url_slowdown_retry {
- continue;
+ if retryable_url.retries > self.max_url_slowdown_retry {
+ return UrlVisit::Skip;
+ }
+
+ if retryable_url.url().host_str().is_none()
+ || !matches!(retryable_url.url().scheme(), "http" | "https")
+ {
+ return UrlVisit::Skip;
+ }
+
+ if !self.robotstxt.is_allowed(retryable_url.url()).await {
+ return UrlVisit::Skip;
+ }
+
+ if let Some(port) = retryable_url.url().port() {
+ if port != 80 && port != 443 {
+ return UrlVisit::Skip;
}
+ }
+
+ UrlVisit::CanCrawl
+ }
- if retryable_url.url().host_str().is_none()
- || !matches!(retryable_url.url().scheme(), "http" | "https")
- {
+ async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {
+ while let Some(retryable_url) = urls.pop_front() {
+ if let UrlVisit::Skip = self.verify_url(&retryable_url).await {
continue;
}
- if !self.robotstxt.is_allowed(retryable_url.url()).await {
- continue;
+ if let Some(delay) = self.robotstxt.crawl_delay(retryable_url.url()).await {
+ if delay > self.min_crawl_delay {
+ self.min_crawl_delay = delay.min(self.max_crawl_delay);
+ }
}
let site = Site(
@@ -262,8 +287,8 @@ impl<S: DatumStream> JobExecutor<S> {
let res = self.process_url(retryable_url.url().clone()).await;
- match res.response {
- UrlResponse::Success { url: _ } => {
+ match res {
+ Ok(res) => {
let weight = retryable_url.weighted_url.weight;
for new_url in res.new_urls {
@@ -278,18 +303,32 @@ impl<S: DatumStream> JobExecutor<S> {
self.wander_prioritiser.inc(new_url, weight);
}
}
- UrlResponse::Failed {
- url: _,
+ Err(Error::FetchFailed {
status_code,
- } => {
- if matches!(status_code, Some(429)) {
- let mut retryable_url = retryable_url;
- retryable_url.retries += 1;
- urls.push_back(retryable_url);
- continue;
+ headers,
+ }) if status_code == 429 => {
+ if headers.contains_key("retry-after") {
+ let retry_after = headers["retry-after"]
+ .to_str()
+ .ok()
+ .and_then(|s| s.parse::<u64>().ok())
+ .map(Duration::from_secs);
+
+ if let Some(retry_after) = retry_after {
+ if retry_after > self.max_crawl_delay {
+ return; // don't crawl anymore from this site
+ }
+
+ tokio::time::sleep(retry_after).await;
+ }
}
+
+ self.increase_politeness();
+ let mut retryable_url = retryable_url;
+ retryable_url.retries += 1;
+ urls.push_back(retryable_url);
}
- UrlResponse::Redirected { url: _, new_url: _ } => {}
+ Err(_) => {}
}
}
}
@@ -297,8 +336,8 @@ impl<S: DatumStream> JobExecutor<S> {
fn increase_politeness(&mut self) {
self.politeness_factor *= 2.0;
- if self.politeness_factor > self.config.max_politeness_factor {
- self.politeness_factor = self.config.max_politeness_factor;
+ if self.politeness_factor > self.max_politeness_factor {
+ self.politeness_factor = self.max_politeness_factor;
}
tracing::warn!("politeness factor increased to {}", self.politeness_factor);
@@ -322,82 +361,25 @@ impl<S: DatumStream> JobExecutor<S> {
.collect()
}
- async fn process_url(&mut self, url: Url) -> ProcessedUrl {
- let fetch = self.crawl_url(url.clone()).await;
-
- match fetch {
- Ok(datum) => match datum.status_code {
- 200 => {
- self.save_datum(datum.clone()).await;
-
- if datum.url != url {
- // todo: we might want to save the redirected url
- }
-
- match Html::parse(&datum.body, datum.url.as_str()) {
- Ok(html) => {
- let new_urls = self.new_urls(&html);
- let url_res = UrlResponse::Success { url: datum.url };
- ProcessedUrl {
- new_urls,
- response: url_res,
- }
- }
- Err(_) => ProcessedUrl {
- new_urls: Vec::new(),
- response: UrlResponse::Failed {
- url,
- status_code: None,
- },
- },
- }
- }
- 301 | 302 => {
- let url_res = UrlResponse::Redirected {
- url,
- new_url: datum.url,
- };
-
- ProcessedUrl {
- new_urls: Vec::new(),
- response: url_res,
- }
- }
-
- _ => {
- if datum.status_code == 429 {
- self.increase_politeness();
- }
-
- tracing::debug!("failed to fetch url ({}): {}", &url, datum.status_code);
- ProcessedUrl {
- new_urls: Vec::new(),
- response: UrlResponse::Failed {
- url,
- status_code: Some(datum.status_code),
- },
- }
- }
- },
- Err(err) => {
- tracing::debug!("failed to fetch url ({}): {}", &url, err);
-
- ProcessedUrl {
- new_urls: Vec::new(),
- response: UrlResponse::Failed {
- url,
- status_code: None,
- },
- }
+ async fn process_url(&mut self, url: Url) -> Result<ProcessedUrl> {
+ let datum = self.crawl_url(url.clone()).await?;
+ self.save_datum(datum.clone()).await;
+
+ match Html::parse(&datum.body, datum.url.as_str()) {
+ Ok(html) => {
+ let root_domain = datum.url.root_domain();
+ let new_urls = self
+ .new_urls(&html)
+ .into_iter()
+ .filter(|new_url| new_url.root_domain() == root_domain)
+ .collect();
+ Ok(ProcessedUrl { new_urls })
}
+ Err(_) => Err(Error::InvalidHtml),
}
}
async fn save_datum(&self, datum: CrawlDatum) {
- if datum.status_code != 200 {
- return;
- }
-
self.writer.write(datum).await.ok();
}
@@ -406,7 +388,7 @@ impl<S: DatumStream> JobExecutor<S> {
.get(url.to_string())
.send()
.await
- .map_err(|e| e.into())
+ .map_err(|e| Error::from(anyhow!(e)))
}
async fn fetch_with_https_priority(&self, url: Url) -> Result<reqwest::Response> {
@@ -419,7 +401,7 @@ impl<S: DatumStream> JobExecutor<S> {
match self.fetch(https).await {
Ok(res) => Ok(res),
Err(_) => {
- tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms)).await;
+ tokio::time::sleep(self.delay_duration()).await;
self.fetch(url.clone()).await
}
}
@@ -428,17 +410,28 @@ impl<S: DatumStream> JobExecutor<S> {
}
}
+ fn delay_duration(&self) -> Duration {
+ let mut delay = self.min_crawl_delay;
+ delay = delay.mul_f32(self.politeness_factor);
+
+ if delay > self.max_crawl_delay {
+ delay = self.max_crawl_delay;
+ }
+
+ delay
+ }
+
async fn politeness_delay(&self, fetch_time: Duration) {
let mut delay = fetch_time;
- if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {
- delay = Duration::from_millis(self.config.min_crawl_delay_ms);
+ if delay < self.min_crawl_delay {
+ delay = self.min_crawl_delay;
}
delay = delay.mul_f32(self.politeness_factor);
- if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {
- delay = Duration::from_millis(self.config.max_crawl_delay_ms);
+ if delay > self.max_crawl_delay {
+ delay = self.max_crawl_delay;
}
tokio::time::sleep(delay).await;
@@ -453,7 +446,7 @@ impl<S: DatumStream> JobExecutor<S> {
.and_then(|value| value.parse::<usize>().ok())
{
if content_length > MAX_CONTENT_LENGTH {
- return Err(Error::ContentTooLarge.into());
+ return Err(Error::ContentTooLarge);
}
}
@@ -466,7 +459,7 @@ impl<S: DatumStream> JobExecutor<S> {
Some(ct) if ct.contains("text/html") => Ok(warc::PayloadType::Html),
Some(ct) if ct.contains("application/rss") => Ok(warc::PayloadType::Rss),
Some(ct) if ct.contains("application/atom") => Ok(warc::PayloadType::Atom),
- ct => Err(Error::InvalidContentType(format!("{ct:?}")).into()),
+ ct => Err(Error::InvalidContentType(format!("{ct:?}"))),
}
}
@@ -493,7 +486,6 @@ impl<S: DatumStream> JobExecutor<S> {
Ok(Some(CrawlDatum {
url,
- status_code,
payload_type,
body: String::new(),
fetch_time_ms: fetch_time.as_millis() as u64,
@@ -520,7 +512,7 @@ impl<S: DatumStream> JobExecutor<S> {
let mut stream = res.bytes_stream();
while let Some(b) = stream.next().await {
if b.is_err() {
- return Err(Error::ContentTooLarge.into());
+ return Err(Error::ContentTooLarge);
}
let b = b.unwrap();
@@ -528,7 +520,7 @@ impl<S: DatumStream> JobExecutor<S> {
bytes.extend_from_slice(&b);
if bytes.len() > MAX_CONTENT_LENGTH {
- return Err(Error::ContentTooLarge.into());
+ return Err(Error::ContentTooLarge);
}
}
@@ -541,7 +533,7 @@ impl<S: DatumStream> JobExecutor<S> {
url.normalize();
if self.crawled_urls.contains(&url) {
- bail!("url already crawled: {}", url);
+ return Err(Error::from(anyhow!("url already crawled: {}", url)));
}
let start = Instant::now();
@@ -559,7 +551,14 @@ impl<S: DatumStream> JobExecutor<S> {
return Ok(datum);
}
- let status_code = res.status().as_u16();
+ let status_code = res.status();
+
+ if status_code != reqwest::StatusCode::OK {
+ return Err(Error::FetchFailed {
+ status_code,
+ headers: res.headers().clone(),
+ });
+ }
let mut res_url = res.url().clone();
res_url.normalize();
@@ -570,7 +569,6 @@ impl<S: DatumStream> JobExecutor<S> {
Ok(CrawlDatum {
url: res_url,
- status_code,
body,
payload_type,
fetch_time_ms: fetch_time.as_millis() as u64,
@@ -587,7 +585,7 @@ impl<S: DatumStream> JobExecutor<S> {
}
let res = self.fetch(url).await;
- tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms)).await;
+ tokio::time::sleep(self.delay_duration()).await;
if res.is_err() {
continue;
diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rs
index 7365c1b5..9e790d40 100644
--- a/crates/core/src/entity_index/mod.rs
+++ b/crates/core/src/entity_index/mod.rs
@@ -430,9 +430,8 @@ impl EntityIndex {
self.image_store.insert(name, image);
}
- pub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {
+ pub fn merge_all_segments(&mut self) -> Result<()> {
self.image_store.merge_all_segments();
- self.prepare_writer();
let base_path = Path::new(&self.path);
let segments: Vec<_> = self.tv_index.load_metas()?.segments.into_iter().collect();
@@ -440,7 +439,7 @@ impl EntityIndex {
self.writer.as_mut().expect("writer has not been prepared"),
segments,
base_path,
- max_num_segments,
+ 1,
)?;
Ok(())
diff --git a/crates/core/src/entrypoint/entity.rs b/crates/core/src/entrypoint/entity.rs
index 48a7105a..05b7f454 100644
--- a/crates/core/src/entrypoint/entity.rs
+++ b/crates/core/src/entrypoint/entity.rs
@@ -196,12 +196,18 @@ impl EntityIndexer {
let mut index = EntityIndex::open(output_path)?;
index.prepare_writer();
+ let mut image_bloom = bloom::BytesBloomFilter::new(1_000_000_000, 0.05);
+
let mut inserts = 0;
for entity in EntityIterator::new(&zim)?
.filter(|e| !e.is_disambiguation)
.filter(|e| !e.article_url.starts_with("Portal:"))
{
+ if let Some(image) = entity.image.as_ref() {
+ image_bloom.insert(image);
+ }
+
index.insert(entity);
inserts += 1;
@@ -216,6 +222,10 @@ impl EntityIndexer {
for image in zim.images()? {
if let Ok(decoded_image) = Image::from_bytes(image.bytes()) {
+ if !image_bloom.contains(&image.url) {
+ continue;
+ }
+
index.insert_image(image.url, decoded_image);
inserts += 1;
@@ -228,7 +238,7 @@ impl EntityIndexer {
}
index.commit();
- index.merge_into_max_segments(1)?;
+ index.merge_all_segments()?;
Ok(())
}
diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rs
index 9ef48f52..b64861b5 100644
--- a/crates/core/src/image_store.rs
+++ b/crates/core/src/image_store.rs
@@ -150,13 +150,7 @@ impl EntityImageStore {
self.store.prepare_writer();
}
pub fn open<P: AsRef<Path>>(path: P) -> Self {
- let store = BaseImageStore::open_with_filters(
- path,
- vec![Box::new(MaxSizeFilter {
- width: 400,
- height: 400,
- })],
- );
+ let store = BaseImageStore::open_with_filters(path, vec![]);
Self { store }
}
@@ -206,8 +200,10 @@ impl Image {
}
pub(crate) fn as_raw_bytes(&self) -> Vec<u8> {
+ let img = self.0.clone().into_rgba8();
+
let mut wrt = BufWriter::new(Cursor::new(Vec::new()));
- self.0.write_to(&mut wrt, image::ImageFormat::Png).unwrap();
+ img.write_to(&mut wrt, image::ImageFormat::WebP).unwrap();
wrt.flush().unwrap();
let mut cursor = wrt.into_inner().unwrap();
@@ -239,7 +235,12 @@ mod tests {
#[test]
fn serialize_deserialize_image() {
let image = Image(
- ImageBuffer::from_pixel(2, 2, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX])).into(),
+ ImageBuffer::from_pixel(
+ 2,
+ 2,
+ image::Rgba::<u8>([u8::MAX, u8::MAX, u8::MAX, u8::MAX]),
+ )
+ .into(),
);
let bytes = bincode::encode_to_vec(&image, bincode::config::standard()).unwrap();
@@ -252,7 +253,12 @@ mod tests {
#[test]
fn store_and_load_image() {
let image = Image(
- ImageBuffer::from_pixel(2, 2, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX])).into(),
+ ImageBuffer::from_pixel(
+ 2,
+ 2,
+ image::Rgba::<u8>([u8::MAX, u8::MAX, u8::MAX, u8::MAX]),
+ )
+ .into(),
);
let key = "test".to_string();
let mut store = BaseImageStore::open(crate::gen_temp_path());
@@ -267,8 +273,12 @@ mod tests {
#[test]
fn resize_filter() {
let image = Image(
- ImageBuffer::from_pixel(32, 32, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]))
- .into(),
+ ImageBuffer::from_pixel(
+ 32,
+ 32,
+ image::Rgba::<u8>([u8::MAX, u8::MAX, u8::MAX, u8::MAX]),
+ )
+ .into(),
);
assert_eq!(image.0.width(), 32);
assert_eq!(image.0.height(), 32);
diff --git a/crates/core/src/live_index/indexer.rs b/crates/core/src/live_index/indexer.rs
index 6ba6fc15..77b00e06 100644
--- a/crates/core/src/live_index/indexer.rs
+++ b/crates/core/src/live_index/indexer.rs
@@ -18,7 +18,7 @@ use std::sync::{Arc, Mutex, RwLock};
use crate::{
config::LiveIndexConfig,
- crawler::{CrawlDatum, DatumStream},
+ crawler::{self, CrawlDatum, DatumStream},
entrypoint::indexer::IndexingWorker,
Result,
};
@@ -72,7 +72,7 @@ impl Indexer {
}
impl DatumStream for Indexer {
- async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<(), crawler::Error> {
self.write_batch
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -83,7 +83,7 @@ impl DatumStream for Indexer {
Ok(())
}
- async fn finish(&self) -> Result<()> {
+ async fn finish(&self) -> Result<(), crawler::Error> {
self.write_batch_to_index();
self.search_index