web search done right
ensure reasonable limit for robotstxt files
Mikkel Denker071e41d1677a
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex ee31da67..ff536468 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -19,6 +19,7 @@ use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc, timtype HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;use anyhow::anyhow;+use futures::StreamExt;use url::Url;use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};@@ -39,11 +40,9 @@ mod worker;pub use coordinator::CrawlCoordinator;pub const MAX_URL_LEN_BYTES: usize = 8192;-pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 256;-/// Number of new domains that can be discovered for each domain crawled.-pub const MAX_DOMAIN_DISCOVERY_FACTOR: usize = 4;-pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 200;+pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 512;+pub const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB#[derive(thiserror::Error, Debug)]pub enum Error {@@ -320,3 +319,37 @@ pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {.build().map_err(|e| Error::from(anyhow!(e)))}++pub async fn encoded_body(res: reqwest::Response) -> Result<String> {+ let content_type = res+ .headers()+ .get(reqwest::header::CONTENT_TYPE)+ .and_then(|value| value.to_str().ok())+ .and_then(|value| value.parse::<mime::Mime>().ok());+ let encoding_name = content_type+ .as_ref()+ .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))+ .unwrap_or("utf-8");+ let encoding =+ encoding_rs::Encoding::for_label(encoding_name.as_bytes()).unwrap_or(encoding_rs::UTF_8);++ let mut bytes = Vec::new();++ let mut stream = res.bytes_stream();+ while let Some(b) = stream.next().await {+ if b.is_err() {+ return Err(Error::ContentTooLarge);+ }++ let b = b.unwrap();++ bytes.extend_from_slice(&b);++ if bytes.len() > MAX_CONTENT_LENGTH {+ return Err(Error::ContentTooLarge);+ }+ }++ let (text, _, _) = encoding.decode(&bytes);+ Ok(text.to_string())+}diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex 40730b4c..188e1d08 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -20,7 +20,7 @@ use url::Url;use crate::{config::CrawlerConfig, crawler};-use super::{Result, Site};+use super::{encoded_body, Result, Site};enum Lookup<T> {Found(T),@@ -90,7 +90,7 @@ impl RobotsTxtManager {}}- let body = match res.text().await {+ let body = match encoded_body(res).await {Ok(body) => body,Err(_) => return Lookup::Unreachable,};diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 31a61e4b..f27f6063 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -14,12 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::anyhow;-use encoding_rs::{Encoding, UTF_8};use hashbrown::HashSet;-use mime::Mime;use quick_xml::events::Event;use rand::seq::SliceRandom;-use tokio_stream::StreamExt;use std::{collections::VecDeque,@@ -40,12 +37,12 @@ use crate::{};use super::{- reqwest_client, robots_txt::RobotsTxtManager, wander_prirotiser::WanderPrioritiser, CrawlDatum,- DatumStream, Domain, Error, Result, RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob,+ encoded_body, reqwest_client, robots_txt::RobotsTxtManager,+ wander_prirotiser::WanderPrioritiser, CrawlDatum, DatumStream, Domain, Error, Result,+ RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob, MAX_CONTENT_LENGTH,+ MAX_OUTGOING_URLS_PER_PAGE,};-const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB-const IGNORED_EXTENSIONS: [&str; 27] = [".pdf", ".jpg", ".zip", ".png", ".css", ".js", ".json", ".jsonp", ".woff2", ".woff", ".ttf",".svg", ".gif", ".jpeg", ".ico", ".mp4", ".mp3", ".avi", ".mov", ".mpeg", ".webm", ".wav",@@ -372,6 +369,7 @@ impl<S: DatumStream> JobExecutor<S> {.new_urls(&html).into_iter().filter(|new_url| new_url.root_domain() == root_domain)+ .take(MAX_OUTGOING_URLS_PER_PAGE).collect();Ok(ProcessedUrl { new_urls })}@@ -495,39 +493,6 @@ impl<S: DatumStream> JobExecutor<S> {}}- async fn encoded_body(&self, res: reqwest::Response) -> Result<String> {- let content_type = res- .headers()- .get(reqwest::header::CONTENT_TYPE)- .and_then(|value| value.to_str().ok())- .and_then(|value| value.parse::<Mime>().ok());- let encoding_name = content_type- .as_ref()- .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))- .unwrap_or("utf-8");- let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8);-- let mut bytes = Vec::new();-- let mut stream = res.bytes_stream();- while let Some(b) = stream.next().await {- if b.is_err() {- return Err(Error::ContentTooLarge);- }-- let b = b.unwrap();-- bytes.extend_from_slice(&b);-- if bytes.len() > MAX_CONTENT_LENGTH {- return Err(Error::ContentTooLarge);- }- }-- let (text, _, _) = encoding.decode(&bytes);- Ok(text.to_string())- }-async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {let mut url = url;url.normalize();@@ -565,7 +530,7 @@ impl<S: DatumStream> JobExecutor<S> {self.crawled_urls.insert(res_url.clone());- let body = self.encoded_body(res).await?;+ let body = encoded_body(res).await?;Ok(CrawlDatum {url: res_url,@@ -584,7 +549,7 @@ impl<S: DatumStream> JobExecutor<S> {continue;}- let res = self.fetch(url).await;+ let res = self.fetch_with_https_priority(url).await;tokio::time::sleep(self.delay_duration()).await;if res.is_err() {