Skip to content

web search done right

ensure reasonable limit for robotstxt files

Mikkel Denker071e41d1677a
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rs
index ee31da67..ff536468 100644
--- a/crates/core/src/crawler/mod.rs
+++ b/crates/core/src/crawler/mod.rs
@@ -19,6 +19,7 @@ use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc, tim
type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;
use anyhow::anyhow;
+use futures::StreamExt;
use url::Url;
use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};
@@ -39,11 +40,9 @@ mod worker;
pub use coordinator::CrawlCoordinator;
pub const MAX_URL_LEN_BYTES: usize = 8192;
-pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 256;
-/// Number of new domains that can be discovered for each domain crawled.
-pub const MAX_DOMAIN_DISCOVERY_FACTOR: usize = 4;
-pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 200;
+pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 512;
+pub const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
#[derive(thiserror::Error, Debug)]
pub enum Error {
@@ -320,3 +319,37 @@ pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {
.build()
.map_err(|e| Error::from(anyhow!(e)))
}
+
+pub async fn encoded_body(res: reqwest::Response) -> Result<String> {
+ let content_type = res
+ .headers()
+ .get(reqwest::header::CONTENT_TYPE)
+ .and_then(|value| value.to_str().ok())
+ .and_then(|value| value.parse::<mime::Mime>().ok());
+ let encoding_name = content_type
+ .as_ref()
+ .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))
+ .unwrap_or("utf-8");
+ let encoding =
+ encoding_rs::Encoding::for_label(encoding_name.as_bytes()).unwrap_or(encoding_rs::UTF_8);
+
+ let mut bytes = Vec::new();
+
+ let mut stream = res.bytes_stream();
+ while let Some(b) = stream.next().await {
+ if b.is_err() {
+ return Err(Error::ContentTooLarge);
+ }
+
+ let b = b.unwrap();
+
+ bytes.extend_from_slice(&b);
+
+ if bytes.len() > MAX_CONTENT_LENGTH {
+ return Err(Error::ContentTooLarge);
+ }
+ }
+
+ let (text, _, _) = encoding.decode(&bytes);
+ Ok(text.to_string())
+}
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rs
index 40730b4c..188e1d08 100644
--- a/crates/core/src/crawler/robots_txt.rs
+++ b/crates/core/src/crawler/robots_txt.rs
@@ -20,7 +20,7 @@ use url::Url;
use crate::{config::CrawlerConfig, crawler};
-use super::{Result, Site};
+use super::{encoded_body, Result, Site};
enum Lookup<T> {
Found(T),
@@ -90,7 +90,7 @@ impl RobotsTxtManager {
}
}
- let body = match res.text().await {
+ let body = match encoded_body(res).await {
Ok(body) => body,
Err(_) => return Lookup::Unreachable,
};
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index 31a61e4b..f27f6063 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -14,12 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::anyhow;
-use encoding_rs::{Encoding, UTF_8};
use hashbrown::HashSet;
-use mime::Mime;
use quick_xml::events::Event;
use rand::seq::SliceRandom;
-use tokio_stream::StreamExt;
use std::{
collections::VecDeque,
@@ -40,12 +37,12 @@ use crate::{
};
use super::{
- reqwest_client, robots_txt::RobotsTxtManager, wander_prirotiser::WanderPrioritiser, CrawlDatum,
- DatumStream, Domain, Error, Result, RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob,
+ encoded_body, reqwest_client, robots_txt::RobotsTxtManager,
+ wander_prirotiser::WanderPrioritiser, CrawlDatum, DatumStream, Domain, Error, Result,
+ RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob, MAX_CONTENT_LENGTH,
+ MAX_OUTGOING_URLS_PER_PAGE,
};
-const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
-
const IGNORED_EXTENSIONS: [&str; 27] = [
".pdf", ".jpg", ".zip", ".png", ".css", ".js", ".json", ".jsonp", ".woff2", ".woff", ".ttf",
".svg", ".gif", ".jpeg", ".ico", ".mp4", ".mp3", ".avi", ".mov", ".mpeg", ".webm", ".wav",
@@ -372,6 +369,7 @@ impl<S: DatumStream> JobExecutor<S> {
.new_urls(&html)
.into_iter()
.filter(|new_url| new_url.root_domain() == root_domain)
+ .take(MAX_OUTGOING_URLS_PER_PAGE)
.collect();
Ok(ProcessedUrl { new_urls })
}
@@ -495,39 +493,6 @@ impl<S: DatumStream> JobExecutor<S> {
}
}
- async fn encoded_body(&self, res: reqwest::Response) -> Result<String> {
- let content_type = res
- .headers()
- .get(reqwest::header::CONTENT_TYPE)
- .and_then(|value| value.to_str().ok())
- .and_then(|value| value.parse::<Mime>().ok());
- let encoding_name = content_type
- .as_ref()
- .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))
- .unwrap_or("utf-8");
- let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8);
-
- let mut bytes = Vec::new();
-
- let mut stream = res.bytes_stream();
- while let Some(b) = stream.next().await {
- if b.is_err() {
- return Err(Error::ContentTooLarge);
- }
-
- let b = b.unwrap();
-
- bytes.extend_from_slice(&b);
-
- if bytes.len() > MAX_CONTENT_LENGTH {
- return Err(Error::ContentTooLarge);
- }
- }
-
- let (text, _, _) = encoding.decode(&bytes);
- Ok(text.to_string())
- }
-
async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {
let mut url = url;
url.normalize();
@@ -565,7 +530,7 @@ impl<S: DatumStream> JobExecutor<S> {
self.crawled_urls.insert(res_url.clone());
- let body = self.encoded_body(res).await?;
+ let body = encoded_body(res).await?;
Ok(CrawlDatum {
url: res_url,
@@ -584,7 +549,7 @@ impl<S: DatumStream> JobExecutor<S> {
continue;
}
- let res = self.fetch(url).await;
+ let res = self.fetch_with_https_priority(url).await;
tokio::time::sleep(self.delay_duration()).await;
if res.is_err() {