web search done right
Fuuuuuuuck had a bug that caused the crawler not to respect robots.txt :(. I'm terribly sorry if this has caused any inconvenience (luckily it was caught before any big crawls)
Mikkel Denker9aba33cb99f9
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.tomlindex 8ec5f1bf..4c1d7907 100644--- a/configs/crawler/crawler.toml+++ b/configs/crawler/crawler.toml@@ -1,9 +1,12 @@num_workers = 10-user_agent = "Mozilla/5.0 (compatible; Stractbot/0.1; +https://trystract.com/bot)"politeness_factor = 1.0timeout_seconds = 30coordinator_host = "0.0.0.0:8080"+[user_agent]+token = "StractSearch"+full = "Mozilla/5.0 (compatible; StractSearch/0.1; +https://trystract.com/webmasters)"+[s3]bucket = "crawl"folder = "test1"diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex c8c50ded..ad9cbbe8 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -180,7 +180,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route.route("/settings/sites", get(sites::route)).route("/settings/privacy", get(improvement::settings)).route("/privacy-and-happy-lawyers", get(privacy::route))- .route("/bot", get(crawler::info_route))+ .route("/webmasters", get(crawler::info_route)).route("/opensearch.xml", get(opensearch::route)).route("/improvement/click", post(improvement::click)).route("/improvement/store", post(improvement::store))diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 96e48a9d..cb30e38b 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -67,6 +67,10 @@ impl CrawlCoordinator {}fn log_crawls_per_second(&self, num_urls: usize) {+ if self.is_done() {+ return;+ }+let mut call_counter = self.call_counter.lock().unwrap();call_counter.count_with_weight(num_urls);diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsindex de800725..b5715b35 100644--- a/core/src/crawler/robots_txt.rs+++ b/core/src/crawler/robots_txt.rs@@ -45,7 +45,7 @@ impl RobotsTxtManager {}async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {- let res = reqwest::get(&format!("https://{}/robots.txt", site.0)).await?;+ let res = reqwest::get(&format!("http://{}/robots.txt", site.0)).await?;if res.status() != reqwest::StatusCode::OK {return Err(Error::FetchFailed(res.status()));@@ -123,3 +123,26 @@ impl RobotsTxt {})}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple() {+ let ua_token = "StractSearch";+ let mut robots_txt = RobotsTxt::new(+ r#"User-agent: StractSearch+ Disallow: /test"#+ .to_string(),+ )+ .unwrap();++ assert!(!robots_txt+ .matcher+ .one_agent_allowed_by_robots(ua_token, "http://example.com/test"));+ assert!(robots_txt+ .matcher+ .one_agent_allowed_by_robots(ua_token, "http://example.com/example"));+ }+}diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex a2f41fe1..fb890e52 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -77,32 +77,42 @@ async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Confiwhile let Ok(message) = rx.recv().await {match message {WarcWriterMessage::Crawl(datum) => {- let is_pdf = datum- .headers- .get("content-type")- .map(|ct| ct.contains("application/pdf"))- .unwrap_or(false);-- let payload = if is_pdf {- Some("application/pdf".to_string())- } else {- None- };-- let warc_record = warc::WarcRecord {- request: warc::Request {- url: datum.url.full(),- },- response: warc::Response {- body: datum.body,- payload_type: payload,- },- metadata: warc::Metadata {- fetch_time_ms: datum.fetch_time_ms as usize,- },- };-- writer.write(&warc_record).unwrap();+ let w = &mut writer;+ let (send, recv) = tokio::sync::oneshot::channel();++ rayon::scope(move |s| {+ s.spawn(move |_| {+ let is_pdf = datum+ .headers+ .get("content-type")+ .map(|ct| ct.contains("application/pdf"))+ .unwrap_or(false);++ let payload = if is_pdf {+ Some("application/pdf".to_string())+ } else {+ None+ };++ let warc_record = warc::WarcRecord {+ request: warc::Request {+ url: datum.url.full(),+ },+ response: warc::Response {+ body: datum.body,+ payload_type: payload,+ },+ metadata: warc::Metadata {+ fetch_time_ms: datum.fetch_time_ms as usize,+ },+ };++ w.write(&warc_record).unwrap();+ send.send(()).unwrap();+ });+ });++ recv.await.unwrap();if writer.num_bytes() > 1_000_000_000 {commit(writer, s3.clone()).await;diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 96295756..9e8e38ef 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -29,6 +29,7 @@ use crate::{crawler::{JobResponse, Request, Response},distributed::{retry_strategy::ExponentialBackoff, sonic},webpage::{Html, Url},+ UserAgent,};use super::{Command, CrawlDatum, Error, Result, Site, UrlResponse, WarcWriter, WorkerJob};@@ -43,7 +44,7 @@ pub struct Worker {pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,client: reqwest::Client,- user_agent: String,+ user_agent: UserAgent,default_politeness_factor: f32,politeness_factor: f32,coordinator_host: SocketAddr,@@ -54,7 +55,7 @@ impl Worker {pub fn new(pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,- user_agent: String,+ user_agent: UserAgent,politeness_factor: f32,timeout: Duration,coordinator_host: SocketAddr,@@ -72,7 +73,7 @@ impl Worker {.timeout(timeout).connect_timeout(timeout).http2_keep_alive_interval(None)- .user_agent(&user_agent)+ .user_agent(&user_agent.full).build()?,user_agent,politeness_factor,@@ -164,7 +165,7 @@ impl Worker {continue;}- if !job.robotstxt.is_allowed(&url, &self.user_agent).await {+ if !job.robotstxt.is_allowed(&url, &self.user_agent.token).await {continue;}diff --git a/core/src/lib.rs b/core/src/lib.rsindex b87bcf85..d74deedb 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -210,11 +210,17 @@ pub struct CrawlCoordinatorConfig {pub seed_urls: Vec<String>,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct UserAgent {+ pub full: String,+ pub token: String,+}+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct CrawlerConfig {pub num_warc_writers: Option<usize>,pub num_workers: usize,- pub user_agent: String,+ pub user_agent: UserAgent,pub politeness_factor: Option<f32>,pub timeout_seconds: u64,pub s3: S3Config,diff --git a/core/src/main.rs b/core/src/main.rsindex 3a74c947..f2a841f7 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -41,31 +41,40 @@ struct Args {#[derive(Subcommand)]enum Commands {+ /// Commands to deploy the Alice serverAlice {#[clap(subcommand)]options: AliceOptions,},+ /// Build the search indexIndexer {#[clap(subcommand)]options: IndexingOptions,},+ /// Calculate centrality metrics that estimates a websites importance. These metrics are used to rank search results.Centrality {#[clap(subcommand)]mode: CentralityType,webgraph_path: String,output_path: String,},+ /// Parse the DMOZ dataset. DMOZ contains a list of websites and their categories.+ /// It can be used to calculate the topic centrality for websites or augments website descriptions during indexing.DmozParser {dmoz_file: String,output_path: String,},+ /// Webgraph specific commands.Webgraph {#[clap(subcommand)]options: WebgraphOptions,},+ /// Deploy the search server.SearchServer {config_path: String,},+ /// Deploy the frontend. The frontend is a web server that serves the search UI and interacts with+ /// the search servers, webgraph servers etc. to provide the necesarry functionality.Frontend {config_path: String,},diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdxindex 813b89b5..2f15f7cd 100644--- a/frontend/src/pages/crawler.mdx+++ b/frontend/src/pages/crawler.mdx@@ -1,13 +1,18 @@---layout: "../layouts/Markdown.astro"-title: StractBot+title: Stract Crawler----# StractBot-StractBot is the name of our crawler that is used to collect pages from the web in order to index them in the search engine. It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler). The crawler uses the user agent `Mozilla/5.0 (compatible; Stractbot/0.1; +https://trystract.com/bot)`.+# Stract Crawler+StractSearch is the name of our crawler that is used to collect pages from the web in order to index them in the search engine. It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler). The crawler uses the user agent `Mozilla/5.0 (compatible; StractSearch/0.1; +https://trystract.com/webmasters)`.## Politeness-StractBot is a polite crawler. This means that it respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server. It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 30 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 30 seconds. This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 128).+StractSearch is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 30 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 30 seconds.+This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 128).++The crawler looks for the token `StractSearch` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.+## Contact us-If you have any concerns or bad experiences with our crawler, please don't hesitate to reach out to us at [[email protected]](mailto:[email protected]).\ No newline at end of file+If you have any concerns or bad experiences with our crawler, please don't hesitate to reach out to us at [[email protected]](mailto:[email protected]). Chances are that others experience the same problems and we would love to fix them.\ No newline at end of file