web search done right
reuse robotstxt manager per worker
Mikkel Denker1f004b094f6c
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 8d2f798b..e9051ef5 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -542,7 +542,7 @@ impl CrawlDb {let mut job = Job {domain: self.domain_ids.value(domain_id.0)?.unwrap(),- fetch_sitemap: true, // todo: make this configureable+ fetch_sitemap: false, // todo: fetch for new sitesurls: VecDeque::with_capacity(urls_per_job),};diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 60729ddd..a6715f79 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -25,7 +25,6 @@ use tokio::sync::Mutex;use crate::{webpage::Url, CrawlerConfig};use self::{- robots_txt::RobotsTxtManager,warc_writer::{WarcWriter, WarcWriterMessage},worker::Worker,};@@ -66,8 +65,8 @@ pub enum Error {#[error("addr parse error: {0}")]AddrParse(#[from] std::net::AddrParseError),- #[error("async channel: {0}")]- SendError(#[from] async_channel::SendError<WarcWriterMessage>),+ #[error("channel: {0}")]+ SendError(#[from] tokio::sync::mpsc::error::SendError<WarcWriterMessage>),#[error("bincode: {0}")]Bincode(#[from] bincode::Error),@@ -123,16 +122,12 @@ pub struct JobResponse {}struct WorkerJob {- robotstxt: RobotsTxtManager,job: Job,}impl From<Job> for WorkerJob {fn from(value: Job) -> Self {- Self {- robotstxt: RobotsTxtManager::new(),- job: value,- }+ Self { job: value }}}diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsindex 981cfbdb..99d0a8dd 100644--- a/core/src/crawler/robots_txt.rs+++ b/core/src/crawler/robots_txt.rs@@ -26,11 +26,13 @@ use super::{Error, Result, Site};pub struct RobotsTxtManager {cache: HashMap<Site, Option<RobotsTxt>>, // None if robots.txt does not exist+ client: reqwest::Client,}impl RobotsTxtManager {- pub fn new() -> Self {+ pub fn new(client: reqwest::Client) -> Self {Self {+ client,cache: HashMap::new(),}}@@ -45,7 +47,11 @@ impl RobotsTxtManager {}async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {- let res = reqwest::get(&format!("http://{}/robots.txt", site.0)).await?;+ let res = self+ .client+ .get(&format!("http://{}/robots.txt", site.0))+ .send()+ .await?;if res.status() != reqwest::StatusCode::OK {return Err(Error::FetchFailed(res.status()));diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex 666701f7..657b2658 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -21,7 +21,7 @@ use super::{CrawlDatum, Result};/// The WarcWriter is responsible for storing the crawl datums/// as WARC files on S3.pub struct WarcWriter {- tx: async_channel::Sender<WarcWriterMessage>,+ tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,}#[derive(Debug, Clone)]@@ -70,10 +70,10 @@ async fn commit(writer: warc::WarcWriter, s3: S3Config) {}}-async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Config) {+async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3: S3Config) {let mut writer = warc::WarcWriter::new();- while let Ok(message) = rx.recv().await {+ while let Some(message) = rx.recv().await {match message {WarcWriterMessage::Crawl(datum) => {let w = &mut writer;@@ -130,7 +130,7 @@ async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Confiimpl WarcWriter {pub fn new(s3: S3Config) -> Self {- let (tx, rx) = async_channel::bounded(1000);+ let (tx, rx) = tokio::sync::mpsc::channel(1000);tokio::spawn(writer_task(rx, s3));@@ -145,11 +145,7 @@ impl WarcWriter {pub async fn finish(&self) -> Result<()> {self.tx.send(WarcWriterMessage::Finish).await?;- self.tx.close();-- while !self.tx.is_empty() {- tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;- }+ self.tx.closed().await;Ok(())}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex bf50fe4d..050b3b65 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -32,7 +32,10 @@ use crate::{UserAgent,};-use super::{Command, CrawlDatum, Error, Result, Site, UrlResponse, WarcWriter, WorkerJob};+use super::{+ robots_txt::RobotsTxtManager, Command, CrawlDatum, Error, Result, Site, UrlResponse,+ WarcWriter, WorkerJob,+};const MIN_CRAWL_DELAY_MS: u64 = 200;const MAX_CRAWL_DELAY_MS: u64 = 30_000;@@ -49,6 +52,7 @@ pub struct Worker {politeness_factor: f32,coordinator_host: SocketAddr,num_jobs_per_fetch: usize,+ robotstxt: RobotsTxtManager,}impl Worker {@@ -65,21 +69,26 @@ impl Worker {return Err(Error::InvalidPolitenessFactor);}+ let client = reqwest::Client::builder()+ .timeout(timeout)+ .connect_timeout(timeout)+ .http2_keep_alive_interval(None)+ .user_agent(&user_agent.full)+ .build()?;++ let robotstxt = RobotsTxtManager::new(client.clone());+Ok(Self {writer,+ client,current_job: None,pending_commands,- client: reqwest::Client::builder()- .timeout(timeout)- .connect_timeout(timeout)- .http2_keep_alive_interval(None)- .user_agent(&user_agent.full)- .build()?,user_agent,politeness_factor,default_politeness_factor: politeness_factor,coordinator_host,num_jobs_per_fetch,+ robotstxt,})}@@ -151,7 +160,7 @@ impl Worker {async fn process_job(&mut self) {self.politeness_factor = self.default_politeness_factor;- let mut job = self.current_job.take().unwrap();+ let job = self.current_job.take().unwrap();tracing::info!("Processing job: {:?}", job.job.domain);let mut url_responses: Vec<UrlResponse> = Vec::new();@@ -164,7 +173,11 @@ impl Worker {continue;}- if !job.robotstxt.is_allowed(&url, &self.user_agent.token).await {+ if !self+ .robotstxt+ .is_allowed(&url, &self.user_agent.token)+ .await+ {continue;}@@ -172,7 +185,7 @@ impl Worker {if job.job.fetch_sitemap && !crawled_sitemaps.contains(&site) {crawled_sitemaps.insert(site.clone());- if let Ok(Some(sitemap)) = job.robotstxt.sitemap(&url).await {+ if let Ok(Some(sitemap)) = self.robotstxt.sitemap(&url).await {let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;discovered_urls.extend(sitemap_urls.into_iter());}