Skip to content

web search done right

reuse robotstxt manager per worker

Mikkel Denker1f004b094f6c
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 8d2f798b..e9051ef5 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -542,7 +542,7 @@ impl CrawlDb {
let mut job = Job {
domain: self.domain_ids.value(domain_id.0)?.unwrap(),
- fetch_sitemap: true, // todo: make this configureable
+ fetch_sitemap: false, // todo: fetch for new sites
urls: VecDeque::with_capacity(urls_per_job),
};
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 60729ddd..a6715f79 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -25,7 +25,6 @@ use tokio::sync::Mutex;
use crate::{webpage::Url, CrawlerConfig};
use self::{
- robots_txt::RobotsTxtManager,
warc_writer::{WarcWriter, WarcWriterMessage},
worker::Worker,
};
@@ -66,8 +65,8 @@ pub enum Error {
#[error("addr parse error: {0}")]
AddrParse(#[from] std::net::AddrParseError),
- #[error("async channel: {0}")]
- SendError(#[from] async_channel::SendError<WarcWriterMessage>),
+ #[error("channel: {0}")]
+ SendError(#[from] tokio::sync::mpsc::error::SendError<WarcWriterMessage>),
#[error("bincode: {0}")]
Bincode(#[from] bincode::Error),
@@ -123,16 +122,12 @@ pub struct JobResponse {
}
struct WorkerJob {
- robotstxt: RobotsTxtManager,
job: Job,
}
impl From<Job> for WorkerJob {
fn from(value: Job) -> Self {
- Self {
- robotstxt: RobotsTxtManager::new(),
- job: value,
- }
+ Self { job: value }
}
}
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 981cfbdb..99d0a8dd 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -26,11 +26,13 @@ use super::{Error, Result, Site};
pub struct RobotsTxtManager {
cache: HashMap<Site, Option<RobotsTxt>>, // None if robots.txt does not exist
+ client: reqwest::Client,
}
impl RobotsTxtManager {
- pub fn new() -> Self {
+ pub fn new(client: reqwest::Client) -> Self {
Self {
+ client,
cache: HashMap::new(),
}
}
@@ -45,7 +47,11 @@ impl RobotsTxtManager {
}
async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {
- let res = reqwest::get(&format!("http://{}/robots.txt", site.0)).await?;
+ let res = self
+ .client
+ .get(&format!("http://{}/robots.txt", site.0))
+ .send()
+ .await?;
if res.status() != reqwest::StatusCode::OK {
return Err(Error::FetchFailed(res.status()));
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index 666701f7..657b2658 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -21,7 +21,7 @@ use super::{CrawlDatum, Result};
/// The WarcWriter is responsible for storing the crawl datums
/// as WARC files on S3.
pub struct WarcWriter {
- tx: async_channel::Sender<WarcWriterMessage>,
+ tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,
}
#[derive(Debug, Clone)]
@@ -70,10 +70,10 @@ async fn commit(writer: warc::WarcWriter, s3: S3Config) {
}
}
-async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Config) {
+async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3: S3Config) {
let mut writer = warc::WarcWriter::new();
- while let Ok(message) = rx.recv().await {
+ while let Some(message) = rx.recv().await {
match message {
WarcWriterMessage::Crawl(datum) => {
let w = &mut writer;
@@ -130,7 +130,7 @@ async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Confi
impl WarcWriter {
pub fn new(s3: S3Config) -> Self {
- let (tx, rx) = async_channel::bounded(1000);
+ let (tx, rx) = tokio::sync::mpsc::channel(1000);
tokio::spawn(writer_task(rx, s3));
@@ -145,11 +145,7 @@ impl WarcWriter {
pub async fn finish(&self) -> Result<()> {
self.tx.send(WarcWriterMessage::Finish).await?;
- self.tx.close();
-
- while !self.tx.is_empty() {
- tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
- }
+ self.tx.closed().await;
Ok(())
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index bf50fe4d..050b3b65 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -32,7 +32,10 @@ use crate::{
UserAgent,
};
-use super::{Command, CrawlDatum, Error, Result, Site, UrlResponse, WarcWriter, WorkerJob};
+use super::{
+ robots_txt::RobotsTxtManager, Command, CrawlDatum, Error, Result, Site, UrlResponse,
+ WarcWriter, WorkerJob,
+};
const MIN_CRAWL_DELAY_MS: u64 = 200;
const MAX_CRAWL_DELAY_MS: u64 = 30_000;
@@ -49,6 +52,7 @@ pub struct Worker {
politeness_factor: f32,
coordinator_host: SocketAddr,
num_jobs_per_fetch: usize,
+ robotstxt: RobotsTxtManager,
}
impl Worker {
@@ -65,21 +69,26 @@ impl Worker {
return Err(Error::InvalidPolitenessFactor);
}
+ let client = reqwest::Client::builder()
+ .timeout(timeout)
+ .connect_timeout(timeout)
+ .http2_keep_alive_interval(None)
+ .user_agent(&user_agent.full)
+ .build()?;
+
+ let robotstxt = RobotsTxtManager::new(client.clone());
+
Ok(Self {
writer,
+ client,
current_job: None,
pending_commands,
- client: reqwest::Client::builder()
- .timeout(timeout)
- .connect_timeout(timeout)
- .http2_keep_alive_interval(None)
- .user_agent(&user_agent.full)
- .build()?,
user_agent,
politeness_factor,
default_politeness_factor: politeness_factor,
coordinator_host,
num_jobs_per_fetch,
+ robotstxt,
})
}
@@ -151,7 +160,7 @@ impl Worker {
async fn process_job(&mut self) {
self.politeness_factor = self.default_politeness_factor;
- let mut job = self.current_job.take().unwrap();
+ let job = self.current_job.take().unwrap();
tracing::info!("Processing job: {:?}", job.job.domain);
let mut url_responses: Vec<UrlResponse> = Vec::new();
@@ -164,7 +173,11 @@ impl Worker {
continue;
}
- if !job.robotstxt.is_allowed(&url, &self.user_agent.token).await {
+ if !self
+ .robotstxt
+ .is_allowed(&url, &self.user_agent.token)
+ .await
+ {
continue;
}
@@ -172,7 +185,7 @@ impl Worker {
if job.job.fetch_sitemap && !crawled_sitemaps.contains(&site) {
crawled_sitemaps.insert(site.clone());
- if let Ok(Some(sitemap)) = job.robotstxt.sitemap(&url).await {
+ if let Ok(Some(sitemap)) = self.robotstxt.sitemap(&url).await {
let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;
discovered_urls.extend(sitemap_urls.into_iter());
}