web search done right
Reduce crawl worker memory usage. We now clear the robotstxt cache for each thread for each new job, since two consecutive jobs are unlikely to be for the same site. If they both happen to be for the same site, we will just fetch robots.txt again.
Mikkel Denkereea1257859e0
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex ffa12ef0..16446b24 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -75,7 +75,6 @@ impl CrawlCoordinator {}pub fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {- let start = Instant::now();let num_crawled_urls = responses.iter().map(|res| res.url_responses.len()).sum();self.log_crawls_per_second(num_crawled_urls);@@ -83,6 +82,7 @@ impl CrawlCoordinator {.fetch_add(num_crawled_urls as u64, Ordering::SeqCst);let mut db = self.db.lock().unwrap();+ let start = Instant::now();let nonempty_domains = db.insert_urls(responses).unwrap();@@ -107,9 +107,8 @@ impl CrawlCoordinator {}pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {- let start = Instant::now();-let mut db = self.db.lock().unwrap();+ let start = Instant::now();let domains = db.sample_domains(num_jobs)?;tracing::debug!("sampled domains: {:?}", domains);diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex afca1247..82a21b6d 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -55,7 +55,20 @@ pub enum Error {type Result<T, E = anyhow::Error> = std::result::Result<T, E>;-#[derive(Clone, Debug, PartialEq, Eq, Hash)]+#[derive(+ Clone,+ Debug,+ PartialEq,+ Eq,+ Hash,+ PartialOrd,+ Ord,+ serde::Serialize,+ serde::Deserialize,+ rkyv::Serialize,+ rkyv::Deserialize,+ rkyv::Archive,+)]struct Site(String);#[derive(diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsindex f76dd4eb..4ee20937 100644--- a/core/src/crawler/robots_txt.rs+++ b/core/src/crawler/robots_txt.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::HashMap, panic, time::Duration};+use std::{collections::BTreeMap, panic, time::Duration};use robotstxt_with_cache::matcher::{CachingRobotsMatcher, LongestMatchRobotsMatchStrategy, RobotsMatcher,@@ -30,7 +30,7 @@ enum Lookup<T> {}pub struct RobotsTxtManager {- cache: HashMap<Site, Lookup<RobotsTxt>>,+ cache: BTreeMap<Site, Lookup<RobotsTxt>>,last_prune: std::time::Instant,client: reqwest::Client,cache_expiration: Duration,@@ -42,10 +42,15 @@ impl RobotsTxtManager {client,cache_expiration,last_prune: std::time::Instant::now(),- cache: HashMap::new(),+ cache: BTreeMap::new(),}}+ pub fn clear(&mut self) {+ self.cache.clear();+ self.last_prune = std::time::Instant::now();+ }+pub async fn is_allowed(&mut self, url: &Url, user_agent: &str) -> bool {match self.get_mut(url).await {Lookup::Found(robots_txt) => robots_txt@@ -78,6 +83,15 @@ impl RobotsTxtManager {return Err(Error::FetchFailed(res.status()).into());}+ if !res+ .headers()+ .get(reqwest::header::CONTENT_TYPE)+ .map(|h| h.to_str().unwrap_or_default().starts_with("text/plain"))+ .unwrap_or(false)+ {+ return Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into());+ }+let body = res.text().await?;match panic::catch_unwind(|| RobotsTxt::new(body)) {diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex e56740ef..4d49fb72 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -137,7 +137,7 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:impl WarcWriter {pub fn new(s3: S3Config) -> Self {- let (tx, rx) = tokio::sync::mpsc::channel(1000);+ let (tx, rx) = tokio::sync::mpsc::channel(1);tokio::spawn(writer_task(rx, s3));diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 58c1550f..fa194e6a 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -44,6 +44,8 @@ use super::{const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB+const MAX_URL_LENGTH: usize = 8192;+pub struct WorkerThread {current_job: Option<WorkerJob>,pending_commands: Arc<Mutex<VecDeque<Command>>>,@@ -188,6 +190,8 @@ impl WorkerThread {let mut job = self.current_job.take().unwrap();tracing::info!("Processing job: {:?}", job.domain);+ self.robotstxt.clear();+let mut url_responses: Vec<UrlResponse> = Vec::new();let mut discovered_urls: HashSet<Url> = HashSet::new();@@ -218,7 +222,11 @@ impl WorkerThread {if let Some(sitemap) = self.robotstxt.sitemap(&retryable_url.url).await {let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;- discovered_urls.extend(sitemap_urls);+ discovered_urls.extend(+ sitemap_urls+ .into_iter()+ .filter(|url| url.as_str().len() < MAX_URL_LENGTH),+ );}}@@ -249,14 +257,21 @@ impl WorkerThread {}}- discovered_urls.extend(res.new_urls);+ discovered_urls.extend(+ res.new_urls+ .into_iter()+ .filter(|url| url.as_str().len() < MAX_URL_LENGTH),+ );url_responses.push(res.response);}let job_response = JobResponse {domain: job.domain,url_responses,- discovered_urls: discovered_urls.into_iter().collect(),+ discovered_urls: discovered_urls+ .into_iter()+ .filter(|url| url.as_str().len() < MAX_URL_LENGTH)+ .collect(),};self.results.lock().await.push(job_response);