web search done right
Remove multiple warc writers again. It didn't seem to actually utilize the warc writers concurrently. It is easier to just spawn multiple workers on the same node in different tmux windows.
Mikkel Denker487552cc792a
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 7c3d4160..8d2f798b 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -495,18 +495,25 @@ impl CrawlDb {pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<DomainId>> {let sampled = weighted_sample(- self.domain_state.iter_mut().filter_map(|(id, state)| {+ self.domain_state.iter().filter_map(|(id, state)| {if state.status == DomainStatus::Pending {- state.status = DomainStatus::CrawlInProgress;Some((id, state.weight))} else {None}}),num_jobs,- );+ )+ .into_iter()+ .copied()+ .collect();++ for id in &sampled {+ let state = self.domain_state.get_mut(id).unwrap();+ state.status = DomainStatus::CrawlInProgress;+ }- Ok(sampled.into_iter().copied().collect())+ Ok(sampled)}pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {@@ -514,17 +521,24 @@ impl CrawlDb {for domain_id in domains {let urls = self.urls.entry(*domain_id).or_default();- let sampled = weighted_sample(+ let sampled: Vec<_> = weighted_sample(urls.iter_mut().filter_map(|(id, state)| {if state.status == UrlStatus::Pending {- state.status = UrlStatus::Crawling;Some((id, state.weight))} else {None}}),urls_per_job,- );+ )+ .into_iter()+ .copied()+ .collect();++ for id in &sampled {+ let state = urls.get_mut(id).unwrap();+ state.status = UrlStatus::Crawling;+ }let mut job = Job {domain: self.domain_ids.value(domain_id.0)?.unwrap(),diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 7b87301b..60729ddd 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -158,7 +158,7 @@ pub struct Crawler {impl Crawler {pub async fn new(config: CrawlerConfig) -> Result<Self> {let pending_commands = Arc::new(Mutex::new(VecDeque::new()));- let writer = Arc::new(WarcWriter::new(config.num_warc_writers, config.s3));+ let writer = Arc::new(WarcWriter::new(config.s3));let timeout = Duration::from_secs(config.timeout_seconds);let mut handles = Vec::new();let coordinator_host = config.coordinator_host.parse()?;diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex fb890e52..666701f7 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -22,7 +22,6 @@ use super::{CrawlDatum, Result};/// as WARC files on S3.pub struct WarcWriter {tx: async_channel::Sender<WarcWriterMessage>,- num_writers: usize,}#[derive(Debug, Clone)]@@ -130,17 +129,12 @@ async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Confi}impl WarcWriter {- pub fn new(num_writers: Option<usize>, s3: S3Config) -> Self {+ pub fn new(s3: S3Config) -> Self {let (tx, rx) = async_channel::bounded(1000);- let num_writers = num_writers.unwrap_or(1);+ tokio::spawn(writer_task(rx, s3));- for _ in 0..num_writers {- let rx = rx.clone();- tokio::spawn(writer_task(rx, s3.clone()));- }-- Self { tx, num_writers }+ Self { tx }}pub async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {@@ -150,9 +144,7 @@ impl WarcWriter {}pub async fn finish(&self) -> Result<()> {- for _ in 0..self.num_writers {- self.tx.send(WarcWriterMessage::Finish).await?;- }+ self.tx.send(WarcWriterMessage::Finish).await?;self.tx.close();while !self.tx.is_empty() {diff --git a/core/src/lib.rs b/core/src/lib.rsindex 7f7144a1..19174508 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -221,7 +221,6 @@ pub struct UserAgent {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct CrawlerConfig {- pub num_warc_writers: Option<usize>,pub num_workers: usize,pub user_agent: UserAgent,pub politeness_factor: Option<f32>,