web search done right
multiple warc writers in crawler
Mikkel Denker0cf20766db48
diff --git a/Cargo.lock b/Cargo.lockindex 57da3d0a..61ba0dbb 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -228,6 +228,17 @@ dependencies = ["toml",]+[[package]]+name = "async-channel"+version = "1.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"+dependencies = [+ "concurrent-queue",+ "event-listener",+ "futures-core",+]+[[package]]name = "async-compression"version = "0.3.15"@@ -848,6 +859,15 @@ dependencies = ["memchr",]+[[package]]+name = "concurrent-queue"+version = "2.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"+dependencies = [+ "crossbeam-utils",+]+[[package]]name = "console"version = "0.15.5"@@ -1341,6 +1361,12 @@ dependencies = ["cc",]+[[package]]+name = "event-listener"+version = "2.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"+[[package]]name = "eventsource-stream"version = "0.2.3"@@ -4096,6 +4122,7 @@ dependencies = ["aes-gcm","anyhow","askama",+ "async-channel","async-stream","async-trait","axum",diff --git a/Cargo.toml b/Cargo.tomlindex d65d146b..1ad7eb95 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -88,4 +88,5 @@ safetensors = "0.3.1"publicsuffix = "2"robotstxt-with-cache = "0.4.0"rust-s3 = "0.33.0"-rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }\ No newline at end of file+rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }+async-channel = "1.8.0"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex 92a9ecee..bf3648cd 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -91,6 +91,7 @@ publicsuffix = { workspace = true }robotstxt-with-cache = { workspace = true }rust-s3 = { workspace = true }rusqlite = { workspace = true }+async-channel = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex d92b26a9..779eff84 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -60,14 +60,14 @@ pub enum Error {#[error("invalid politeness factor")]InvalidPolitenessFactor,- #[error("send error")]- SendError(#[from] tokio::sync::mpsc::error::SendError<WarcWriterMessage>),-#[error("sqlite error")]Sqlite(#[from] rusqlite::Error),#[error("addr parse error: {0}")]AddrParse(#[from] std::net::AddrParseError),++ #[error("async channel: {0}")]+ SendError(#[from] async_channel::SendError<WarcWriterMessage>),}type Result<T> = std::result::Result<T, Error>;@@ -137,7 +137,7 @@ pub struct Crawler {impl Crawler {pub async fn new(config: CrawlerConfig) -> Result<Self> {let pending_commands = Arc::new(Mutex::new(VecDeque::new()));- let writer = Arc::new(WarcWriter::new(config.s3));+ let writer = Arc::new(WarcWriter::new(config.num_warc_writers, config.s3));let timeout = Duration::from_secs(config.timeout_seconds);let mut handles = Vec::new();let coordinator_host = config.coordinator_host.parse()?;@@ -165,7 +165,7 @@ impl Crawler {pub async fn wait(self) {for handle in self.handles {- handle.await.unwrap();+ handle.await.ok();}self.writer.finish().await.unwrap();diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex d4f544f8..a2f41fe1 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -21,7 +21,8 @@ use super::{CrawlDatum, Result};/// The WarcWriter is responsible for storing the crawl datums/// as WARC files on S3.pub struct WarcWriter {- tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,+ tx: async_channel::Sender<WarcWriterMessage>,+ num_writers: usize,}#[derive(Debug, Clone)]@@ -70,10 +71,10 @@ async fn commit(writer: warc::WarcWriter, s3: S3Config) {}}-async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3: S3Config) {+async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Config) {let mut writer = warc::WarcWriter::new();- while let Some(message) = rx.recv().await {+ while let Ok(message) = rx.recv().await {match message {WarcWriterMessage::Crawl(datum) => {let is_pdf = datum@@ -119,12 +120,17 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:}impl WarcWriter {- pub fn new(s3: S3Config) -> Self {- let (tx, rx) = tokio::sync::mpsc::channel(1000);+ pub fn new(num_writers: Option<usize>, s3: S3Config) -> Self {+ let (tx, rx) = async_channel::bounded(1000);- tokio::spawn(writer_task(rx, s3));+ let num_writers = num_writers.unwrap_or(1);- Self { tx }+ for _ in 0..num_writers {+ let rx = rx.clone();+ tokio::spawn(writer_task(rx, s3.clone()));+ }++ Self { tx, num_writers }}pub async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {@@ -134,8 +140,14 @@ impl WarcWriter {}pub async fn finish(&self) -> Result<()> {- self.tx.send(WarcWriterMessage::Finish).await?;- self.tx.closed().await;+ for _ in 0..self.num_writers {+ self.tx.send(WarcWriterMessage::Finish).await?;+ }+ self.tx.close();++ while !self.tx.is_empty() {+ tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;+ }Ok(())}diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex f3be97ea..75bc6320 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -47,6 +47,7 @@ pub async fn coordinator(config: CrawlCoordinatorConfig) -> Result<()> {match &req.body {crate::crawler::Request::NewJobs { num_jobs } => {if coordinator.is_done() {+ tracing::info!("Crawl is done. Waiting for workers to finish.");req.respond(sonic::Response::Content(crate::crawler::Response::Done)).await.ok();diff --git a/core/src/lib.rs b/core/src/lib.rsindex 1c3ce7e1..ec03b89b 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -213,6 +213,7 @@ pub struct CrawlCoordinatorConfig {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct CrawlerConfig {+ pub num_warc_writers: Option<usize>,pub num_workers: usize,pub user_agent: String,pub politeness_factor: Option<f32>,