Skip to content

web search done right

multiple warc writers in crawler

Mikkel Denker0cf20766db48
diff --git a/Cargo.lock b/Cargo.lock
index 57da3d0a..61ba0dbb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -228,6 +228,17 @@ dependencies = [
"toml",
]
+[[package]]
+name = "async-channel"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"
+dependencies = [
+ "concurrent-queue",
+ "event-listener",
+ "futures-core",
+]
+
[[package]]
name = "async-compression"
version = "0.3.15"
@@ -848,6 +859,15 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "concurrent-queue"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
+dependencies = [
+ "crossbeam-utils",
+]
+
[[package]]
name = "console"
version = "0.15.5"
@@ -1341,6 +1361,12 @@ dependencies = [
"cc",
]
+[[package]]
+name = "event-listener"
+version = "2.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
+
[[package]]
name = "eventsource-stream"
version = "0.2.3"
@@ -4096,6 +4122,7 @@ dependencies = [
"aes-gcm",
"anyhow",
"askama",
+ "async-channel",
"async-stream",
"async-trait",
"axum",
diff --git a/Cargo.toml b/Cargo.toml
index d65d146b..1ad7eb95 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -88,4 +88,5 @@ safetensors = "0.3.1"
publicsuffix = "2"
robotstxt-with-cache = "0.4.0"
rust-s3 = "0.33.0"
-rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
\ No newline at end of file
+rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
+async-channel = "1.8.0"
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 92a9ecee..bf3648cd 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -91,6 +91,7 @@ publicsuffix = { workspace = true }
robotstxt-with-cache = { workspace = true }
rust-s3 = { workspace = true }
rusqlite = { workspace = true }
+async-channel = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index d92b26a9..779eff84 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -60,14 +60,14 @@ pub enum Error {
#[error("invalid politeness factor")]
InvalidPolitenessFactor,
- #[error("send error")]
- SendError(#[from] tokio::sync::mpsc::error::SendError<WarcWriterMessage>),
-
#[error("sqlite error")]
Sqlite(#[from] rusqlite::Error),
#[error("addr parse error: {0}")]
AddrParse(#[from] std::net::AddrParseError),
+
+ #[error("async channel: {0}")]
+ SendError(#[from] async_channel::SendError<WarcWriterMessage>),
}
type Result<T> = std::result::Result<T, Error>;
@@ -137,7 +137,7 @@ pub struct Crawler {
impl Crawler {
pub async fn new(config: CrawlerConfig) -> Result<Self> {
let pending_commands = Arc::new(Mutex::new(VecDeque::new()));
- let writer = Arc::new(WarcWriter::new(config.s3));
+ let writer = Arc::new(WarcWriter::new(config.num_warc_writers, config.s3));
let timeout = Duration::from_secs(config.timeout_seconds);
let mut handles = Vec::new();
let coordinator_host = config.coordinator_host.parse()?;
@@ -165,7 +165,7 @@ impl Crawler {
pub async fn wait(self) {
for handle in self.handles {
- handle.await.unwrap();
+ handle.await.ok();
}
self.writer.finish().await.unwrap();
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index d4f544f8..a2f41fe1 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -21,7 +21,8 @@ use super::{CrawlDatum, Result};
/// The WarcWriter is responsible for storing the crawl datums
/// as WARC files on S3.
pub struct WarcWriter {
- tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,
+ tx: async_channel::Sender<WarcWriterMessage>,
+ num_writers: usize,
}
#[derive(Debug, Clone)]
@@ -70,10 +71,10 @@ async fn commit(writer: warc::WarcWriter, s3: S3Config) {
}
}
-async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3: S3Config) {
+async fn writer_task(rx: async_channel::Receiver<WarcWriterMessage>, s3: S3Config) {
let mut writer = warc::WarcWriter::new();
- while let Some(message) = rx.recv().await {
+ while let Ok(message) = rx.recv().await {
match message {
WarcWriterMessage::Crawl(datum) => {
let is_pdf = datum
@@ -119,12 +120,17 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:
}
impl WarcWriter {
- pub fn new(s3: S3Config) -> Self {
- let (tx, rx) = tokio::sync::mpsc::channel(1000);
+ pub fn new(num_writers: Option<usize>, s3: S3Config) -> Self {
+ let (tx, rx) = async_channel::bounded(1000);
- tokio::spawn(writer_task(rx, s3));
+ let num_writers = num_writers.unwrap_or(1);
- Self { tx }
+ for _ in 0..num_writers {
+ let rx = rx.clone();
+ tokio::spawn(writer_task(rx, s3.clone()));
+ }
+
+ Self { tx, num_writers }
}
pub async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
@@ -134,8 +140,14 @@ impl WarcWriter {
}
pub async fn finish(&self) -> Result<()> {
- self.tx.send(WarcWriterMessage::Finish).await?;
- self.tx.closed().await;
+ for _ in 0..self.num_writers {
+ self.tx.send(WarcWriterMessage::Finish).await?;
+ }
+ self.tx.close();
+
+ while !self.tx.is_empty() {
+ tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
+ }
Ok(())
}
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index f3be97ea..75bc6320 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -47,6 +47,7 @@ pub async fn coordinator(config: CrawlCoordinatorConfig) -> Result<()> {
match &req.body {
crate::crawler::Request::NewJobs { num_jobs } => {
if coordinator.is_done() {
+ tracing::info!("Crawl is done. Waiting for workers to finish.");
req.respond(sonic::Response::Content(crate::crawler::Response::Done))
.await
.ok();
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 1c3ce7e1..ec03b89b 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -213,6 +213,7 @@ pub struct CrawlCoordinatorConfig {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CrawlerConfig {
+ pub num_warc_writers: Option<usize>,
pub num_workers: usize,
pub user_agent: String,
pub politeness_factor: Option<f32>,