web search done right
support WARC-Date in .warc files
Mikkel Denker38362b723c6a
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex e8aaddf7..020ae449 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -267,6 +267,7 @@ pub struct CrawlDatum {pub payload_type: warc::PayloadType,pub body: String,pub fetch_time_ms: u64,+ pub date: chrono::DateTime<chrono::Utc>,}pub struct Crawler {diff --git a/crates/core/src/crawler/warc_writer.rs b/crates/core/src/crawler/warc_writer.rsindex d3aa1d08..5dda021c 100644--- a/crates/core/src/crawler/warc_writer.rs+++ b/crates/core/src/crawler/warc_writer.rs@@ -114,6 +114,7 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:let warc_record = warc::WarcRecord {request: warc::Request {url: datum.url.to_string(),+ date: Some(datum.date),},response: warc::Response {body: datum.body,diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 28f9cc18..bca7805d 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -567,6 +567,7 @@ impl<S: DatumSink> JobExecutor<S> {payload_type,body: String::new(),fetch_time_ms: 0,+ date: chrono::Utc::now(),}))} else {Ok(None)@@ -607,6 +608,7 @@ impl<S: DatumSink> JobExecutor<S> {body: body?,payload_type: payload_type?,fetch_time_ms: 0,+ date: chrono::Utc::now(),})}diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rsindex 71b09997..1640daa1 100644--- a/crates/core/src/warc.rs+++ b/crates/core/src/warc.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -236,6 +236,8 @@ impl Arbitrary for WarcRecord {pub struct Request {// WARC-Target-URIpub url: String,+ // WARC-Date+ pub date: Option<chrono::DateTime<chrono::Utc>>,}impl Request {@@ -246,6 +248,11 @@ impl Request {.get("WARC-TARGET-URI").ok_or(Error::WarcParse("No target url".to_string()))?.to_owned(),+ date: record.header.get("WARC-DATE").and_then(|d| {+ chrono::DateTime::parse_from_rfc3339(d)+ .ok()+ .map(|d| d.with_timezone(&chrono::Utc))+ }),})}}@@ -256,7 +263,17 @@ impl Arbitrary for Request {type Strategy = BoxedStrategy<Self>;fn arbitrary_with(_args: ()) -> Self::Strategy {- ".+".prop_map(|url| Self { url }).boxed()+ (".+", any::<u64>())+ .prop_map(|(url, timestamp)| {+ let date = match chrono::TimeZone::timestamp_opt(&chrono::Utc, timestamp as i64, 0)+ {+ chrono::LocalResult::Single(date) => Some(date),+ _ => None,+ };++ Self { url, date }+ })+ .boxed()}}@@ -664,6 +681,12 @@ impl WarcWriter {self.writer.write_all("WARC/1.0\r\n".as_bytes())?;self.writer.write_all("WARC-Type: request\r\n".as_bytes())?;++ if let Some(date) = &record.request.date {+ self.writer+ .write_all(format!("WARC-Date: {}\r\n", date.to_rfc3339()).as_bytes())?;+ }+self.writer.write_all(format!("WARC-Target-URI: {}\r\n", record.request.url).as_bytes())?;self.writer.write_all("Content-Length: 0\r\n".as_bytes())?;@@ -804,9 +827,11 @@ mod tests {#[test]fn writer_reader_invariant() {let mut writer = WarcWriter::new();+ let date = chrono::Utc::now();let record1 = WarcRecord {request: Request {url: "https://a.com".to_string(),+ date: Some(date),},response: Response {body: "body of a".to_string(),@@ -821,6 +846,7 @@ mod tests {let record2 = WarcRecord {request: Request {url: "https://b.com".to_string(),+ date: Some(date),},response: Response {body: "body of b".to_string(),@@ -841,10 +867,12 @@ mod tests {assert_eq!(records.len(), 2);assert_eq!(&records[0].request.url, "https://a.com");+ assert_eq!(records[0].request.date, Some(date));assert_eq!(&records[0].response.body, "body of a");assert_eq!(records[0].metadata.fetch_time_ms, 1337);assert_eq!(&records[1].request.url, "https://b.com");+ assert_eq!(records[1].request.date, Some(date));assert_eq!(&records[1].response.body, "body of b");assert_eq!(records[1].metadata.fetch_time_ms, 4242);}@@ -857,6 +885,7 @@ mod tests {let record = WarcRecord {request: Request {url: "https://a.com".to_string(),+ date: None,},response: Response {body: utf8.to_string(),@@ -889,6 +918,7 @@ mod tests {let record = WarcRecord {request: Request {url: "https://a.com".to_string(),+ date: None,},response: Response {body: body.to_string(),