web search done right
saving content length before replacing some characters causes an offset in the written warcfile
Mikkel Denker293527008f18
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex 657b2658..30f6ca90 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -14,6 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::time::Duration;+use crate::{warc, S3Config};use super::{CrawlDatum, Result};@@ -53,7 +55,9 @@ async fn commit(writer: warc::WarcWriter, s3: S3Config) {},) {Ok(bucket) => {- let bucket = bucket.with_path_style();+ let bucket = bucket+ .with_path_style()+ .with_request_timeout(Duration::from_secs(30 * 60));if let Err(err) = bucket.put_object_with_content_type(diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 62c761c1..05814790 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -308,7 +308,7 @@ impl Indexer {let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(warc_paths.into_iter()- .skip(config.skip_num_warc_files.unwrap_or(0))+ .skip(config.skip_warc_files.unwrap_or(0)).take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1)).into_iter()@@ -382,7 +382,7 @@ impl Indexer {|job| -> IndexPointer { job.map(&worker) },warc_paths.into_iter()- .skip(config.skip_num_warc_files.unwrap_or(0))+ .skip(config.skip_warc_files.unwrap_or(0)).take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1)).into_iter()diff --git a/core/src/lib.rs b/core/src/lib.rsindex c953d146..94a02291 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -79,7 +79,7 @@ pub mod summarizer;mod tokenizer;#[allow(unused)]mod ttl_cache;-mod warc;+pub mod warc;pub mod webgraph;pub mod webpage;mod widgets;@@ -87,7 +87,7 @@ mod widgets;#[derive(Debug, Deserialize, Clone)]pub struct IndexingMasterConfig {limit_warc_files: Option<usize>,- skip_num_warc_files: Option<usize>,+ skip_warc_files: Option<usize>,warc_source: WarcSource,workers: Vec<String>,batch_size: Option<usize>,@@ -99,7 +99,7 @@ pub struct IndexingMasterConfig {#[derive(Debug, Deserialize, Clone)]pub struct IndexingLocalConfig {limit_warc_files: Option<usize>,- skip_num_warc_files: Option<usize>,+ skip_warc_files: Option<usize>,warc_source: WarcSource,batch_size: Option<usize>,webgraph_path: Option<String>,diff --git a/core/src/warc.rs b/core/src/warc.rsindex b3085e31..ec425833 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -26,10 +26,9 @@ use std::time::Duration;use flate2::read::MultiGzDecoder;use flate2::write::GzEncoder;use flate2::Compression;-use regex::Regex;use tracing::{debug, trace};-pub(crate) struct WarcFile {+pub struct WarcFile {bytes: Vec<u8>,}@@ -303,7 +302,6 @@ impl<R: Read> RecordIterator<R> {loop {let mut line_buf = String::new();-if let Err(io) = self.reader.read_line(&mut line_buf) {return Some(Err(Error::IOError(io)));}@@ -473,18 +471,15 @@ impl WarcWriter {)?;}- let content_len = record.response.body.len() + 4; // +4 is for the \r\n\r\n between http header and body+ let body = record.response.body.as_bytes();+ let content_len = body.len() + 4; // +4 is for the \r\n\r\n between http header and bodywriter.write_all(format!("Content-Length: {content_len}\r\n").as_bytes())?;writer.write_all("\r\n".as_bytes())?;// write the http-header here if we want to in the futurewriter.write_all("\r\n\r\n".as_bytes())?;- let re = Regex::new(r"(\r\n\r\n)+").unwrap();- re.replace_all(&record.response.body, "");- let body = re.replace(&record.response.body, "");-- writer.write_all(body.as_bytes())?;+ writer.write_all(body)?;writer.write_all("\r\n\r\n".as_bytes())?;writer.write_all("WARC/1.0\r\n".as_bytes())?;@@ -521,6 +516,12 @@ impl WarcWriter {}}+impl Default for WarcWriter {+ fn default() -> Self {+ Self::new()+ }+}+#[cfg(test)]mod tests {use super::*;@@ -649,4 +650,36 @@ mod tests {assert_eq!(&records[0].response.body, utf8);assert_eq!(records[0].metadata.fetch_time_ms, 0);}++ #[test]+ fn writer_tabs() {+ let body = r#"+ this+ is+ a+ test "#;+ let mut writer = WarcWriter::new();+ let record = WarcRecord {+ request: Request {+ url: "https://a.com".to_string(),+ },+ response: Response {+ body: body.to_string(),+ payload_type: Some("text/html".to_string()),+ },+ metadata: Metadata { fetch_time_ms: 0 },+ };+ writer.write(&record).unwrap();++ let compressed = writer.finish().unwrap();+ let records: Vec<WarcRecord> = WarcFile::new(compressed)+ .records()+ .map(|res| res.unwrap())+ .collect();++ assert_eq!(records.len(), 1);+ assert_eq!(&records[0].request.url, "https://a.com");+ assert_eq!(&records[0].response.body, body);+ assert_eq!(records[0].metadata.fetch_time_ms, 0);+ }}