Skip to content

web search done right

saving content length before replacing some characters causes an offset in the written warcfile

Mikkel Denker293527008f18
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index 657b2658..30f6ca90 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -14,6 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::time::Duration;
+
use crate::{warc, S3Config};
use super::{CrawlDatum, Result};
@@ -53,7 +55,9 @@ async fn commit(writer: warc::WarcWriter, s3: S3Config) {
},
) {
Ok(bucket) => {
- let bucket = bucket.with_path_style();
+ let bucket = bucket
+ .with_path_style()
+ .with_request_timeout(Duration::from_secs(30 * 60));
if let Err(err) = bucket
.put_object_with_content_type(
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 62c761c1..05814790 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -308,7 +308,7 @@ impl Indexer {
let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
warc_paths
.into_iter()
- .skip(config.skip_num_warc_files.unwrap_or(0))
+ .skip(config.skip_warc_files.unwrap_or(0))
.take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
.into_iter()
@@ -382,7 +382,7 @@ impl Indexer {
|job| -> IndexPointer { job.map(&worker) },
warc_paths
.into_iter()
- .skip(config.skip_num_warc_files.unwrap_or(0))
+ .skip(config.skip_warc_files.unwrap_or(0))
.take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
.into_iter()
diff --git a/core/src/lib.rs b/core/src/lib.rs
index c953d146..94a02291 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -79,7 +79,7 @@ pub mod summarizer;
mod tokenizer;
#[allow(unused)]
mod ttl_cache;
-mod warc;
+pub mod warc;
pub mod webgraph;
pub mod webpage;
mod widgets;
@@ -87,7 +87,7 @@ mod widgets;
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingMasterConfig {
limit_warc_files: Option<usize>,
- skip_num_warc_files: Option<usize>,
+ skip_warc_files: Option<usize>,
warc_source: WarcSource,
workers: Vec<String>,
batch_size: Option<usize>,
@@ -99,7 +99,7 @@ pub struct IndexingMasterConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
limit_warc_files: Option<usize>,
- skip_num_warc_files: Option<usize>,
+ skip_warc_files: Option<usize>,
warc_source: WarcSource,
batch_size: Option<usize>,
webgraph_path: Option<String>,
diff --git a/core/src/warc.rs b/core/src/warc.rs
index b3085e31..ec425833 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -26,10 +26,9 @@ use std::time::Duration;
use flate2::read::MultiGzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
-use regex::Regex;
use tracing::{debug, trace};
-pub(crate) struct WarcFile {
+pub struct WarcFile {
bytes: Vec<u8>,
}
@@ -303,7 +302,6 @@ impl<R: Read> RecordIterator<R> {
loop {
let mut line_buf = String::new();
-
if let Err(io) = self.reader.read_line(&mut line_buf) {
return Some(Err(Error::IOError(io)));
}
@@ -473,18 +471,15 @@ impl WarcWriter {
)?;
}
- let content_len = record.response.body.len() + 4; // +4 is for the \r\n\r\n between http header and body
+ let body = record.response.body.as_bytes();
+ let content_len = body.len() + 4; // +4 is for the \r\n\r\n between http header and body
writer.write_all(format!("Content-Length: {content_len}\r\n").as_bytes())?;
writer.write_all("\r\n".as_bytes())?;
// write the http-header here if we want to in the future
writer.write_all("\r\n\r\n".as_bytes())?;
- let re = Regex::new(r"(\r\n\r\n)+").unwrap();
- re.replace_all(&record.response.body, "");
- let body = re.replace(&record.response.body, "");
-
- writer.write_all(body.as_bytes())?;
+ writer.write_all(body)?;
writer.write_all("\r\n\r\n".as_bytes())?;
writer.write_all("WARC/1.0\r\n".as_bytes())?;
@@ -521,6 +516,12 @@ impl WarcWriter {
}
}
+impl Default for WarcWriter {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -649,4 +650,36 @@ mod tests {
assert_eq!(&records[0].response.body, utf8);
assert_eq!(records[0].metadata.fetch_time_ms, 0);
}
+
+ #[test]
+ fn writer_tabs() {
+ let body = r#"
+ this
+ is
+ a
+ test "#;
+ let mut writer = WarcWriter::new();
+ let record = WarcRecord {
+ request: Request {
+ url: "https://a.com".to_string(),
+ },
+ response: Response {
+ body: body.to_string(),
+ payload_type: Some("text/html".to_string()),
+ },
+ metadata: Metadata { fetch_time_ms: 0 },
+ };
+ writer.write(&record).unwrap();
+
+ let compressed = writer.finish().unwrap();
+ let records: Vec<WarcRecord> = WarcFile::new(compressed)
+ .records()
+ .map(|res| res.unwrap())
+ .collect();
+
+ assert_eq!(records.len(), 1);
+ assert_eq!(&records[0].request.url, "https://a.com");
+ assert_eq!(&records[0].response.body, body);
+ assert_eq!(records[0].metadata.fetch_time_ms, 0);
+ }
}