Skip to content

web search done right

read warc files

Mikkel Denkerd844a8594ca4
diff --git a/Cargo.toml b/Cargo.toml
index 622c6365..c3334055 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ edition = "2021"
name = "cuely"
[dependencies]
+flate2 = "1.0.23"
rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
serde = "1.0.137"
diff --git a/src/indexer.rs b/src/indexer.rs
new file mode 100644
index 00000000..3aa0c1c6
--- /dev/null
+++ b/src/indexer.rs
@@ -0,0 +1,78 @@
+use rusoto_core::Region;
+use rusoto_s3::{GetObjectRequest, S3Client, S3};
+use std::fs::File;
+use std::io::{self, BufRead};
+use tokio::io::AsyncReadExt;
+
+use crate::warc::WarcFile;
+use crate::{Config, Error, Result};
+
+pub struct Indexer {
+ warc_paths: Vec<String>,
+ config: Config,
+}
+
+impl Indexer {
+ pub fn from_config(config: Config) -> Self {
+ let file = File::open(&config.warc_paths_file).unwrap();
+ let mut warc_paths = Vec::new();
+
+ for line in io::BufReader::new(file).lines() {
+ warc_paths.push(line.unwrap());
+ }
+
+ Self { warc_paths, config }
+ }
+
+ pub async fn run(self) -> Result<()> {
+ for warc_s3_path in self.warc_paths {
+ println!("{}", warc_s3_path);
+ let raw_object = Indexer::download_from_s3(
+ warc_s3_path,
+ self.config.s3.name.clone(),
+ self.config.s3.endpoint.clone(),
+ self.config.s3.bucket.clone(),
+ )
+ .await?;
+
+ println!("Downloaded {} bytes", raw_object.len());
+ let warc = WarcFile::new(&raw_object[..]);
+ for record in warc {
+ println!("TEST: {:?}", record);
+ }
+ }
+
+ Ok(())
+ }
+
+ async fn download_from_s3(
+ key: String,
+ region_name: String,
+ region_endpoint: String,
+ bucket: String,
+ ) -> Result<Vec<u8>> {
+ let region = Region::Custom {
+ name: region_name,
+ endpoint: region_endpoint,
+ };
+
+ let client = S3Client::new(region);
+
+ let obj = client
+ .get_object(GetObjectRequest {
+ bucket,
+ key,
+ ..Default::default()
+ })
+ .await?;
+
+ let mut res = Vec::new();
+ obj.body
+ .ok_or(Error::S3DownloadError)?
+ .into_async_read()
+ .read_to_end(&mut res)
+ .await?;
+
+ Ok(res)
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index e8513a7d..3ca5c69d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,10 +1,11 @@
-use rusoto_core::Region;
-use rusoto_s3::{GetObjectRequest, S3Client, S3};
use serde::Deserialize;
-use std::fs::File;
-use std::io::{self, BufRead};
+use std::io;
use thiserror::Error;
-use tokio::io::AsyncReadExt;
+
+mod indexer;
+mod warc;
+
+pub use indexer::Indexer;
#[derive(Debug, Deserialize)]
pub struct Config {
@@ -26,7 +27,7 @@ pub struct S3Config {
}
#[derive(Error, Debug)]
-pub enum CuelyError {
+pub enum Error {
#[error("Failed to download object from S3")]
S3DownloadError,
@@ -35,70 +36,12 @@ pub enum CuelyError {
#[error("Got an IO error")]
IOError(#[from] io::Error),
-}
-
-pub struct Indexer {
- warc_paths: Vec<String>,
- config: Config,
-}
-
-type Result<T> = std::result::Result<T, CuelyError>;
-
-impl Indexer {
- pub fn from_config(config: Config) -> Self {
- let file = File::open(&config.warc_paths_file).unwrap();
- let mut warc_paths = Vec::new();
-
- for line in io::BufReader::new(file).lines() {
- warc_paths.push(line.unwrap());
- }
- Self { warc_paths, config }
- }
+ #[error("Not valid UTF8")]
+ FromUTF8(#[from] std::string::FromUtf8Error),
- pub async fn run(self) -> Result<()> {
- for warc_s3_path in self.warc_paths {
- println!("{}", warc_s3_path);
- Indexer::download_from_s3(
- warc_s3_path,
- self.config.s3.name.clone(),
- self.config.s3.endpoint.clone(),
- self.config.s3.bucket.clone(),
- )
- .await?;
- }
-
- Ok(())
- }
-
- async fn download_from_s3(
- key: String,
- region_name: String,
- region_endpoint: String,
- bucket: String,
- ) -> Result<Vec<u8>> {
- let region = Region::Custom {
- name: region_name,
- endpoint: region_endpoint,
- };
-
- let client = S3Client::new(region);
-
- let obj = client
- .get_object(GetObjectRequest {
- bucket,
- key,
- ..Default::default()
- })
- .await?;
-
- let mut res = Vec::new();
- obj.body
- .ok_or(CuelyError::S3DownloadError)?
- .into_async_read()
- .read_to_end(&mut res)
- .await?;
-
- Ok(res)
- }
+ #[error("Failed to parse WARC file")]
+ WarcParse,
}
+
+pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/warc.rs b/src/warc.rs
new file mode 100644
index 00000000..0db84072
--- /dev/null
+++ b/src/warc.rs
@@ -0,0 +1,256 @@
+use crate::{Error, Result};
+use std::collections::BTreeMap;
+use std::io::{BufRead, BufReader, Read};
+
+use flate2::read::MultiGzDecoder;
+
+pub(crate) struct WarcFile<R: Read> {
+ bytes: BufReader<MultiGzDecoder<R>>,
+ num_reads: usize,
+}
+
+fn rtrim(s: &mut String) {
+ s.truncate(s.trim_end().len());
+}
+
+impl<R: Read> WarcFile<R> {
+ pub(crate) fn new(bytes: R) -> Self {
+ Self {
+ bytes: BufReader::new(MultiGzDecoder::new(bytes)),
+ num_reads: 0,
+ }
+ }
+
+ fn next_raw(&mut self) -> Option<Result<RawWarcRecord>> {
+ let mut version = String::new();
+
+ if let Err(_io) = self.bytes.read_line(&mut version) {
+ return None;
+ }
+
+ if version.is_empty() {
+ return None;
+ }
+
+ rtrim(&mut version);
+
+ if !version.to_uppercase().starts_with("WARC/1.") {
+ return Some(Err(Error::WarcParse));
+ }
+
+ let mut header = BTreeMap::<String, String>::new();
+
+ loop {
+ let mut line_buf = String::new();
+
+ if let Err(io) = self.bytes.read_line(&mut line_buf) {
+ return Some(Err(Error::IOError(io)));
+ }
+
+ rtrim(&mut line_buf);
+
+ if &line_buf == "\r\n" || line_buf.is_empty() {
+ // end of header
+ break;
+ }
+ if let Some(semi) = line_buf.find(':') {
+ let value = line_buf.split_off(semi + 1).trim().to_string();
+ line_buf.pop(); // remove colon
+ let key = line_buf;
+
+ header.insert(key.to_ascii_uppercase(), value);
+ } else {
+ return Some(Err(Error::WarcParse));
+ }
+ }
+
+ let content_len = header.get("CONTENT-LENGTH");
+ if content_len.is_none() {
+ return Some(Err(Error::WarcParse));
+ }
+
+ let content_len = content_len.unwrap().parse::<usize>();
+ if content_len.is_err() {
+ return Some(Err(Error::WarcParse));
+ }
+
+ let content_len = content_len.unwrap();
+ let mut content = vec![0; content_len];
+ if let Err(io) = self.bytes.read_exact(&mut content) {
+ return Some(Err(Error::IOError(io)));
+ }
+
+ let mut linefeed = [0u8; 4];
+ if let Err(io) = self.bytes.read_exact(&mut linefeed) {
+ return Some(Err(Error::IOError(io)));
+ }
+
+ if linefeed != [13, 10, 13, 10] {
+ return Some(Err(Error::WarcParse));
+ }
+
+ let record = RawWarcRecord { header, content };
+
+ Some(Ok(record))
+ }
+}
+
+#[derive(Debug)]
+struct RawWarcRecord {
+ header: BTreeMap<String, String>,
+ content: Vec<u8>,
+}
+
+#[derive(Debug)]
+pub(crate) struct WarcRecord {
+ request: Request,
+ response: Response,
+}
+
+#[derive(Debug)]
+struct Request {
+ // WARC-Target-URI
+ url: String,
+}
+
+impl Request {
+ fn from_raw(record: RawWarcRecord) -> Result<Self> {
+ Ok(Self {
+ url: record
+ .header
+ .get("WARC-TARGET-URI")
+ .ok_or(Error::WarcParse)?
+ .to_owned(),
+ })
+ }
+}
+
+#[derive(Debug)]
+struct Response {
+ body: String,
+}
+
+impl Response {
+ fn from_raw(record: RawWarcRecord) -> Result<Self> {
+ Ok(Self {
+ body: String::from_utf8_lossy(&record.content).to_string(),
+ })
+ }
+}
+
+#[derive(Debug)]
+struct Metadata {
+ // fetchTimeMs
+ fetch_time_ms: usize,
+ languages: Vec<Language>,
+}
+
+#[derive(Debug)]
+struct Language {
+ name: String,
+ score: f64,
+}
+
+impl<R: BufRead> Iterator for WarcFile<R> {
+ type Item = Result<WarcRecord>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.num_reads == 0 {
+ self.next_raw().unwrap().unwrap(); // skip warc_info
+ }
+ self.num_reads += 1;
+
+ let items = [self.next_raw(), self.next_raw(), self.next_raw()];
+
+ let mut response = None;
+ let mut request = None;
+
+ for item in items {
+ if item.is_none() {
+ return None;
+ }
+ let item = item.unwrap();
+
+ if item.is_err() {
+ return Some(Err(Error::WarcParse));
+ }
+ let item = item.unwrap();
+
+ if let Some(warc_type) = item.header.get("WARC-TYPE") {
+ match warc_type.as_str() {
+ "request" => request = Some(Request::from_raw(item)),
+ "response" => response = Some(Response::from_raw(item)),
+ "metadata" => {}
+ _ => {
+ return Some(Err(Error::WarcParse));
+ }
+ }
+ }
+ }
+
+ if request.is_none() || response.is_none() {
+ return Some(Err(Error::WarcParse));
+ }
+
+ let request = request.unwrap();
+ let response = response.unwrap();
+
+ if request.is_err() || response.is_err() {
+ return Some(Err(Error::WarcParse));
+ }
+
+ let request = request.unwrap();
+ let response = response.unwrap();
+
+ Some(Ok(WarcRecord { request, response }))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use flate2::write::GzEncoder;
+ use flate2::Compression;
+ use std::io::Write;
+
+ #[test]
+ fn it_works() {
+ let raw = b"\
+ warc/1.0\r\n\
+ warc-tYPE: WARCINFO\r\n\
+ cONTENT-lENGTH: 25\r\n\
+ \r\n\
+ ISpARToF: cc-main-2022-05\r\n\
+ \r\n\
+ warc/1.0\r\n\
+ WARC-Target-URI: http://0575ls.cn/news-52300.htm\r\n\
+ warc-tYPE: request\r\n\
+ cONTENT-lENGTH: 15\r\n\
+ \r\n\
+ body of request\r\n\
+ \r\n\
+ warc/1.0\r\n\
+ warc-tYPE: response\r\n\
+ cONTENT-lENGTH: 16\r\n\
+ \r\n\
+ body of response\r\n\
+ \r\n\
+ warc/1.0\r\n\
+ warc-tYPE: metadata\r\n\
+ cONTENT-lENGTH: 16\r\n\
+ \r\n\
+ body of metadata\r\n\
+ \r\n";
+ let mut e = GzEncoder::new(Vec::new(), Compression::default());
+ e.write_all(raw).unwrap();
+ let compressed = e.finish().unwrap();
+
+ let records: Vec<WarcRecord> = WarcFile::new(&compressed[..])
+ .map(|res| res.unwrap())
+ .collect();
+
+ assert_eq!(records.len(), 1);
+ assert_eq!(&records[0].request.url, "http://0575ls.cn/news-52300.htm");
+ assert_eq!(&records[0].response.body, "body of response");
+ }
+}