Skip to content

web search done right

support internet archive warc files. the ordering seems to be a bit different than the ones from commoncrawl. these changes should hopefully make the parser overall more robust.

Mikkel Denker91ffe15cfdb1
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index a6a19bfe..1dac8629 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -46,6 +46,7 @@ fn download_files() {
"lambdamart.txt",
"test.zim",
"mistral-7b-instruct-v0.2.Q4_K_M.gguf",
+ "internet_archive.warc.gz",
] {
let path = Path::new(DATA_PATH).join(name);
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 1016c076..fb47a263 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -81,8 +81,8 @@ mod widgets;
#[derive(Error, Debug)]
pub enum Error {
- #[error("Failed to parse WARC file")]
- WarcParse(&'static str),
+ #[error("Failed to parse WARC file: {0}")]
+ WarcParse(String),
#[error("Encountered an empty required field ({0}) when converting to tantivy")]
EmptyField(&'static str),
diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rs
index e9f5d945..9f627d9a 100644
--- a/crates/core/src/warc.rs
+++ b/crates/core/src/warc.rs
@@ -230,7 +230,7 @@ impl Request {
url: record
.header
.get("WARC-TARGET-URI")
- .ok_or(Error::WarcParse("No target url"))?
+ .ok_or(Error::WarcParse("No target url".to_string()))?
.to_owned(),
})
}
@@ -257,7 +257,7 @@ impl FromStr for PayloadType {
"application/rss+xml" => Ok(Self::Rss),
"application/atom" => Ok(Self::Atom),
"application/atom+xml" => Ok(Self::Atom),
- _ => Err(Error::WarcParse("Unknown payload type")),
+ _ => Err(Error::WarcParse("Unknown payload type".to_string())),
}
}
}
@@ -286,7 +286,7 @@ impl Response {
let (_header, content) = content
.split_once("\r\n\r\n")
- .ok_or(Error::WarcParse("Invalid http body"))?;
+ .ok_or(Error::WarcParse("Invalid http body".to_string()))?;
Ok(Self {
body: content.to_string(),
@@ -322,7 +322,7 @@ impl Metadata {
}
}
- Err(Error::WarcParse("Failed to parse metadata").into())
+ Err(Error::WarcParse("Failed to parse metadata".to_string()).into())
}
}
@@ -346,7 +346,9 @@ impl<R: Read> RecordIterator<R> {
rtrim(&mut version);
if !version.to_uppercase().starts_with("WARC/1.") {
- return Some(Err(Error::WarcParse("Unknown WARC version").into()));
+ return Some(Err(
+ Error::WarcParse("Unknown WARC version".to_string()).into()
+ ));
}
let mut header = BTreeMap::<String, String>::new();
@@ -377,10 +379,10 @@ impl<R: Read> RecordIterator<R> {
line_buf.pop(); // remove colon
let key = line_buf;
- header.insert(key.to_ascii_uppercase(), value);
+ header.insert(key.to_uppercase(), value);
} else {
return Some(Err(Error::WarcParse(
- "All header lines must contain a colon",
+ "All header lines must contain a colon".to_string(),
)
.into()));
}
@@ -388,14 +390,18 @@ impl<R: Read> RecordIterator<R> {
let content_len = header.get("CONTENT-LENGTH");
if content_len.is_none() {
- return Some(Err(Error::WarcParse("Record has no content-length").into()));
+ return Some(Err(Error::WarcParse(
+ "Record has no content-length".to_string(),
+ )
+ .into()));
}
let content_len = content_len.unwrap().parse::<usize>();
if content_len.is_err() {
- return Some(Err(
- Error::WarcParse("Could not parse content length").into()
- ));
+ return Some(Err(Error::WarcParse(
+ "Could not parse content length".to_string(),
+ )
+ .into()));
}
let content_len = content_len.unwrap();
@@ -410,7 +416,9 @@ impl<R: Read> RecordIterator<R> {
}
if linefeed != [13, 10, 13, 10] {
- return Some(Err(Error::WarcParse("Invalid record ending").into()));
+ return Some(Err(
+ Error::WarcParse("Invalid record ending".to_string()).into()
+ ));
}
let record = RawWarcRecord { header, content };
@@ -429,6 +437,8 @@ impl<R: Read> Iterator for RecordIterator<R> {
self.num_reads += 1;
let mut request = None;
+ let mut response = None;
+ let mut metadata = None;
while let Some(item) = self.next_raw() {
if item.is_err() {
@@ -439,84 +449,73 @@ impl<R: Read> Iterator for RecordIterator<R> {
if let Some(warc_type) = item.header.get("WARC-TYPE") {
if warc_type.as_str() == "request" {
- request = Some(Request::from_raw(item));
- break;
+ if request.is_some() {
+ return Some(Err(Error::WarcParse(
+ "Already have a request but got another.".to_string(),
+ )
+ .into()));
+ }
+
+ match Request::from_raw(item) {
+ Ok(req) => {
+ request = Some(req);
+ }
+ Err(err) => return Some(Err(Error::WarcParse(err.to_string()).into())),
+ };
+ } else if warc_type.as_str() == "response" || warc_type.as_str() == "revisit" {
+ if let Some(content_type) = item.header.get("CONTENT-TYPE") {
+ if !content_type.starts_with("application/http") {
+ continue;
+ }
+ }
+
+ if response.is_some() {
+ return Some(Err(Error::WarcParse(
+ "Already have a response but got another.".to_string(),
+ )
+ .into()));
+ }
+
+ match Response::from_raw(item) {
+ Ok(res) => {
+ response = Some(res);
+ }
+ Err(err) => {
+ return Some(Err(Error::WarcParse(err.to_string()).into()));
+ }
+ };
+ } else if warc_type.as_str() == "metadata" {
+ if let Some(content_type) = item.header.get("CONTENT-TYPE") {
+ if !content_type.starts_with("application/warc-fields") {
+ continue;
+ }
+ }
+
+ if metadata.is_some() {
+ return Some(Err(Error::WarcParse(
+ "Already have metadata but got another.".to_string(),
+ )
+ .into()));
+ }
+
+ match Metadata::from_raw(item) {
+ Ok(met) => {
+ metadata = Some(met);
+ }
+ Err(err) => return Some(Err(Error::WarcParse(err.to_string()).into())),
+ }
}
}
- }
-
- let request = request?;
-
- // next 2 should be response and metadata
- let response = self.next_raw()?;
-
- if response.is_err() {
- return Some(Err(response.err().unwrap()));
- }
-
- let response = response.unwrap();
-
- match response.header.get("WARC-TYPE") {
- Some(warc_type) => {
- if warc_type.as_str() != "response" {
- return Some(Err(Error::WarcParse(
- "Expected response, got something else",
- )
- .into()));
- }
- }
- None => {
- return Some(Err(Error::WarcParse(
- "Expected response, got something else",
- )
- .into()));
- }
- }
-
- let response = Response::from_raw(response);
- let metadata = self.next_raw()?;
-
- if metadata.is_err() {
- return Some(Err(metadata.err().unwrap()));
- }
-
- let metadata = metadata.unwrap();
-
- match metadata.header.get("WARC-TYPE") {
- Some(warc_type) => {
- if warc_type.as_str() != "metadata" {
- return Some(Err(Error::WarcParse(
- "Expected metadata, got something else",
- )
- .into()));
- }
- }
- None => {
- return Some(Err(Error::WarcParse(
- "Expected metadata, got something else",
- )
- .into()));
+ if request.is_some() && response.is_some() && metadata.is_some() {
+ break;
}
}
- let metadata = Metadata::from_raw(metadata);
-
- if request.is_err() || response.is_err() || metadata.is_err() {
- return Some(Err(Error::WarcParse(
- "Request, response or metadata is error",
- )
- .into()));
- }
-
- let request = request.unwrap();
- let response = response.unwrap();
- let metadata = metadata.unwrap();
-
Some(Ok(WarcRecord {
- request,
- response,
- metadata,
+ request: request?,
+ response: response?,
+ metadata: metadata?,
}))
}
}
@@ -675,6 +674,7 @@ impl Default for WarcWriter {
#[cfg(test)]
mod tests {
use super::*;
+ use core::panic;
use flate2::write::GzEncoder;
use flate2::Compression;
use proptest::prelude::*;
@@ -725,6 +725,25 @@ mod tests {
assert_eq!(records[0].metadata.fetch_time_ms, 937);
}
+ #[test]
+ fn internet_archive_parse() {
+ if !Path::new("../../../data/internet_archive.warc.gz").exists() {
+ return;
+ }
+
+ let file = File::open("../../../data/internet_archive.warc.gz").unwrap();
+ let mut reader = BufReader::new(MultiGzDecoder::new(BufReader::new(file)));
+
+ let mut bytes = Vec::new();
+ reader.read_to_end(&mut bytes).unwrap();
+
+ for record in WarcFile::new(bytes).records() {
+ if let Err(err) = record {
+ panic!("Error: {:?}", err);
+ }
+ }
+ }
+
#[test]
fn writer_reader_invariant() {
let mut writer = WarcWriter::new();