Skip to content

web search done right

28.3 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::distributed::retry_strategy::ExponentialBackoff;
use crate::{config::S3Config, config::WarcSource, Error, Result};
use std::collections::BTreeMap;
use std::fmt::Display;
use std::fs::File;
use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};
use std::path::Path;
use std::str::FromStr;
use std::thread::sleep;
use std::time::Duration;

use flate2::read::MultiGzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
use fnv::FnvHashSet;
#[cfg(test)]
use proptest::prelude::*;

use tracing::{debug, trace};

pub struct WarcFile {
    bytes: Vec<u8>,
}

fn rtrim(s: &mut String) {
    s.truncate(s.trim_end().len());
}

fn decode_string(raw: &[u8]) -> String {
    if let Ok(res) = String::from_utf8(raw.to_owned()) {
        res
    } else {
        let mut detector = chardetng::EncodingDetector::new();
        let end = std::cmp::min(64, raw.len());
        detector.feed(&raw[..end], false);
        let (enc, conf) = detector.guess_assess(None, true);

        if conf {
            let (cow, _, had_errors) = enc.decode(raw);
            if !had_errors {
                return cow.to_string();
            }
        }

        String::from_utf8_lossy(raw).to_string()
    }
}

impl WarcFile {
    pub fn new(bytes: Vec<u8>) -> Self {
        Self { bytes }
    }

    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
        let file = File::open(path)?;
        let mut reader = BufReader::new(file);

        let mut bytes = Vec::new();
        reader.read_to_end(&mut bytes)?;

        Ok(Self::new(bytes))
    }

    pub fn records(&self) -> RecordIterator<&[u8]> {
        RecordIterator {
            reader: BufReader::new(MultiGzDecoder::new(&self.bytes[..])),
            num_reads: 0,
        }
    }

    pub(crate) fn download(source: &WarcSource, warc_path: &str) -> Result<Self> {
        let mut cursor = Cursor::new(Vec::new());
        Self::download_into_buf(source, warc_path, &mut cursor)?;
        cursor.rewind()?;

        let mut buf = Vec::new();
        cursor.read_to_end(&mut buf)?;

        Ok(Self::new(buf))
    }

    pub(crate) fn download_into_buf<W: Write + Seek>(
        source: &WarcSource,
        warc_path: &str,
        buf: &mut W,
    ) -> Result<()> {
        for dur in ExponentialBackoff::from_millis(10)
            .with_limit(Duration::from_secs(30))
            .take(35)
        {
            let res = match source.clone() {
                WarcSource::HTTP(config) => {
                    WarcFile::download_from_http(warc_path, config.base_url, buf)
                }
                WarcSource::Local(config) => {
                    WarcFile::load_from_folder(warc_path, &config.folder, buf)
                }
                WarcSource::S3(config) => WarcFile::download_from_s3(warc_path, &config, buf),
            };

            if res.is_ok() {
                return Ok(());
            } else {
                trace!("Error {:?}", res);
            }

            debug!("warc download failed: {:?}", res.err().unwrap());
            debug!("retrying in {} ms", dur.as_millis());

            sleep(dur);
        }

        Err(Error::DownloadFailed.into())
    }

    fn load_from_folder<W: Write + Seek>(name: &str, folder: &str, buf: &mut W) -> Result<()> {
        let f = File::open(Path::new(folder).join(name))?;
        let mut reader = BufReader::new(f);

        buf.rewind()?;

        std::io::copy(&mut reader, buf)?;

        Ok(())
    }

    fn download_from_http<W: Write + Seek>(
        warc_path: &str,
        base_url: String,
        buf: &mut W,
    ) -> Result<()> {
        let mut url = base_url;
        if !url.ends_with('/') {
            url += "/";
        }
        url += warc_path;

        let client = reqwest::blocking::ClientBuilder::new()
            .tcp_keepalive(None)
            .pool_idle_timeout(Duration::from_secs(30 * 60))
            .timeout(Duration::from_secs(30 * 60))
            .connect_timeout(Duration::from_secs(30 * 60))
            .build()?;
        let res = client.get(url).send()?;

        if res.status().as_u16() != 200 {
            return Err(Error::DownloadFailed.into());
        }

        let bytes = res.bytes()?;

        buf.rewind()?;
        std::io::copy(&mut &bytes[..], buf)?;

        Ok(())
    }

    fn download_from_s3<W: Write + Seek>(
        warc_path: &str,
        config: &S3Config,
        buf: &mut W,
    ) -> Result<()> {
        let bucket = s3::Bucket::new(
            &config.bucket,
            s3::Region::Custom {
                region: "".to_string(),
                endpoint: config.endpoint.clone(),
            },
            s3::creds::Credentials {
                access_key: Some(config.access_key.clone()),
                secret_key: Some(config.secret_key.clone()),
                security_token: None,
                session_token: None,
                expiration: None,
            },
        )?
        .with_path_style()
        .with_request_timeout(Duration::from_secs(30 * 60))?;

        let res = bucket.get_object_blocking(warc_path)?;

        buf.write_all(res.bytes())?;

        Ok(())
    }
}

#[derive(Debug)]
struct RawWarcRecord {
    header: BTreeMap<String, String>,
    content: Vec<u8>,
}

#[derive(Debug)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct WarcRecord {
    pub request: Request,
    pub response: Response,
    pub metadata: Metadata,
}

#[cfg(test)]
impl Arbitrary for WarcRecord {
    type Parameters = ();
    type Strategy = BoxedStrategy<Self>;

    fn arbitrary_with(_args: ()) -> Self::Strategy {
        (any::<Request>(), any::<Response>(), any::<Metadata>())
            .prop_map(|(request, response, metadata)| Self {
                request,
                response,
                metadata,
            })
            .boxed()
    }
}

#[derive(Debug)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct Request {
    // WARC-Target-URI
    pub url: String,
    // WARC-Date
    pub date: Option<chrono::DateTime<chrono::Utc>>,
}

impl Request {
    fn from_raw(record: RawWarcRecord) -> Result<Self> {
        Ok(Self {
            url: record
                .header
                .get("WARC-TARGET-URI")
                .ok_or(Error::WarcParse("No target url".to_string()))?
                .to_owned(),
            date: record.header.get("WARC-DATE").and_then(|d| {
                chrono::DateTime::parse_from_rfc3339(d)
                    .ok()
                    .map(|d| d.with_timezone(&chrono::Utc))
            }),
        })
    }
}

#[cfg(test)]
impl Arbitrary for Request {
    type Parameters = ();
    type Strategy = BoxedStrategy<Self>;

    fn arbitrary_with(_args: ()) -> Self::Strategy {
        (".+", any::<u64>())
            .prop_map(|(url, timestamp)| {
                let timestamp = timestamp % 1_733_000_000;

                let date = match chrono::TimeZone::timestamp_opt(&chrono::Utc, timestamp as i64, 0)
                {
                    chrono::LocalResult::Single(date) => Some(date),
                    _ => None,
                };

                Self { url, date }
            })
            .boxed()
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PayloadType {
    Html,
    Pdf,
    Rss,
    Atom,
}

#[cfg(test)]
impl Arbitrary for PayloadType {
    type Parameters = ();
    type Strategy = BoxedStrategy<Self>;

    fn arbitrary_with(_args: ()) -> Self::Strategy {
        prop_oneof![
            Just(Self::Html),
            Just(Self::Pdf),
            Just(Self::Rss),
            Just(Self::Atom),
        ]
        .boxed()
    }
}

impl FromStr for PayloadType {
    type Err = Error;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s {
            "application/html" => Ok(Self::Html),
            "text/html" => Ok(Self::Html),
            "application/pdf" => Ok(Self::Pdf),
            "application/rss" => Ok(Self::Rss),
            "application/rss+xml" => Ok(Self::Rss),
            "application/atom" => Ok(Self::Atom),
            "application/atom+xml" => Ok(Self::Atom),
            _ => Err(Error::WarcParse("Unknown payload type".to_string())),
        }
    }
}

impl Display for PayloadType {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let str = match self {
            Self::Html => "text/html",
            Self::Pdf => "application/pdf",
            Self::Rss => "application/rss",
            Self::Atom => "application/atom",
        };
        write!(f, "{}", str)
    }
}

#[derive(Debug)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct Response {
    pub body: String,
    pub payload_type: Option<PayloadType>,
}

impl Response {
    fn from_raw(record: RawWarcRecord) -> Result<Self> {
        let content = decode_string(&record.content[..]);

        let (_header, content) = content
            .split_once("\r\n\r\n")
            .ok_or(Error::WarcParse("Invalid http body".to_string()))?;

        Ok(Self {
            body: content.to_string(),
            payload_type: record
                .header
                .get("WARC-IDENTIFIED-PAYLOAD-TYPE")
                .and_then(|p| PayloadType::from_str(p).ok()),
        })
    }
}

#[cfg(test)]
impl Arbitrary for Response {
    type Parameters = ();
    type Strategy = BoxedStrategy<Self>;

    fn arbitrary_with(_args: ()) -> Self::Strategy {
        (".+", any::<Option<PayloadType>>())
            .prop_map(|(body, payload_type)| Self { body, payload_type })
            .boxed()
    }
}

#[derive(Debug)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct Metadata {
    // fetchTimeMs
    pub fetch_time_ms: u64,
}

impl Metadata {
    fn from_raw(record: RawWarcRecord) -> Result<Self> {
        let r = BufReader::new(&record.content[..]);

        for line in r.lines() {
            let mut line = line?;
            if let Some(semi) = line.find(':') {
                let value = line.split_off(semi + 1).trim().to_string();
                line.pop(); // remove colon
                let key = line;
                if key == "fetchTimeMs" {
                    let fetch_time_ms = value.parse::<u64>()?;
                    return Ok(Self { fetch_time_ms });
                }
            }
        }

        Err(Error::WarcParse("Failed to parse metadata".to_string()).into())
    }
}

#[cfg(test)]
impl Arbitrary for Metadata {
    type Parameters = ();
    type Strategy = BoxedStrategy<Self>;

    fn arbitrary_with(_args: ()) -> Self::Strategy {
        (0..10000u64)
            .prop_map(|fetch_time_ms| Self { fetch_time_ms })
            .boxed()
    }
}

pub struct RecordIterator<R: Read> {
    reader: BufReader<MultiGzDecoder<R>>,
    num_reads: usize,
}

impl<R: Read> RecordIterator<R> {
    fn next_raw(&mut self) -> Option<Result<RawWarcRecord>> {
        let mut version = String::new();

        if let Err(_io) = self.reader.read_line(&mut version) {
            return None;
        }

        if version.is_empty() {
            return None;
        }

        rtrim(&mut version);

        if !version.to_uppercase().starts_with("WARC/1.") {
            return Some(Err(
                Error::WarcParse("Unknown WARC version".to_string()).into()
            ));
        }

        let mut header = BTreeMap::<String, String>::new();

        loop {
            let mut line_buf = String::new();
            if let Err(io) = self.reader.read_line(&mut line_buf) {
                return Some(Err(io.into()));
            }

            if &line_buf == "\r\n" || line_buf.is_empty() {
                // end of header
                break;
            }
            if let Some(semi) = line_buf.find(':') {
                let mut value = line_buf.split_off(semi + 1).to_string();

                if let Some(stripped) = value.strip_suffix("\r\n") {
                    value = stripped.to_string();
                } else if let Some(stripped) = value.strip_suffix('\n') {
                    value = stripped.to_string();
                }

                if let Some(stripped) = value.strip_prefix(' ') {
                    value = stripped.to_string();
                }

                line_buf.pop(); // remove colon
                let key = line_buf;

                header.insert(key.to_uppercase(), value);
            } else {
                return Some(Err(Error::WarcParse(
                    "All header lines must contain a colon".to_string(),
                )
                .into()));
            }
        }

        let content_len = header.get("CONTENT-LENGTH");
        if content_len.is_none() {
            return Some(Err(Error::WarcParse(
                "Record has no content-length".to_string(),
            )
            .into()));
        }

        let content_len = content_len.unwrap().parse::<usize>();
        if content_len.is_err() {
            return Some(Err(Error::WarcParse(
                "Could not parse content length".to_string(),
            )
            .into()));
        }

        let content_len = content_len.unwrap();
        let mut content = vec![0; content_len];
        if let Err(io) = self.reader.read_exact(&mut content) {
            return Some(Err(io.into()));
        }

        let mut linefeed = [0u8; 4];
        if let Err(io) = self.reader.read_exact(&mut linefeed) {
            return Some(Err(io.into()));
        }

        if linefeed != [13, 10, 13, 10] {
            return Some(Err(
                Error::WarcParse("Invalid record ending".to_string()).into()
            ));
        }

        let record = RawWarcRecord { header, content };

        Some(Ok(record))
    }
}

impl<R: Read> Iterator for RecordIterator<R> {
    type Item = Result<WarcRecord>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.num_reads == 0 {
            self.next_raw()?.ok()?; // skip warc_info
        }
        self.num_reads += 1;

        let mut request = None;
        let mut response = None;
        let mut metadata = None;

        while let Some(item) = self.next_raw() {
            if item.is_err() {
                return Some(Err(item.err().unwrap()));
            }

            let item = item.unwrap();

            if let Some(warc_type) = item.header.get("WARC-TYPE") {
                if warc_type.as_str() == "request" {
                    if request.is_some() {
                        return Some(Err(Error::WarcParse(
                            "Already have a request but got another.".to_string(),
                        )
                        .into()));
                    }

                    match Request::from_raw(item) {
                        Ok(req) => {
                            request = Some(req);
                        }
                        Err(err) => return Some(Err(Error::WarcParse(err.to_string()).into())),
                    };
                } else if warc_type.as_str() == "response" || warc_type.as_str() == "revisit" {
                    if let Some(content_type) = item.header.get("CONTENT-TYPE") {
                        if !content_type.starts_with("application/http") {
                            continue;
                        }
                    }

                    if response.is_some() {
                        return Some(Err(Error::WarcParse(
                            "Already have a response but got another.".to_string(),
                        )
                        .into()));
                    }

                    match Response::from_raw(item) {
                        Ok(res) => {
                            response = Some(res);
                        }
                        Err(err) => {
                            return Some(Err(Error::WarcParse(err.to_string()).into()));
                        }
                    };
                } else if warc_type.as_str() == "metadata" {
                    if let Some(content_type) = item.header.get("CONTENT-TYPE") {
                        if !content_type.starts_with("application/warc-fields") {
                            continue;
                        }
                    }

                    if metadata.is_some() {
                        return Some(Err(Error::WarcParse(
                            "Already have metadata but got another.".to_string(),
                        )
                        .into()));
                    }

                    match Metadata::from_raw(item) {
                        Ok(met) => {
                            metadata = Some(met);
                        }
                        Err(err) => return Some(Err(Error::WarcParse(err.to_string()).into())),
                    }
                }
            }

            if request.is_some() && response.is_some() && metadata.is_some() {
                break;
            }
        }

        Some(Ok(WarcRecord {
            request: request?,
            response: response?,
            metadata: metadata?,
        }))
    }
}

pub struct DeduplicatedWarcWriter {
    writer: WarcWriter,
    seen_url_hashes: FnvHashSet<md5::Digest>,
}

impl Default for DeduplicatedWarcWriter {
    fn default() -> Self {
        Self::new()
    }
}

impl DeduplicatedWarcWriter {
    pub fn new() -> Self {
        Self {
            writer: WarcWriter::new(),
            seen_url_hashes: FnvHashSet::default(),
        }
    }

    pub fn write(&mut self, record: &WarcRecord) -> Result<()> {
        let url_hash = md5::compute(&record.request.url);
        if self.seen_url_hashes.contains(&url_hash) {
            return Ok(());
        }

        self.seen_url_hashes.insert(url_hash);

        self.writer.write(record)
    }

    pub fn finish(self) -> Result<Vec<u8>> {
        self.writer.finish()
    }

    pub fn num_bytes(&self) -> usize {
        self.writer.num_bytes()
    }

    pub fn num_writes(&self) -> usize {
        self.writer.num_writes()
    }
}

pub struct WarcWriter {
    num_writes: usize,
    writer: GzEncoder<Vec<u8>>,
}

impl WarcWriter {
    pub fn new() -> Self {
        let mut writer = GzEncoder::new(Default::default(), Compression::best());

        writer.write_all("WARC/1.0\r\n".as_bytes()).unwrap();
        writer
            .write_all("WARC-Type: warcinfo\r\n".as_bytes())
            .unwrap();

        let date = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
        let content = format!("ISPARTOF: crawl[{}]", date);
        let content_len = content.len();

        writer
            .write_all(format!("Content-Length: {content_len}\r\n").as_bytes())
            .unwrap();
        writer.write_all("\r\n".as_bytes()).unwrap();
        writer.write_all(content.as_bytes()).unwrap();
        writer.write_all("\r\n\r\n".as_bytes()).unwrap();

        writer.flush().unwrap();

        Self {
            num_writes: 0,
            writer,
        }
    }

    pub fn write(&mut self, record: &WarcRecord) -> Result<()> {
        self.writer.write_all("WARC/1.0\r\n".as_bytes())?;

        self.writer.write_all("WARC-Type: request\r\n".as_bytes())?;

        if let Some(date) = &record.request.date {
            self.writer
                .write_all(format!("WARC-Date: {}\r\n", date.to_rfc3339()).as_bytes())?;
        }

        self.writer
            .write_all(format!("WARC-Target-URI: {}\r\n", record.request.url).as_bytes())?;
        self.writer.write_all("Content-Length: 0\r\n".as_bytes())?;
        self.writer.write_all("\r\n".as_bytes())?;
        self.writer.write_all("\r\n\r\n".as_bytes())?;

        self.writer.write_all("WARC/1.0\r\n".as_bytes())?;
        self.writer
            .write_all("WARC-Type: response\r\n".as_bytes())?;

        if let Some(payload_type) = &record.response.payload_type {
            self.writer.write_all(
                format!("WARC-Identified-Payload-Type: {payload_type}\r\n").as_bytes(),
            )?;
        }

        let body = record.response.body.as_bytes();
        let content_len = body.len() + 4; // +4 is for the \r\n\r\n between http header and body
        self.writer
            .write_all(format!("Content-Length: {content_len}\r\n").as_bytes())?;

        self.writer.write_all("\r\n".as_bytes())?;
        // write the http-header here if we want to in the future
        self.writer.write_all("\r\n\r\n".as_bytes())?;

        self.writer.write_all(body)?;
        self.writer.write_all("\r\n\r\n".as_bytes())?;

        self.writer.write_all("WARC/1.0\r\n".as_bytes())?;
        self.writer
            .write_all("WARC-Type: metadata\r\n".as_bytes())?;

        let body = format!("fetchTimeMs: {}", record.metadata.fetch_time_ms);
        let content_len = body.len();

        self.writer
            .write_all(format!("Content-Length: {content_len}\r\n").as_bytes())?;
        self.writer.write_all("\r\n".as_bytes())?;
        self.writer.write_all(body.as_bytes())?;
        self.writer.write_all("\r\n\r\n".as_bytes())?;

        self.writer.flush().unwrap();

        self.num_writes += 1;

        Ok(())
    }

    pub fn finish(self) -> Result<Vec<u8>> {
        Ok(self.writer.finish()?)
    }

    pub fn num_bytes(&self) -> usize {
        self.writer.get_ref().len()
    }

    pub fn num_writes(&self) -> usize {
        self.num_writes
    }
}

impl Default for WarcWriter {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use core::panic;

    #[test]
    fn it_works() {
        let raw = b"\
                warc/1.0\r\n\
                warc-tYPE: WARCINFO\r\n\
                cONTENT-lENGTH: 25\r\n\
                \r\n\
                ISpARToF: cc-main-2022-05\r\n\
                \r\n\
                warc/1.0\r\n\
                WARC-Target-URI: http://0575ls.cn/news-52300.htm\r\n\
                warc-tYPE: request\r\n\
                cONTENT-lENGTH: 15\r\n\
                \r\n\
                body of request\r\n\
                \r\n\
                warc/1.0\r\n\
                warc-tYPE: response\r\n\
                cONTENT-lENGTH: 29\r\n\
                \r\n\
                http-body\r\n\
                \r\n\
                body of response\r\n\
                \r\n\
                warc/1.0\r\n\
                warc-tYPE: metadata\r\n\
                cONTENT-lENGTH: 16\r\n\
                \r\n\
                fetchTimeMs: 937\r\n\
                \r\n";
        let mut e = GzEncoder::new(Vec::new(), Compression::default());
        e.write_all(raw).unwrap();
        let compressed = e.finish().unwrap();

        let records: Vec<WarcRecord> = WarcFile::new(compressed)
            .records()
            .map(|res| res.unwrap())
            .collect();

        assert_eq!(records.len(), 1);
        assert_eq!(&records[0].request.url, "http://0575ls.cn/news-52300.htm");
        assert_eq!(&records[0].response.body, "body of response");
        assert_eq!(records[0].metadata.fetch_time_ms, 937);
    }

    #[test]
    fn internet_archive_parse() {
        let data_path = Path::new("../../data/internet_archive.warc.gz");
        if !data_path.exists() {
            // Skip the test if the test data is not available
            return;
        }

        let mut records = 0;

        for record in WarcFile::open(data_path).unwrap().records() {
            records += 1;
            if let Err(err) = record {
                panic!("Error: {:?}", err);
            }
        }

        assert!(records > 0);
    }

    #[test]
    fn writer_reader_invariant() {
        let mut writer = WarcWriter::new();
        let date = chrono::Utc::now();
        let record1 = WarcRecord {
            request: Request {
                url: "https://a.com".to_string(),
                date: Some(date),
            },
            response: Response {
                body: "body of a".to_string(),
                payload_type: Some(PayloadType::Html),
            },
            metadata: Metadata {
                fetch_time_ms: 1337,
            },
        };
        writer.write(&record1).unwrap();

        let record2 = WarcRecord {
            request: Request {
                url: "https://b.com".to_string(),
                date: Some(date),
            },
            response: Response {
                body: "body of b".to_string(),
                payload_type: None,
            },
            metadata: Metadata {
                fetch_time_ms: 4242,
            },
        };
        writer.write(&record2).unwrap();

        let compressed = writer.finish().unwrap();

        let records: Vec<WarcRecord> = WarcFile::new(compressed)
            .records()
            .map(|res| res.unwrap())
            .collect();

        assert_eq!(records.len(), 2);
        assert_eq!(&records[0].request.url, "https://a.com");
        assert_eq!(records[0].request.date, Some(date));
        assert_eq!(&records[0].response.body, "body of a");
        assert_eq!(records[0].metadata.fetch_time_ms, 1337);

        assert_eq!(&records[1].request.url, "https://b.com");
        assert_eq!(records[1].request.date, Some(date));
        assert_eq!(&records[1].response.body, "body of b");
        assert_eq!(records[1].metadata.fetch_time_ms, 4242);
    }

    #[test]
    fn writer_utf8() {
        let utf8 = "๐Ÿฆ€";

        let mut writer = WarcWriter::new();
        let record = WarcRecord {
            request: Request {
                url: "https://a.com".to_string(),
                date: None,
            },
            response: Response {
                body: utf8.to_string(),
                payload_type: Some(PayloadType::Html),
            },
            metadata: Metadata { fetch_time_ms: 0 },
        };
        writer.write(&record).unwrap();

        let compressed = writer.finish().unwrap();
        let records: Vec<WarcRecord> = WarcFile::new(compressed)
            .records()
            .map(|res| res.unwrap())
            .collect();

        assert_eq!(records.len(), 1);
        assert_eq!(&records[0].request.url, "https://a.com");
        assert_eq!(&records[0].response.body, utf8);
        assert_eq!(records[0].metadata.fetch_time_ms, 0);
    }

    #[test]
    fn writer_tabs() {
        let body = r#"
               this
            is
            a
            test             "#;
        let mut writer = WarcWriter::new();
        let record = WarcRecord {
            request: Request {
                url: "https://a.com".to_string(),
                date: None,
            },
            response: Response {
                body: body.to_string(),
                payload_type: Some(PayloadType::Html),
            },
            metadata: Metadata { fetch_time_ms: 0 },
        };
        writer.write(&record).unwrap();

        let compressed = writer.finish().unwrap();
        let records: Vec<WarcRecord> = WarcFile::new(compressed)
            .records()
            .map(|res| res.unwrap())
            .collect();

        assert_eq!(records.len(), 1);
        assert_eq!(&records[0].request.url, "https://a.com");
        assert_eq!(&records[0].response.body, body);
        assert_eq!(records[0].metadata.fetch_time_ms, 0);
    }

    #[test]
    fn character_encodings() {
        for (encoding, s) in [
            (
                encoding_rs::WINDOWS_1252,
                "Groupe CROISEUR LEGER aprรจs 10 courses",
            ),
            (encoding_rs::EUC_JP, "ใ‚ใ„ใ†ใˆใŠ"),
            (encoding_rs::EUC_KR, "์•ˆ๋…•ํ•˜์„ธ์š”"),
        ] {
            let encoded = encoding.encode(s).0;
            let string = decode_string(&encoded);

            assert_eq!(s, string, "Failed for encoding {:?}", encoding.name());
        }
    }

    proptest! {
        #[test]
        fn write_read_invariant_prop(records: Vec<WarcRecord>) {
            let mut writer = WarcWriter::new();
            for record in records.iter() {
                writer.write(record).unwrap();
            }
            let compressed = writer.finish().unwrap();

            let read_records: Vec<WarcRecord> = WarcFile::new(compressed)
                .records()
                .map(|res| res.unwrap())
                .collect();

            prop_assert_eq!(records, read_records);
        }
    }
}