web search done right
moved warc download into warcfile
Mikkel Denker4433cc8965ee
diff --git a/src/indexer.rs b/src/indexer.rsindex b0bacbc8..37c84dd1 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -54,46 +54,29 @@ impl Indexer {stream::iter(self.warc_paths.into_iter().progress_with(pb)).map(|warc_path| {- let download_config = self+ let source = self.config.warc_source.as_ref().expect("Indexing needs a warc source").clone();- tokio::spawn(async move {- let raw_object = match download_config {- WarcSource::S3(config) => {- Indexer::download_from_s3(- warc_path.clone(),- config.name.clone(),- config.endpoint.clone(),- config.bucket.clone(),- )- .await- }- WarcSource::HTTP(config) => {- Indexer::download_from_http(warc_path.clone(), config.base_url.clone())- .await- }- };-- raw_object- })+ tokio::spawn(async move { WarcFile::download(source, &warc_path).await })}).buffer_unordered(20)- .map(|raw_bytes| {+ .map(|warc| {tokio::task::spawn_blocking(move || {- if raw_bytes.is_err() {+ if warc.is_err() {return;}- let raw_bytes = raw_bytes.unwrap();- if raw_bytes.is_err() {+ let warc = warc.unwrap();++ if warc.is_err() {return;}- let raw_bytes = raw_bytes.unwrap();- let warc = WarcFile::new(&raw_bytes[..]);- for record in warc.flatten() {+ let warc = warc.unwrap();++ for record in warc.records().flatten() {let _webpage = Html::parse(&record.response.body, &record.request.url);// println!("TEST: {:?}", webpage.title());// println!();diff --git a/src/warc.rs b/src/warc.rsindex 548191c8..cacf1c8c 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -13,15 +13,17 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{Error, Result};+use crate::{Error, Result, WarcSource};use std::collections::BTreeMap;use std::io::{BufRead, BufReader, Read};+use tokio::io::AsyncReadExt;use flate2::read::MultiGzDecoder;+use rusoto_core::Region;+use rusoto_s3::{GetObjectRequest, S3Client, S3};-pub(crate) struct WarcFile<R: Read> {- bytes: BufReader<MultiGzDecoder<R>>,- num_reads: usize,+pub(crate) struct WarcFile {+ bytes: Vec<u8>,}fn rtrim(s: &mut String) {@@ -52,87 +54,80 @@ fn decode(raw: Vec<u8>) -> String {}}-impl<R: Read> WarcFile<R> {- pub(crate) fn new(bytes: R) -> Self {- Self {- bytes: BufReader::new(MultiGzDecoder::new(bytes)),- num_reads: 0,- }+impl WarcFile {+ pub(crate) fn new(bytes: Vec<u8>) -> Self {+ Self { bytes }}- fn next_raw(&mut self) -> Option<Result<RawWarcRecord>> {- let mut version = String::new();-- if let Err(_io) = self.bytes.read_line(&mut version) {- return None;- }-- if version.is_empty() {- return None;- }-- rtrim(&mut version);-- if !version.to_uppercase().starts_with("WARC/1.") {- return Some(Err(Error::WarcParse("Unknown WARC version")));+ pub fn records(&self) -> RecordIterator<&[u8]> {+ RecordIterator {+ reader: BufReader::new(MultiGzDecoder::new(&self.bytes[..])),+ num_reads: 0,}+ }- let mut header = BTreeMap::<String, String>::new();-- loop {- let mut line_buf = String::new();-- if let Err(io) = self.bytes.read_line(&mut line_buf) {- return Some(Err(Error::IOError(io)));- }-- rtrim(&mut line_buf);-- if &line_buf == "\r\n" || line_buf.is_empty() {- // end of header- break;+ pub(crate) async fn download(source: WarcSource, path: &str) -> Result<Self> {+ match source {+ WarcSource::S3(config) => {+ WarcFile::download_from_s3(+ path,+ config.name.clone(),+ config.endpoint.clone(),+ config.bucket.clone(),+ )+ .await}- if let Some(semi) = line_buf.find(':') {- let value = line_buf.split_off(semi + 1).trim().to_string();- line_buf.pop(); // remove colon- let key = line_buf;-- header.insert(key.to_ascii_uppercase(), value);- } else {- return Some(Err(Error::WarcParse(- "All header lines must contain a colon",- )));+ WarcSource::HTTP(config) => {+ WarcFile::download_from_http(path, config.base_url.clone()).await}}+ }- let content_len = header.get("CONTENT-LENGTH");- if content_len.is_none() {- return Some(Err(Error::WarcParse("Record has no content-length")));+ async fn download_from_http(path: &str, base_url: String) -> Result<Self> {+ let mut url = base_url;+ if !url.ends_with('/') {+ url += "/";}+ url += path;- let content_len = content_len.unwrap().parse::<usize>();- if content_len.is_err() {- return Some(Err(Error::WarcParse("Could not parse content length")));- }+ let client = reqwest::Client::new();+ let res = client.get(url).send().await?;- let content_len = content_len.unwrap();- let mut content = vec![0; content_len];- if let Err(io) = self.bytes.read_exact(&mut content) {- return Some(Err(Error::IOError(io)));- }-- let mut linefeed = [0u8; 4];- if let Err(io) = self.bytes.read_exact(&mut linefeed) {- return Some(Err(Error::IOError(io)));- }-- if linefeed != [13, 10, 13, 10] {- return Some(Err(Error::WarcParse("Invalid record ending")));- }+ let bytes = Vec::from(&res.bytes().await?[..]);- let record = RawWarcRecord { header, content };+ Ok(WarcFile::new(bytes))+ }- Some(Ok(record))+ async fn download_from_s3(+ path: &str,+ region_name: String,+ region_endpoint: String,+ bucket: String,+ ) -> Result<Self> {+ let path = path.to_string();+ let region = Region::Custom {+ name: region_name,+ endpoint: region_endpoint,+ };++ let client = S3Client::new(region);++ let obj = client+ .get_object(GetObjectRequest {+ bucket,+ key: path,+ ..Default::default()+ })+ .await?;++ let mut bytes = Vec::new();+ obj.body+ .ok_or(Error::S3DownloadError)?+ .into_async_read()+ .read_to_end(&mut bytes)+ .await?;++ Ok(WarcFile::new(bytes))}}@@ -213,7 +208,89 @@ impl Metadata {}}-impl<R: BufRead> Iterator for WarcFile<R> {+pub(crate) struct RecordIterator<R: Read> {+ reader: BufReader<MultiGzDecoder<R>>,+ num_reads: usize,+}++impl<R: Read> RecordIterator<R> {+ fn next_raw(&mut self) -> Option<Result<RawWarcRecord>> {+ let mut version = String::new();++ if let Err(_io) = self.reader.read_line(&mut version) {+ return None;+ }++ if version.is_empty() {+ return None;+ }++ rtrim(&mut version);++ if !version.to_uppercase().starts_with("WARC/1.") {+ return Some(Err(Error::WarcParse("Unknown WARC version")));+ }++ let mut header = BTreeMap::<String, String>::new();++ loop {+ let mut line_buf = String::new();++ if let Err(io) = self.reader.read_line(&mut line_buf) {+ return Some(Err(Error::IOError(io)));+ }++ rtrim(&mut line_buf);++ if &line_buf == "\r\n" || line_buf.is_empty() {+ // end of header+ break;+ }+ if let Some(semi) = line_buf.find(':') {+ let value = line_buf.split_off(semi + 1).trim().to_string();+ line_buf.pop(); // remove colon+ let key = line_buf;++ header.insert(key.to_ascii_uppercase(), value);+ } else {+ return Some(Err(Error::WarcParse(+ "All header lines must contain a colon",+ )));+ }+ }++ let content_len = header.get("CONTENT-LENGTH");+ if content_len.is_none() {+ return Some(Err(Error::WarcParse("Record has no content-length")));+ }++ let content_len = content_len.unwrap().parse::<usize>();+ if content_len.is_err() {+ return Some(Err(Error::WarcParse("Could not parse content length")));+ }++ let content_len = content_len.unwrap();+ let mut content = vec![0; content_len];+ if let Err(io) = self.reader.read_exact(&mut content) {+ return Some(Err(Error::IOError(io)));+ }++ let mut linefeed = [0u8; 4];+ if let Err(io) = self.reader.read_exact(&mut linefeed) {+ return Some(Err(Error::IOError(io)));+ }++ if linefeed != [13, 10, 13, 10] {+ return Some(Err(Error::WarcParse("Invalid record ending")));+ }++ let record = RawWarcRecord { header, content };++ Some(Ok(record))+ }+}++impl<R: Read> Iterator for RecordIterator<R> {type Item = Result<WarcRecord>;fn next(&mut self) -> Option<Self::Item> {@@ -318,7 +395,8 @@ mod tests {e.write_all(raw).unwrap();let compressed = e.finish().unwrap();- let records: Vec<WarcRecord> = WarcFile::new(&compressed[..])+ let records: Vec<WarcRecord> = WarcFile::new(compressed)+ .records().map(|res| res.unwrap()).collect();