web search done right
added some tests that represents some of the features we want
Mikkel Denkerd6d10c6efb2d
diff --git a/Cargo.toml b/Cargo.tomlindex cabd9585..da938651 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -13,6 +13,7 @@ bincode = "1.3.3"encoding_rs = "0.8.31"flate2 = "1.0.23"futures = "0.3.21"+indicatif = "0.16.2"itertools = "0.10.3"reqwest = "0.11.10"rusoto_core = "0.48.0"diff --git a/src/indexer.rs b/src/indexer.rsindex e7f458e3..b74d1f70 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -1,11 +1,12 @@use futures::prelude::*;+use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};use rusoto_core::Region;use rusoto_s3::{GetObjectRequest, S3Client, S3};use std::fs::File;use std::io::{self, BufRead};use tokio::io::AsyncReadExt;-use crate::warc::WarcFile;+use crate::warc::{self, WarcFile};use crate::webpage::Webpage;use crate::{Config, Error, Result, WarcSource};@@ -27,44 +28,65 @@ impl Indexer {}pub async fn run(self) -> Result<()> {- let warc_source = &self.config.warc_source;-- stream::iter(self.warc_paths)- .map(|warc_s3_path| async move {- println!("{}", warc_s3_path);- let download_config = warc_source.as_ref().expect("Indexing needs a warc source");-- let raw_object = match download_config {- WarcSource::S3(config) => {- Indexer::download_from_s3(- warc_s3_path,- config.name.clone(),- config.endpoint.clone(),- config.bucket.clone(),- )- .await+ let pb = ProgressBar::new(self.warc_paths.len() as u64);+ pb.set_style(+ ProgressStyle::default_bar()+ .template(+ "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",+ )+ .progress_chars("#>-"),+ );++ stream::iter(self.warc_paths.into_iter().progress_with(pb))+ .map(|warc_path| {+ let download_config = self+ .config+ .warc_source+ .as_ref()+ .expect("Indexing needs a warc source")+ .clone();++ tokio::spawn(async move {+ let raw_object = match download_config {+ WarcSource::S3(config) => {+ Indexer::download_from_s3(+ warc_path.clone(),+ config.name.clone(),+ config.endpoint.clone(),+ config.bucket.clone(),+ )+ .await+ }+ WarcSource::HTTP(config) => {+ Indexer::download_from_http(warc_path.clone(), config.base_url.clone())+ .await+ }+ };++ raw_object+ })+ })+ .buffer_unordered(10)+ .map(|raw_bytes| {+ tokio::task::spawn_blocking(move || {+ if raw_bytes.is_err() {+ return;+ }+ let raw_bytes = raw_bytes.unwrap();+ if raw_bytes.is_err() {+ return;}- WarcSource::HTTP(config) => {- Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await+ let raw_bytes = raw_bytes.unwrap();+ let warc = WarcFile::new(&raw_bytes[..]);+ for record in warc.flatten() {+ let webpage = Webpage::parse(&record.response.body, &record.request.url);+ // println!("TEST: {:?}", webpage.title());+ // println!();}- };-- if raw_object.is_err() {- return;- }- let raw_object = raw_object.unwrap();-- println!("Downloaded {} bytes", raw_object.len());- let warc = WarcFile::new(&raw_object[..]);- for record in warc.flatten() {- let webpage = Webpage::parse(&record.response.body);- println!("TEST: {:?}", webpage.text());- println!();- }-- panic!();+ // panic!();+ })})- .buffer_unordered(1)+ .buffer_unordered(10).collect::<Vec<_>>().await;diff --git a/src/lib.rs b/src/lib.rsindex b4f23127..5fafcc58 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -4,13 +4,17 @@ use std::num::ParseIntError;use thiserror::Error;mod indexer;+mod query;+mod ranker;+mod search_index;+mod searcher;mod warc;mod webgraph;mod webpage;pub use indexer::Indexer;-#[derive(Debug, Deserialize)]+#[derive(Debug, Deserialize, Clone)]pub struct Config {pub mode: Mode,warc_source: Option<WarcSource>,@@ -24,7 +28,7 @@ pub enum WarcSource {HTTP(HttpConfig),}-#[derive(Debug, Deserialize)]+#[derive(Debug, Deserialize, Clone)]pub enum Mode {Indexer,}diff --git a/src/query.rs b/src/query.rsnew file mode 100644index 00000000..9a60f0ab--- /dev/null+++ b/src/query.rs@@ -0,0 +1,7 @@+pub struct Query {}++impl Query {+ pub fn parse(query: &str) -> Query {+ todo!();+ }+}diff --git a/src/ranker.rs b/src/ranker.rsnew file mode 100644index 00000000..9e7ef8b2--- /dev/null+++ b/src/ranker.rs@@ -0,0 +1 @@+pub(crate) struct Ranker {}diff --git a/src/search_index.rs b/src/search_index.rsnew file mode 100644index 00000000..1e6e61e2--- /dev/null+++ b/src/search_index.rs@@ -0,0 +1,124 @@+use crate::query::Query;+use crate::{webpage::Webpage, Result};+use std::path::Path;++fn create_schema() -> tantivy::schema::Schema {+ todo!();+}+pub struct Index {+ tantivy_index: tantivy::Index,+}++impl Index {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ todo!();+ }++ fn temporary() -> Self {+ let schema = create_schema();+ let tantivy_index = tantivy::Index::create_in_ram(schema);++ Self { tantivy_index }+ }++ pub fn insert(&mut self, webpage: Webpage) {+ todo!();+ }++ pub fn commit(&mut self) {+ todo!();+ }++ pub fn search(&self, query: &Query) -> Vec<FastWebpage> {+ todo!();+ }++ pub fn retrieve(&self, webpages: Vec<FastWebpage>) -> Vec<RetrievedWebpage> {+ todo!();+ }+}++pub struct FastWebpage {+ url: String,+}+pub struct RetrievedWebpage {+ title: String,+ url: String,+ snippet: String,+ body: String,+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple_search() {+ let mut index = Index::temporary();+ let query = Query::parse("website");++ let result = index.search(&query);+ assert_eq!(result.len(), 0);++ index.insert(Webpage::parse(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ ));+ index.commit();++ let result = index.search(&query);+ assert_eq!(result.len(), 1);+ assert_eq!(result[0].url, "https://www.example.com");+ }++ #[test]+ fn retrieve_doc() {+ let mut index = Index::temporary();+ let query = Query::parse("website");++ index.insert(Webpage::parse(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ ));+ index.commit();++ let result = index.search(&query);+ let full_docs = index.retrieve(result);+ assert_eq!(full_docs.len(), 1);+ assert_eq!(full_docs[0].url, "https://www.example.com");+ assert_eq!(full_docs[0].title, "Test website");+ }++ #[test]+ fn document_not_matching() {+ let mut index = Index::temporary();+ let query = Query::parse("this query should not match");++ index.insert(Webpage::parse(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ ));+ index.commit();++ let result = index.search(&query);+ assert_eq!(result.len(), 0);+ }+}diff --git a/src/searcher.rs b/src/searcher.rsnew file mode 100644index 00000000..4c2a62e7--- /dev/null+++ b/src/searcher.rs@@ -0,0 +1,6 @@+use crate::{ranker::Ranker, search_index::Index};++pub struct Searcher {+ index: Index,+ ranker: Ranker,+}diff --git a/src/webpage.rs b/src/webpage.rsindex 83679be1..bd921270 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -3,18 +3,20 @@ use scraper::{Html, Node, Selector};use std::collections::BTreeMap;#[derive(Debug)]-pub(crate) struct Webpage {+pub struct Webpage {dom: Html,+ url: String,}impl Webpage {- pub(crate) fn parse(html: &str) -> Self {+ pub fn parse(html: &str, url: &str) -> Self {Self {dom: Html::parse_document(html),+ url: url.to_string(),}}- pub(crate) fn links(&self) -> Vec<Link> {+ pub fn links(&self) -> Vec<Link> {let selector = Selector::parse("a").expect("Failed to parse selector");self.dom.select(&selector)@@ -52,7 +54,7 @@ impl Webpage {.collect::<Vec<String>>()}- pub(crate) fn text(&self) -> String {+ pub fn text(&self) -> String {let selector = Selector::parse("body a,body div,@@ -76,12 +78,20 @@ impl Webpage {.to_string()}- pub(crate) fn title(&self) -> Option<String> {+ pub fn title(&self) -> Option<String> {let selector = Selector::parse("title").expect("Failed to parse selector");self.grab_texts(&selector).get(0).cloned()}- pub(crate) fn metadata(&self) -> Vec<Meta> {+ pub fn url(&self) -> &str {+ &self.url+ }++ pub fn host(&self) -> &str {+ todo!();+ }++ pub fn metadata(&self) -> Vec<Meta> {let selector = Selector::parse("meta").expect("Failed to parse selector");self.dom.select(&selector)@@ -97,12 +107,12 @@ impl Webpage {}#[derive(Debug, PartialEq)]-pub(crate) struct Link {+pub struct Link {destination: String,text: String,}-pub(crate) type Meta = BTreeMap<String, String>;+pub type Meta = BTreeMap<String, String>;#[cfg(test)]mod tests {@@ -122,7 +132,7 @@ mod tests {</html>"#;- let webpage = Webpage::parse(raw);+ let webpage = Webpage::parse(raw, "https://www.example.com/whatever");assert_eq!(&webpage.text(), "Best example website ever");assert_eq!(webpage.title(), Some("Best website".to_string()));@@ -139,6 +149,8 @@ mod tests {expected_meta.insert("content".to_string(), "value".to_string());assert_eq!(webpage.metadata(), vec![expected_meta]);+ assert_eq!(webpage.url(), "https://www.example.com/whatever");+ assert_eq!(webpage.host(), "www.example.com");}#[test]@@ -162,7 +174,7 @@ mod tests {</html>"#;- let webpage = Webpage::parse(raw);+ let webpage = Webpage::parse(raw, "https://www.example.com");assert_eq!(webpage.text(),@@ -191,7 +203,7 @@ mod tests {</html>"#;- let webpage = Webpage::parse(raw);+ let webpage = Webpage::parse(raw, "https://www.example.com");assert_eq!(webpage.text(),