Skip to content

web search done right

added some tests that represents some of the features we want

Mikkel Denkerd6d10c6efb2d
diff --git a/Cargo.toml b/Cargo.toml
index cabd9585..da938651 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,7 @@ bincode = "1.3.3"
encoding_rs = "0.8.31"
flate2 = "1.0.23"
futures = "0.3.21"
+indicatif = "0.16.2"
itertools = "0.10.3"
reqwest = "0.11.10"
rusoto_core = "0.48.0"
diff --git a/src/indexer.rs b/src/indexer.rs
index e7f458e3..b74d1f70 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -1,11 +1,12 @@
use futures::prelude::*;
+use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
use rusoto_core::Region;
use rusoto_s3::{GetObjectRequest, S3Client, S3};
use std::fs::File;
use std::io::{self, BufRead};
use tokio::io::AsyncReadExt;
-use crate::warc::WarcFile;
+use crate::warc::{self, WarcFile};
use crate::webpage::Webpage;
use crate::{Config, Error, Result, WarcSource};
@@ -27,44 +28,65 @@ impl Indexer {
}
pub async fn run(self) -> Result<()> {
- let warc_source = &self.config.warc_source;
-
- stream::iter(self.warc_paths)
- .map(|warc_s3_path| async move {
- println!("{}", warc_s3_path);
- let download_config = warc_source.as_ref().expect("Indexing needs a warc source");
-
- let raw_object = match download_config {
- WarcSource::S3(config) => {
- Indexer::download_from_s3(
- warc_s3_path,
- config.name.clone(),
- config.endpoint.clone(),
- config.bucket.clone(),
- )
- .await
+ let pb = ProgressBar::new(self.warc_paths.len() as u64);
+ pb.set_style(
+ ProgressStyle::default_bar()
+ .template(
+ "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",
+ )
+ .progress_chars("#>-"),
+ );
+
+ stream::iter(self.warc_paths.into_iter().progress_with(pb))
+ .map(|warc_path| {
+ let download_config = self
+ .config
+ .warc_source
+ .as_ref()
+ .expect("Indexing needs a warc source")
+ .clone();
+
+ tokio::spawn(async move {
+ let raw_object = match download_config {
+ WarcSource::S3(config) => {
+ Indexer::download_from_s3(
+ warc_path.clone(),
+ config.name.clone(),
+ config.endpoint.clone(),
+ config.bucket.clone(),
+ )
+ .await
+ }
+ WarcSource::HTTP(config) => {
+ Indexer::download_from_http(warc_path.clone(), config.base_url.clone())
+ .await
+ }
+ };
+
+ raw_object
+ })
+ })
+ .buffer_unordered(10)
+ .map(|raw_bytes| {
+ tokio::task::spawn_blocking(move || {
+ if raw_bytes.is_err() {
+ return;
+ }
+ let raw_bytes = raw_bytes.unwrap();
+ if raw_bytes.is_err() {
+ return;
}
- WarcSource::HTTP(config) => {
- Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await
+ let raw_bytes = raw_bytes.unwrap();
+ let warc = WarcFile::new(&raw_bytes[..]);
+ for record in warc.flatten() {
+ let webpage = Webpage::parse(&record.response.body, &record.request.url);
+ // println!("TEST: {:?}", webpage.title());
+ // println!();
}
- };
-
- if raw_object.is_err() {
- return;
- }
- let raw_object = raw_object.unwrap();
-
- println!("Downloaded {} bytes", raw_object.len());
- let warc = WarcFile::new(&raw_object[..]);
- for record in warc.flatten() {
- let webpage = Webpage::parse(&record.response.body);
- println!("TEST: {:?}", webpage.text());
- println!();
- }
-
- panic!();
+ // panic!();
+ })
})
- .buffer_unordered(1)
+ .buffer_unordered(10)
.collect::<Vec<_>>()
.await;
diff --git a/src/lib.rs b/src/lib.rs
index b4f23127..5fafcc58 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -4,13 +4,17 @@ use std::num::ParseIntError;
use thiserror::Error;
mod indexer;
+mod query;
+mod ranker;
+mod search_index;
+mod searcher;
mod warc;
mod webgraph;
mod webpage;
pub use indexer::Indexer;
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
pub struct Config {
pub mode: Mode,
warc_source: Option<WarcSource>,
@@ -24,7 +28,7 @@ pub enum WarcSource {
HTTP(HttpConfig),
}
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
pub enum Mode {
Indexer,
}
diff --git a/src/query.rs b/src/query.rs
new file mode 100644
index 00000000..9a60f0ab
--- /dev/null
+++ b/src/query.rs
@@ -0,0 +1,7 @@
+pub struct Query {}
+
+impl Query {
+ pub fn parse(query: &str) -> Query {
+ todo!();
+ }
+}
diff --git a/src/ranker.rs b/src/ranker.rs
new file mode 100644
index 00000000..9e7ef8b2
--- /dev/null
+++ b/src/ranker.rs
@@ -0,0 +1 @@
+pub(crate) struct Ranker {}
diff --git a/src/search_index.rs b/src/search_index.rs
new file mode 100644
index 00000000..1e6e61e2
--- /dev/null
+++ b/src/search_index.rs
@@ -0,0 +1,124 @@
+use crate::query::Query;
+use crate::{webpage::Webpage, Result};
+use std::path::Path;
+
+fn create_schema() -> tantivy::schema::Schema {
+ todo!();
+}
+pub struct Index {
+ tantivy_index: tantivy::Index,
+}
+
+impl Index {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ todo!();
+ }
+
+ fn temporary() -> Self {
+ let schema = create_schema();
+ let tantivy_index = tantivy::Index::create_in_ram(schema);
+
+ Self { tantivy_index }
+ }
+
+ pub fn insert(&mut self, webpage: Webpage) {
+ todo!();
+ }
+
+ pub fn commit(&mut self) {
+ todo!();
+ }
+
+ pub fn search(&self, query: &Query) -> Vec<FastWebpage> {
+ todo!();
+ }
+
+ pub fn retrieve(&self, webpages: Vec<FastWebpage>) -> Vec<RetrievedWebpage> {
+ todo!();
+ }
+}
+
+pub struct FastWebpage {
+ url: String,
+}
+pub struct RetrievedWebpage {
+ title: String,
+ url: String,
+ snippet: String,
+ body: String,
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple_search() {
+ let mut index = Index::temporary();
+ let query = Query::parse("website");
+
+ let result = index.search(&query);
+ assert_eq!(result.len(), 0);
+
+ index.insert(Webpage::parse(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ ));
+ index.commit();
+
+ let result = index.search(&query);
+ assert_eq!(result.len(), 1);
+ assert_eq!(result[0].url, "https://www.example.com");
+ }
+
+ #[test]
+ fn retrieve_doc() {
+ let mut index = Index::temporary();
+ let query = Query::parse("website");
+
+ index.insert(Webpage::parse(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ ));
+ index.commit();
+
+ let result = index.search(&query);
+ let full_docs = index.retrieve(result);
+ assert_eq!(full_docs.len(), 1);
+ assert_eq!(full_docs[0].url, "https://www.example.com");
+ assert_eq!(full_docs[0].title, "Test website");
+ }
+
+ #[test]
+ fn document_not_matching() {
+ let mut index = Index::temporary();
+ let query = Query::parse("this query should not match");
+
+ index.insert(Webpage::parse(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ ));
+ index.commit();
+
+ let result = index.search(&query);
+ assert_eq!(result.len(), 0);
+ }
+}
diff --git a/src/searcher.rs b/src/searcher.rs
new file mode 100644
index 00000000..4c2a62e7
--- /dev/null
+++ b/src/searcher.rs
@@ -0,0 +1,6 @@
+use crate::{ranker::Ranker, search_index::Index};
+
+pub struct Searcher {
+ index: Index,
+ ranker: Ranker,
+}
diff --git a/src/webpage.rs b/src/webpage.rs
index 83679be1..bd921270 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -3,18 +3,20 @@ use scraper::{Html, Node, Selector};
use std::collections::BTreeMap;
#[derive(Debug)]
-pub(crate) struct Webpage {
+pub struct Webpage {
dom: Html,
+ url: String,
}
impl Webpage {
- pub(crate) fn parse(html: &str) -> Self {
+ pub fn parse(html: &str, url: &str) -> Self {
Self {
dom: Html::parse_document(html),
+ url: url.to_string(),
}
}
- pub(crate) fn links(&self) -> Vec<Link> {
+ pub fn links(&self) -> Vec<Link> {
let selector = Selector::parse("a").expect("Failed to parse selector");
self.dom
.select(&selector)
@@ -52,7 +54,7 @@ impl Webpage {
.collect::<Vec<String>>()
}
- pub(crate) fn text(&self) -> String {
+ pub fn text(&self) -> String {
let selector = Selector::parse(
"body a,
body div,
@@ -76,12 +78,20 @@ impl Webpage {
.to_string()
}
- pub(crate) fn title(&self) -> Option<String> {
+ pub fn title(&self) -> Option<String> {
let selector = Selector::parse("title").expect("Failed to parse selector");
self.grab_texts(&selector).get(0).cloned()
}
- pub(crate) fn metadata(&self) -> Vec<Meta> {
+ pub fn url(&self) -> &str {
+ &self.url
+ }
+
+ pub fn host(&self) -> &str {
+ todo!();
+ }
+
+ pub fn metadata(&self) -> Vec<Meta> {
let selector = Selector::parse("meta").expect("Failed to parse selector");
self.dom
.select(&selector)
@@ -97,12 +107,12 @@ impl Webpage {
}
#[derive(Debug, PartialEq)]
-pub(crate) struct Link {
+pub struct Link {
destination: String,
text: String,
}
-pub(crate) type Meta = BTreeMap<String, String>;
+pub type Meta = BTreeMap<String, String>;
#[cfg(test)]
mod tests {
@@ -122,7 +132,7 @@ mod tests {
</html>
"#;
- let webpage = Webpage::parse(raw);
+ let webpage = Webpage::parse(raw, "https://www.example.com/whatever");
assert_eq!(&webpage.text(), "Best example website ever");
assert_eq!(webpage.title(), Some("Best website".to_string()));
@@ -139,6 +149,8 @@ mod tests {
expected_meta.insert("content".to_string(), "value".to_string());
assert_eq!(webpage.metadata(), vec![expected_meta]);
+ assert_eq!(webpage.url(), "https://www.example.com/whatever");
+ assert_eq!(webpage.host(), "www.example.com");
}
#[test]
@@ -162,7 +174,7 @@ mod tests {
</html>
"#;
- let webpage = Webpage::parse(raw);
+ let webpage = Webpage::parse(raw, "https://www.example.com");
assert_eq!(
webpage.text(),
@@ -191,7 +203,7 @@ mod tests {
</html>
"#;
- let webpage = Webpage::parse(raw);
+ let webpage = Webpage::parse(raw, "https://www.example.com");
assert_eq!(
webpage.text(),