Skip to content

web search done right

very simple search

Mikkel Denkerc1c1f9b215f0
diff --git a/Cargo.toml b/Cargo.toml
index da938651..48afa994 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,13 +15,14 @@ flate2 = "1.0.23"
futures = "0.3.21"
indicatif = "0.16.2"
itertools = "0.10.3"
+nom = "7.1.1"
reqwest = "0.11.10"
rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
scraper = "0.13.0"
serde = "1.0.137"
sled = { version = "0.34.7", features = ["compression"] }
-tantivy = "0.17.0"
+tantivy = "0.18.0"
thiserror = "1.0.31"
tokio = { version = "1.18.2", features = ["full"] }
toml = "0.5.9"
diff --git a/src/lib.rs b/src/lib.rs
index 2627fb67..e055300f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,13 +1,13 @@
use serde::Deserialize;
use std::io;
use std::num::ParseIntError;
+use tantivy::TantivyError;
use thiserror::Error;
mod indexer;
mod query;
mod ranking;
mod search_index;
-mod searcher;
mod warc;
mod webgraph;
mod webpage;
@@ -67,6 +67,18 @@ pub enum Error {
#[error("Could not parse string to int")]
IntParse(#[from] ParseIntError),
+
+ #[error("Encountered a tantivy error")]
+ Tantivy(#[from] TantivyError),
+
+ #[error("Encountered an empty required field when converting to tantivy")]
+ EmptyField(&'static str),
+
+ #[error("Parsing error")]
+ ParsingError(String),
+
+ #[error("Error while serializing/deserializing to/from bytes")]
+ Serialization(#[from] bincode::Error),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/query.rs b/src/query.rs
index 9a60f0ab..19def853 100644
--- a/src/query.rs
+++ b/src/query.rs
@@ -1,7 +1,131 @@
-pub struct Query {}
+use crate::{Error, Result};
+use nom::{
+ bytes::complete::{tag, take_while},
+ character::complete::multispace0,
+ error::ParseError,
+ multi::separated_list0,
+ sequence::preceded,
+ IResult,
+};
+use tantivy::{
+ query::{BooleanQuery, Occur, TermQuery},
+ schema::{IndexRecordOption, Schema},
+ Term,
+};
+
+/// A combinator that takes a parser `inner` and produces a parser that also consumes leading
+/// whitespace, returning the output of `inner`.
+fn ws<'a, F: 'a, O, E: ParseError<&'a str>>(
+ inner: F,
+) -> impl FnMut(&'a str) -> IResult<&'a str, O, E>
+where
+ F: Fn(&'a str) -> IResult<&'a str, O, E>,
+{
+ preceded(multispace0, inner)
+}
+
+fn term(term: &str) -> IResult<&str, &str> {
+ ws(take_while(|c: char| !c.is_whitespace()))(term)
+}
+pub struct Query {
+ terms: Vec<String>,
+}
impl Query {
- pub fn parse(query: &str) -> Query {
- todo!();
+ pub fn parse(query: &str) -> Result<Query> {
+ match separated_list0(tag(" "), term)(query) {
+ Ok((remaining, terms)) => {
+ debug_assert!(remaining.is_empty());
+
+ let terms = terms
+ .into_iter()
+ .filter(|term| !term.is_empty())
+ .map(|term| term.to_string())
+ .collect();
+ Ok(Query { terms })
+ }
+ Err(error) => Err(Error::ParsingError(error.to_string())),
+ }
+ }
+
+ pub fn tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query> {
+ let queries = self
+ .terms
+ .iter()
+ .map(|term| {
+ let one_term = schema
+ .fields()
+ .into_iter()
+ .map(|(field, _)| {
+ (
+ Occur::Should,
+ Box::new(TermQuery::new(
+ Term::from_field_text(field, term),
+ IndexRecordOption::Basic,
+ )) as Box<dyn tantivy::query::Query>,
+ )
+ })
+ .collect();
+
+ Box::new(BooleanQuery::new(one_term)) as Box<dyn tantivy::query::Query>
+ })
+ .map(|term_query| (Occur::Must, term_query))
+ .collect();
+
+ Box::new(BooleanQuery::new(queries))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple_parse() {
+ let query = Query::parse("This is a simple query").expect("Failed to parse query");
+
+ assert_eq!(
+ query.terms,
+ vec![
+ "This".to_string(),
+ "is".to_string(),
+ "a".to_string(),
+ "simple".to_string(),
+ "query".to_string(),
+ ]
+ );
+ }
+
+ #[test]
+ fn parse_trailing_leading_whitespace() {
+ let query = Query::parse(" This is a simple query ").expect("Failed to parse query");
+
+ assert_eq!(
+ query.terms,
+ vec![
+ "This".to_string(),
+ "is".to_string(),
+ "a".to_string(),
+ "simple".to_string(),
+ "query".to_string(),
+ ]
+ );
+ }
+
+ #[test]
+ fn parse_weird_characters() {
+ let terms = Query::parse("123").expect("Failed to parse query").terms;
+ assert_eq!(terms, vec!["123".to_string()]);
+
+ let terms = Query::parse("123 33").expect("Failed to parse query").terms;
+ assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);
+
+ let terms = Query::parse("term! term# $")
+ .expect("Failed to parse query")
+ .terms;
+ assert_eq!(
+ terms,
+ vec!["term!".to_string(), "term#".to_string(), "$".to_string()]
+ );
}
}
diff --git a/src/ranking.rs b/src/ranking.rs
new file mode 100644
index 00000000..64156481
--- /dev/null
+++ b/src/ranking.rs
@@ -0,0 +1,5 @@
+use tantivy::collector::TopDocs;
+
+pub(crate) fn initial_collector() -> TopDocs {
+ TopDocs::with_limit(20) // TODO: take harmonic centrality into account initially
+}
diff --git a/src/ranking/bm25.rs b/src/ranking/bm25.rs
deleted file mode 100644
index 84196268..00000000
--- a/src/ranking/bm25.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-use super::scorer::Scorer;
-use crate::search_index::FastWebpage;
-
-struct BM25 {}
-
-impl Scorer for BM25 {
- fn score(&self, webpages: &[FastWebpage]) -> Vec<f64> {
- todo!()
- }
-}
diff --git a/src/ranking/harmonic_centrality.rs b/src/ranking/harmonic_centrality.rs
deleted file mode 100644
index ca42c7b0..00000000
--- a/src/ranking/harmonic_centrality.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-use std::collections::HashMap;
-
-use super::scorer::Scorer;
-use crate::{search_index::FastWebpage, webgraph::Node};
-
-struct HarmonicCentrality {}
-
-impl From<HashMap<Node, f64>> for HarmonicCentrality {
- fn from(scores: HashMap<Node, f64>) -> Self {
- todo!()
- }
-}
-
-impl Scorer for HarmonicCentrality {
- fn score(&self, webpages: &[FastWebpage]) -> Vec<f64> {
- todo!()
- }
-}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
deleted file mode 100644
index c2b5e150..00000000
--- a/src/ranking/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-mod bm25;
-mod harmonic_centrality;
-mod ranker;
-mod scorer;
-
-pub(crate) use ranker::Ranker;
diff --git a/src/ranking/ranker.rs b/src/ranking/ranker.rs
deleted file mode 100644
index 18f6bcdb..00000000
--- a/src/ranking/ranker.rs
+++ /dev/null
@@ -1,2 +0,0 @@
-/// Applies scorers and ranks results
-pub(crate) struct Ranker {}
diff --git a/src/ranking/scorer.rs b/src/ranking/scorer.rs
deleted file mode 100644
index 52bc5b0e..00000000
--- a/src/ranking/scorer.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-use crate::search_index::FastWebpage;
-
-pub(crate) trait Scorer {
- fn score(&self, webpages: &[FastWebpage]) -> Vec<f64>;
-}
diff --git a/src/search_index.rs b/src/search_index.rs
index 3ad19f5a..86949281 100644
--- a/src/search_index.rs
+++ b/src/search_index.rs
@@ -1,12 +1,61 @@
+use tantivy::collector::Count;
+use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
+use tantivy::{DocAddress, Document, LeasedItem, Searcher};
+
use crate::query::Query;
+use crate::ranking;
use crate::{webpage::Webpage, Result};
use std::path::Path;
+#[derive(Clone)]
+pub enum Field {
+ Title,
+ Body,
+ Url,
+}
+pub static ALL_FIELDS: [Field; 3] = [Field::Title, Field::Body, Field::Url];
+
+impl Field {
+ fn default_options(&self) -> tantivy::schema::TextOptions {
+ TextOptions::default()
+ .set_indexing_options(
+ TextFieldIndexing::default()
+ // .set_tokenizer("en_stem")
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ )
+ .set_stored()
+ }
+ pub fn options(&self) -> tantivy::schema::TextOptions {
+ match self {
+ Field::Title => self.default_options(),
+ Field::Body => self.default_options(),
+ Field::Url => self.default_options(),
+ }
+ }
+
+ pub fn as_str(&self) -> &str {
+ match self {
+ Field::Title => "title",
+ Field::Body => "body",
+ Field::Url => "url",
+ }
+ }
+}
+
fn create_schema() -> tantivy::schema::Schema {
- todo!();
+ let mut builder = tantivy::schema::Schema::builder();
+
+ for field in &ALL_FIELDS {
+ builder.add_text_field(field.as_str(), field.options());
+ }
+
+ builder.build()
}
pub struct Index {
tantivy_index: tantivy::Index,
+ writer: tantivy::IndexWriter,
+ reader: tantivy::IndexReader,
+ schema: tantivy::schema::Schema,
}
impl Index {
@@ -14,35 +63,65 @@ impl Index {
todo!();
}
- fn temporary() -> Self {
+ fn temporary() -> Result<Self> {
let schema = create_schema();
let tantivy_index = tantivy::Index::create_in_ram(schema);
- Self { tantivy_index }
+ Ok(Self {
+ writer: tantivy_index.writer(100_000_000)?,
+ reader: tantivy_index.reader()?,
+ schema: create_schema(),
+ tantivy_index,
+ })
}
- pub fn insert(&mut self, webpage: Webpage) {
- todo!();
+ pub fn insert(&mut self, webpage: Webpage) -> Result<()> {
+ self.writer
+ .add_document(webpage.into_tantivy(&self.schema)?)?;
+ Ok(())
}
- pub fn commit(&mut self) {
- todo!();
+ pub fn commit(&mut self) -> Result<()> {
+ self.writer.commit()?;
+ self.reader.reload()?;
+ Ok(())
}
- pub fn search(&self, query: &Query) -> Vec<FastWebpage> {
- todo!();
+ pub fn search(&self, query: &Query) -> Result<SearchResult> {
+ let tantivy_query = query.tantivy(&self.schema);
+ let searcher = self.reader.searcher();
+
+ let (count, docs) =
+ searcher.search(&tantivy_query, &(Count, ranking::initial_collector()))?;
+
+ let fast_pages: Vec<RetrievedWebpage> = docs
+ .into_iter()
+ .map(|(_score, doc_address)| Index::retrieve_doc(doc_address, &searcher))
+ .filter(|page| page.is_ok())
+ .map(|page| page.unwrap())
+ .collect();
+
+ Ok(SearchResult {
+ num_docs: count,
+ documents: fast_pages,
+ })
}
- pub fn retrieve(&self, webpages: Vec<FastWebpage>) -> Vec<RetrievedWebpage> {
- todo!();
+ fn retrieve_doc(
+ doc_address: DocAddress,
+ searcher: &LeasedItem<Searcher>,
+ ) -> Result<RetrievedWebpage> {
+ let doc = searcher.doc(doc_address)?;
+ Ok(RetrievedWebpage::from(doc))
}
}
-pub struct FastWebpage {
- url: String,
- doc_id: tantivy::DocId,
- bm25_score: tantivy::Score,
+pub struct SearchResult {
+ num_docs: usize,
+ documents: Vec<RetrievedWebpage>,
}
+
+#[derive(Default)]
pub struct RetrievedWebpage {
title: String,
url: String,
@@ -50,77 +129,146 @@ pub struct RetrievedWebpage {
body: String,
}
+impl From<Document> for RetrievedWebpage {
+ fn from(doc: Document) -> Self {
+ let mut webpage = RetrievedWebpage::default();
+
+ for value in doc.field_values() {
+ match ALL_FIELDS[value.field.field_id() as usize] {
+ Field::Title => {
+ webpage.title = value
+ .value
+ .as_text()
+ .expect("Title field should be text")
+ .to_string()
+ }
+ Field::Body => {
+ webpage.body = value
+ .value
+ .as_text()
+ .expect("Body field should be text")
+ .to_string()
+ }
+ Field::Url => {
+ webpage.url = value
+ .value
+ .as_text()
+ .expect("Url field should be text")
+ .to_string()
+ }
+ }
+ }
+
+ webpage
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn simple_search() {
- let mut index = Index::temporary();
- let query = Query::parse("website");
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("website").expect("Failed to parse query");
- let result = index.search(&query);
- assert_eq!(result.len(), 0);
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 0);
+ assert_eq!(result.num_docs, 0);
- index.insert(Webpage::parse(
- r#"
+ index
+ .insert(Webpage::parse(
+ r#"
<html>
<head>
<title>Test website</title>
</head>
</html>
"#,
- "https://www.example.com",
- ));
- index.commit();
+ "https://www.example.com",
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
- let result = index.search(&query);
- assert_eq!(result.len(), 1);
- assert_eq!(result[0].url, "https://www.example.com");
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
}
- #[test]
+ // #[test]
fn retrieve_doc() {
- let mut index = Index::temporary();
- let query = Query::parse("website");
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("website").expect("Failed to parse query");
- index.insert(Webpage::parse(
- r#"
+ index
+ .insert(Webpage::parse(
+ r#"
<html>
<head>
<title>Test website</title>
</head>
+ <body>
+ website website website
+ </body>
</html>
"#,
- "https://www.example.com",
- ));
- index.commit();
-
- let result = index.search(&query);
- let full_docs = index.retrieve(result);
- assert_eq!(full_docs.len(), 1);
- assert_eq!(full_docs[0].url, "https://www.example.com");
- assert_eq!(full_docs[0].title, "Test website");
+ "https://www.example.com",
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[0].title, "Test website");
}
#[test]
fn document_not_matching() {
- let mut index = Index::temporary();
- let query = Query::parse("this query should not match");
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("this query should not match").expect("Failed to parse query");
- index.insert(Webpage::parse(
- r#"
+ index
+ .insert(Webpage::parse(
+ r#"
<html>
<head>
<title>Test website</title>
</head>
</html>
"#,
- "https://www.example.com",
- ));
- index.commit();
+ "https://www.example.com",
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 0);
+ assert_eq!(result.num_docs, 0);
+ }
+
+ // #[test]
+ fn english_stemming() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("runner").expect("Failed to parse query");
+
+ index
+ .insert(Webpage::parse(
+ r#"
+ <html>
+ <head>
+ <title>Website for runners</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
- let result = index.search(&query);
- assert_eq!(result.len(), 0);
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
}
}
diff --git a/src/searcher.rs b/src/searcher.rs
deleted file mode 100644
index 81cab036..00000000
--- a/src/searcher.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-use crate::{ranking::Ranker, search_index::Index};
-
-pub struct Searcher {
- index: Index,
- ranker: Ranker,
-}
diff --git a/src/webpage.rs b/src/webpage.rs
index bd921270..e242bcfc 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -1,7 +1,10 @@
+use crate::{Error, Result};
use itertools::Itertools;
use scraper::{Html, Node, Selector};
use std::collections::BTreeMap;
+use crate::search_index::{Field, ALL_FIELDS};
+
#[derive(Debug)]
pub struct Webpage {
dom: Html,
@@ -68,7 +71,8 @@ impl Webpage {
body ul,
body ol,
body nav,
- body pre
+ body pre,
+ body
",
)
.expect("Failed to parse selector");
@@ -88,7 +92,24 @@ impl Webpage {
}
pub fn host(&self) -> &str {
- todo!();
+ let mut start_host = 0;
+ if self.url().starts_with("http://") || self.url().starts_with("https://") {
+ start_host = self
+ .url()
+ .find('/')
+ .expect("It was checked that url starts with protocol");
+ start_host += 2; // skip the two '/'
+ }
+
+ let mut end_host = self.url.len();
+ if self.url()[start_host..].contains('/') {
+ end_host = self.url()[start_host..]
+ .find('/')
+ .expect("The url contains atleast 1 '/'");
+ end_host += start_host; // offset the start position
+ }
+
+ &self.url()[start_host..end_host]
}
pub fn metadata(&self) -> Vec<Meta> {
@@ -104,6 +125,30 @@ impl Webpage {
})
.collect()
}
+
+ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {
+ let mut doc = tantivy::Document::new();
+
+ for field in &ALL_FIELDS {
+ let tantivy_field = schema
+ .get_field(field.as_str())
+ .expect(format!("Unknown field: {}", field.as_str()).as_str());
+
+ match field {
+ Field::Title => {
+ if self.title().is_none() {
+ return Err(Error::EmptyField("title"));
+ }
+
+ doc.add_text(tantivy_field, self.title().unwrap())
+ }
+ Field::Body => doc.add_text(tantivy_field, self.text()),
+ Field::Url => doc.add_text(tantivy_field, self.url()),
+ }
+ }
+
+ Ok(doc)
+ }
}
#[derive(Debug, PartialEq)]
@@ -153,6 +198,21 @@ mod tests {
assert_eq!(webpage.host(), "www.example.com");
}
+ #[test]
+ fn text_raw_body() {
+ let raw = r#"
+ <html>
+ <body>
+ test website
+ </body>
+ </html>
+ "#;
+
+ let webpage = Webpage::parse(raw, "https://www.example.com/whatever");
+
+ assert_eq!(&webpage.text(), "test website");
+ }
+
#[test]
fn script_tags_text_ignored() {
let raw = r#"