web search done right
very simple search
Mikkel Denkerc1c1f9b215f0
diff --git a/Cargo.toml b/Cargo.tomlindex da938651..48afa994 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -15,13 +15,14 @@ flate2 = "1.0.23"futures = "0.3.21"indicatif = "0.16.2"itertools = "0.10.3"+nom = "7.1.1"reqwest = "0.11.10"rusoto_core = "0.48.0"rusoto_s3 = "0.48.0"scraper = "0.13.0"serde = "1.0.137"sled = { version = "0.34.7", features = ["compression"] }-tantivy = "0.17.0"+tantivy = "0.18.0"thiserror = "1.0.31"tokio = { version = "1.18.2", features = ["full"] }toml = "0.5.9"diff --git a/src/lib.rs b/src/lib.rsindex 2627fb67..e055300f 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -1,13 +1,13 @@use serde::Deserialize;use std::io;use std::num::ParseIntError;+use tantivy::TantivyError;use thiserror::Error;mod indexer;mod query;mod ranking;mod search_index;-mod searcher;mod warc;mod webgraph;mod webpage;@@ -67,6 +67,18 @@ pub enum Error {#[error("Could not parse string to int")]IntParse(#[from] ParseIntError),++ #[error("Encountered a tantivy error")]+ Tantivy(#[from] TantivyError),++ #[error("Encountered an empty required field when converting to tantivy")]+ EmptyField(&'static str),++ #[error("Parsing error")]+ ParsingError(String),++ #[error("Error while serializing/deserializing to/from bytes")]+ Serialization(#[from] bincode::Error),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/query.rs b/src/query.rsindex 9a60f0ab..19def853 100644--- a/src/query.rs+++ b/src/query.rs@@ -1,7 +1,131 @@-pub struct Query {}+use crate::{Error, Result};+use nom::{+ bytes::complete::{tag, take_while},+ character::complete::multispace0,+ error::ParseError,+ multi::separated_list0,+ sequence::preceded,+ IResult,+};+use tantivy::{+ query::{BooleanQuery, Occur, TermQuery},+ schema::{IndexRecordOption, Schema},+ Term,+};++/// A combinator that takes a parser `inner` and produces a parser that also consumes leading+/// whitespace, returning the output of `inner`.+fn ws<'a, F: 'a, O, E: ParseError<&'a str>>(+ inner: F,+) -> impl FnMut(&'a str) -> IResult<&'a str, O, E>+where+ F: Fn(&'a str) -> IResult<&'a str, O, E>,+{+ preceded(multispace0, inner)+}++fn term(term: &str) -> IResult<&str, &str> {+ ws(take_while(|c: char| !c.is_whitespace()))(term)+}+pub struct Query {+ terms: Vec<String>,+}impl Query {- pub fn parse(query: &str) -> Query {- todo!();+ pub fn parse(query: &str) -> Result<Query> {+ match separated_list0(tag(" "), term)(query) {+ Ok((remaining, terms)) => {+ debug_assert!(remaining.is_empty());++ let terms = terms+ .into_iter()+ .filter(|term| !term.is_empty())+ .map(|term| term.to_string())+ .collect();+ Ok(Query { terms })+ }+ Err(error) => Err(Error::ParsingError(error.to_string())),+ }+ }++ pub fn tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query> {+ let queries = self+ .terms+ .iter()+ .map(|term| {+ let one_term = schema+ .fields()+ .into_iter()+ .map(|(field, _)| {+ (+ Occur::Should,+ Box::new(TermQuery::new(+ Term::from_field_text(field, term),+ IndexRecordOption::Basic,+ )) as Box<dyn tantivy::query::Query>,+ )+ })+ .collect();++ Box::new(BooleanQuery::new(one_term)) as Box<dyn tantivy::query::Query>+ })+ .map(|term_query| (Occur::Must, term_query))+ .collect();++ Box::new(BooleanQuery::new(queries))+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple_parse() {+ let query = Query::parse("This is a simple query").expect("Failed to parse query");++ assert_eq!(+ query.terms,+ vec![+ "This".to_string(),+ "is".to_string(),+ "a".to_string(),+ "simple".to_string(),+ "query".to_string(),+ ]+ );+ }++ #[test]+ fn parse_trailing_leading_whitespace() {+ let query = Query::parse(" This is a simple query ").expect("Failed to parse query");++ assert_eq!(+ query.terms,+ vec![+ "This".to_string(),+ "is".to_string(),+ "a".to_string(),+ "simple".to_string(),+ "query".to_string(),+ ]+ );+ }++ #[test]+ fn parse_weird_characters() {+ let terms = Query::parse("123").expect("Failed to parse query").terms;+ assert_eq!(terms, vec!["123".to_string()]);++ let terms = Query::parse("123 33").expect("Failed to parse query").terms;+ assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);++ let terms = Query::parse("term! term# $")+ .expect("Failed to parse query")+ .terms;+ assert_eq!(+ terms,+ vec!["term!".to_string(), "term#".to_string(), "$".to_string()]+ );}}diff --git a/src/ranking.rs b/src/ranking.rsnew file mode 100644index 00000000..64156481--- /dev/null+++ b/src/ranking.rs@@ -0,0 +1,5 @@+use tantivy::collector::TopDocs;++pub(crate) fn initial_collector() -> TopDocs {+ TopDocs::with_limit(20) // TODO: take harmonic centrality into account initially+}diff --git a/src/ranking/bm25.rs b/src/ranking/bm25.rsdeleted file mode 100644index 84196268..00000000--- a/src/ranking/bm25.rs+++ /dev/null@@ -1,10 +0,0 @@-use super::scorer::Scorer;-use crate::search_index::FastWebpage;--struct BM25 {}--impl Scorer for BM25 {- fn score(&self, webpages: &[FastWebpage]) -> Vec<f64> {- todo!()- }-}diff --git a/src/ranking/harmonic_centrality.rs b/src/ranking/harmonic_centrality.rsdeleted file mode 100644index ca42c7b0..00000000--- a/src/ranking/harmonic_centrality.rs+++ /dev/null@@ -1,18 +0,0 @@-use std::collections::HashMap;--use super::scorer::Scorer;-use crate::{search_index::FastWebpage, webgraph::Node};--struct HarmonicCentrality {}--impl From<HashMap<Node, f64>> for HarmonicCentrality {- fn from(scores: HashMap<Node, f64>) -> Self {- todo!()- }-}--impl Scorer for HarmonicCentrality {- fn score(&self, webpages: &[FastWebpage]) -> Vec<f64> {- todo!()- }-}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsdeleted file mode 100644index c2b5e150..00000000--- a/src/ranking/mod.rs+++ /dev/null@@ -1,6 +0,0 @@-mod bm25;-mod harmonic_centrality;-mod ranker;-mod scorer;--pub(crate) use ranker::Ranker;diff --git a/src/ranking/ranker.rs b/src/ranking/ranker.rsdeleted file mode 100644index 18f6bcdb..00000000--- a/src/ranking/ranker.rs+++ /dev/null@@ -1,2 +0,0 @@-/// Applies scorers and ranks results-pub(crate) struct Ranker {}diff --git a/src/ranking/scorer.rs b/src/ranking/scorer.rsdeleted file mode 100644index 52bc5b0e..00000000--- a/src/ranking/scorer.rs+++ /dev/null@@ -1,5 +0,0 @@-use crate::search_index::FastWebpage;--pub(crate) trait Scorer {- fn score(&self, webpages: &[FastWebpage]) -> Vec<f64>;-}diff --git a/src/search_index.rs b/src/search_index.rsindex 3ad19f5a..86949281 100644--- a/src/search_index.rs+++ b/src/search_index.rs@@ -1,12 +1,61 @@+use tantivy::collector::Count;+use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};+use tantivy::{DocAddress, Document, LeasedItem, Searcher};+use crate::query::Query;+use crate::ranking;use crate::{webpage::Webpage, Result};use std::path::Path;+#[derive(Clone)]+pub enum Field {+ Title,+ Body,+ Url,+}+pub static ALL_FIELDS: [Field; 3] = [Field::Title, Field::Body, Field::Url];++impl Field {+ fn default_options(&self) -> tantivy::schema::TextOptions {+ TextOptions::default()+ .set_indexing_options(+ TextFieldIndexing::default()+ // .set_tokenizer("en_stem")+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ )+ .set_stored()+ }+ pub fn options(&self) -> tantivy::schema::TextOptions {+ match self {+ Field::Title => self.default_options(),+ Field::Body => self.default_options(),+ Field::Url => self.default_options(),+ }+ }++ pub fn as_str(&self) -> &str {+ match self {+ Field::Title => "title",+ Field::Body => "body",+ Field::Url => "url",+ }+ }+}+fn create_schema() -> tantivy::schema::Schema {- todo!();+ let mut builder = tantivy::schema::Schema::builder();++ for field in &ALL_FIELDS {+ builder.add_text_field(field.as_str(), field.options());+ }++ builder.build()}pub struct Index {tantivy_index: tantivy::Index,+ writer: tantivy::IndexWriter,+ reader: tantivy::IndexReader,+ schema: tantivy::schema::Schema,}impl Index {@@ -14,35 +63,65 @@ impl Index {todo!();}- fn temporary() -> Self {+ fn temporary() -> Result<Self> {let schema = create_schema();let tantivy_index = tantivy::Index::create_in_ram(schema);- Self { tantivy_index }+ Ok(Self {+ writer: tantivy_index.writer(100_000_000)?,+ reader: tantivy_index.reader()?,+ schema: create_schema(),+ tantivy_index,+ })}- pub fn insert(&mut self, webpage: Webpage) {- todo!();+ pub fn insert(&mut self, webpage: Webpage) -> Result<()> {+ self.writer+ .add_document(webpage.into_tantivy(&self.schema)?)?;+ Ok(())}- pub fn commit(&mut self) {- todo!();+ pub fn commit(&mut self) -> Result<()> {+ self.writer.commit()?;+ self.reader.reload()?;+ Ok(())}- pub fn search(&self, query: &Query) -> Vec<FastWebpage> {- todo!();+ pub fn search(&self, query: &Query) -> Result<SearchResult> {+ let tantivy_query = query.tantivy(&self.schema);+ let searcher = self.reader.searcher();++ let (count, docs) =+ searcher.search(&tantivy_query, &(Count, ranking::initial_collector()))?;++ let fast_pages: Vec<RetrievedWebpage> = docs+ .into_iter()+ .map(|(_score, doc_address)| Index::retrieve_doc(doc_address, &searcher))+ .filter(|page| page.is_ok())+ .map(|page| page.unwrap())+ .collect();++ Ok(SearchResult {+ num_docs: count,+ documents: fast_pages,+ })}- pub fn retrieve(&self, webpages: Vec<FastWebpage>) -> Vec<RetrievedWebpage> {- todo!();+ fn retrieve_doc(+ doc_address: DocAddress,+ searcher: &LeasedItem<Searcher>,+ ) -> Result<RetrievedWebpage> {+ let doc = searcher.doc(doc_address)?;+ Ok(RetrievedWebpage::from(doc))}}-pub struct FastWebpage {- url: String,- doc_id: tantivy::DocId,- bm25_score: tantivy::Score,+pub struct SearchResult {+ num_docs: usize,+ documents: Vec<RetrievedWebpage>,}++#[derive(Default)]pub struct RetrievedWebpage {title: String,url: String,@@ -50,77 +129,146 @@ pub struct RetrievedWebpage {body: String,}+impl From<Document> for RetrievedWebpage {+ fn from(doc: Document) -> Self {+ let mut webpage = RetrievedWebpage::default();++ for value in doc.field_values() {+ match ALL_FIELDS[value.field.field_id() as usize] {+ Field::Title => {+ webpage.title = value+ .value+ .as_text()+ .expect("Title field should be text")+ .to_string()+ }+ Field::Body => {+ webpage.body = value+ .value+ .as_text()+ .expect("Body field should be text")+ .to_string()+ }+ Field::Url => {+ webpage.url = value+ .value+ .as_text()+ .expect("Url field should be text")+ .to_string()+ }+ }+ }++ webpage+ }+}+#[cfg(test)]mod tests {use super::*;#[test]fn simple_search() {- let mut index = Index::temporary();- let query = Query::parse("website");+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("website").expect("Failed to parse query");- let result = index.search(&query);- assert_eq!(result.len(), 0);+ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 0);+ assert_eq!(result.num_docs, 0);- index.insert(Webpage::parse(- r#"+ index+ .insert(Webpage::parse(+ r#"<html><head><title>Test website</title></head></html>"#,- "https://www.example.com",- ));- index.commit();+ "https://www.example.com",+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");- let result = index.search(&query);- assert_eq!(result.len(), 1);- assert_eq!(result[0].url, "https://www.example.com");+ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.example.com");}- #[test]+ // #[test]fn retrieve_doc() {- let mut index = Index::temporary();- let query = Query::parse("website");+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("website").expect("Failed to parse query");- index.insert(Webpage::parse(- r#"+ index+ .insert(Webpage::parse(+ r#"<html><head><title>Test website</title></head>+ <body>+ website website website+ </body></html>"#,- "https://www.example.com",- ));- index.commit();-- let result = index.search(&query);- let full_docs = index.retrieve(result);- assert_eq!(full_docs.len(), 1);- assert_eq!(full_docs[0].url, "https://www.example.com");- assert_eq!(full_docs[0].title, "Test website");+ "https://www.example.com",+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.example.com");+ assert_eq!(result.documents[0].title, "Test website");}#[test]fn document_not_matching() {- let mut index = Index::temporary();- let query = Query::parse("this query should not match");+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("this query should not match").expect("Failed to parse query");- index.insert(Webpage::parse(- r#"+ index+ .insert(Webpage::parse(+ r#"<html><head><title>Test website</title></head></html>"#,- "https://www.example.com",- ));- index.commit();+ "https://www.example.com",+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 0);+ assert_eq!(result.num_docs, 0);+ }++ // #[test]+ fn english_stemming() {+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("runner").expect("Failed to parse query");++ index+ .insert(Webpage::parse(+ r#"+ <html>+ <head>+ <title>Website for runners</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");- let result = index.search(&query);- assert_eq!(result.len(), 0);+ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.example.com");}}diff --git a/src/searcher.rs b/src/searcher.rsdeleted file mode 100644index 81cab036..00000000--- a/src/searcher.rs+++ /dev/null@@ -1,6 +0,0 @@-use crate::{ranking::Ranker, search_index::Index};--pub struct Searcher {- index: Index,- ranker: Ranker,-}diff --git a/src/webpage.rs b/src/webpage.rsindex bd921270..e242bcfc 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -1,7 +1,10 @@+use crate::{Error, Result};use itertools::Itertools;use scraper::{Html, Node, Selector};use std::collections::BTreeMap;+use crate::search_index::{Field, ALL_FIELDS};+#[derive(Debug)]pub struct Webpage {dom: Html,@@ -68,7 +71,8 @@ impl Webpage {body ul,body ol,body nav,- body pre+ body pre,+ body",).expect("Failed to parse selector");@@ -88,7 +92,24 @@ impl Webpage {}pub fn host(&self) -> &str {- todo!();+ let mut start_host = 0;+ if self.url().starts_with("http://") || self.url().starts_with("https://") {+ start_host = self+ .url()+ .find('/')+ .expect("It was checked that url starts with protocol");+ start_host += 2; // skip the two '/'+ }++ let mut end_host = self.url.len();+ if self.url()[start_host..].contains('/') {+ end_host = self.url()[start_host..]+ .find('/')+ .expect("The url contains atleast 1 '/'");+ end_host += start_host; // offset the start position+ }++ &self.url()[start_host..end_host]}pub fn metadata(&self) -> Vec<Meta> {@@ -104,6 +125,30 @@ impl Webpage {}).collect()}++ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {+ let mut doc = tantivy::Document::new();++ for field in &ALL_FIELDS {+ let tantivy_field = schema+ .get_field(field.as_str())+ .expect(format!("Unknown field: {}", field.as_str()).as_str());++ match field {+ Field::Title => {+ if self.title().is_none() {+ return Err(Error::EmptyField("title"));+ }++ doc.add_text(tantivy_field, self.title().unwrap())+ }+ Field::Body => doc.add_text(tantivy_field, self.text()),+ Field::Url => doc.add_text(tantivy_field, self.url()),+ }+ }++ Ok(doc)+ }}#[derive(Debug, PartialEq)]@@ -153,6 +198,21 @@ mod tests {assert_eq!(webpage.host(), "www.example.com");}+ #[test]+ fn text_raw_body() {+ let raw = r#"+ <html>+ <body>+ test website+ </body>+ </html>+ "#;++ let webpage = Webpage::parse(raw, "https://www.example.com/whatever");++ assert_eq!(&webpage.text(), "test website");+ }+#[test]fn script_tags_text_ignored() {let raw = r#"