web search done right
Stemmed terms now show up in the generated snippets. We use the language of the webpage to determine which stemmer to use.
Mikkel Denker0ed6bc236db6
diff --git a/src/index.rs b/src/index.rsindex ad2d85c3..3d47126a 100644--- a/src/index.rs+++ b/src/index.rs@@ -215,7 +215,7 @@ impl From<Document> for RetrievedWebpage {.expect("Title field should be text").to_string()}- Field::Body => {+ Field::StemmedBody => {webpage.body = value.value.as_text()@@ -233,7 +233,7 @@ impl From<Document> for RetrievedWebpage {| Field::Centrality| Field::Host| Field::StemmedTitle- | Field::StemmedBody+ | Field::Body| Field::Domain| Field::DomainIfHomepage| Field::IsHomepagediff --git a/src/schema.rs b/src/schema.rsindex 21f1037e..b370911c 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -71,7 +71,7 @@ impl Field {pub fn options(&self) -> IndexingOption {match self {Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),- Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),+ Field::Body => IndexingOption::Text(self.default_text_options()),Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),Field::Host => IndexingOption::Text(self.default_text_options()),Field::Domain => IndexingOption::Text(self.default_text_options()),@@ -91,7 +91,8 @@ impl Field {self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),),Field::StemmedBody => IndexingOption::Text(- self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),+ self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())+ .set_stored(),),Field::FetchTimeMs => IndexingOption::Numeric(NumericOptions::default()diff --git a/src/snippet.rs b/src/snippet.rsindex d1ea49d3..c35d7b53 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -15,12 +15,14 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::frontend::search::html_escape;+use crate::tokenizer::StemmedTokenizer;use std::cmp::Ordering;use std::collections::BTreeMap;use std::ops::Range;use tantivy::tokenizer::{TextAnalyzer, Token};use tantivy::{Score, Searcher};+use whatlang::Lang;/// For now we use the snippet generator from tantivy with a minor modification to support our TokenStreamMerger./// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf.@@ -197,26 +199,33 @@ impl SnippetGenerator {searcher: &Searcher,query: &dyn tantivy::query::Query,field: tantivy::schema::Field,+ tokenizer: tantivy::tokenizer::TextAnalyzer,) -> crate::Result<SnippetGenerator> {let mut terms = BTreeMap::new();query.query_terms(&mut terms);let mut terms_text: BTreeMap<String, Score> = Default::default();- for (term, _) in terms {++ for (mut term, _) in terms {if term.field() != field {continue;}let term_str = if let Some(term_str) = term.as_str() {- term_str+ tokenizer+ .token_stream(term_str)+ .next()+ .unwrap()+ .text+ .clone()} else {continue;};+ term.set_text(&term_str);let doc_freq = searcher.doc_freq(&term)?;if doc_freq > 0 {let score = 1.0 / (1.0 + doc_freq as Score);- terms_text.insert(term_str.to_string(), score);+ terms_text.insert(term_str, score);}}- let tokenizer = searcher.index().tokenizer_for_field(field)?;Ok(SnippetGenerator {terms_text,tokenizer,@@ -240,14 +249,14 @@ impl SnippetGenerator {use crate::{query::Query, schema::Field, Result};pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Result<String> {- let generator = SnippetGenerator::create(- searcher,- query,- searcher- .schema()- .get_field(Field::Body.as_str())- .expect("Failed to get body field"),- )?;+ let field = searcher+ .schema()+ .get_field(Field::StemmedBody.as_str())+ .expect("Failed to get body field");+ let tokenizer =+ StemmedTokenizer::with_forced_language(whatlang::detect_lang(text).unwrap_or(Lang::Eng))+ .into();+ let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;let mut snippet = generator.snippet(text);@@ -315,6 +324,42 @@ Survey in 2016, 2017, and 2018."#;assert_eq!(result.documents[0].snippet, "<b>Rust</b> is a systems programming <b>language</b> sponsored by Mozilla which describes it as a \"safe, concurrent, practical <b>language</b>\", supporting functional and imperative-procedural paradigms. <b>Rust</b> is...".to_string());}+ #[test]+ fn stemmed_words_snippet_highlight() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Website for runners</title>+ </head>+ <body>+ {}+ </body>+ </html>+ "#,+ TEST_TEXT+ ),+ "https://www.example.com",+ vec![],+ 1.0,+ 500,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let searcher = Searcher::from(index);++ let result = searcher.search("describe").expect("Search failed");++ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].snippet, "Rust is a systems programming language sponsored by Mozilla which <b>describes</b> it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is...".to_string());+ }+#[test]fn test_snippet() {let terms = btreemap! {@@ -476,6 +521,7 @@ Survey in 2016, 2017, and 2018."#;let text_field = schema_builder.add_text_field("text", tantivy::schema::TEXT);let schema = schema_builder.build();let index = tantivy::Index::create_in_ram(schema);+ let tokenizer: TextAnalyzer = SimpleTokenizer.into();{// writing the segmentlet mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();@@ -495,13 +541,15 @@ Survey in 2016, 2017, and 2018."#;{let query = query_parser.parse_query("e").unwrap();let snippet_generator =- SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ SnippetGenerator::create(&searcher, &*query, text_field, tokenizer.clone())+ .unwrap();assert!(snippet_generator.terms_text().is_empty());}{let query = query_parser.parse_query("a").unwrap();let snippet_generator =- SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ SnippetGenerator::create(&searcher, &*query, text_field, tokenizer.clone())+ .unwrap();assert_eq!(&btreemap!("a".to_string() => 0.25),snippet_generator.terms_text()@@ -510,7 +558,8 @@ Survey in 2016, 2017, and 2018."#;{let query = query_parser.parse_query("a b").unwrap();let snippet_generator =- SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ SnippetGenerator::create(&searcher, &*query, text_field, tokenizer.clone())+ .unwrap();assert_eq!(&btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),snippet_generator.terms_text()@@ -519,7 +568,7 @@ Survey in 2016, 2017, and 2018."#;{let query = query_parser.parse_query("a b c").unwrap();let snippet_generator =- SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ SnippetGenerator::create(&searcher, &*query, text_field, tokenizer).unwrap();assert_eq!(&btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),snippet_generator.terms_text()@@ -546,9 +595,11 @@ Survey in 2016, 2017, and 2018."#;index_writer.commit()?;}let searcher = index.reader().unwrap().searcher();+ let tokenizer = searcher.index().tokenizer_for_field(text_field).unwrap();let query_parser = tantivy::query::QueryParser::for_index(&index, vec![text_field]);let query = query_parser.parse_query("rust design").unwrap();- let snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ let snippet_generator =+ SnippetGenerator::create(&searcher, &*query, text_field, tokenizer).unwrap();{let snippet = snippet_generator.snippet(TEST_TEXT);assert_eq!(diff --git a/src/tokenizer.rs b/src/tokenizer.rsindex 3dc2aba5..db3426d6 100644--- a/src/tokenizer.rs+++ b/src/tokenizer.rs@@ -77,12 +77,19 @@ impl NormalTokenizer {}#[derive(Clone, Default)]-pub struct StemmedTokenizer {}+pub struct StemmedTokenizer {+ force_language: Option<Lang>,+}impl StemmedTokenizer {pub fn as_str() -> &'static str {"stemmed_tokenizer"}+ pub fn with_forced_language(lang: Lang) -> Self {+ Self {+ force_language: Some(lang),+ }+ }}impl tantivy::tokenizer::Tokenizer for Tokenizer {@@ -105,10 +112,15 @@ impl tantivy::tokenizer::Tokenizer for NormalTokenizer {impl tantivy::tokenizer::Tokenizer for StemmedTokenizer {fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {let analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);- if let Some(lang) = whatlang::detect_lang(text) {- analyzer.filter(MyStemmer::from(lang).0).token_stream(text)- } else {- analyzer.token_stream(text)++ let lang = match self.force_language {+ Some(lang) => Some(lang),+ None => whatlang::detect_lang(text),+ };++ match lang {+ Some(lang) => analyzer.filter(MyStemmer::from(lang).0).token_stream(text),+ None => analyzer.token_stream(text),}}}