web search done right
Custom tokenizer that stems multiple languages and fixed snippet generation as a result of the changed tokenizer
Mikkel Denkerc8b51db9efa0
diff --git a/src/snippet.rs b/src/snippet.rsindex f83ede79..99d6b514 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -14,14 +14,270 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::SnippetGenerator;+use std::cmp::Ordering;+use std::collections::BTreeMap;+use std::ops::Range;-use crate::{query::Query, schema::Field, Result};+use htmlescape::encode_minimal;++use tantivy::tokenizer::{TextAnalyzer, Token};+use tantivy::{Document, Score, Searcher};-/// For now we use the snippet generator from tantivy.+/// For now we use the snippet generator from tantivy with a minor modification to support our TokenStreamMerger./// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf./// This will require us to store each paragraph of the webpage separately to get adequate performance./// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.++const DEFAULT_MAX_NUM_CHARS: usize = 150;++#[derive(Debug)]+struct FragmentCandidate {+ score: Score,+ start_offset: usize,+ stop_offset: usize,+ highlighted: Vec<Range<usize>>,+}++impl FragmentCandidate {+ /// Create a basic `FragmentCandidate`+ ///+ /// `score`, `num_chars` are set to 0+ /// and `highlighted` is set to empty vec+ /// stop_offset is set to start_offset, which is taken as a param.+ fn new(start_offset: usize) -> FragmentCandidate {+ FragmentCandidate {+ score: 0.0,+ start_offset,+ stop_offset: start_offset,+ highlighted: vec![],+ }+ }++ /// Updates `score` and `highlighted` fields of the objects.+ ///+ /// taking the token and terms, the token is added to the fragment.+ /// if the token is one of the terms, the score+ /// and highlighted fields are updated in the fragment.+ fn try_add_token(&mut self, token: &Token, terms: &BTreeMap<String, Score>) {+ self.stop_offset = token.offset_to;++ if let Some(&score) = terms.get(&token.text.to_lowercase()) {+ self.score += score;+ self.highlighted.push(token.offset_from..token.offset_to);+ }+ }+}++/// `Snippet`+/// Contains a fragment of a document, and some highlighed parts inside it.+#[derive(Debug)]+struct Snippet {+ fragment: String,+ highlighted: Vec<Range<usize>>,+}++const HIGHLIGHTEN_PREFIX: &str = "<b>";+const HIGHLIGHTEN_POSTFIX: &str = "</b>";++impl Snippet {+ /// Create a new, empty, `Snippet`+ fn empty() -> Snippet {+ Snippet {+ fragment: String::new(),+ highlighted: Vec::new(),+ }+ }++ /// Returns a hignlightned html from the `Snippet`.+ fn to_html(&self) -> String {+ let mut html = String::new();+ let mut start_from: usize = 0;++ for item in self.highlighted.iter() {+ if item.start < start_from {+ start_from = item.end;+ continue;+ }+ html.push_str(&encode_minimal(&self.fragment[start_from..item.start]));+ html.push_str(HIGHLIGHTEN_PREFIX);+ html.push_str(&encode_minimal(&self.fragment[item.clone()]));+ html.push_str(HIGHLIGHTEN_POSTFIX);+ start_from = item.end;+ }+ html.push_str(&encode_minimal(+ &self.fragment[start_from..self.fragment.len()],+ ));+ html+ }++ /// Returns the fragment of text used in the snippet.+ fn fragment(&self) -> &str {+ &self.fragment+ }++ /// Returns a list of higlighted positions from the `Snippet`.+ fn highlighted(&self) -> &[Range<usize>] {+ &self.highlighted+ }+}++/// Returns a non-empty list of "good" fragments.+///+/// If no target term is within the text, then the function+/// should return an empty Vec.+///+/// If a target term is within the text, then the returned+/// list is required to be non-empty.+///+/// The returned list is non-empty and contain less+/// than 12 possibly overlapping fragments.+///+/// All fragments should contain at least one target term+/// and have at most `max_num_chars` characters (not bytes).+///+/// It is ok to emit non-overlapping fragments, for instance,+/// one short and one long containing the same keyword, in order+/// to leave optimization opportunity to the fragment selector+/// upstream.+///+/// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\+/// has to be a valid string.+fn search_fragments<'a>(+ tokenizer: &TextAnalyzer,+ text: &'a str,+ terms: &BTreeMap<String, Score>,+ max_num_chars: usize,+) -> Vec<FragmentCandidate> {+ let mut token_stream = tokenizer.token_stream(text);+ let mut fragment = FragmentCandidate::new(0);+ let mut fragments: Vec<FragmentCandidate> = vec![];+ while let Some(next) = token_stream.next() {+ if (next.offset_to - fragment.start_offset) > max_num_chars {+ if fragment.score > 0.0 {+ fragments.push(fragment)+ };+ fragment = FragmentCandidate::new(next.offset_from);+ }+ fragment.try_add_token(next, terms);+ }+ if fragment.score > 0.0 {+ fragments.push(fragment)+ }++ fragments+}++/// Returns a Snippet+///+/// Takes a vector of `FragmentCandidate`s and the text.+/// Figures out the best fragment from it and creates a snippet.+fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str) -> Snippet {+ let best_fragment_opt = fragments.iter().max_by(|left, right| {+ let cmp_score = left+ .score+ .partial_cmp(&right.score)+ .unwrap_or(Ordering::Equal);+ if cmp_score == Ordering::Equal {+ (right.start_offset, right.stop_offset).cmp(&(left.start_offset, left.stop_offset))+ } else {+ cmp_score+ }+ });+ if let Some(fragment) = best_fragment_opt {+ let fragment_text = &text[fragment.start_offset..fragment.stop_offset];+ let highlighted = fragment+ .highlighted+ .iter()+ .map(|item| item.start - fragment.start_offset..item.end - fragment.start_offset)+ .collect();+ Snippet {+ fragment: fragment_text.to_string(),+ highlighted,+ }+ } else {+ // when there no fragments to chose from,+ // for now create a empty snippet+ Snippet {+ fragment: String::new(),+ highlighted: vec![],+ }+ }+}++struct SnippetGenerator {+ terms_text: BTreeMap<String, Score>,+ tokenizer: TextAnalyzer,+ field: tantivy::schema::Field,+ max_num_chars: usize,+}++impl SnippetGenerator {+ /// Creates a new snippet generator+ fn create(+ searcher: &Searcher,+ query: &dyn tantivy::query::Query,+ field: tantivy::schema::Field,+ ) -> crate::Result<SnippetGenerator> {+ let mut terms = BTreeMap::new();+ query.query_terms(&mut terms);+ let mut terms_text: BTreeMap<String, Score> = Default::default();+ for (term, _) in terms {+ if term.field() != field {+ continue;+ }+ let term_str = if let Some(term_str) = term.as_str() {+ term_str+ } else {+ continue;+ };+ let doc_freq = searcher.doc_freq(&term)?;+ if doc_freq > 0 {+ let score = 1.0 / (1.0 + doc_freq as Score);+ terms_text.insert(term_str.to_string(), score);+ }+ }+ let tokenizer = searcher.index().tokenizer_for_field(field)?;+ Ok(SnippetGenerator {+ terms_text,+ tokenizer,+ field,+ max_num_chars: DEFAULT_MAX_NUM_CHARS,+ })+ }++ /// Sets a maximum number of chars.+ fn set_max_num_chars(&mut self, max_num_chars: usize) {+ self.max_num_chars = max_num_chars;+ }++ #[cfg(test)]+ fn terms_text(&self) -> &BTreeMap<String, Score> {+ &self.terms_text+ }++ /// Generates a snippet for the given `Document`.+ ///+ /// This method extract the text associated to the `SnippetGenerator`'s field+ /// and computes a snippet.+ fn snippet_from_doc(&self, doc: &Document) -> Snippet {+ let text: String = doc+ .get_all(self.field)+ .flat_map(tantivy::schema::Value::as_text)+ .collect::<Vec<&str>>()+ .join(" ");+ self.snippet(&text)+ }++ /// Generates a snippet for the given text.+ fn snippet(&self, text: &str) -> Snippet {+ let fragment_candidates =+ search_fragments(&self.tokenizer, text, &self.terms_text, self.max_num_chars);+ select_best_fragment_combination(&fragment_candidates[..], text)+ }+}++use crate::{query::Query, schema::Field, Result};+pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Result<String> {let generator = SnippetGenerator::create(searcher,@@ -40,7 +296,10 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resu#[cfg(test)]mod tests {+ use super::*;use crate::{index::Index, searcher::Searcher, webpage::Webpage};+ use maplit::btreemap;+ use tantivy::tokenizer::SimpleTokenizer;const TEST_TEXT: &str = r#"Rust is a systems programming language sponsored byMozilla which describes it as a "safe, concurrent, practical language", supporting functional and@@ -90,4 +349,260 @@ Survey in 2016, 2017, and 2018."#;.to_string());}++ #[test]+ fn test_snippet() {+ let terms = btreemap! {+ String::from("rust") => 1.0,+ String::from("language") => 0.9+ };+ let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 100);+ assert_eq!(fragments.len(), 7);+ {+ let first = &fragments[0];+ assert_eq!(first.score, 1.9);+ assert_eq!(first.stop_offset, 89);+ }+ let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);+ assert_eq!(+ snippet.fragment,+ "Rust is a systems programming language sponsored by\nMozilla which describes it as a \+ \"safe"+ );+ assert_eq!(+ snippet.to_html(),+ "<b>Rust</b> is a systems programming <b>language</b> sponsored by\nMozilla which \+ describes it as a "safe"+ )+ }++ #[test]+ fn test_snippet_scored_fragment() {+ {+ let terms = btreemap! {+ String::from("rust") =>1.0,+ String::from("language") => 0.9+ };+ let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);+ {+ let first = &fragments[0];+ assert_eq!(first.score, 1.0);+ assert_eq!(first.stop_offset, 17);+ }+ let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);+ assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")+ }+ {+ let terms = btreemap! {+ String::from("rust") =>0.9,+ String::from("language") => 1.0+ };+ let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);+ // assert_eq!(fragments.len(), 7);+ {+ let first = &fragments[0];+ assert_eq!(first.score, 0.9);+ assert_eq!(first.stop_offset, 17);+ }+ let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);+ assert_eq!(snippet.to_html(), "programming <b>language</b>")+ }+ }++ #[test]+ fn test_snippet_in_second_fragment() {+ let text = "a b c d e f g";++ let mut terms = BTreeMap::new();+ terms.insert(String::from("c"), 1.0);++ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);++ assert_eq!(fragments.len(), 1);+ {+ let first = &fragments[0];+ assert_eq!(first.score, 1.0);+ assert_eq!(first.start_offset, 4);+ assert_eq!(first.stop_offset, 7);+ }++ let snippet = select_best_fragment_combination(&fragments[..], text);+ assert_eq!(snippet.fragment, "c d");+ assert_eq!(snippet.to_html(), "<b>c</b> d");+ }++ #[test]+ fn test_snippet_with_term_at_the_end_of_fragment() {+ let text = "a b c d e f f g";++ let mut terms = BTreeMap::new();+ terms.insert(String::from("f"), 1.0);++ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);++ assert_eq!(fragments.len(), 2);+ {+ let first = &fragments[0];+ assert_eq!(first.score, 1.0);+ assert_eq!(first.stop_offset, 11);+ assert_eq!(first.start_offset, 8);+ }++ let snippet = select_best_fragment_combination(&fragments[..], text);+ assert_eq!(snippet.fragment, "e f");+ assert_eq!(snippet.to_html(), "e <b>f</b>");+ }++ #[test]+ fn test_snippet_with_second_fragment_has_the_highest_score() {+ let text = "a b c d e f g";++ let mut terms = BTreeMap::new();+ terms.insert(String::from("f"), 1.0);+ terms.insert(String::from("a"), 0.9);++ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 7);++ assert_eq!(fragments.len(), 2);+ {+ let first = &fragments[0];+ assert_eq!(first.score, 0.9);+ assert_eq!(first.stop_offset, 7);+ assert_eq!(first.start_offset, 0);+ }++ let snippet = select_best_fragment_combination(&fragments[..], text);+ assert_eq!(snippet.fragment, "e f g");+ assert_eq!(snippet.to_html(), "e <b>f</b> g");+ }++ #[test]+ fn test_snippet_with_term_not_in_text() {+ let text = "a b c d";++ let mut terms = BTreeMap::new();+ terms.insert(String::from("z"), 1.0);++ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);++ assert_eq!(fragments.len(), 0);++ let snippet = select_best_fragment_combination(&fragments[..], text);+ assert_eq!(snippet.fragment, "");+ assert_eq!(snippet.to_html(), "");+ }++ #[test]+ fn test_snippet_with_no_terms() {+ let text = "a b c d";++ let terms = BTreeMap::new();+ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);+ assert_eq!(fragments.len(), 0);++ let snippet = select_best_fragment_combination(&fragments[..], text);+ assert_eq!(snippet.fragment, "");+ assert_eq!(snippet.to_html(), "");+ }++ #[test]+ fn test_snippet_generator_term_score() {+ let mut schema_builder = tantivy::schema::Schema::builder();+ let text_field = schema_builder.add_text_field("text", tantivy::schema::TEXT);+ let schema = schema_builder.build();+ let index = tantivy::Index::create_in_ram(schema);+ {+ // writing the segment+ let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();+ index_writer+ .add_document(tantivy::doc!(text_field => "a"))+ .unwrap();+ index_writer+ .add_document(tantivy::doc!(text_field => "a"))+ .unwrap();+ index_writer+ .add_document(tantivy::doc!(text_field => "a b"))+ .unwrap();+ index_writer.commit().unwrap();+ }+ let searcher = index.reader().unwrap().searcher();+ let query_parser = tantivy::query::QueryParser::for_index(&index, vec![text_field]);+ {+ let query = query_parser.parse_query("e").unwrap();+ let snippet_generator =+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ assert!(snippet_generator.terms_text().is_empty());+ }+ {+ let query = query_parser.parse_query("a").unwrap();+ let snippet_generator =+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ assert_eq!(+ &btreemap!("a".to_string() => 0.25),+ snippet_generator.terms_text()+ );+ }+ {+ let query = query_parser.parse_query("a b").unwrap();+ let snippet_generator =+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ assert_eq!(+ &btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),+ snippet_generator.terms_text()+ );+ }+ {+ let query = query_parser.parse_query("a b c").unwrap();+ let snippet_generator =+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ assert_eq!(+ &btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),+ snippet_generator.terms_text()+ );+ }+ }++ #[test]+ fn test_snippet_generator() -> crate::Result<()> {+ let mut schema_builder = tantivy::schema::Schema::builder();+ let text_options = tantivy::schema::TextOptions::default().set_indexing_options(+ tantivy::schema::TextFieldIndexing::default()+ .set_tokenizer("en_stem")+ .set_index_option(tantivy::schema::IndexRecordOption::Basic),+ );+ let text_field = schema_builder.add_text_field("text", text_options);+ let schema = schema_builder.build();+ let index = tantivy::Index::create_in_ram(schema);+ {+ // writing the segment+ let mut index_writer = index.writer_with_num_threads(1, 10_000_000)?;+ let doc = tantivy::doc!(text_field => TEST_TEXT);+ index_writer.add_document(doc)?;+ index_writer.commit()?;+ }+ let searcher = index.reader().unwrap().searcher();+ let query_parser = tantivy::query::QueryParser::for_index(&index, vec![text_field]);+ let query = query_parser.parse_query("rust design").unwrap();+ let mut snippet_generator =+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();+ {+ let snippet = snippet_generator.snippet(TEST_TEXT);+ assert_eq!(+ snippet.to_html(),+ "imperative-procedural paradigms. <b>Rust</b> is syntactically similar to \+ C++[according to whom?],\nbut its <b>designers</b> intend it to provide better \+ memory safety"+ );+ }+ {+ snippet_generator.set_max_num_chars(90);+ let snippet = snippet_generator.snippet(TEST_TEXT);+ assert_eq!(+ snippet.to_html(),+ "<b>Rust</b> is syntactically similar to C++[according to whom?],\nbut its \+ <b>designers</b> intend it to"+ );+ }+ Ok(())+ }}diff --git a/src/tokenizer.rs b/src/tokenizer.rsindex 6df7ccf3..12c8d2e8 100644--- a/src/tokenizer.rs+++ b/src/tokenizer.rs@@ -14,24 +14,108 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::tokenizer::{Language, LowerCaser, SimpleTokenizer, Stemmer, TextAnalyzer};+use tantivy::tokenizer::{+ Language, LowerCaser, PreTokenizedStream, PreTokenizedString, SimpleTokenizer, Stemmer,+ TextAnalyzer, TokenStream,+};+use whatlang::Lang;++struct TokenStreamMerger<'a> {+ streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>,+ current_stream: usize,+}++impl<'a> TokenStreamMerger<'a> {+ fn merge(streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>) -> Self {+ Self {+ streams,+ current_stream: 0,+ }+ }+}++impl<'a> tantivy::tokenizer::TokenStream for TokenStreamMerger<'a> {+ fn advance(&mut self) -> bool {+ if let Some(stream) = self.streams.get_mut(self.current_stream) {+ let mut has_more = stream.advance();++ if !has_more {+ self.current_stream += 1;+ has_more = self.current_stream < self.streams.len()+ && self.streams[self.current_stream].advance();+ }++ has_more+ } else {+ false+ }+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ self.streams[self.current_stream].token()+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ self.streams[self.current_stream].token_mut()+ }+}++struct MyStemmer(Stemmer);++impl From<Lang> for MyStemmer {+ fn from(lang: Lang) -> Self {+ match lang {+ Lang::Dan => MyStemmer(Stemmer::new(Language::Danish)),+ Lang::Ara => MyStemmer(Stemmer::new(Language::Arabic)),+ Lang::Nld => MyStemmer(Stemmer::new(Language::Dutch)),+ Lang::Eng => MyStemmer(Stemmer::new(Language::English)),+ Lang::Fin => MyStemmer(Stemmer::new(Language::Finnish)),+ Lang::Fra => MyStemmer(Stemmer::new(Language::French)),+ Lang::Deu => MyStemmer(Stemmer::new(Language::German)),+ Lang::Hun => MyStemmer(Stemmer::new(Language::Hungarian)),+ Lang::Ita => MyStemmer(Stemmer::new(Language::Italian)),+ Lang::Por => MyStemmer(Stemmer::new(Language::Portuguese)),+ Lang::Ron => MyStemmer(Stemmer::new(Language::Romanian)),+ Lang::Rus => MyStemmer(Stemmer::new(Language::Russian)),+ Lang::Spa => MyStemmer(Stemmer::new(Language::Spanish)),+ Lang::Swe => MyStemmer(Stemmer::new(Language::Swedish)),+ Lang::Tam => MyStemmer(Stemmer::new(Language::Tamil)),+ Lang::Tur => MyStemmer(Stemmer::new(Language::Turkish)),+ _ => MyStemmer(Stemmer::new(Language::English)),+ }+ }+}+#[derive(Clone, Default)]pub struct Tokenizer {}impl tantivy::tokenizer::Tokenizer for Tokenizer {fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let mut analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);-- // TODO:- // Use whatlang to determine which stemmer to use.- // We will somehow need to search for the non-stemmed version- // of the string, as whatlang might be wrong which will result in- // wrong stemming. I tried merging multiple token streams but it- // caused tantivy to crash. A solution might be to create a separate index- // for stem and non-stemmed and do merging in the searcher. This- // might use too much space.- analyzer = analyzer.filter(Stemmer::new(Language::English));-- analyzer.token_stream(text.clone())+ let analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);++ let mut streams = vec![analyzer.token_stream(text)];++ if let Some(lang) = whatlang::detect_lang(text) {+ let stemmed_analyser = analyzer.clone().filter(MyStemmer::from(lang).0);+ streams.push(stemmed_analyser.token_stream(text));+ }++ let mut merger = TokenStreamMerger::merge(streams);++ let mut tokens = Vec::new();+ while let Some(token) = merger.next() {+ tokens.push(token.clone());+ }++ tokens.sort_by(|a, b| {+ a.offset_from+ .partial_cmp(&b.offset_from)+ .unwrap_or(a.offset_to.cmp(&b.offset_to))+ });++ tantivy::tokenizer::BoxTokenStream::from(PreTokenizedStream::from(PreTokenizedString {+ text: text.to_string(),+ tokens,+ }))}}