web search done right
Tokenizer can produce multiple tokens for each term. Should search for stemmed and non-stemmed version.
Mikkel Denker11953e8bdbd6
diff --git a/src/index.rs b/src/index.rsindex 479fb97f..b8120968 100644--- a/src/index.rs+++ b/src/index.rs@@ -244,6 +244,35 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com");}+ #[test]+ fn stemmed_query_english() {+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("runners").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Fast runner</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ vec![],+ 1.0,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.example.com");+ }+#[test]fn searchable_backlinks() {let mut index = Index::temporary().expect("Unable to open index");diff --git a/src/query.rs b/src/query.rsindex c0a169d7..9be68219 100644--- a/src/query.rs+++ b/src/query.rs@@ -86,9 +86,9 @@ impl Query {term: &str,analyzer: Option<TextAnalyzer>,tantivy_field: Field,- ) -> Term {+ ) -> impl Iterator<Item = Term> {match analyzer {- None => Term::from_field_text(tantivy_field, term),+ None => vec![Term::from_field_text(tantivy_field, term)].into_iter(),Some(tokenizer) => {let mut terms: Vec<Term> = Vec::new();let mut token_stream = tokenizer.token_stream(term);@@ -97,7 +97,7 @@ impl Query {terms.push(term);});- terms.into_iter().next().unwrap()+ terms.into_iter()}}}@@ -116,14 +116,26 @@ impl Query {.into_iter().map(|(field, entry)| {let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);- let term = Query::process_tantivy_term(term, analyzer, field);-- let term_query = Box::new(TermQuery::new(- term,- IndexRecordOption::WithFreqsAndPositions,- ))- as Box<dyn tantivy::query::Query>;- (Occur::Should, term_query)+ let processed_terms = Query::process_tantivy_term(term, analyzer, field);++ let processed_queries = processed_terms+ .map(|term| {+ (+ Occur::Should,+ Box::new(TermQuery::new(+ term,+ IndexRecordOption::WithFreqsAndPositions,+ ))+ as Box<dyn tantivy::query::Query>,+ )+ })+ .collect();++ (+ Occur::Should,+ Box::new(BooleanQuery::new(processed_queries))+ as Box<dyn tantivy::query::Query>,+ )}).collect();