Skip to content

web search done right

Tokenizer can produce multiple tokens for each term. Should search for stemmed and non-stemmed version.

Mikkel Denker11953e8bdbd6
diff --git a/src/index.rs b/src/index.rs
index 479fb97f..b8120968 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -244,6 +244,35 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
}
+ #[test]
+ fn stemmed_query_english() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("runners").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Fast runner</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ }
+
#[test]
fn searchable_backlinks() {
let mut index = Index::temporary().expect("Unable to open index");
diff --git a/src/query.rs b/src/query.rs
index c0a169d7..9be68219 100644
--- a/src/query.rs
+++ b/src/query.rs
@@ -86,9 +86,9 @@ impl Query {
term: &str,
analyzer: Option<TextAnalyzer>,
tantivy_field: Field,
- ) -> Term {
+ ) -> impl Iterator<Item = Term> {
match analyzer {
- None => Term::from_field_text(tantivy_field, term),
+ None => vec![Term::from_field_text(tantivy_field, term)].into_iter(),
Some(tokenizer) => {
let mut terms: Vec<Term> = Vec::new();
let mut token_stream = tokenizer.token_stream(term);
@@ -97,7 +97,7 @@ impl Query {
terms.push(term);
});
- terms.into_iter().next().unwrap()
+ terms.into_iter()
}
}
}
@@ -116,14 +116,26 @@ impl Query {
.into_iter()
.map(|(field, entry)| {
let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);
- let term = Query::process_tantivy_term(term, analyzer, field);
-
- let term_query = Box::new(TermQuery::new(
- term,
- IndexRecordOption::WithFreqsAndPositions,
- ))
- as Box<dyn tantivy::query::Query>;
- (Occur::Should, term_query)
+ let processed_terms = Query::process_tantivy_term(term, analyzer, field);
+
+ let processed_queries = processed_terms
+ .map(|term| {
+ (
+ Occur::Should,
+ Box::new(TermQuery::new(
+ term,
+ IndexRecordOption::WithFreqsAndPositions,
+ ))
+ as Box<dyn tantivy::query::Query>,
+ )
+ })
+ .collect();
+
+ (
+ Occur::Should,
+ Box::new(BooleanQuery::new(processed_queries))
+ as Box<dyn tantivy::query::Query>,
+ )
})
.collect();