Skip to content

web search done right

english stemming

Mikkel Denker05a438a64dad
diff --git a/src/indexer.rs b/src/indexer.rs
index b74d1f70..289c47ef 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -66,7 +66,7 @@ impl Indexer {
raw_object
})
})
- .buffer_unordered(10)
+ .buffer_unordered(20)
.map(|raw_bytes| {
tokio::task::spawn_blocking(move || {
if raw_bytes.is_err() {
@@ -86,7 +86,7 @@ impl Indexer {
// panic!();
})
})
- .buffer_unordered(10)
+ .buffer_unordered(20)
.collect::<Vec<_>>()
.await;
diff --git a/src/query.rs b/src/query.rs
index 19def853..a0d80dda 100644
--- a/src/query.rs
+++ b/src/query.rs
@@ -9,7 +9,8 @@ use nom::{
};
use tantivy::{
query::{BooleanQuery, Occur, TermQuery},
- schema::{IndexRecordOption, Schema},
+ schema::{Field, FieldEntry, IndexRecordOption, Schema},
+ tokenizer::{TextAnalyzer, TokenizerManager},
Term,
};
@@ -48,7 +49,48 @@ impl Query {
}
}
- pub fn tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query> {
+ fn get_tantivy_analyzer(
+ entry: &FieldEntry,
+ tokenizer_manager: &TokenizerManager,
+ ) -> Option<TextAnalyzer> {
+ match entry.field_type() {
+ tantivy::schema::FieldType::Str(options) => {
+ options.get_indexing_options().map(|indexing_options| {
+ let tokenizer_name = indexing_options.tokenizer();
+ tokenizer_manager
+ .get(tokenizer_name)
+ .expect("Unknown tokenizer")
+ })
+ }
+ _ => None,
+ }
+ }
+
+ fn process_tantivy_term(
+ term: &str,
+ analyzer: Option<TextAnalyzer>,
+ tantivy_field: Field,
+ ) -> Term {
+ match analyzer {
+ None => Term::from_field_text(tantivy_field, term),
+ Some(tokenizer) => {
+ let mut terms: Vec<Term> = Vec::new();
+ let mut token_stream = tokenizer.token_stream(term);
+ token_stream.process(&mut |token| {
+ let term = Term::from_field_text(tantivy_field, &token.text);
+ terms.push(term);
+ });
+
+ terms.into_iter().next().unwrap()
+ }
+ }
+ }
+
+ pub fn tantivy(
+ &self,
+ schema: &Schema,
+ tokenizer_manager: &TokenizerManager,
+ ) -> Box<dyn tantivy::query::Query> {
let queries = self
.terms
.iter()
@@ -56,14 +98,16 @@ impl Query {
let one_term = schema
.fields()
.into_iter()
- .map(|(field, _)| {
- (
- Occur::Should,
- Box::new(TermQuery::new(
- Term::from_field_text(field, term),
- IndexRecordOption::Basic,
- )) as Box<dyn tantivy::query::Query>,
- )
+ .map(|(field, entry)| {
+ let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);
+ let term = Query::process_tantivy_term(term, analyzer, field);
+
+ let term_query = Box::new(TermQuery::new(
+ term,
+ IndexRecordOption::WithFreqsAndPositions,
+ ))
+ as Box<dyn tantivy::query::Query>;
+ (Occur::Should, term_query)
})
.collect();
diff --git a/src/search_index.rs b/src/search_index.rs
index 86949281..86cd1593 100644
--- a/src/search_index.rs
+++ b/src/search_index.rs
@@ -20,7 +20,7 @@ impl Field {
TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
- // .set_tokenizer("en_stem")
+ .set_tokenizer("en_stem")
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored()
@@ -88,7 +88,7 @@ impl Index {
}
pub fn search(&self, query: &Query) -> Result<SearchResult> {
- let tantivy_query = query.tantivy(&self.schema);
+ let tantivy_query = query.tantivy(&self.schema, self.tantivy_index.tokenizers());
let searcher = self.reader.searcher();
let (count, docs) =
@@ -196,34 +196,6 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
}
- // #[test]
- fn retrieve_doc() {
- let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("website").expect("Failed to parse query");
-
- index
- .insert(Webpage::parse(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- website website website
- </body>
- </html>
- "#,
- "https://www.example.com",
- ))
- .expect("failed to parse webpage");
- index.commit().expect("failed to commit index");
-
- let result = index.search(&query).expect("Search failed");
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
- assert_eq!(result.documents[0].title, "Test website");
- }
-
#[test]
fn document_not_matching() {
let mut index = Index::temporary().expect("Unable to open index");
@@ -248,7 +220,7 @@ mod tests {
assert_eq!(result.num_docs, 0);
}
- // #[test]
+ #[test]
fn english_stemming() {
let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("runner").expect("Failed to parse query");
@@ -271,4 +243,9 @@ mod tests {
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
+
+ // #[test]
+ // fn snippet() {
+ // todo!();
+ // }
}