web search done right
english stemming
Mikkel Denker05a438a64dad
diff --git a/src/indexer.rs b/src/indexer.rsindex b74d1f70..289c47ef 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -66,7 +66,7 @@ impl Indexer {raw_object})})- .buffer_unordered(10)+ .buffer_unordered(20).map(|raw_bytes| {tokio::task::spawn_blocking(move || {if raw_bytes.is_err() {@@ -86,7 +86,7 @@ impl Indexer {// panic!();})})- .buffer_unordered(10)+ .buffer_unordered(20).collect::<Vec<_>>().await;diff --git a/src/query.rs b/src/query.rsindex 19def853..a0d80dda 100644--- a/src/query.rs+++ b/src/query.rs@@ -9,7 +9,8 @@ use nom::{};use tantivy::{query::{BooleanQuery, Occur, TermQuery},- schema::{IndexRecordOption, Schema},+ schema::{Field, FieldEntry, IndexRecordOption, Schema},+ tokenizer::{TextAnalyzer, TokenizerManager},Term,};@@ -48,7 +49,48 @@ impl Query {}}- pub fn tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query> {+ fn get_tantivy_analyzer(+ entry: &FieldEntry,+ tokenizer_manager: &TokenizerManager,+ ) -> Option<TextAnalyzer> {+ match entry.field_type() {+ tantivy::schema::FieldType::Str(options) => {+ options.get_indexing_options().map(|indexing_options| {+ let tokenizer_name = indexing_options.tokenizer();+ tokenizer_manager+ .get(tokenizer_name)+ .expect("Unknown tokenizer")+ })+ }+ _ => None,+ }+ }++ fn process_tantivy_term(+ term: &str,+ analyzer: Option<TextAnalyzer>,+ tantivy_field: Field,+ ) -> Term {+ match analyzer {+ None => Term::from_field_text(tantivy_field, term),+ Some(tokenizer) => {+ let mut terms: Vec<Term> = Vec::new();+ let mut token_stream = tokenizer.token_stream(term);+ token_stream.process(&mut |token| {+ let term = Term::from_field_text(tantivy_field, &token.text);+ terms.push(term);+ });++ terms.into_iter().next().unwrap()+ }+ }+ }++ pub fn tantivy(+ &self,+ schema: &Schema,+ tokenizer_manager: &TokenizerManager,+ ) -> Box<dyn tantivy::query::Query> {let queries = self.terms.iter()@@ -56,14 +98,16 @@ impl Query {let one_term = schema.fields().into_iter()- .map(|(field, _)| {- (- Occur::Should,- Box::new(TermQuery::new(- Term::from_field_text(field, term),- IndexRecordOption::Basic,- )) as Box<dyn tantivy::query::Query>,- )+ .map(|(field, entry)| {+ let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);+ let term = Query::process_tantivy_term(term, analyzer, field);++ let term_query = Box::new(TermQuery::new(+ term,+ IndexRecordOption::WithFreqsAndPositions,+ ))+ as Box<dyn tantivy::query::Query>;+ (Occur::Should, term_query)}).collect();diff --git a/src/search_index.rs b/src/search_index.rsindex 86949281..86cd1593 100644--- a/src/search_index.rs+++ b/src/search_index.rs@@ -20,7 +20,7 @@ impl Field {TextOptions::default().set_indexing_options(TextFieldIndexing::default()- // .set_tokenizer("en_stem")+ .set_tokenizer("en_stem").set_index_option(IndexRecordOption::WithFreqsAndPositions),).set_stored()@@ -88,7 +88,7 @@ impl Index {}pub fn search(&self, query: &Query) -> Result<SearchResult> {- let tantivy_query = query.tantivy(&self.schema);+ let tantivy_query = query.tantivy(&self.schema, self.tantivy_index.tokenizers());let searcher = self.reader.searcher();let (count, docs) =@@ -196,34 +196,6 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com");}- // #[test]- fn retrieve_doc() {- let mut index = Index::temporary().expect("Unable to open index");- let query = Query::parse("website").expect("Failed to parse query");-- index- .insert(Webpage::parse(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- website website website- </body>- </html>- "#,- "https://www.example.com",- ))- .expect("failed to parse webpage");- index.commit().expect("failed to commit index");-- let result = index.search(&query).expect("Search failed");- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.example.com");- assert_eq!(result.documents[0].title, "Test website");- }-#[test]fn document_not_matching() {let mut index = Index::temporary().expect("Unable to open index");@@ -248,7 +220,7 @@ mod tests {assert_eq!(result.num_docs, 0);}- // #[test]+ #[test]fn english_stemming() {let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("runner").expect("Failed to parse query");@@ -271,4 +243,9 @@ mod tests {assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com");}++ // #[test]+ // fn snippet() {+ // todo!();+ // }}