web search done right
add '.' char as a stopword for bigram and trigram tokenizers. searching for 'example.com' would put too much weight on the '.com' bigram part
Mikkel Denkerd88c277e4528
diff --git a/core/src/tokenizer/mod.rs b/core/src/tokenizer/mod.rsindex 690ee285..85992016 100644--- a/core/src/tokenizer/mod.rs+++ b/core/src/tokenizer/mod.rs@@ -109,22 +109,38 @@ impl Normal {}}-#[derive(Clone, Default)]+#[derive(Clone)]pub struct BigramTokenizer {inner_tokenizer: Normal,}+impl Default for BigramTokenizer {+ fn default() -> Self {+ Self {+ inner_tokenizer: Normal::with_stopwords(vec![".".to_string()]),+ }+ }+}+impl BigramTokenizer {pub fn as_str() -> &'static str {"bigram_tokenizer"}}-#[derive(Clone, Default)]+#[derive(Clone)]pub struct TrigramTokenizer {inner_tokenizer: Normal,}+impl Default for TrigramTokenizer {+ fn default() -> Self {+ Self {+ inner_tokenizer: Normal::with_stopwords(vec![".".to_string()]),+ }+ }+}+impl TrigramTokenizer {pub fn as_str() -> &'static str {"trigram_tokenizer"@@ -1001,10 +1017,8 @@ key1.key2="this\" is @ a # test""#;vec!["thisis".to_string(), "isa".to_string(), "atest".to_string(),]);- assert_eq!(- tokenize_bigram("this.is"),- vec!["this.".to_string(), ".is".to_string(),]- );+ // '.' is a stopword+ assert_eq!(tokenize_bigram("this.is"), vec!["thisis".to_string()]);}#[test]