Skip to content

web search done right

add '.' char as a stopword for bigram and trigram tokenizers. searching for 'example.com' would put too much weight on the '.com' bigram part

Mikkel Denkerd88c277e4528
diff --git a/core/src/tokenizer/mod.rs b/core/src/tokenizer/mod.rs
index 690ee285..85992016 100644
--- a/core/src/tokenizer/mod.rs
+++ b/core/src/tokenizer/mod.rs
@@ -109,22 +109,38 @@ impl Normal {
}
}
-#[derive(Clone, Default)]
+#[derive(Clone)]
pub struct BigramTokenizer {
inner_tokenizer: Normal,
}
+impl Default for BigramTokenizer {
+ fn default() -> Self {
+ Self {
+ inner_tokenizer: Normal::with_stopwords(vec![".".to_string()]),
+ }
+ }
+}
+
impl BigramTokenizer {
pub fn as_str() -> &'static str {
"bigram_tokenizer"
}
}
-#[derive(Clone, Default)]
+#[derive(Clone)]
pub struct TrigramTokenizer {
inner_tokenizer: Normal,
}
+impl Default for TrigramTokenizer {
+ fn default() -> Self {
+ Self {
+ inner_tokenizer: Normal::with_stopwords(vec![".".to_string()]),
+ }
+ }
+}
+
impl TrigramTokenizer {
pub fn as_str() -> &'static str {
"trigram_tokenizer"
@@ -1001,10 +1017,8 @@ key1.key2="this\" is @ a # test""#;
vec!["thisis".to_string(), "isa".to_string(), "atest".to_string(),]
);
- assert_eq!(
- tokenize_bigram("this.is"),
- vec!["this.".to_string(), ".is".to_string(),]
- );
+ // '.' is a stopword
+ assert_eq!(tokenize_bigram("this.is"), vec!["thisis".to_string()]);
}
#[test]