web search done right
make sure 'DomainIfHomepageNoTokenizer' is (not) tokenized correctly
Mikkel Denker5d4183c57a5b
diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rsindex fe49358b..b03ed771 100644--- a/crates/core/src/tokenizer/mod.rs+++ b/crates/core/src/tokenizer/mod.rs@@ -804,6 +804,18 @@ mod tests {res}+ fn tokenize_identity(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut tokenizer = Identity {};+ let mut stream = tokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }+#[test]fn simple_tokenization() {assert_eq!(@@ -1057,4 +1069,10 @@ key1.key2="this\" is @ a # test""#;assert_eq!(tokenize_url(".com"), vec![".", "com ",])}++ #[test]+ fn identity() {+ assert_eq!(tokenize_identity("this is a test"), vec!["this is a test"]);+ assert_eq!(tokenize_identity("a-b"), vec!["a-b"]);+ }}diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex 4ca386cb..69293c4f 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -41,7 +41,7 @@ impl Html {}let title = title.unwrap();- Ok(self.pretokenize_string(title))+ Ok(self.pretokenize_string(title, TextField::Title))}fn pretokenize_all_text(&self) -> Result<PreTokenizedString> {@@ -52,35 +52,35 @@ impl Html {}let all_text = all_text.unwrap();- Ok(self.pretokenize_string(all_text))+ Ok(self.pretokenize_string(all_text, TextField::AllBody))}fn pretokenize_clean_text(&self) -> PreTokenizedString {let clean_text = self.clean_text().cloned().unwrap_or_default();- self.pretokenize_string(clean_text)+ self.pretokenize_string(clean_text, TextField::CleanBody)}fn pretokenize_url(&self) -> PreTokenizedString {let url = self.url().to_string();- self.pretokenize_string(url)+ self.pretokenize_string(url, TextField::Url)}fn pretokenize_domain(&self) -> PreTokenizedString {let domain = self.url().root_domain().unwrap_or_default().to_string();- self.pretokenize_string(domain)+ self.pretokenize_string(domain, TextField::Domain)}fn pretokenize_site(&self) -> PreTokenizedString {let site = self.url().normalized_host().unwrap_or_default().to_string();- self.pretokenize_string(site)+ self.pretokenize_string(site, TextField::SiteWithout)}fn pretokenize_description(&self) -> PreTokenizedString {let text = self.description().unwrap_or_default();- self.pretokenize_string(text)+ self.pretokenize_string(text, TextField::Description)}fn pretokenize_microformats(&self) -> PreTokenizedString {@@ -91,11 +91,11 @@ impl Html {text.push(' ');}- self.pretokenize_string(text)+ self.pretokenize_string(text, TextField::MicroformatTags)}- fn pretokenize_string(&self, text: String) -> PreTokenizedString {- self.pretokenize_string_with(text, tokenizer::Tokenizer::default())+ fn pretokenize_string(&self, text: String, field: TextField) -> PreTokenizedString {+ self.pretokenize_string_with(text, field.indexing_tokenizer())}fn pretokenize_string_with(@@ -381,7 +381,19 @@ impl Html {}Field::Text(TextField::DomainIfHomepageNoTokenizer) => {if self.is_homepage() {- doc.add_pre_tokenized_text(tantivy_field, domain.clone());+ doc.add_pre_tokenized_text(+ tantivy_field,+ PreTokenizedString {+ text: domain.text.clone(),+ tokens: vec![tantivy::tokenizer::Token {+ offset_from: 0,+ offset_to: domain.text.len(),+ position: 0,+ text: domain.text.clone(),+ position_length: 1,+ }],+ },+ );} else {doc.add_text(tantivy_field, "");}