web search done right
normalize diacritics/accents
Mikkel Denkerf1b72a897d77
diff --git a/Cargo.lock b/Cargo.lockindex 4c1b75ac..ee9e396b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4996,6 +4996,7 @@ dependencies = ["tracing","tracing-subscriber","tracing-test",+ "unicode-normalization","unicode-segmentation","url","utoipa",diff --git a/Cargo.toml b/Cargo.tomlindex 7e9ad95a..27cd984c 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -165,6 +165,7 @@ tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}tracing = {version = "0.1.34", features = ["release_max_level_info"]}tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}tracing-test = "0.2.4"+unicode-normalization = "0.1.23"unicode-segmentation = "1.11.0"url = {version = "2.4.0", features = ["serde"]}utoipa = {version = "4.2.3", features = ["axum_extras"]}diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 1941940f..e729659a 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -102,6 +102,7 @@ tower-http.workspace = truetower.workspace = truetracing-subscriber.workspace = truetracing.workspace = true+unicode-normalization.workspace = trueunicode-segmentation.workspace = trueurl.workspace = trueutoipa-swagger-ui.workspace = truediff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex c2e9495f..1bd1741b 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -1284,4 +1284,68 @@ mod tests {assert_eq!(webpages[0].title, "C++");assert_eq!(webpages[0].url, "https://www.a.com/");}++ #[test]+ fn test_unicode_normalization() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");++ let webpage = Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>æble café</title>+ </head>+ <body>+ {CONTENT} test+ </body>+ </html>+ "#,+ CONTENT = crate::rand_words(100)+ ),+ "https://www.a.com",+ )+ .unwrap();++ index.insert(&webpage).unwrap();++ index.commit().expect("failed to commit index");++ let test = |q: &str| {+ let ctx = index.local_search_ctx();++ let query = Query::parse(+ &ctx,+ &SearchQuery {+ query: q.to_string(),+ ..Default::default()+ },+ &index,+ )+ .expect("Failed to parse query");++ let ranker = Ranker::new(+ SignalComputer::new(Some(&query)),+ ctx.columnfield_reader.clone(),+ CollectorConfig::default(),+ );++ let res = index+ .search_initial(&query, &ctx, ranker.collector(ctx.clone()))+ .unwrap();++ assert_eq!(res.top_websites.len(), 1, "query: {}", q);++ let webpages = index.retrieve_websites(&res.top_websites, &query).unwrap();++ assert_eq!(webpages.len(), 1);+ assert_eq!(webpages[0].title, "æble café");+ assert_eq!(webpages[0].url, "https://www.a.com/");+ };++ test("cafe");+ test("café");+ test("æble");+ test("æble café");+ }}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 0498c2a7..58bba358 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -81,7 +81,7 @@ pub mod similar_hosts;mod snippet;mod stopwords;pub mod summarizer;-mod tokenizer;+pub mod tokenizer;#[allow(unused)]mod ttl_cache;pub mod warc;diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex c1282b28..9ce581f3 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -569,6 +569,18 @@ Survey in 2016, 2017, and 2018."#;);}+ #[test]+ fn test_fuzz_infinite_loop() {+ let text = "\u{a0}";+ let query = "";++ let terms = query+ .split_whitespace()+ .map(|s| s.to_string())+ .collect::<Vec<_>>();+ let _ = snippet_string(text, &terms, whatlang::Lang::Eng, SnippetConfig::default());+ }+proptest! {#[test]fn prop_snippet_gen(text in ".{1,100}", query in ".{1,100}") {diff --git a/crates/core/src/tokenizer/fields/default.rs b/crates/core/src/tokenizer/fields/default.rsindex 1f299d2f..1d012cab 100644--- a/crates/core/src/tokenizer/fields/default.rs+++ b/crates/core/src/tokenizer/fields/default.rs@@ -14,9 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::tokenizer::{BoxTokenStream, LowerCaser, StopWordFilter, TextAnalyzer};+use tantivy::tokenizer::{BoxTokenStream, StopWordFilter, TextAnalyzer};-use crate::tokenizer::{self, Tokenize};+use crate::tokenizer::{self, normalizer, Normalize, Tokenize};#[derive(Clone, Default)]pub struct DefaultTokenizer {@@ -40,7 +40,7 @@ impl tantivy::tokenizer::Tokenizer for DefaultTokenizer {type TokenStream<'a> = BoxTokenStream<'a>;fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {- let builder = TextAnalyzer::builder(Normal).filter(LowerCaser);+ let builder = TextAnalyzer::builder(Normal);self.analyzer = if let Some(stopwords) = &self.stopwords {Some(@@ -69,7 +69,12 @@ impl tantivy::tokenizer::Tokenizer for Normal {type TokenStream<'a> = BoxTokenStream<'a>;fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {- let stream = Box::new(text.tokenize());+ let stream = Box::new(+ text.tokenize()+ .normalize(&normalizer::Lowercase)+ .normalize(&normalizer::UnicodeNFKD)+ .normalize(&normalizer::UnicodeDiacritics),+ );BoxTokenStream::new(NormalTokenStream::new_boxed(stream))}@@ -190,7 +195,7 @@ mod tests {fn katakana() {assert_eq!(tokenize_default("test ダ.com"),- vec!["test", "ダ", ".", "com"]+ vec!["test", "タ\u{3099}", ".", "com"]);}diff --git a/crates/core/src/tokenizer/fields/url.rs b/crates/core/src/tokenizer/fields/url.rsindex e2560dc6..d127fa15 100644--- a/crates/core/src/tokenizer/fields/url.rs+++ b/crates/core/src/tokenizer/fields/url.rs@@ -196,6 +196,7 @@ impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {#[cfg(test)]mod tests {use super::*;+ use proptest::prelude::*;use tantivy::tokenizer::Tokenizer as _;fn tokenize_url(s: &str) -> Vec<String> {@@ -261,4 +262,16 @@ mod tests {]);}++ proptest! {+ #![proptest_config(ProptestConfig::with_cases(4096))]++ #[test]+ fn test_single_space(url: String) {+ let tokens = tokenize_url(&url);++ let num_spaces = tokens.iter().filter(|s| s.contains(' ')).count();+ prop_assert!(num_spaces <= 1);+ }+ }}diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rsindex a9107ec5..769489c5 100644--- a/crates/core/src/tokenizer/mod.rs+++ b/crates/core/src/tokenizer/mod.rs@@ -16,6 +16,7 @@mod add_space_last;pub mod fields;+pub mod normalizer;mod script;mod script_tokenizer;mod segmenter;@@ -29,7 +30,7 @@ pub use fields::FieldTokenizer;use self::segmenter::Segmenter;-#[derive(Debug)]+#[derive(Clone, Debug)]pub struct Token<'a> {text: Cow<'a, str>,span: std::ops::Range<usize>,@@ -50,6 +51,10 @@ impl<'a> Token<'a> {self.text.borrow()}+ pub fn mut_text(&mut self) -> &mut String {+ self.text.to_mut()+ }+pub fn span(&self) -> std::ops::Range<usize> {self.span.clone()}@@ -69,9 +74,33 @@ impl Tokenize for str {}}+pub trait Normalize<'a, N>+where+ N: normalizer::Normalizer<'a>,+{+ fn normalize(self, normalizer: &'a N) -> impl Iterator<Item = Token<'a>> + 'a;+}++impl<'a, T, N> Normalize<'a, N> for T+where+ T: Iterator<Item = Token<'a>> + 'a,+ N: normalizer::Normalizer<'a>,+{+ fn normalize(self, normalizer: &'a N) -> impl Iterator<Item = Token<'a>> + 'a {+ self.map(move |token| {+ if normalizer.should_normalize(&token) {+ normalizer.normalize(token)+ } else {+ token+ }+ })+ }+}+#[cfg(test)]mod tests {use super::*;+ use normalizer::{Lowercase, UnicodeDiacritics, UnicodeNFKD};use proptest::prelude::*;#[test]@@ -92,6 +121,29 @@ mod tests {assert_eq!(tokens[8].text(), ".");}+ #[test]+ fn test_normalizer() {+ let input = "Hello, world!";+ let tokens: Vec<_> = input.tokenize().normalize(&Lowercase).collect();++ assert_eq!(tokens.len(), 4);+ assert_eq!(tokens[0].text(), "hello");+ assert_eq!(tokens[1].text(), ",");+ assert_eq!(tokens[2].text(), "world");+ assert_eq!(tokens[3].text(), "!");++ let input = "café";+ let tokens = input+ .tokenize()+ .normalize(&Lowercase)+ .normalize(&UnicodeNFKD)+ .normalize(&UnicodeDiacritics)+ .collect::<Vec<_>>();++ assert_eq!(tokens.len(), 1);+ assert_eq!(tokens[0].text(), "cafe");+ }+proptest! {#[test]fn prop_tokenizer_correct_span(txt: String) {diff --git a/crates/core/src/tokenizer/normalizer/lowercase.rs b/crates/core/src/tokenizer/normalizer/lowercase.rsnew file mode 100644index 00000000..49d8a2f0--- /dev/null+++ b/crates/core/src/tokenizer/normalizer/lowercase.rs@@ -0,0 +1,58 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::Normalizer;+use crate::tokenizer::Token;++pub struct Lowercase;++impl<'a> Normalizer<'a> for Lowercase {+ fn normalize(&self, mut token: Token<'a>) -> Token<'a> {+ if token.text().is_ascii() {+ token.mut_text().make_ascii_lowercase();+ token+ } else {+ Token::new(token.text().to_lowercase(), token.span())+ }+ }++ fn should_normalize(&self, token: &Token<'a>) -> bool {+ token.text().chars().any(|c| c.is_uppercase())+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_lowercase() {+ let normalizer = Lowercase;+ let token = Token::new("Hello", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "hello");+ assert_eq!(normalized.span(), 0..5);+ assert!(normalizer.should_normalize(&token));++ let token = Token::new("hello", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "hello");++ let token = Token::new("HÈLLÖ", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "hèllö");+ }+}diff --git a/crates/core/src/tokenizer/normalizer/mod.rs b/crates/core/src/tokenizer/normalizer/mod.rsnew file mode 100644index 00000000..796777c4--- /dev/null+++ b/crates/core/src/tokenizer/normalizer/mod.rs@@ -0,0 +1,55 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod lowercase;+mod unicode;++pub use lowercase::Lowercase;+pub use unicode::diacritics::UnicodeDiacritics;+pub use unicode::nfd::UnicodeNFD;+pub use unicode::nfkc::UnicodeNFKC;+pub use unicode::nfkd::UnicodeNFKD;++use super::Token;++pub trait Normalizer<'a> {+ fn normalize(&self, token: Token<'a>) -> Token<'a>;+ fn should_normalize(&self, token: &Token<'a>) -> bool;+}++pub trait CharNormalizer {+ fn normalize(&self, c: char) -> impl Iterator<Item = char>;+ fn should_normalize(&self, c: char) -> bool;+}++impl<'a, T> Normalizer<'a> for T+where+ T: CharNormalizer,+{+ fn normalize(&self, token: Token<'a>) -> Token<'a> {+ let text = token+ .text()+ .chars()+ .flat_map(|c| self.normalize(c))+ .collect::<String>();++ Token::new(text, token.span())+ }++ fn should_normalize(&self, token: &Token<'a>) -> bool {+ token.text().chars().any(|c| self.should_normalize(c))+ }+}diff --git a/crates/core/src/tokenizer/normalizer/unicode/diacritics.rs b/crates/core/src/tokenizer/normalizer/unicode/diacritics.rsnew file mode 100644index 00000000..63c7e175--- /dev/null+++ b/crates/core/src/tokenizer/normalizer/unicode/diacritics.rs@@ -0,0 +1,43 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::tokenizer::normalizer::CharNormalizer;++#[inline]+fn is_diacritic(c: char) -> bool {+ matches!(c,+ '\u{0300}'..='\u{036F}'+ | '\u{1AB0}'..='\u{1AFF}'+ | '\u{1DC0}'..='\u{1DFF}'+ | '\u{20D0}'..='\u{20FF}'+ | '\u{FE20}'..='\u{FE2F}')+}++pub struct UnicodeDiacritics;++impl CharNormalizer for UnicodeDiacritics {+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {+ if is_diacritic(c) {+ None.into_iter()+ } else {+ Some(c).into_iter()+ }+ }++ fn should_normalize(&self, c: char) -> bool {+ is_diacritic(c)+ }+}diff --git a/crates/core/src/tokenizer/normalizer/unicode/mod.rs b/crates/core/src/tokenizer/normalizer/unicode/mod.rsnew file mode 100644index 00000000..1a3cafe4--- /dev/null+++ b/crates/core/src/tokenizer/normalizer/unicode/mod.rs@@ -0,0 +1,20 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub mod diacritics;+pub mod nfd;+pub mod nfkc;+pub mod nfkd;diff --git a/crates/core/src/tokenizer/normalizer/unicode/nfd.rs b/crates/core/src/tokenizer/normalizer/unicode/nfd.rsnew file mode 100644index 00000000..32e5e7dd--- /dev/null+++ b/crates/core/src/tokenizer/normalizer/unicode/nfd.rs@@ -0,0 +1,67 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::iter;++use crate::tokenizer::normalizer::CharNormalizer;++pub struct UnicodeNFD;++impl CharNormalizer for UnicodeNFD {+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {+ unicode_normalization::UnicodeNormalization::nfd(c)+ }++ fn should_normalize(&self, c: char) -> bool {+ match unicode_normalization::is_nfd_quick(iter::once(c)) {+ unicode_normalization::IsNormalized::No+ | unicode_normalization::IsNormalized::Maybe => true,+ unicode_normalization::IsNormalized::Yes => false,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::UnicodeNFD;+ use crate::tokenizer::normalizer::Normalizer;+ use crate::tokenizer::Token;++ #[test]+ fn test_nfd() {+ let normalizer = UnicodeNFD;+ let token = Token::new("ffi", 0..3);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "ffi");+ assert_eq!(normalized.span(), 0..3);++ let token = Token::new("ffi", 0..3);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "ffi");++ let token = Token::new("HÈLLÖ", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "HE\u{300}LLO\u{308}");++ let token = Token::new("hello", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "hello");++ let token = Token::new("café", 0..4);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "cafe\u{301}");+ }+}diff --git a/crates/core/src/tokenizer/normalizer/unicode/nfkc.rs b/crates/core/src/tokenizer/normalizer/unicode/nfkc.rsnew file mode 100644index 00000000..95d0b4b8--- /dev/null+++ b/crates/core/src/tokenizer/normalizer/unicode/nfkc.rs@@ -0,0 +1,68 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::iter;++use crate::tokenizer::normalizer::CharNormalizer;++pub struct UnicodeNFKC;++impl CharNormalizer for UnicodeNFKC {+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {+ unicode_normalization::UnicodeNormalization::nfkc(c)+ }++ fn should_normalize(&self, c: char) -> bool {+ match unicode_normalization::is_nfkc_quick(iter::once(c)) {+ unicode_normalization::IsNormalized::No+ | unicode_normalization::IsNormalized::Maybe => true,+ unicode_normalization::IsNormalized::Yes => false,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::UnicodeNFKC;+ use crate::tokenizer::normalizer::Normalizer;+ use crate::tokenizer::Token;++ #[test]+ fn test_nfkc() {+ let normalizer = UnicodeNFKC;+ let token = Token::new("ffi", 0..3);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "ffi");+ assert_eq!(normalized.span(), 0..3);+ assert!(normalizer.should_normalize(&token));++ let token = Token::new("ffi", 0..3);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "ffi");++ let token = Token::new("HÈLLÖ", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "HÈLLÖ");++ let token = Token::new("hello", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "hello");++ let token = Token::new("café", 0..4);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "café");+ }+}diff --git a/crates/core/src/tokenizer/normalizer/unicode/nfkd.rs b/crates/core/src/tokenizer/normalizer/unicode/nfkd.rsnew file mode 100644index 00000000..99fd9e9a--- /dev/null+++ b/crates/core/src/tokenizer/normalizer/unicode/nfkd.rs@@ -0,0 +1,64 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::iter;++use crate::tokenizer::normalizer::CharNormalizer;++pub struct UnicodeNFKD;++impl CharNormalizer for UnicodeNFKD {+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {+ unicode_normalization::UnicodeNormalization::nfkd(c)+ }++ fn should_normalize(&self, c: char) -> bool {+ match unicode_normalization::is_nfkd_quick(iter::once(c)) {+ unicode_normalization::IsNormalized::No+ | unicode_normalization::IsNormalized::Maybe => true,+ unicode_normalization::IsNormalized::Yes => false,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::UnicodeNFKD;+ use crate::tokenizer::normalizer::Normalizer;+ use crate::tokenizer::Token;++ #[test]+ fn test_nfkd() {+ let normalizer = UnicodeNFKD;+ let token = Token::new("ffi", 0..3);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "ffi");+ assert_eq!(normalized.span(), 0..3);+ assert!(normalizer.should_normalize(&token));++ let token = Token::new("ffi", 0..3);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "ffi");++ let token = Token::new("HÈLLÖ", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "HE\u{300}LLO\u{308}");++ let token = Token::new("hello", 0..5);+ let normalized = normalizer.normalize(token.clone());+ assert_eq!(normalized.text(), "hello");+ }+}diff --git a/crates/core/src/tokenizer/segmenter.rs b/crates/core/src/tokenizer/segmenter.rsindex 42409f14..bb414370 100644--- a/crates/core/src/tokenizer/segmenter.rs+++ b/crates/core/src/tokenizer/segmenter.rs@@ -87,7 +87,7 @@ impl<'a> Iterator for SegmentIterator<'a> {let next_script = Script::from(c);if let Some(script) = &script {- if &next_script != script {+ if &next_script != script && next_script != Script::Other {break;}} else {@@ -130,11 +130,11 @@ mod tests {let txt = "Hello, こんにちは、世界!";let segments: Vec<_> = txt.segments().collect();- assert_eq!(segments.len(), 2);- assert_eq!(segments[0].text(), "Hello, ");+ // TODO: this should be split into multiple segments+ // when we have more script tokenizers than just latin+ assert_eq!(segments.len(), 1);+ assert_eq!(segments[0].text(), "Hello, こんにちは、世界!");assert_eq!(segments[0].script, Script::Latin);- assert_eq!(segments[1].text(), "こんにちは、世界!");- assert_eq!(segments[1].script, Script::Other);}proptest! {diff --git a/crates/core/src/tokenizer/split_whitespace_with_range.rs b/crates/core/src/tokenizer/split_whitespace_with_range.rsindex 001aeafe..150f4deb 100644--- a/crates/core/src/tokenizer/split_whitespace_with_range.rs+++ b/crates/core/src/tokenizer/split_whitespace_with_range.rs@@ -100,6 +100,22 @@ mod tests {assert_eq!(tokens[2], ("123", 16..19));}+ #[test]+ fn unicode() {+ let txt = "best café";+ let tokens: Vec<_> = txt.split_whitespace_with_range().collect();+ assert_eq!(tokens.len(), 2);+ assert_eq!(tokens[0], ("best", 0..4));+ assert_eq!(tokens[1], ("café", 5..10));++ let txt = "Hello, 世界! 123";+ let tokens: Vec<_> = txt.split_whitespace_with_range().collect();+ assert_eq!(tokens.len(), 3);+ assert_eq!(tokens[0], ("Hello,", 0..6));+ assert_eq!(tokens[1], ("世界!", 7..14));+ assert_eq!(tokens[2], ("123", 15..18));+ }+proptest! {#[test]fn prop_split_whitespace_with_range(s: String) {@@ -108,5 +124,13 @@ mod tests {assert_eq!(&s[range.clone()], txt);}}++ #[test]+ fn consistent_with_std(s: String) {+ let tokens: Vec<_> = s.split_whitespace().collect();+ let tokens_with_range: Vec<_> = s.split_whitespace_with_range().collect();+ let tokens_with_range: Vec<_> = tokens_with_range.into_iter().map(|(txt, _)| txt).collect();+ assert_eq!(tokens, tokens_with_range);+ }}}diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rsindex 9b047c9b..fdcdba51 100644--- a/crates/core/src/web_spell/mod.rs+++ b/crates/core/src/web_spell/mod.rs@@ -35,7 +35,6 @@ use fst::Streamer;use std::ops::Range;use crate::ceil_char_boundary;-use crate::floor_char_boundary;use itertools::intersperse;#[derive(Debug, thiserror::Error)]@@ -170,7 +169,7 @@ pub fn sentence_ranges(text: &str) -> Vec<Range<usize>> {let mut start = last_start;while start < text.len() && text[start..].starts_with(|c: char| c.is_whitespace()) {- start = floor_char_boundary(&text, start + 1);+ start = ceil_char_boundary(&text, start + 1);}res.push(start..text.len());