Skip to content

web search done right

normalize diacritics/accents

Mikkel Denkerf1b72a897d77
diff --git a/Cargo.lock b/Cargo.lock
index 4c1b75ac..ee9e396b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4996,6 +4996,7 @@ dependencies = [
"tracing",
"tracing-subscriber",
"tracing-test",
+ "unicode-normalization",
"unicode-segmentation",
"url",
"utoipa",
diff --git a/Cargo.toml b/Cargo.toml
index 7e9ad95a..27cd984c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -165,6 +165,7 @@ tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}
tracing = {version = "0.1.34", features = ["release_max_level_info"]}
tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}
tracing-test = "0.2.4"
+unicode-normalization = "0.1.23"
unicode-segmentation = "1.11.0"
url = {version = "2.4.0", features = ["serde"]}
utoipa = {version = "4.2.3", features = ["axum_extras"]}
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 1941940f..e729659a 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -102,6 +102,7 @@ tower-http.workspace = true
tower.workspace = true
tracing-subscriber.workspace = true
tracing.workspace = true
+unicode-normalization.workspace = true
unicode-segmentation.workspace = true
url.workspace = true
utoipa-swagger-ui.workspace = true
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index c2e9495f..1bd1741b 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -1284,4 +1284,68 @@ mod tests {
assert_eq!(webpages[0].title, "C++");
assert_eq!(webpages[0].url, "https://www.a.com/");
}
+
+ #[test]
+ fn test_unicode_normalization() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ let webpage = Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>æble café</title>
+ </head>
+ <body>
+ {CONTENT} test
+ </body>
+ </html>
+ "#,
+ CONTENT = crate::rand_words(100)
+ ),
+ "https://www.a.com",
+ )
+ .unwrap();
+
+ index.insert(&webpage).unwrap();
+
+ index.commit().expect("failed to commit index");
+
+ let test = |q: &str| {
+ let ctx = index.local_search_ctx();
+
+ let query = Query::parse(
+ &ctx,
+ &SearchQuery {
+ query: q.to_string(),
+ ..Default::default()
+ },
+ &index,
+ )
+ .expect("Failed to parse query");
+
+ let ranker = Ranker::new(
+ SignalComputer::new(Some(&query)),
+ ctx.columnfield_reader.clone(),
+ CollectorConfig::default(),
+ );
+
+ let res = index
+ .search_initial(&query, &ctx, ranker.collector(ctx.clone()))
+ .unwrap();
+
+ assert_eq!(res.top_websites.len(), 1, "query: {}", q);
+
+ let webpages = index.retrieve_websites(&res.top_websites, &query).unwrap();
+
+ assert_eq!(webpages.len(), 1);
+ assert_eq!(webpages[0].title, "æble café");
+ assert_eq!(webpages[0].url, "https://www.a.com/");
+ };
+
+ test("cafe");
+ test("café");
+ test("æble");
+ test("æble café");
+ }
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 0498c2a7..58bba358 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -81,7 +81,7 @@ pub mod similar_hosts;
mod snippet;
mod stopwords;
pub mod summarizer;
-mod tokenizer;
+pub mod tokenizer;
#[allow(unused)]
mod ttl_cache;
pub mod warc;
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index c1282b28..9ce581f3 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -569,6 +569,18 @@ Survey in 2016, 2017, and 2018."#;
);
}
+ #[test]
+ fn test_fuzz_infinite_loop() {
+ let text = "\u{a0}";
+ let query = "";
+
+ let terms = query
+ .split_whitespace()
+ .map(|s| s.to_string())
+ .collect::<Vec<_>>();
+ let _ = snippet_string(text, &terms, whatlang::Lang::Eng, SnippetConfig::default());
+ }
+
proptest! {
#[test]
fn prop_snippet_gen(text in ".{1,100}", query in ".{1,100}") {
diff --git a/crates/core/src/tokenizer/fields/default.rs b/crates/core/src/tokenizer/fields/default.rs
index 1f299d2f..1d012cab 100644
--- a/crates/core/src/tokenizer/fields/default.rs
+++ b/crates/core/src/tokenizer/fields/default.rs
@@ -14,9 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::tokenizer::{BoxTokenStream, LowerCaser, StopWordFilter, TextAnalyzer};
+use tantivy::tokenizer::{BoxTokenStream, StopWordFilter, TextAnalyzer};
-use crate::tokenizer::{self, Tokenize};
+use crate::tokenizer::{self, normalizer, Normalize, Tokenize};
#[derive(Clone, Default)]
pub struct DefaultTokenizer {
@@ -40,7 +40,7 @@ impl tantivy::tokenizer::Tokenizer for DefaultTokenizer {
type TokenStream<'a> = BoxTokenStream<'a>;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
- let builder = TextAnalyzer::builder(Normal).filter(LowerCaser);
+ let builder = TextAnalyzer::builder(Normal);
self.analyzer = if let Some(stopwords) = &self.stopwords {
Some(
@@ -69,7 +69,12 @@ impl tantivy::tokenizer::Tokenizer for Normal {
type TokenStream<'a> = BoxTokenStream<'a>;
fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {
- let stream = Box::new(text.tokenize());
+ let stream = Box::new(
+ text.tokenize()
+ .normalize(&normalizer::Lowercase)
+ .normalize(&normalizer::UnicodeNFKD)
+ .normalize(&normalizer::UnicodeDiacritics),
+ );
BoxTokenStream::new(NormalTokenStream::new_boxed(stream))
}
@@ -190,7 +195,7 @@ mod tests {
fn katakana() {
assert_eq!(
tokenize_default("test ダ.com"),
- vec!["test", "ダ", ".", "com"]
+ vec!["test", "タ\u{3099}", ".", "com"]
);
}
diff --git a/crates/core/src/tokenizer/fields/url.rs b/crates/core/src/tokenizer/fields/url.rs
index e2560dc6..d127fa15 100644
--- a/crates/core/src/tokenizer/fields/url.rs
+++ b/crates/core/src/tokenizer/fields/url.rs
@@ -196,6 +196,7 @@ impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {
#[cfg(test)]
mod tests {
use super::*;
+ use proptest::prelude::*;
use tantivy::tokenizer::Tokenizer as _;
fn tokenize_url(s: &str) -> Vec<String> {
@@ -261,4 +262,16 @@ mod tests {
]
);
}
+
+ proptest! {
+ #![proptest_config(ProptestConfig::with_cases(4096))]
+
+ #[test]
+ fn test_single_space(url: String) {
+ let tokens = tokenize_url(&url);
+
+ let num_spaces = tokens.iter().filter(|s| s.contains(' ')).count();
+ prop_assert!(num_spaces <= 1);
+ }
+ }
}
diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rs
index a9107ec5..769489c5 100644
--- a/crates/core/src/tokenizer/mod.rs
+++ b/crates/core/src/tokenizer/mod.rs
@@ -16,6 +16,7 @@
mod add_space_last;
pub mod fields;
+pub mod normalizer;
mod script;
mod script_tokenizer;
mod segmenter;
@@ -29,7 +30,7 @@ pub use fields::FieldTokenizer;
use self::segmenter::Segmenter;
-#[derive(Debug)]
+#[derive(Clone, Debug)]
pub struct Token<'a> {
text: Cow<'a, str>,
span: std::ops::Range<usize>,
@@ -50,6 +51,10 @@ impl<'a> Token<'a> {
self.text.borrow()
}
+ pub fn mut_text(&mut self) -> &mut String {
+ self.text.to_mut()
+ }
+
pub fn span(&self) -> std::ops::Range<usize> {
self.span.clone()
}
@@ -69,9 +74,33 @@ impl Tokenize for str {
}
}
+pub trait Normalize<'a, N>
+where
+ N: normalizer::Normalizer<'a>,
+{
+ fn normalize(self, normalizer: &'a N) -> impl Iterator<Item = Token<'a>> + 'a;
+}
+
+impl<'a, T, N> Normalize<'a, N> for T
+where
+ T: Iterator<Item = Token<'a>> + 'a,
+ N: normalizer::Normalizer<'a>,
+{
+ fn normalize(self, normalizer: &'a N) -> impl Iterator<Item = Token<'a>> + 'a {
+ self.map(move |token| {
+ if normalizer.should_normalize(&token) {
+ normalizer.normalize(token)
+ } else {
+ token
+ }
+ })
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
+ use normalizer::{Lowercase, UnicodeDiacritics, UnicodeNFKD};
use proptest::prelude::*;
#[test]
@@ -92,6 +121,29 @@ mod tests {
assert_eq!(tokens[8].text(), ".");
}
+ #[test]
+ fn test_normalizer() {
+ let input = "Hello, world!";
+ let tokens: Vec<_> = input.tokenize().normalize(&Lowercase).collect();
+
+ assert_eq!(tokens.len(), 4);
+ assert_eq!(tokens[0].text(), "hello");
+ assert_eq!(tokens[1].text(), ",");
+ assert_eq!(tokens[2].text(), "world");
+ assert_eq!(tokens[3].text(), "!");
+
+ let input = "café";
+ let tokens = input
+ .tokenize()
+ .normalize(&Lowercase)
+ .normalize(&UnicodeNFKD)
+ .normalize(&UnicodeDiacritics)
+ .collect::<Vec<_>>();
+
+ assert_eq!(tokens.len(), 1);
+ assert_eq!(tokens[0].text(), "cafe");
+ }
+
proptest! {
#[test]
fn prop_tokenizer_correct_span(txt: String) {
diff --git a/crates/core/src/tokenizer/normalizer/lowercase.rs b/crates/core/src/tokenizer/normalizer/lowercase.rs
new file mode 100644
index 00000000..49d8a2f0
--- /dev/null
+++ b/crates/core/src/tokenizer/normalizer/lowercase.rs
@@ -0,0 +1,58 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::Normalizer;
+use crate::tokenizer::Token;
+
+pub struct Lowercase;
+
+impl<'a> Normalizer<'a> for Lowercase {
+ fn normalize(&self, mut token: Token<'a>) -> Token<'a> {
+ if token.text().is_ascii() {
+ token.mut_text().make_ascii_lowercase();
+ token
+ } else {
+ Token::new(token.text().to_lowercase(), token.span())
+ }
+ }
+
+ fn should_normalize(&self, token: &Token<'a>) -> bool {
+ token.text().chars().any(|c| c.is_uppercase())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_lowercase() {
+ let normalizer = Lowercase;
+ let token = Token::new("Hello", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "hello");
+ assert_eq!(normalized.span(), 0..5);
+ assert!(normalizer.should_normalize(&token));
+
+ let token = Token::new("hello", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "hello");
+
+ let token = Token::new("HÈLLÖ", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "hèllö");
+ }
+}
diff --git a/crates/core/src/tokenizer/normalizer/mod.rs b/crates/core/src/tokenizer/normalizer/mod.rs
new file mode 100644
index 00000000..796777c4
--- /dev/null
+++ b/crates/core/src/tokenizer/normalizer/mod.rs
@@ -0,0 +1,55 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod lowercase;
+mod unicode;
+
+pub use lowercase::Lowercase;
+pub use unicode::diacritics::UnicodeDiacritics;
+pub use unicode::nfd::UnicodeNFD;
+pub use unicode::nfkc::UnicodeNFKC;
+pub use unicode::nfkd::UnicodeNFKD;
+
+use super::Token;
+
+pub trait Normalizer<'a> {
+ fn normalize(&self, token: Token<'a>) -> Token<'a>;
+ fn should_normalize(&self, token: &Token<'a>) -> bool;
+}
+
+pub trait CharNormalizer {
+ fn normalize(&self, c: char) -> impl Iterator<Item = char>;
+ fn should_normalize(&self, c: char) -> bool;
+}
+
+impl<'a, T> Normalizer<'a> for T
+where
+ T: CharNormalizer,
+{
+ fn normalize(&self, token: Token<'a>) -> Token<'a> {
+ let text = token
+ .text()
+ .chars()
+ .flat_map(|c| self.normalize(c))
+ .collect::<String>();
+
+ Token::new(text, token.span())
+ }
+
+ fn should_normalize(&self, token: &Token<'a>) -> bool {
+ token.text().chars().any(|c| self.should_normalize(c))
+ }
+}
diff --git a/crates/core/src/tokenizer/normalizer/unicode/diacritics.rs b/crates/core/src/tokenizer/normalizer/unicode/diacritics.rs
new file mode 100644
index 00000000..63c7e175
--- /dev/null
+++ b/crates/core/src/tokenizer/normalizer/unicode/diacritics.rs
@@ -0,0 +1,43 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::tokenizer::normalizer::CharNormalizer;
+
+#[inline]
+fn is_diacritic(c: char) -> bool {
+ matches!(c,
+ '\u{0300}'..='\u{036F}'
+ | '\u{1AB0}'..='\u{1AFF}'
+ | '\u{1DC0}'..='\u{1DFF}'
+ | '\u{20D0}'..='\u{20FF}'
+ | '\u{FE20}'..='\u{FE2F}')
+}
+
+pub struct UnicodeDiacritics;
+
+impl CharNormalizer for UnicodeDiacritics {
+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {
+ if is_diacritic(c) {
+ None.into_iter()
+ } else {
+ Some(c).into_iter()
+ }
+ }
+
+ fn should_normalize(&self, c: char) -> bool {
+ is_diacritic(c)
+ }
+}
diff --git a/crates/core/src/tokenizer/normalizer/unicode/mod.rs b/crates/core/src/tokenizer/normalizer/unicode/mod.rs
new file mode 100644
index 00000000..1a3cafe4
--- /dev/null
+++ b/crates/core/src/tokenizer/normalizer/unicode/mod.rs
@@ -0,0 +1,20 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub mod diacritics;
+pub mod nfd;
+pub mod nfkc;
+pub mod nfkd;
diff --git a/crates/core/src/tokenizer/normalizer/unicode/nfd.rs b/crates/core/src/tokenizer/normalizer/unicode/nfd.rs
new file mode 100644
index 00000000..32e5e7dd
--- /dev/null
+++ b/crates/core/src/tokenizer/normalizer/unicode/nfd.rs
@@ -0,0 +1,67 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::iter;
+
+use crate::tokenizer::normalizer::CharNormalizer;
+
+pub struct UnicodeNFD;
+
+impl CharNormalizer for UnicodeNFD {
+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {
+ unicode_normalization::UnicodeNormalization::nfd(c)
+ }
+
+ fn should_normalize(&self, c: char) -> bool {
+ match unicode_normalization::is_nfd_quick(iter::once(c)) {
+ unicode_normalization::IsNormalized::No
+ | unicode_normalization::IsNormalized::Maybe => true,
+ unicode_normalization::IsNormalized::Yes => false,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::UnicodeNFD;
+ use crate::tokenizer::normalizer::Normalizer;
+ use crate::tokenizer::Token;
+
+ #[test]
+ fn test_nfd() {
+ let normalizer = UnicodeNFD;
+ let token = Token::new("ffi", 0..3);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "ffi");
+ assert_eq!(normalized.span(), 0..3);
+
+ let token = Token::new("ffi", 0..3);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "ffi");
+
+ let token = Token::new("HÈLLÖ", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "HE\u{300}LLO\u{308}");
+
+ let token = Token::new("hello", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "hello");
+
+ let token = Token::new("café", 0..4);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "cafe\u{301}");
+ }
+}
diff --git a/crates/core/src/tokenizer/normalizer/unicode/nfkc.rs b/crates/core/src/tokenizer/normalizer/unicode/nfkc.rs
new file mode 100644
index 00000000..95d0b4b8
--- /dev/null
+++ b/crates/core/src/tokenizer/normalizer/unicode/nfkc.rs
@@ -0,0 +1,68 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::iter;
+
+use crate::tokenizer::normalizer::CharNormalizer;
+
+pub struct UnicodeNFKC;
+
+impl CharNormalizer for UnicodeNFKC {
+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {
+ unicode_normalization::UnicodeNormalization::nfkc(c)
+ }
+
+ fn should_normalize(&self, c: char) -> bool {
+ match unicode_normalization::is_nfkc_quick(iter::once(c)) {
+ unicode_normalization::IsNormalized::No
+ | unicode_normalization::IsNormalized::Maybe => true,
+ unicode_normalization::IsNormalized::Yes => false,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::UnicodeNFKC;
+ use crate::tokenizer::normalizer::Normalizer;
+ use crate::tokenizer::Token;
+
+ #[test]
+ fn test_nfkc() {
+ let normalizer = UnicodeNFKC;
+ let token = Token::new("ffi", 0..3);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "ffi");
+ assert_eq!(normalized.span(), 0..3);
+ assert!(normalizer.should_normalize(&token));
+
+ let token = Token::new("ffi", 0..3);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "ffi");
+
+ let token = Token::new("HÈLLÖ", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "HÈLLÖ");
+
+ let token = Token::new("hello", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "hello");
+
+ let token = Token::new("café", 0..4);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "café");
+ }
+}
diff --git a/crates/core/src/tokenizer/normalizer/unicode/nfkd.rs b/crates/core/src/tokenizer/normalizer/unicode/nfkd.rs
new file mode 100644
index 00000000..99fd9e9a
--- /dev/null
+++ b/crates/core/src/tokenizer/normalizer/unicode/nfkd.rs
@@ -0,0 +1,64 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::iter;
+
+use crate::tokenizer::normalizer::CharNormalizer;
+
+pub struct UnicodeNFKD;
+
+impl CharNormalizer for UnicodeNFKD {
+ fn normalize(&self, c: char) -> impl Iterator<Item = char> {
+ unicode_normalization::UnicodeNormalization::nfkd(c)
+ }
+
+ fn should_normalize(&self, c: char) -> bool {
+ match unicode_normalization::is_nfkd_quick(iter::once(c)) {
+ unicode_normalization::IsNormalized::No
+ | unicode_normalization::IsNormalized::Maybe => true,
+ unicode_normalization::IsNormalized::Yes => false,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::UnicodeNFKD;
+ use crate::tokenizer::normalizer::Normalizer;
+ use crate::tokenizer::Token;
+
+ #[test]
+ fn test_nfkd() {
+ let normalizer = UnicodeNFKD;
+ let token = Token::new("ffi", 0..3);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "ffi");
+ assert_eq!(normalized.span(), 0..3);
+ assert!(normalizer.should_normalize(&token));
+
+ let token = Token::new("ffi", 0..3);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "ffi");
+
+ let token = Token::new("HÈLLÖ", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "HE\u{300}LLO\u{308}");
+
+ let token = Token::new("hello", 0..5);
+ let normalized = normalizer.normalize(token.clone());
+ assert_eq!(normalized.text(), "hello");
+ }
+}
diff --git a/crates/core/src/tokenizer/segmenter.rs b/crates/core/src/tokenizer/segmenter.rs
index 42409f14..bb414370 100644
--- a/crates/core/src/tokenizer/segmenter.rs
+++ b/crates/core/src/tokenizer/segmenter.rs
@@ -87,7 +87,7 @@ impl<'a> Iterator for SegmentIterator<'a> {
let next_script = Script::from(c);
if let Some(script) = &script {
- if &next_script != script {
+ if &next_script != script && next_script != Script::Other {
break;
}
} else {
@@ -130,11 +130,11 @@ mod tests {
let txt = "Hello, こんにちは、世界!";
let segments: Vec<_> = txt.segments().collect();
- assert_eq!(segments.len(), 2);
- assert_eq!(segments[0].text(), "Hello, ");
+ // TODO: this should be split into multiple segments
+ // when we have more script tokenizers than just latin
+ assert_eq!(segments.len(), 1);
+ assert_eq!(segments[0].text(), "Hello, こんにちは、世界!");
assert_eq!(segments[0].script, Script::Latin);
- assert_eq!(segments[1].text(), "こんにちは、世界!");
- assert_eq!(segments[1].script, Script::Other);
}
proptest! {
diff --git a/crates/core/src/tokenizer/split_whitespace_with_range.rs b/crates/core/src/tokenizer/split_whitespace_with_range.rs
index 001aeafe..150f4deb 100644
--- a/crates/core/src/tokenizer/split_whitespace_with_range.rs
+++ b/crates/core/src/tokenizer/split_whitespace_with_range.rs
@@ -100,6 +100,22 @@ mod tests {
assert_eq!(tokens[2], ("123", 16..19));
}
+ #[test]
+ fn unicode() {
+ let txt = "best café";
+ let tokens: Vec<_> = txt.split_whitespace_with_range().collect();
+ assert_eq!(tokens.len(), 2);
+ assert_eq!(tokens[0], ("best", 0..4));
+ assert_eq!(tokens[1], ("café", 5..10));
+
+ let txt = "Hello, 世界! 123";
+ let tokens: Vec<_> = txt.split_whitespace_with_range().collect();
+ assert_eq!(tokens.len(), 3);
+ assert_eq!(tokens[0], ("Hello,", 0..6));
+ assert_eq!(tokens[1], ("世界!", 7..14));
+ assert_eq!(tokens[2], ("123", 15..18));
+ }
+
proptest! {
#[test]
fn prop_split_whitespace_with_range(s: String) {
@@ -108,5 +124,13 @@ mod tests {
assert_eq!(&s[range.clone()], txt);
}
}
+
+ #[test]
+ fn consistent_with_std(s: String) {
+ let tokens: Vec<_> = s.split_whitespace().collect();
+ let tokens_with_range: Vec<_> = s.split_whitespace_with_range().collect();
+ let tokens_with_range: Vec<_> = tokens_with_range.into_iter().map(|(txt, _)| txt).collect();
+ assert_eq!(tokens, tokens_with_range);
+ }
}
}
diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rs
index 9b047c9b..fdcdba51 100644
--- a/crates/core/src/web_spell/mod.rs
+++ b/crates/core/src/web_spell/mod.rs
@@ -35,7 +35,6 @@ use fst::Streamer;
use std::ops::Range;
use crate::ceil_char_boundary;
-use crate::floor_char_boundary;
use itertools::intersperse;
#[derive(Debug, thiserror::Error)]
@@ -170,7 +169,7 @@ pub fn sentence_ranges(text: &str) -> Vec<Range<usize>> {
let mut start = last_start;
while start < text.len() && text[start..].starts_with(|c: char| c.is_whitespace()) {
- start = floor_char_boundary(&text, start + 1);
+ start = ceil_char_boundary(&text, start + 1);
}
res.push(start..text.len());