web search done right
re-write tokenizer to not use logos anymore
Mikkel Denkerc4192af997c0
diff --git a/Cargo.lock b/Cargo.lockindex 5bb40029..4c1b75ac 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4996,6 +4996,7 @@ dependencies = ["tracing","tracing-subscriber","tracing-test",+ "unicode-segmentation","url","utoipa","utoipa-swagger-ui",diff --git a/Cargo.toml b/Cargo.tomlindex e9ae94dc..7e9ad95a 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -165,6 +165,7 @@ tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}tracing = {version = "0.1.34", features = ["release_max_level_info"]}tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}tracing-test = "0.2.4"+unicode-segmentation = "1.11.0"url = {version = "2.4.0", features = ["serde"]}utoipa = {version = "4.2.3", features = ["axum_extras"]}utoipa-swagger-ui = {version = "7.0.0", features = ["axum"]}diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 37b636c7..1941940f 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -102,6 +102,7 @@ tower-http.workspace = truetower.workspace = truetracing-subscriber.workspace = truetracing.workspace = true+unicode-segmentation.workspace = trueurl.workspace = trueutoipa-swagger-ui.workspace = trueutoipa.workspace = truediff --git a/crates/core/src/ampc/dht/mod.rs b/crates/core/src/ampc/dht/mod.rsindex 9498ef8b..92036334 100644--- a/crates/core/src/ampc/dht/mod.rs+++ b/crates/core/src/ampc/dht/mod.rs@@ -146,7 +146,6 @@ pub mod tests {use openraft::{error::InitializeError, Config};use proptest::prelude::*;- use proptest_derive::Arbitrary;use futures::{pin_mut, TryStreamExt};use rand::seq::SliceRandom;@@ -557,7 +556,6 @@ pub mod tests {bincode::Encode,bincode::Decode,PartialEq,- Arbitrary,)]enum Action {Set { key: String, value: String },@@ -566,6 +564,21 @@ pub mod tests {Get { prev_key: usize },}+ impl Arbitrary for Action {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: ()) -> Self::Strategy {+ prop_oneof![+ (".{1,10}", ".{1,10}").prop_map(|(key, value)| Action::Set { key, value }),+ (0..1000).prop_map(|prev_key| Action::Get {+ prev_key: prev_key as usize+ }),+ ]+ .boxed()+ }+ }+proptest! {#![proptest_config(ProptestConfig::with_cases(10))]diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rsindex bfe63b15..84a686ec 100644--- a/crates/core/src/distributed/sonic/mod.rs+++ b/crates/core/src/distributed/sonic/mod.rs@@ -306,7 +306,6 @@ mod tests {use std::{collections::HashMap, future::Future};use proptest::prelude::*;- use proptest_derive::Arbitrary;use crate::free_socket_addr;@@ -343,12 +342,26 @@ mod tests {})}- #[derive(Debug, Clone, bincode::Encode, bincode::Decode, PartialEq, Arbitrary)]+ #[derive(Debug, Clone, bincode::Encode, bincode::Decode, PartialEq)]struct Message {text: String,other: HashMap<String, f32>,}+ impl Arbitrary for Message {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: ()) -> Self::Strategy {+ (+ any::<String>(),+ prop::collection::hash_map(".*", 0.0f32..100.0f32, 0..10),+ )+ .prop_map(|(text, other)| Message { text, other })+ .boxed()+ }+ }+proptest! {#[test]fn basic_arb(a1: Message, b1: Message) {diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rsindex b7d3e3a4..489a0276 100644--- a/crates/core/src/distributed/sonic/service.rs+++ b/crates/core/src/distributed/sonic/service.rs@@ -335,9 +335,8 @@ mod tests {mod counter_service {use std::sync::atomic::AtomicI32;- use proptest_derive::Arbitrary;-use super::super::Message;+ use proptest::prelude::*;pub struct CounterService {pub counter: AtomicI32,@@ -346,17 +345,21 @@ mod tests {sonic_service!(CounterService, [Change, Reset]);#[derive(- Debug,- Clone,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- Arbitrary,+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,)]pub struct Change {pub amount: i32,}++ impl Arbitrary for Change {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {+ (0..100).prop_map(|amount| Change { amount }).boxed()+ }+ }+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,)]diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rsindex 24619b46..c27022dd 100644--- a/crates/core/src/entity_index/mod.rs+++ b/crates/core/src/entity_index/mod.rs@@ -34,7 +34,7 @@ use tantivy::{use crate::{image_store::{EntityImageStore, Image, ImageStore},inverted_index::merge_tantivy_segments,- tokenizer::Normal,+ tokenizer::fields::DefaultTokenizer,Result,};@@ -49,7 +49,7 @@ fn schema() -> Schema {TextOptions::default().set_indexing_options(TextFieldIndexing::default()- .set_tokenizer(Normal::as_str())+ .set_tokenizer(DefaultTokenizer::as_str()).set_index_option(IndexRecordOption::WithFreqsAndPositions),).set_stored(),@@ -59,7 +59,7 @@ fn schema() -> Schema {TextOptions::default().set_indexing_options(TextFieldIndexing::default()- .set_tokenizer(Normal::as_str())+ .set_tokenizer(DefaultTokenizer::as_str()).set_index_option(IndexRecordOption::WithFreqsAndPositions),).set_stored(),@@ -164,8 +164,8 @@ impl EntityIndex {.collect();tantivy_index.tokenizers().register(- Normal::as_str(),- Normal::with_stopwords(stopwords.clone().into_iter().collect()),+ DefaultTokenizer::as_str(),+ DefaultTokenizer::with_stopwords(stopwords.clone().into_iter().collect()),);let image_store = EntityImageStore::open(path.as_ref().join("images"));@@ -270,7 +270,7 @@ impl EntityIndex {let entity_abstract = self.schema.get_field("abstract").unwrap();let mut term_queries = Vec::new();- let mut tokenizer = Normal::default();+ let mut tokenizer = DefaultTokenizer::default();let mut stream = tokenizer.token_stream(query);while let Some(token) = stream.next() {if self.stopwords.contains(&token.text) {diff --git a/crates/core/src/feed/index.rs b/crates/core/src/feed/index.rsindex 9ae1ecf1..9593d23b 100644--- a/crates/core/src/feed/index.rs+++ b/crates/core/src/feed/index.rs@@ -21,7 +21,7 @@ use std::{use crate::{inverted_index::merge_tantivy_segments,- tokenizer::{Tokenizer, UrlTokenizer},+ tokenizer::fields::{FieldTokenizer, UrlTokenizer},};use anyhow::Result;use hashbrown::HashSet;@@ -45,8 +45,8 @@ pub struct FeedIndex {impl FeedIndex {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let url_tokenizer = Tokenizer::Url(UrlTokenizer);- let kind_tokenizer = Tokenizer::default();+ let url_tokenizer = FieldTokenizer::Url(UrlTokenizer);+ let kind_tokenizer = FieldTokenizer::default();let mut builder = tantivy::schema::Schema::builder();diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex e810702d..c2e9495f 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -48,14 +48,14 @@ use crate::ranking::initial::Score;use crate::schema::text_field::TextField;use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum, TextFieldEnum};use crate::snippet::TextSnippet;-use crate::tokenizer::{+use crate::tokenizer::fields::{BigramTokenizer, Identity, JsonField, Stemmed, TrigramTokenizer, UrlTokenizer,};use crate::webpage::region::Region;use crate::webpage::schema_org;use crate::Result;-use crate::{schema::create_schema, tokenizer::Tokenizer};+use crate::{schema::create_schema, tokenizer::FieldTokenizer};use std::fs;use std::path::Path;use std::sync::Arc;@@ -109,25 +109,25 @@ impl From<DocAddress> for tantivy::DocAddress {}fn register_tokenizers(manager: &TokenizerManager) {- let tokenizer = Tokenizer::default();+ let tokenizer = FieldTokenizer::default();manager.register(tokenizer.as_str(), tokenizer);- let tokenizer = Tokenizer::Stemmed(Stemmed::default());+ let tokenizer = FieldTokenizer::Stemmed(Stemmed::default());manager.register(tokenizer.as_str(), tokenizer);- let tokenizer = Tokenizer::Identity(Identity::default());+ let tokenizer = FieldTokenizer::Identity(Identity::default());manager.register(tokenizer.as_str(), tokenizer);- let tokenizer = Tokenizer::Bigram(BigramTokenizer::default());+ let tokenizer = FieldTokenizer::Bigram(BigramTokenizer::default());manager.register(tokenizer.as_str(), tokenizer);- let tokenizer = Tokenizer::Trigram(TrigramTokenizer::default());+ let tokenizer = FieldTokenizer::Trigram(TrigramTokenizer::default());manager.register(tokenizer.as_str(), tokenizer);- let tokenizer = Tokenizer::Url(UrlTokenizer);+ let tokenizer = FieldTokenizer::Url(UrlTokenizer);manager.register(tokenizer.as_str(), tokenizer);- let tokenizer = Tokenizer::Json(JsonField);+ let tokenizer = FieldTokenizer::Json(JsonField);manager.register(tokenizer.as_str(), tokenizer);}diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 9ccb61ec..b7c4d456 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -29,8 +29,9 @@ use crate::{enum_dispatch_from_discriminant,enum_map::InsertEnumMapKey,ranking::bm25::Bm25Constants,- tokenizer::{- self, BigramTokenizer, Identity, JsonField, Tokenizer, TrigramTokenizer, UrlTokenizer,+ tokenizer,+ tokenizer::fields::{+ BigramTokenizer, FieldTokenizer, Identity, JsonField, TrigramTokenizer, UrlTokenizer,},webpage::Html,Result,@@ -64,11 +65,11 @@ pub trait TextField:}#[allow(unused_variables)]- fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::default()+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::default()}- fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {+ fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {self.tokenizer(lang)}@@ -372,10 +373,10 @@ impl TextField for StemmedTitle {"stemmed_title"}- fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {match lang {- Some(lang) => tokenizer::Stemmed::with_forced_language(*lang).into(),- None => tokenizer::Stemmed::default().into(),+ Some(lang) => tokenizer::fields::Stemmed::with_forced_language(*lang).into(),+ None => tokenizer::fields::Stemmed::default().into(),}}@@ -418,10 +419,10 @@ impl TextField for StemmedCleanBody {"stemmed_body"}- fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {match lang {- Some(lang) => tokenizer::Stemmed::with_forced_language(*lang).into(),- None => tokenizer::Stemmed::default().into(),+ Some(lang) => tokenizer::fields::Stemmed::with_forced_language(*lang).into(),+ None => tokenizer::fields::Stemmed::default().into(),}}@@ -543,8 +544,8 @@ impl TextField for UrlNoTokenizer {"url_no_tokenizer"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn is_searchable(&self) -> bool {@@ -590,8 +591,8 @@ impl TextField for UrlForSiteOperator {true}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Url(UrlTokenizer)+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Url(UrlTokenizer)}fn add_html_tantivy(@@ -673,8 +674,8 @@ impl TextField for SiteNoTokenizer {"site_no_tokenizer"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn is_searchable(&self) -> bool {@@ -716,8 +717,8 @@ impl TextField for DomainNoTokenizer {"domain_no_tokenizer"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn is_searchable(&self) -> bool {@@ -759,8 +760,8 @@ impl TextField for DomainNameNoTokenizer {"domain_name_no_tokenizer"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn is_searchable(&self) -> bool {@@ -802,8 +803,8 @@ impl TextField for SiteIfHomepageNoTokenizer {"site_if_homepage_no_tokenizer"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn add_html_tantivy(@@ -882,8 +883,8 @@ impl TextField for DomainNameIfHomepageNoTokenizer {"domain_name_if_homepage_no_tokenizer"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn add_html_tantivy(@@ -929,8 +930,8 @@ impl TextField for DomainIfHomepageNoTokenizer {"domain_if_homepage_no_tokenizer"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn add_html_tantivy(@@ -1125,8 +1126,8 @@ impl TextField for SchemaOrgJson {"schema_org_json"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn is_stored(&self) -> bool {@@ -1161,8 +1162,8 @@ impl TextField for FlattenedSchemaOrgJson {true}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Json(JsonField)+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Json(JsonField)}fn add_html_tantivy(@@ -1197,12 +1198,12 @@ impl TextField for CleanBodyBigrams {CleanBody.into()}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Bigram(BigramTokenizer::default())+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Bigram(BigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::default()+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::default()}fn is_searchable(&self) -> bool {@@ -1241,12 +1242,12 @@ impl TextField for TitleBigrams {Title.into()}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Bigram(BigramTokenizer::default())+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Bigram(BigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::default()+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::default()}fn is_searchable(&self) -> bool {@@ -1291,12 +1292,12 @@ impl TextField for CleanBodyTrigrams {CleanBody.into()}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Trigram(TrigramTokenizer::default())+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Trigram(TrigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::default()+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::default()}fn is_searchable(&self) -> bool {@@ -1335,12 +1336,12 @@ impl TextField for TitleTrigrams {Title.into()}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Trigram(TrigramTokenizer::default())+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Trigram(TrigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::default()+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::default()}fn is_searchable(&self) -> bool {@@ -1405,8 +1406,8 @@ impl TextField for SafetyClassification {"safety_classification"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn add_html_tantivy(@@ -1447,8 +1448,8 @@ impl TextField for InsertionTimestamp {"insertion_timestamp"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn indexing_option(&self) -> IndexingOption {@@ -1490,8 +1491,8 @@ impl TextField for RecipeFirstIngredientTagId {"recipe_first_ingredient_tag_id"}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Identity(Identity {})+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Identity(Identity {})}fn is_stored(&self) -> bool {@@ -1563,8 +1564,8 @@ impl TextField for Links {true}- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {- Tokenizer::Url(UrlTokenizer)+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Url(UrlTokenizer)}fn add_html_tantivy(diff --git a/crates/core/src/simhash.rs b/crates/core/src/simhash.rsindex 08b275b4..20e15561 100644--- a/crates/core/src/simhash.rs+++ b/crates/core/src/simhash.rs@@ -19,7 +19,7 @@ use std::{hash::{Hash, Hasher},};-use crate::tokenizer::Tokenizer;+use crate::tokenizer::FieldTokenizer;pub type HashType = u64;@@ -30,7 +30,7 @@ fn hash_token(token: &tantivy::tokenizer::Token) -> HashType {}pub fn hash(text: &str) -> HashType {- let mut tokenizer = Tokenizer::default();+ let mut tokenizer = FieldTokenizer::default();let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, text);diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex aed2b2ab..8dfac08a 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -19,7 +19,9 @@ use std::ops::Range;use crate::config::SnippetConfig;use crate::highlighted::{HighlightedFragment, HighlightedKind};use crate::query::Query;-use crate::tokenizer::{BigramTokenizer, Normal, Stemmed, Tokenizer, TrigramTokenizer};+use crate::tokenizer::fields::{+ BigramTokenizer, DefaultTokenizer, FieldTokenizer, Stemmed, TrigramTokenizer,+};use crate::web_spell::sentence_ranges;use crate::webpage::region::Region;use hashbrown::{HashMap, HashSet};@@ -81,10 +83,10 @@ struct SnippetBuilder {impl SnippetBuilder {fn highlight(&mut self, terms: &HashSet<String>, lang: whatlang::Lang) {for mut tokenizer in [- Tokenizer::Stemmed(Stemmed::with_forced_language(lang)),- Tokenizer::Normal(Normal::default()),- Tokenizer::Bigram(BigramTokenizer::default()),- Tokenizer::Trigram(TrigramTokenizer::default()),+ FieldTokenizer::Stemmed(Stemmed::with_forced_language(lang)),+ FieldTokenizer::Default(DefaultTokenizer::default()),+ FieldTokenizer::Bigram(BigramTokenizer::default()),+ FieldTokenizer::Trigram(TrigramTokenizer::default()),] {let mut stream =tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &self.fragment);@@ -143,7 +145,11 @@ impl SnippetBuilder {}}-fn passages(text: &str, mut tokenizer: Tokenizer, config: &SnippetConfig) -> Vec<PassageCandidate> {+fn passages(+ text: &str,+ mut tokenizer: FieldTokenizer,+ config: &SnippetConfig,+) -> Vec<PassageCandidate> {sentence_ranges(text).into_iter().filter(|offset| offset.end - offset.start > config.min_passage_width)@@ -216,7 +222,7 @@ fn snippet_string_builder(terms: &[String],lang: whatlang::Lang,config: SnippetConfig,- mut tokenizer: Tokenizer,+ mut tokenizer: FieldTokenizer,) -> SnippetBuilder {let terms: HashSet<String> = terms.iter()@@ -288,7 +294,7 @@ fn snippet_string(lang: whatlang::Lang,config: SnippetConfig,) -> TextSnippet {- let tokenizer = Tokenizer::Normal(Normal::default());+ let tokenizer = FieldTokenizer::Default(DefaultTokenizer::default());let snip = snippet_string_builder(text, terms, lang, config.clone(), tokenizer).build();if !snip.fragments.is_empty()@@ -300,7 +306,7 @@ fn snippet_string(return snip;}- let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));+ let tokenizer = FieldTokenizer::Stemmed(Stemmed::with_forced_language(lang));snippet_string_builder(text, terms, lang, config, tokenizer).build()}@@ -346,6 +352,7 @@ mod tests {searcher::{LocalSearcher, SearchQuery},webpage::Webpage,};+ use proptest::prelude::*;const TEST_TEXT: &str = r#"Rust is a systems programming language sponsored byMozilla which describes it as a "safe, concurrent, practical language", supporting functional and@@ -546,7 +553,7 @@ Survey in 2016, 2017, and 2018."#;&["thisis".to_string()],whatlang::Lang::Eng,SnippetConfig::default(),- Tokenizer::Normal(Normal::default()),+ FieldTokenizer::Default(DefaultTokenizer::default()),);let mut terms = HashSet::new();@@ -561,4 +568,12 @@ Survey in 2016, 2017, and 2018."#;"<b>this is</b> a test");}++ proptest! {+ #[test]+ fn prop_snippet_gen(text: String, query: String) {+ let terms = query.split_whitespace().map(|s| s.to_string()).collect::<Vec<_>>();+ let _ = snippet_string(&text, &terms, whatlang::Lang::Eng, SnippetConfig::default());+ }+ }}diff --git a/crates/core/src/tokenizer/fields/bigram.rs b/crates/core/src/tokenizer/fields/bigram.rsnew file mode 100644index 00000000..548ff74e--- /dev/null+++ b/crates/core/src/tokenizer/fields/bigram.rs@@ -0,0 +1,80 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::BoxTokenStream;++use super::{default::DefaultTokenizer, ngram::NGramTokenStream};++#[derive(Clone)]+pub struct BigramTokenizer {+ inner_tokenizer: DefaultTokenizer,+}++impl Default for BigramTokenizer {+ fn default() -> Self {+ Self {+ inner_tokenizer: DefaultTokenizer::with_stopwords(vec![]),+ }+ }+}++impl BigramTokenizer {+ pub fn as_str() -> &'static str {+ "bigram_tokenizer"+ }+}+impl tantivy::tokenizer::Tokenizer for BigramTokenizer {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let inner_stream = self.inner_tokenizer.token_stream(text);+ let stream: NGramTokenStream<2> = NGramTokenStream::new(inner_stream);+ BoxTokenStream::new(stream)+ }+}++#[cfg(test)]+mod tests {+ use tantivy::tokenizer::Tokenizer;++ use super::*;+ fn tokenize_bigram(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut tokenizer = BigramTokenizer::default();+ let mut stream = tokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn bigram_tokenizer() {+ assert!(tokenize_bigram("").is_empty());+ assert!(tokenize_bigram("test").is_empty());++ assert_eq!(tokenize_bigram("this is"), vec!["thisis"]);+ assert_eq!(tokenize_bigram("this is a"), vec!["thisis", "isa",]);+ assert_eq!(+ tokenize_bigram("this is a test"),+ vec!["thisis", "isa", "atest",]+ );++ assert_eq!(tokenize_bigram("this.is"), vec!["this.", ".is"]);+ }+}diff --git a/crates/core/src/tokenizer/fields/default.rs b/crates/core/src/tokenizer/fields/default.rsnew file mode 100644index 00000000..5c0476bd--- /dev/null+++ b/crates/core/src/tokenizer/fields/default.rs@@ -0,0 +1,217 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::{BoxTokenStream, LowerCaser, StopWordFilter, TextAnalyzer};++use crate::tokenizer::{self, Tokenize};++#[derive(Clone, Default)]+pub struct DefaultTokenizer {+ stopwords: Option<Vec<String>>,+ analyzer: Option<TextAnalyzer>,+}++impl DefaultTokenizer {+ pub fn as_str() -> &'static str {+ "tokenizer"+ }++ pub fn with_stopwords(stopwords: Vec<String>) -> Self {+ Self {+ stopwords: Some(stopwords),+ analyzer: None,+ }+ }+}+impl tantivy::tokenizer::Tokenizer for DefaultTokenizer {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let builder = TextAnalyzer::builder(Normal).filter(LowerCaser);++ self.analyzer = if let Some(stopwords) = &self.stopwords {+ Some(+ builder+ .filter(StopWordFilter::remove(stopwords.clone()))+ .build(),+ )+ } else {+ Some(builder.build())+ };++ self.analyzer.as_mut().unwrap().token_stream(text)+ }+}++#[derive(Clone)]+pub struct Normal;++pub struct NormalTokenStream<'a> {+ stream: Box<dyn Iterator<Item = tokenizer::Token> + 'a>,+ token: Option<tantivy::tokenizer::Token>,+ next_position: usize,+}++impl tantivy::tokenizer::Tokenizer for Normal {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {+ let stream = Box::new(text.tokenize());++ BoxTokenStream::new(NormalTokenStream::new_boxed(stream))+ }+}++impl<'a> tantivy::tokenizer::TokenStream for NormalTokenStream<'a> {+ fn advance(&mut self) -> bool {+ self.token = self.stream.next().map(|token| {+ let span = token.span();+ let pos = self.next_position;+ self.next_position += 1;+ tantivy::tokenizer::Token {+ offset_from: span.start,+ offset_to: span.end,+ position: pos,+ text: token.text().to_string(),+ ..Default::default()+ }+ });++ self.token.is_some()+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ self.token.as_ref().unwrap()+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ self.token.as_mut().unwrap()+ }+}++impl<'a> NormalTokenStream<'a> {+ fn new_boxed(stream: Box<dyn Iterator<Item = tokenizer::Token> + 'a>) -> BoxTokenStream<'a> {+ BoxTokenStream::new(Self {+ stream,+ token: None,+ next_position: 0,+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use proptest::prelude::*;+ use tantivy::tokenizer::Tokenizer as _;++ fn tokenize_default(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut tokenizer = DefaultTokenizer::default();+ let mut stream = tokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn default_tokenization() {+ assert_eq!(+ tokenize_default("this is a relatively simple123 test string"),+ vec![+ "this",+ "is",+ "a",+ "relatively",+ "simple123",+ "test",+ "string"+ ]+ );+ }++ #[test]+ fn special_character_tokenization() {+ assert_eq!(+ tokenize_default("example.com"),+ vec!["example", ".", "com",]+ );+ assert_eq!(+ tokenize_default("example. com"),+ vec!["example", ".", "com",]+ );+ assert_eq!(+ tokenize_default("example . com"),+ vec!["example", ".", "com",]+ );++ assert_eq!(+ tokenize_default("a c++ blog post"),+ vec!["a", "c", "+", "+", "blog", "post"]+ );+ assert_eq!(tokenize_default("path/test"), vec!["path", "/", "test",]);+ }++ #[test]+ fn han() {+ assert_eq!(+ tokenize_default("test 漢.com"),+ vec!["test", "漢", ".", "com"]+ );+ }++ #[test]+ fn hiragana() {+ assert_eq!(+ tokenize_default("test あ.com"),+ vec!["test", "あ", ".", "com"]+ );+ }++ #[test]+ fn katakana() {+ assert_eq!(+ tokenize_default("test ダ.com"),+ vec!["test", "ダ", ".", "com"]+ );+ }++ #[test]+ fn cyrillic() {+ assert_eq!(+ tokenize_default("test б.com"),+ vec!["test", "б", ".", "com"]+ );+ }++ #[test]+ fn arabic() {+ assert_eq!(+ tokenize_default("test ب.com"),+ vec!["test", "ب", ".", "com"]+ );+ }++ proptest! {+ #[test]+ fn prop_default_tokenization(s: String) {+ let _ = tokenize_default(&s);+ }+ }+}diff --git a/crates/core/src/tokenizer/fields/identity.rs b/crates/core/src/tokenizer/fields/identity.rsnew file mode 100644index 00000000..957b28e6--- /dev/null+++ b/crates/core/src/tokenizer/fields/identity.rs@@ -0,0 +1,95 @@+use tantivy::tokenizer::BoxTokenStream;++// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+#[derive(Clone, Default, Debug)]+pub struct Identity {}++impl Identity {+ pub fn as_str() -> &'static str {+ "identity_tokenizer"+ }+}+impl tantivy::tokenizer::Tokenizer for Identity {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {+ BoxTokenStream::new(IdentityTokenStream::from(text.to_string()))+ }+}+pub struct IdentityTokenStream {+ num_advances: usize,+ token: Option<tantivy::tokenizer::Token>,+}++impl From<String> for IdentityTokenStream {+ fn from(text: String) -> Self {+ Self {+ num_advances: 0,+ token: Some(tantivy::tokenizer::Token {+ offset_from: 0,+ offset_to: text.len(),+ position: 0,+ text,+ ..Default::default()+ }),+ }+ }+}+impl tantivy::tokenizer::TokenStream for IdentityTokenStream {+ fn advance(&mut self) -> bool {+ self.num_advances += 1;++ if self.num_advances == 1 {+ true+ } else {+ self.token = None;+ false+ }+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ self.token.as_ref().unwrap()+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ self.token.as_mut().unwrap()+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use tantivy::tokenizer::Tokenizer as _;++ fn tokenize_identity(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut tokenizer = Identity {};+ let mut stream = tokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn identity() {+ assert_eq!(tokenize_identity("this is a test"), vec!["this is a test"]);+ assert_eq!(tokenize_identity("a-b"), vec!["a-b"]);+ }+}diff --git a/crates/core/src/tokenizer/fields/json.rs b/crates/core/src/tokenizer/fields/json.rsnew file mode 100644index 00000000..5dd551d1--- /dev/null+++ b/crates/core/src/tokenizer/fields/json.rs@@ -0,0 +1,356 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::str::CharIndices;++use tantivy::tokenizer::BoxTokenStream;++use crate::{ceil_char_boundary, floor_char_boundary};++pub struct FlattenedJson {+ flattened_json: String,+ inner_tokenizer: JsonField,+}++struct IntermediateFlatValue {+ parent_keys: Vec<String>,+ val: serde_json::Value,+}++fn flatten(val: serde_json::Value) -> Vec<String> {+ let mut res = Vec::new();++ let mut stack = Vec::new();+ stack.push(IntermediateFlatValue {+ parent_keys: Vec::new(),+ val,+ });++ while let Some(elem) = stack.pop() {+ match elem.val {+ serde_json::Value::Null => {+ res.push(itertools::intersperse(elem.parent_keys, ".".to_string()).collect())+ }+ serde_json::Value::Bool(b) => {+ let key: String =+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();+ res.push(format!("{key}=\"{b}\""))+ }+ serde_json::Value::Number(n) => {+ let key: String =+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();+ res.push(format!("{key}=\"{n}\""))+ }+ serde_json::Value::String(s) => {+ let key: String =+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();+ res.push(format!("{key}=\"{}\"", s.replace('"', "\\\"")))+ }+ serde_json::Value::Array(arr) => {+ for item in arr {+ stack.push(IntermediateFlatValue {+ parent_keys: elem.parent_keys.clone(),+ val: item,+ });+ }+ }+ serde_json::Value::Object(map) => {+ for (key, val) in map {+ let mut parent_keys = elem.parent_keys.clone();+ parent_keys.push(key);++ stack.push(IntermediateFlatValue { parent_keys, val });+ }+ }+ }+ }++ res.reverse();++ res+}++impl FlattenedJson {+ pub fn new<T>(value: &T) -> crate::Result<Self>+ where+ T: serde::Serialize,+ {+ let json = serde_json::to_string(value)?;+ let val: serde_json::Value = serde_json::from_str(&json)?;++ let flattened_json = itertools::intersperse(flatten(val), "\n".to_string()).collect();++ Ok(Self {+ flattened_json,+ inner_tokenizer: JsonField,+ })+ }++ pub fn token_stream(&mut self) -> BoxTokenStream {+ tantivy::tokenizer::Tokenizer::token_stream(&mut self.inner_tokenizer, &self.flattened_json)+ }++ pub fn text(&self) -> &str {+ &self.flattened_json+ }+}++#[derive(Clone, Debug)]+pub struct JsonField;++impl JsonField {+ pub fn as_str() -> &'static str {+ "json_tokenizer"+ }+}++impl tantivy::tokenizer::Tokenizer for JsonField {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {+ BoxTokenStream::new(JsonFieldTokenStream {+ text,+ chars: text.char_indices(),+ token: tantivy::tokenizer::Token::default(),+ })+ }+}++pub struct JsonFieldTokenStream<'a> {+ text: &'a str,+ chars: CharIndices<'a>,+ token: tantivy::tokenizer::Token,+}++impl<'a> JsonFieldTokenStream<'a> {+ // search for the end of the current token.+ fn search_token_end(&mut self, is_quote: bool) -> usize {+ let mut escaped = false;+ for (offset, c) in self.chars.by_ref() {+ if is_quote {+ if c == '\\' {+ escaped = true;+ } else {+ if c == '"' && !escaped {+ return offset;+ }++ escaped = false;+ }+ } else if !c.is_alphanumeric() {+ return offset;+ }+ }++ self.text.len()+ }+}++impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {+ fn advance(&mut self) -> bool {+ self.token.text.clear();+ self.token.position = self.token.position.wrapping_add(1);+ let mut prev_was_quote = false;++ while let Some((offset_from, c)) = self.chars.next() {+ if !matches!(c, '.' | '\n' | '"') {+ let offset_to = self.search_token_end(prev_was_quote);+ self.token.offset_from = offset_from;+ self.token.offset_to = offset_to;++ if prev_was_quote {+ self.token.offset_from -= 1;+ self.token.offset_to += 1;++ self.token.offset_from = floor_char_boundary(self.text, self.token.offset_from);+ self.token.offset_to =+ ceil_char_boundary(self.text, self.token.offset_to).min(self.text.len());+ }++ self.token+ .text+ .push_str(&self.text[self.token.offset_from..self.token.offset_to]);+ return true;+ }++ prev_was_quote = c == '"';+ }+ false+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ &mut self.token+ }+}++#[cfg(test)]+mod tests {+ use tantivy::tokenizer::Tokenizer;++ use super::*;++ fn tokenize_json(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut tokenizer = JsonField;+ let mut stream = tokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn tokenize_json_field() {+ assert_eq!(+ tokenize_json(r#"Test.field="value""#),+ vec!["Test", "field", "\"value\"",]+ );+ assert_eq!(+ tokenize_json(r#"Test.field="this is the value""#),+ vec!["Test", "field", "\"this is the value\"",]+ );+ assert_eq!(+ tokenize_json(r#"Test.field="this is\" the value""#),+ vec!["Test", "field", "\"this is\\\" the value\"",]+ );+ assert_eq!(+ tokenize_json("Test.field=\"this*@# is\\\" the\\\" \nvalue\""),+ vec!["Test", "field", "\"this*@# is\\\" the\\\" \nvalue\"",]+ );+ }++ fn flattened_json_helper(json: &str, expected: &str) {+ let parsed: serde_json::Value = serde_json::from_str(json).unwrap();+ let flattened = FlattenedJson::new(&parsed).unwrap();+ let flat = flattened.text();++ assert_eq!(flat, expected);+ }++ #[test]+ fn flatten_json_object() {+ let json = r#"+ {+ "key1": "val1",+ "key2": "val2"+ }+ "#;+ let expected = r#"key1="val1"+key2="val2""#;++ flattened_json_helper(json, expected);++ let json = r#"+ {+ "key1": 1,+ "key2": 2+ }+ "#;+ let expected = r#"key1="1"+key2="2""#;++ flattened_json_helper(json, expected);++ let json = r#"+ {+ "key1": {+ "key2": "value1",+ "key3": "value2"+ }+ }+ "#;+ let expected = r#"key1.key2="value1"+key1.key3="value2""#;++ flattened_json_helper(json, expected);++ let json = r#"+ {+ "$key1": {+ "$key2": "value1",+ "key3": "value2"+ }+ }+ "#;+ let expected = r#"$key1.$key2="value1"+$key1.key3="value2""#;++ flattened_json_helper(json, expected);++ let json = r#"+ {+ "key1": [+ "value1",+ "value2"+ ]+ }+ "#;+ let expected = r#"key1="value1"+key1="value2""#;++ flattened_json_helper(json, expected);++ let json = r#"+ {+ "key1": [+ "value1",+ {+ "key2": "value2",+ "key3": 123+ }+ ]+ }+ "#;+ let expected = r#"key1="value1"+key1.key2="value2"+key1.key3="123""#;++ flattened_json_helper(json, expected);++ let json = r#"+ {+ "key1": [+ "value1",+ {+ "key2": "this\" is @ a # test"+ }+ ]+ }+ "#;+ let expected = r#"key1="value1"+key1.key2="this\" is @ a # test""#;++ flattened_json_helper(json, expected);+ }++ #[test]+ fn out_of_bounds_crash() {+ tokenize_json(+ r#"+Breadcrumb.title="Home"+Breadcrumb.url="https://www.eurotecnicaservice.it/?lang=en"+Breadcrumb.title="Fuser Pur"+Breadcrumb.url="https://www.eurotecnicaservice.it/testing\"+"#,+ );+ }+}diff --git a/crates/core/src/tokenizer/fields/mod.rs b/crates/core/src/tokenizer/fields/mod.rsnew file mode 100644index 00000000..1558c708--- /dev/null+++ b/crates/core/src/tokenizer/fields/mod.rs@@ -0,0 +1,84 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::BoxTokenStream;++pub use self::{+ bigram::BigramTokenizer, default::DefaultTokenizer, identity::Identity, json::FlattenedJson,+ json::JsonField, stemmed::Stemmed, trigram::TrigramTokenizer, url::UrlTokenizer,+};++mod default;+mod identity;+mod json;+mod stemmed;+mod url;++mod bigram;+mod ngram;+mod trigram;++#[derive(Clone)]+pub enum FieldTokenizer {+ Default(DefaultTokenizer),+ Identity(Identity),+ Stemmed(Stemmed),+ Bigram(BigramTokenizer),+ Trigram(TrigramTokenizer),+ Json(JsonField),+ Url(UrlTokenizer),+}++impl FieldTokenizer {+ pub fn as_str(&self) -> &'static str {+ match self {+ FieldTokenizer::Default(_) => DefaultTokenizer::as_str(),+ FieldTokenizer::Stemmed(_) => Stemmed::as_str(),+ FieldTokenizer::Identity(_) => Identity::as_str(),+ FieldTokenizer::Bigram(_) => BigramTokenizer::as_str(),+ FieldTokenizer::Trigram(_) => TrigramTokenizer::as_str(),+ FieldTokenizer::Json(_) => JsonField::as_str(),+ FieldTokenizer::Url(_) => UrlTokenizer::as_str(),+ }+ }+}++impl From<Stemmed> for FieldTokenizer {+ fn from(stemmed: Stemmed) -> Self {+ Self::Stemmed(stemmed)+ }+}++impl Default for FieldTokenizer {+ fn default() -> Self {+ Self::Default(DefaultTokenizer::default())+ }+}+impl tantivy::tokenizer::Tokenizer for FieldTokenizer {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ match self {+ FieldTokenizer::Default(tokenizer) => tokenizer.token_stream(text),+ FieldTokenizer::Stemmed(tokenizer) => tokenizer.token_stream(text),+ FieldTokenizer::Identity(tokenizer) => tokenizer.token_stream(text),+ FieldTokenizer::Json(tokenizer) => tokenizer.token_stream(text),+ FieldTokenizer::Bigram(tokenizer) => tokenizer.token_stream(text),+ FieldTokenizer::Trigram(tokenizer) => tokenizer.token_stream(text),+ FieldTokenizer::Url(tokenizer) => tokenizer.token_stream(text),+ }+ }+}diff --git a/crates/core/src/tokenizer/fields/ngram.rs b/crates/core/src/tokenizer/fields/ngram.rsnew file mode 100644index 00000000..369972b2--- /dev/null+++ b/crates/core/src/tokenizer/fields/ngram.rs@@ -0,0 +1,92 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::BoxTokenStream;++pub struct NGramTokenStream<'a, const N: usize> {+ inner: BoxTokenStream<'a>,+ token: tantivy::tokenizer::Token,+ token_window: [tantivy::tokenizer::Token; N],+ next_pos: usize,+}++impl<'a, const N: usize> NGramTokenStream<'a, N> {+ pub fn new(inner: BoxTokenStream<'a>) -> Self {+ Self {+ inner,+ token: tantivy::tokenizer::Token::default(),+ token_window: std::array::from_fn(|_| tantivy::tokenizer::Token::default()),+ next_pos: 0,+ }+ }+}++fn reuse_token_alloc(token: &mut tantivy::tokenizer::Token, new_token: &tantivy::tokenizer::Token) {+ token.text.clear();+ token.text += new_token.text.as_str();+ token.offset_from = new_token.offset_from;+ token.offset_to = new_token.offset_to;+ token.position = new_token.position;+ token.position_length = new_token.position_length;+}++impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, N> {+ fn advance(&mut self) -> bool {+ if !self.inner.advance() {+ return false;+ }++ self.token_window.rotate_left(1);+ reuse_token_alloc(&mut self.token_window[N - 1], self.inner.token());++ while self.token_window[0].text.is_empty() {+ if !self.inner.advance() {+ return false;+ }++ self.token_window.rotate_left(1);+ reuse_token_alloc(&mut self.token_window[N - 1], self.inner.token());+ }++ self.next_pos += 1;++ let begin = self+ .token_window+ .iter()+ .position(|token| !token.text.is_empty())+ .unwrap_or(N - 1);++ self.token.position = self.next_pos;+ self.token.offset_from = self.token_window[begin].offset_from;+ self.token.offset_to = self.token_window[N - 1].offset_to;+ self.token.position_length = N - begin;++ self.token.text.clear();+ for token in &self.token_window {+ self.token.text += token.text.as_str();+ }++ true+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ &mut self.token+ }+}diff --git a/crates/core/src/tokenizer/fields/stemmed.rs b/crates/core/src/tokenizer/fields/stemmed.rsnew file mode 100644index 00000000..77af3df8--- /dev/null+++ b/crates/core/src/tokenizer/fields/stemmed.rs@@ -0,0 +1,59 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::{BoxTokenStream, LowerCaser, TextAnalyzer};+use whatlang::Lang;++use crate::tokenizer::stemmer::Stemmer;++use super::default::Normal;++#[derive(Clone, Default)]+pub struct Stemmed {+ force_language: Option<Lang>,+ analyzer: Option<TextAnalyzer>,+}++impl Stemmed {+ pub fn as_str() -> &'static str {+ "stemmed_tokenizer"+ }+ pub fn with_forced_language(lang: Lang) -> Self {+ Self {+ force_language: Some(lang),+ analyzer: None,+ }+ }+}+impl tantivy::tokenizer::Tokenizer for Stemmed {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let builder = TextAnalyzer::builder(Normal).filter(LowerCaser);++ let lang = match self.force_language {+ Some(lang) => Some(lang),+ None => whatlang::detect_lang(text),+ };++ self.analyzer = match lang {+ Some(lang) => Some(builder.filter(Stemmer::from(lang).into_tantivy()).build()),+ None => Some(builder.build()),+ };++ self.analyzer.as_mut().unwrap().token_stream(text)+ }+}diff --git a/crates/core/src/tokenizer/fields/trigram.rs b/crates/core/src/tokenizer/fields/trigram.rsnew file mode 100644index 00000000..54fc0884--- /dev/null+++ b/crates/core/src/tokenizer/fields/trigram.rs@@ -0,0 +1,79 @@+use tantivy::tokenizer::BoxTokenStream;++use super::{default::DefaultTokenizer, ngram::NGramTokenStream};++// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+#[derive(Clone)]+pub struct TrigramTokenizer {+ inner_tokenizer: DefaultTokenizer,+}++impl Default for TrigramTokenizer {+ fn default() -> Self {+ Self {+ inner_tokenizer: DefaultTokenizer::with_stopwords(vec![]),+ }+ }+}++impl TrigramTokenizer {+ pub fn as_str() -> &'static str {+ "trigram_tokenizer"+ }+}+impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let inner = self.inner_tokenizer.token_stream(text);+ let stream: NGramTokenStream<3> = NGramTokenStream::new(inner);+ BoxTokenStream::new(stream)+ }+}++#[cfg(test)]+mod tests {+ use tantivy::tokenizer::Tokenizer;++ use super::*;++ fn tokenize_trigram(s: &str) -> Vec<String> {+ let mut res = Vec::new();++ let mut tokenizer = TrigramTokenizer::default();+ let mut stream = tokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn trigram_tokenizer() {+ assert!(tokenize_trigram("").is_empty());+ assert!(tokenize_trigram("test").is_empty());+ assert!(tokenize_trigram("this is").is_empty());++ assert_eq!(tokenize_trigram("this is a"), vec!["thisisa",]);+ assert_eq!(+ tokenize_trigram("this is a test"),+ vec!["thisisa", "isatest"]+ );+ }+}diff --git a/crates/core/src/tokenizer/fields/url.rs b/crates/core/src/tokenizer/fields/url.rsnew file mode 100644index 00000000..e2560dc6--- /dev/null+++ b/crates/core/src/tokenizer/fields/url.rs@@ -0,0 +1,264 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::collections::VecDeque;++use tantivy::tokenizer::BoxTokenStream;++use crate::tokenizer::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};++#[derive(Clone, Default)]+struct ParsedUrl {+ protocol: Option<VecDeque<String>>,+ domain: Option<VecDeque<String>>,+ path: VecDeque<String>,+}++#[derive(Debug, Clone)]+pub struct UrlTokenizer;++impl UrlTokenizer {+ pub fn as_str() -> &'static str {+ "url_tokenizer"+ }++ fn parse_url(text: &str) -> ParsedUrl {+ url::Url::parse(text)+ .or_else(|_| url::Url::parse(&format!("http://{}", text)))+ .map(|url| {+ let domain = Some(+ url.host_str()+ .unwrap_or("")+ .split_preserve(|c| matches!(c, '.'))+ .filter(|s| !(*s).is_empty())+ .map(|s| s.to_string())+ .add_space_last()+ .collect(),+ );+ let path: VecDeque<_> = url+ .path()+ .split_preserve(|c| matches!(c, '/' | '-' | '_'))+ .filter(|s| !(*s).is_empty())+ .map(|s| s.to_string())+ .collect();++ if matches!(url.scheme(), "http" | "https") {+ ParsedUrl {+ protocol: None,+ domain,+ path,+ }+ } else {+ let mut v = VecDeque::new();+ v.push_back(url.scheme().to_string());++ ParsedUrl {+ protocol: Some(v),+ domain,+ path,+ }+ }+ })+ .unwrap_or_default()+ }+}++impl tantivy::tokenizer::Tokenizer for UrlTokenizer {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {+ let text = text.replace(' ', "%20");++ let urls = text+ .split('\n')+ .filter(|s| !s.is_empty())+ .map(|s| s.to_lowercase())+ .map(|s| Self::parse_url(&s))+ .collect();++ BoxTokenStream::new(SiteOperatorUrlTokenStream::new(urls))+ }+}++pub struct SiteOperatorUrlTokenStream {+ urls: VecDeque<ParsedUrl>,+ current_url: ParsedUrl,+ token: tantivy::tokenizer::Token,+}++impl SiteOperatorUrlTokenStream {+ fn new(mut urls: VecDeque<ParsedUrl>) -> Self {+ let current_url = urls.pop_front().unwrap_or_default();++ Self {+ urls,+ current_url,+ token: tantivy::tokenizer::Token::default(),+ }+ }++ fn advance_current_url(&mut self) -> bool {+ if let Some(protocol) = self.current_url.protocol.as_mut() {+ self.token.position = self.token.position.wrapping_add(1);+ self.token.text.clear();++ if let Some(s) = protocol.pop_front() {+ self.token.text.push_str(&s);+ self.token.offset_from = 0;+ self.token.offset_to = s.len();+ } else {+ self.token.offset_from = self.token.offset_to;+ self.token.text.push_str("://");+ self.token.offset_to += self.token.text.len();++ self.current_url.protocol = None;+ }++ return true;+ }++ if let Some(domain) = self.current_url.domain.as_mut() {+ if let Some(s) = domain.pop_front() {+ self.token.text.clear();+ self.token.position = self.token.position.wrapping_add(1);++ self.token.text.push_str(&s);++ self.token.offset_from = self.token.offset_to;+ self.token.offset_to += self.token.text.len();+ return true;+ }+ }++ if let Some(s) = self.current_url.path.pop_front() {+ self.token.text.clear();+ self.token.position = self.token.position.wrapping_add(1);++ self.token.text.push_str(&s);+ self.token.offset_from = self.token.offset_to;+ self.token.offset_to += self.token.text.len();++ return true;+ }++ false+ }++ fn next_url(&mut self) -> bool {+ if let Some(url) = self.urls.pop_front() {+ self.current_url = url;++ self.token.position = self.token.position.wrapping_add(1);+ self.token.text.clear();+ self.token.text.push('\n');++ self.token.offset_from = self.token.offset_to;+ self.token.offset_to += self.token.text.len();++ true+ } else {+ false+ }+ }+}++impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {+ fn advance(&mut self) -> bool {+ if self.advance_current_url() {+ return true;+ }++ self.next_url()+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ &mut self.token+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use tantivy::tokenizer::Tokenizer as _;++ fn tokenize_url(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut tokenizer = UrlTokenizer;+ let mut stream = tokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn url() {+ assert_eq!(+ tokenize_url("https://www.example.com"),+ vec!["www", ".", "example", ".", "com ", "/"]+ );++ assert_eq!(+ tokenize_url("https://www.example.com/test"),+ vec!["www", ".", "example", ".", "com ", "/", "test",]+ );++ assert_eq!(+ tokenize_url("example.com"),+ vec!["example", ".", "com ", "/"]+ );++ assert_eq!(+ tokenize_url("example.com/another/path"),+ vec!["example", ".", "com ", "/", "another", "/", "path",]+ );++ assert_eq!(tokenize_url(".com"), vec![".", "com ", "/"])+ }++ #[test]+ fn multiple_urls() {+ assert_eq!(+ tokenize_url("https://www.example.com\nhttps://www.example.com"),+ vec![+ "www", ".", "example", ".", "com ", "/", "\n", "www", ".", "example", ".", "com ",+ "/"+ ]+ );++ assert_eq!(+ tokenize_url("https://www.example.com/test\nhttps://www.abcd.com"),+ vec![+ "www", ".", "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".",+ "com ", "/"+ ]+ );++ assert_eq!(+ tokenize_url("https://example.com/test\nhttps://www.abcd.com/test"),+ vec![+ "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".", "com ", "/",+ "test",+ ]+ );+ }+}diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rsindex 402d5406..74e2fb3a 100644--- a/crates/core/src/tokenizer/mod.rs+++ b/crates/core/src/tokenizer/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,1138 +14,83 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{array, collections::VecDeque, str::CharIndices};--use logos::{Lexer, Logos};-use tantivy::tokenizer::{- BoxTokenStream, Language, LowerCaser, Stemmer, StopWordFilter, TextAnalyzer,-};--use whatlang::Lang;--use crate::{ceil_char_boundary, floor_char_boundary};--use self::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};-mod add_space_last;+pub mod fields;+mod script;+mod script_tokenizer;+mod segmenter;mod split_preserve;+mod split_whitespace_with_range;+mod stemmer;-struct MyStemmer(Stemmer);--impl From<Lang> for MyStemmer {- fn from(lang: Lang) -> Self {- match lang {- Lang::Dan => MyStemmer(Stemmer::new(Language::Danish)),- Lang::Ara => MyStemmer(Stemmer::new(Language::Arabic)),- Lang::Nld => MyStemmer(Stemmer::new(Language::Dutch)),- Lang::Fin => MyStemmer(Stemmer::new(Language::Finnish)),- Lang::Fra => MyStemmer(Stemmer::new(Language::French)),- Lang::Deu => MyStemmer(Stemmer::new(Language::German)),- Lang::Hun => MyStemmer(Stemmer::new(Language::Hungarian)),- Lang::Ita => MyStemmer(Stemmer::new(Language::Italian)),- Lang::Por => MyStemmer(Stemmer::new(Language::Portuguese)),- Lang::Ron => MyStemmer(Stemmer::new(Language::Romanian)),- Lang::Rus => MyStemmer(Stemmer::new(Language::Russian)),- Lang::Spa => MyStemmer(Stemmer::new(Language::Spanish)),- Lang::Swe => MyStemmer(Stemmer::new(Language::Swedish)),- Lang::Tam => MyStemmer(Stemmer::new(Language::Tamil)),- Lang::Tur => MyStemmer(Stemmer::new(Language::Turkish)),- _ => MyStemmer(Stemmer::new(Language::English)),- }- }-}--#[derive(Clone)]-pub enum Tokenizer {- Normal(Normal),- Identity(Identity),- Stemmed(Stemmed),- Bigram(BigramTokenizer),- Trigram(TrigramTokenizer),- Json(JsonField),- Url(UrlTokenizer),-}--impl Tokenizer {- pub fn as_str(&self) -> &'static str {- match self {- Tokenizer::Normal(_) => Normal::as_str(),- Tokenizer::Stemmed(_) => Stemmed::as_str(),- Tokenizer::Identity(_) => Identity::as_str(),- Tokenizer::Bigram(_) => BigramTokenizer::as_str(),- Tokenizer::Trigram(_) => TrigramTokenizer::as_str(),- Tokenizer::Json(_) => JsonField::as_str(),- Tokenizer::Url(_) => UrlTokenizer::as_str(),- }- }-}--impl From<Stemmed> for Tokenizer {- fn from(stemmed: Stemmed) -> Self {- Self::Stemmed(stemmed)- }-}--impl Default for Tokenizer {- fn default() -> Self {- Self::Normal(Normal::default())- }-}--#[derive(Clone, Default)]-pub struct Normal {- stopwords: Option<Vec<String>>,- analyzer: Option<TextAnalyzer>,-}--impl Normal {- pub fn as_str() -> &'static str {- "tokenizer"- }-- pub fn with_stopwords(stopwords: Vec<String>) -> Self {- Self {- stopwords: Some(stopwords),- analyzer: None,- }- }-}--#[derive(Clone)]-pub struct BigramTokenizer {- inner_tokenizer: Normal,-}--impl Default for BigramTokenizer {- fn default() -> Self {- Self {- inner_tokenizer: Normal::with_stopwords(vec![".".to_string()]),- }- }-}--impl BigramTokenizer {- pub fn as_str() -> &'static str {- "bigram_tokenizer"- }-}--#[derive(Clone)]-pub struct TrigramTokenizer {- inner_tokenizer: Normal,-}--impl Default for TrigramTokenizer {- fn default() -> Self {- Self {- inner_tokenizer: Normal::with_stopwords(vec![".".to_string()]),- }- }-}--impl TrigramTokenizer {- pub fn as_str() -> &'static str {- "trigram_tokenizer"- }-}--#[derive(Clone, Default)]-pub struct Stemmed {- force_language: Option<Lang>,- analyzer: Option<TextAnalyzer>,-}--impl Stemmed {- pub fn as_str() -> &'static str {- "stemmed_tokenizer"- }- pub fn with_forced_language(lang: Lang) -> Self {- Self {- force_language: Some(lang),- analyzer: None,- }- }-}--#[derive(Clone, Default, Debug)]-pub struct Identity {}--impl Identity {- pub fn as_str() -> &'static str {- "identity_tokenizer"- }-}--impl tantivy::tokenizer::Tokenizer for Tokenizer {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {- match self {- Tokenizer::Normal(tokenizer) => tokenizer.token_stream(text),- Tokenizer::Stemmed(tokenizer) => tokenizer.token_stream(text),- Tokenizer::Identity(tokenizer) => tokenizer.token_stream(text),- Tokenizer::Json(tokenizer) => tokenizer.token_stream(text),- Tokenizer::Bigram(tokenizer) => tokenizer.token_stream(text),- Tokenizer::Trigram(tokenizer) => tokenizer.token_stream(text),- Tokenizer::Url(tokenizer) => tokenizer.token_stream(text),- }- }-}--impl tantivy::tokenizer::Tokenizer for Normal {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {- let builder = TextAnalyzer::builder(Simple).filter(LowerCaser);-- self.analyzer = if let Some(stopwords) = &self.stopwords {- Some(- builder- .filter(StopWordFilter::remove(stopwords.clone()))- .build(),- )- } else {- Some(builder.build())- };-- self.analyzer.as_mut().unwrap().token_stream(text)- }-}--impl tantivy::tokenizer::Tokenizer for Stemmed {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {- let builder = TextAnalyzer::builder(Simple).filter(LowerCaser);-- let lang = match self.force_language {- Some(lang) => Some(lang),- None => whatlang::detect_lang(text),- };-- self.analyzer = match lang {- Some(lang) => Some(builder.filter(MyStemmer::from(lang).0).build()),- None => Some(builder.build()),- };-- self.analyzer.as_mut().unwrap().token_stream(text)- }-}--impl tantivy::tokenizer::Tokenizer for Identity {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {- BoxTokenStream::new(IdentityTokenStream::from(text.to_string()))- }-}--impl tantivy::tokenizer::Tokenizer for BigramTokenizer {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {- let inner_stream = self.inner_tokenizer.token_stream(text);- let stream: NGramTokenStream<2> = NGramTokenStream::new(inner_stream);- BoxTokenStream::new(stream)- }-}--impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {- let inner = self.inner_tokenizer.token_stream(text);- let stream: NGramTokenStream<3> = NGramTokenStream::new(inner);- BoxTokenStream::new(stream)- }-}--pub struct IdentityTokenStream {- num_advances: usize,- token: Option<tantivy::tokenizer::Token>,-}--impl From<String> for IdentityTokenStream {- fn from(text: String) -> Self {- Self {- num_advances: 0,- token: Some(tantivy::tokenizer::Token {- offset_from: 0,- offset_to: text.len(),- position: 0,- text,- ..Default::default()- }),- }- }-}--impl tantivy::tokenizer::TokenStream for IdentityTokenStream {- fn advance(&mut self) -> bool {- self.num_advances += 1;-- if self.num_advances == 1 {- true- } else {- self.token = None;- false- }- }-- fn token(&self) -> &tantivy::tokenizer::Token {- self.token.as_ref().unwrap()- }-- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {- self.token.as_mut().unwrap()- }-}--#[derive(Logos, Debug, PartialEq)]-#[logos(skip r"[ \t\n\f]+")]-enum Token {- #[regex("[\\w|\\p{Han}|\\p{Hiragana}|\\p{Katakana}|\\p{Cyrillic}|\\p{Arabic}]+")]- Text,-}--#[derive(Clone)]-pub struct Simple;--pub struct SimpleTokenStream<'a> {- lexer: Lexer<'a, Token>,- token: Option<tantivy::tokenizer::Token>,- next_position: usize,-}--impl tantivy::tokenizer::Tokenizer for Simple {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {- let lexer = Token::lexer(text);- BoxTokenStream::new(SimpleTokenStream {- lexer,- token: None,- next_position: 0,- })- }-}--impl<'a> tantivy::tokenizer::TokenStream for SimpleTokenStream<'a> {- fn advance(&mut self) -> bool {- self.token = self.lexer.next().map(|_| {- let span = self.lexer.span();- let pos = self.next_position;- self.next_position += 1;- tantivy::tokenizer::Token {- offset_from: span.start,- offset_to: span.end,- position: pos,- text: self.lexer.slice().to_string(),- ..Default::default()- }- });-- self.token.is_some()- }-- fn token(&self) -> &tantivy::tokenizer::Token {- self.token.as_ref().unwrap()- }-- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {- self.token.as_mut().unwrap()- }-}--pub struct NGramTokenStream<'a, const N: usize> {- inner: BoxTokenStream<'a>,- token: tantivy::tokenizer::Token,- token_window: [tantivy::tokenizer::Token; N],- next_pos: usize,-}--impl<'a, const N: usize> NGramTokenStream<'a, N> {- pub fn new(inner: BoxTokenStream<'a>) -> Self {- Self {- inner,- token: tantivy::tokenizer::Token::default(),- token_window: array::from_fn(|_| tantivy::tokenizer::Token::default()),- next_pos: 0,- }- }-}--fn reuse_token_alloc(token: &mut tantivy::tokenizer::Token, new_token: &tantivy::tokenizer::Token) {- token.text.clear();- token.text += new_token.text.as_str();- token.offset_from = new_token.offset_from;- token.offset_to = new_token.offset_to;- token.position = new_token.position;- token.position_length = new_token.position_length;-}--impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, N> {- fn advance(&mut self) -> bool {- if !self.inner.advance() {- return false;- }-- self.token_window.rotate_left(1);- reuse_token_alloc(&mut self.token_window[N - 1], self.inner.token());+pub use fields::FieldTokenizer;- while self.token_window[0].text.is_empty() {- if !self.inner.advance() {- return false;- }-- self.token_window.rotate_left(1);- reuse_token_alloc(&mut self.token_window[N - 1], self.inner.token());- }-- self.next_pos += 1;-- let begin = self- .token_window- .iter()- .position(|token| !token.text.is_empty())- .unwrap_or(N - 1);-- self.token.position = self.next_pos;- self.token.offset_from = self.token_window[begin].offset_from;- self.token.offset_to = self.token_window[N - 1].offset_to;- self.token.position_length = N - begin;-- self.token.text.clear();- for token in &self.token_window {- self.token.text += token.text.as_str();- }-- true- }-- fn token(&self) -> &tantivy::tokenizer::Token {- &self.token- }-- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {- &mut self.token- }-}--pub struct FlattenedJson {- flattened_json: String,- inner_tokenizer: JsonField,-}--struct IntermediateFlatValue {- parent_keys: Vec<String>,- val: serde_json::Value,-}--fn flatten(val: serde_json::Value) -> Vec<String> {- let mut res = Vec::new();-- let mut stack = Vec::new();- stack.push(IntermediateFlatValue {- parent_keys: Vec::new(),- val,- });-- while let Some(elem) = stack.pop() {- match elem.val {- serde_json::Value::Null => {- res.push(itertools::intersperse(elem.parent_keys, ".".to_string()).collect())- }- serde_json::Value::Bool(b) => {- let key: String =- itertools::intersperse(elem.parent_keys, ".".to_string()).collect();- res.push(format!("{key}=\"{b}\""))- }- serde_json::Value::Number(n) => {- let key: String =- itertools::intersperse(elem.parent_keys, ".".to_string()).collect();- res.push(format!("{key}=\"{n}\""))- }- serde_json::Value::String(s) => {- let key: String =- itertools::intersperse(elem.parent_keys, ".".to_string()).collect();- res.push(format!("{key}=\"{}\"", s.replace('"', "\\\"")))- }- serde_json::Value::Array(arr) => {- for item in arr {- stack.push(IntermediateFlatValue {- parent_keys: elem.parent_keys.clone(),- val: item,- });- }- }- serde_json::Value::Object(map) => {- for (key, val) in map {- let mut parent_keys = elem.parent_keys.clone();- parent_keys.push(key);+use self::segmenter::Segmenter;- stack.push(IntermediateFlatValue { parent_keys, val });- }- }- }- }-- res.reverse();-- res+#[derive(Debug)]+pub struct Token {+ text: String,+ span: std::ops::Range<usize>,}-impl FlattenedJson {- pub fn new<T>(value: &T) -> crate::Result<Self>- where- T: serde::Serialize,- {- let json = serde_json::to_string(value)?;- let val: serde_json::Value = serde_json::from_str(&json)?;-- let flattened_json = itertools::intersperse(flatten(val), "\n".to_string()).collect();-- Ok(Self {- flattened_json,- inner_tokenizer: JsonField,- })- }-- pub fn token_stream(&mut self) -> BoxTokenStream {- tantivy::tokenizer::Tokenizer::token_stream(&mut self.inner_tokenizer, &self.flattened_json)+impl Token {+ pub fn new(text: String, span: std::ops::Range<usize>) -> Self {+ Token { text, span }}pub fn text(&self) -> &str {- &self.flattened_json- }-}--#[derive(Clone, Debug)]-pub struct JsonField;--impl JsonField {- pub fn as_str() -> &'static str {- "json_tokenizer"- }-}--impl tantivy::tokenizer::Tokenizer for JsonField {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {- BoxTokenStream::new(JsonFieldTokenStream {- text,- chars: text.char_indices(),- token: tantivy::tokenizer::Token::default(),- })+ &self.text}-}--pub struct JsonFieldTokenStream<'a> {- text: &'a str,- chars: CharIndices<'a>,- token: tantivy::tokenizer::Token,-}--impl<'a> JsonFieldTokenStream<'a> {- // search for the end of the current token.- fn search_token_end(&mut self, is_quote: bool) -> usize {- let mut escaped = false;- for (offset, c) in self.chars.by_ref() {- if is_quote {- if c == '\\' {- escaped = true;- } else {- if c == '"' && !escaped {- return offset;- }-- escaped = false;- }- } else if !c.is_alphanumeric() {- return offset;- }- }-- self.text.len()- }-}--impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {- fn advance(&mut self) -> bool {- self.token.text.clear();- self.token.position = self.token.position.wrapping_add(1);- let mut prev_was_quote = false;-- while let Some((offset_from, c)) = self.chars.next() {- if !matches!(c, '.' | '\n' | '"') {- let offset_to = self.search_token_end(prev_was_quote);- self.token.offset_from = offset_from;- self.token.offset_to = offset_to;-- if prev_was_quote {- self.token.offset_from -= 1;- self.token.offset_to += 1;-- self.token.offset_from = floor_char_boundary(self.text, self.token.offset_from);- self.token.offset_to =- ceil_char_boundary(self.text, self.token.offset_to).min(self.text.len());- }-- self.token- .text- .push_str(&self.text[self.token.offset_from..self.token.offset_to]);- return true;- }-- prev_was_quote = c == '"';- }- false- }-- fn token(&self) -> &tantivy::tokenizer::Token {- &self.token- }-- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {- &mut self.token- }-}--#[derive(Clone, Default)]-struct ParsedUrl {- protocol: Option<VecDeque<String>>,- domain: Option<VecDeque<String>>,- path: VecDeque<String>,-}--#[derive(Debug, Clone)]-pub struct UrlTokenizer;-impl UrlTokenizer {- pub fn as_str() -> &'static str {- "url_tokenizer"+ pub fn span(&self) -> std::ops::Range<usize> {+ self.span.clone()}- fn parse_url(text: &str) -> ParsedUrl {- url::Url::parse(text)- .or_else(|_| url::Url::parse(&format!("http://{}", text)))- .map(|url| {- let domain = Some(- url.host_str()- .unwrap_or("")- .split_preserve(|c| matches!(c, '.'))- .filter(|s| !(*s).is_empty())- .map(|s| s.to_string())- .add_space_last()- .collect(),- );- let path: VecDeque<_> = url- .path()- .split_preserve(|c| matches!(c, '/' | '-' | '_'))- .filter(|s| !(*s).is_empty())- .map(|s| s.to_string())- .collect();-- if matches!(url.scheme(), "http" | "https") {- ParsedUrl {- protocol: None,- domain,- path,- }- } else {- let mut v = VecDeque::new();- v.push_back(url.scheme().to_string());-- ParsedUrl {- protocol: Some(v),- domain,- path,- }- }- })- .unwrap_or_default()- }-}--impl tantivy::tokenizer::Tokenizer for UrlTokenizer {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {- debug_assert_eq!(text.chars().filter(|c| *c == ' ').count(), 0);-- let urls = text- .split('\n')- .filter(|s| !s.is_empty())- .map(|s| s.to_lowercase())- .map(|s| Self::parse_url(&s))- .collect();-- BoxTokenStream::new(SiteOperatorUrlTokenStream::new(urls))+ pub fn offset(&mut self, offset: usize) {+ self.span = self.span.start + offset..self.span.end + offset;}}-pub struct SiteOperatorUrlTokenStream {- urls: VecDeque<ParsedUrl>,- current_url: ParsedUrl,- token: tantivy::tokenizer::Token,-}--impl SiteOperatorUrlTokenStream {- fn new(mut urls: VecDeque<ParsedUrl>) -> Self {- let current_url = urls.pop_front().unwrap_or_default();-- Self {- urls,- current_url,- token: tantivy::tokenizer::Token::default(),- }- }-- fn advance_current_url(&mut self) -> bool {- if let Some(protocol) = self.current_url.protocol.as_mut() {- self.token.position = self.token.position.wrapping_add(1);- self.token.text.clear();-- if let Some(s) = protocol.pop_front() {- self.token.text.push_str(&s);- self.token.offset_from = 0;- self.token.offset_to = s.len();- } else {- self.token.offset_from = self.token.offset_to;- self.token.text.push_str("://");- self.token.offset_to += self.token.text.len();-- self.current_url.protocol = None;- }-- return true;- }-- if let Some(domain) = self.current_url.domain.as_mut() {- if let Some(s) = domain.pop_front() {- self.token.text.clear();- self.token.position = self.token.position.wrapping_add(1);-- self.token.text.push_str(&s);-- self.token.offset_from = self.token.offset_to;- self.token.offset_to += self.token.text.len();- return true;- }- }-- if let Some(s) = self.current_url.path.pop_front() {- self.token.text.clear();- self.token.position = self.token.position.wrapping_add(1);-- self.token.text.push_str(&s);- self.token.offset_from = self.token.offset_to;- self.token.offset_to += self.token.text.len();-- return true;- }-- false- }-- fn next_url(&mut self) -> bool {- if let Some(url) = self.urls.pop_front() {- self.current_url = url;-- self.token.position = self.token.position.wrapping_add(1);- self.token.text.clear();- self.token.text.push('\n');-- self.token.offset_from = self.token.offset_to;- self.token.offset_to += self.token.text.len();-- true- } else {- false- }- }+pub trait Tokenize {+ fn tokenize(&self) -> impl Iterator<Item = Token> + '_;}-impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {- fn advance(&mut self) -> bool {- if self.advance_current_url() {- return true;- }-- self.next_url()- }-- fn token(&self) -> &tantivy::tokenizer::Token {- &self.token- }-- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {- &mut self.token+impl Tokenize for str {+ fn tokenize(&self) -> impl Iterator<Item = Token> + '_ {+ self.segments().flat_map(|segment| segment.tokenize())}}#[cfg(test)]mod tests {- use tantivy::tokenizer::Tokenizer as _;-use super::*;-- fn tokenize_simple(s: &str) -> Vec<String> {- let mut res = Vec::new();- let mut tokenizer = Normal::default();- let mut stream = tokenizer.token_stream(s);-- while let Some(token) = stream.next() {- res.push(token.text.clone());- }-- res- }-- fn tokenize_json(s: &str) -> Vec<String> {- let mut res = Vec::new();- let mut tokenizer = JsonField;- let mut stream = tokenizer.token_stream(s);-- while let Some(token) = stream.next() {- res.push(token.text.clone());- }-- res- }-- fn tokenize_bigram(s: &str) -> Vec<String> {- let mut res = Vec::new();- let mut tokenizer = Tokenizer::Bigram(BigramTokenizer::default());- let mut stream = tokenizer.token_stream(s);-- while let Some(token) = stream.next() {- res.push(token.text.clone());- }-- res- }-- fn tokenize_trigram(s: &str) -> Vec<String> {- let mut res = Vec::new();-- let mut tokenizer = Tokenizer::Trigram(TrigramTokenizer::default());- let mut stream = tokenizer.token_stream(s);-- while let Some(token) = stream.next() {- res.push(token.text.clone());- }-- res- }-- fn tokenize_url(s: &str) -> Vec<String> {- let mut res = Vec::new();- let mut tokenizer = UrlTokenizer;- let mut stream = tokenizer.token_stream(s);-- while let Some(token) = stream.next() {- res.push(token.text.clone());- }-- res- }-- fn tokenize_identity(s: &str) -> Vec<String> {- let mut res = Vec::new();- let mut tokenizer = Identity {};- let mut stream = tokenizer.token_stream(s);-- while let Some(token) = stream.next() {- res.push(token.text.clone());- }-- res- }-- #[test]- fn simple_tokenization() {- assert_eq!(- tokenize_simple("this is a relatively simple123 test string"),- vec![- "this",- "is",- "a",- "relatively",- "simple123",- "test",- "string"- ]- );- }-- #[test]- fn out_of_bounds_crash() {- tokenize_json(- r#"-Breadcrumb.title="Home"-Breadcrumb.url="https://www.eurotecnicaservice.it/?lang=en"-Breadcrumb.title="Fuser Pur"-Breadcrumb.url="https://www.eurotecnicaservice.it/testing\"-"#,- );- }-- #[test]- fn special_character_tokenization() {- assert_eq!(tokenize_simple("example.com"), vec!["example", ".", "com",]);- assert_eq!(- tokenize_simple("example. com"),- vec!["example", ".", "com",]- );- assert_eq!(- tokenize_simple("example . com"),- vec!["example", ".", "com",]- );-- assert_eq!(- tokenize_simple("a c++ blog post"),- vec!["a", "c", "+", "+", "blog", "post"]- );- assert_eq!(tokenize_simple("path/test"), vec!["path", "/", "test",]);- }+ use proptest::prelude::*;#[test]- fn tokenize_json_field() {- assert_eq!(- tokenize_json(r#"Test.field="value""#),- vec!["Test", "field", "\"value\"",]- );- assert_eq!(- tokenize_json(r#"Test.field="this is the value""#),- vec!["Test", "field", "\"this is the value\"",]- );- assert_eq!(- tokenize_json(r#"Test.field="this is\" the value""#),- vec!["Test", "field", "\"this is\\\" the value\"",]- );- assert_eq!(- tokenize_json("Test.field=\"this*@# is\\\" the\\\" \nvalue\""),- vec!["Test", "field", "\"this*@# is\\\" the\\\" \nvalue\"",]- );- }-- fn flattened_json_helper(json: &str, expected: &str) {- let parsed: serde_json::Value = serde_json::from_str(json).unwrap();- let flat = &FlattenedJson::new(&parsed).unwrap().flattened_json;-- assert_eq!(flat, expected);- }-- #[test]- fn flatten_json_object() {- let json = r#"- {- "key1": "val1",- "key2": "val2"- }- "#;- let expected = r#"key1="val1"-key2="val2""#;-- flattened_json_helper(json, expected);-- let json = r#"- {- "key1": 1,- "key2": 2- }- "#;- let expected = r#"key1="1"-key2="2""#;-- flattened_json_helper(json, expected);-- let json = r#"- {- "key1": {- "key2": "value1",- "key3": "value2"- }- }- "#;- let expected = r#"key1.key2="value1"-key1.key3="value2""#;-- flattened_json_helper(json, expected);-- let json = r#"- {- "$key1": {- "$key2": "value1",- "key3": "value2"+ fn test_tokenizer() {+ let input = "Hello, world! This is a test.";++ let tokens: Vec<_> = input.tokenize().collect();+ assert_eq!(tokens.len(), 9);++ assert_eq!(tokens[0].text(), "Hello");+ assert_eq!(tokens[1].text(), ",");+ assert_eq!(tokens[2].text(), "world");+ assert_eq!(tokens[3].text(), "!");+ assert_eq!(tokens[4].text(), "This");+ assert_eq!(tokens[5].text(), "is");+ assert_eq!(tokens[6].text(), "a");+ assert_eq!(tokens[7].text(), "test");+ assert_eq!(tokens[8].text(), ".");+ }++ proptest! {+ #[test]+ fn prop_tokenizer_correct_span(txt: String) {+ let tokens: Vec<_> = txt.tokenize().collect();+ for token in tokens {+ assert_eq!(&txt[token.span()], token.text());}}- "#;- let expected = r#"$key1.$key2="value1"-$key1.key3="value2""#;-- flattened_json_helper(json, expected);-- let json = r#"- {- "key1": [- "value1",- "value2"- ]- }- "#;- let expected = r#"key1="value1"-key1="value2""#;-- flattened_json_helper(json, expected);-- let json = r#"- {- "key1": [- "value1",- {- "key2": "value2",- "key3": 123- }- ]- }- "#;- let expected = r#"key1="value1"-key1.key2="value2"-key1.key3="123""#;-- flattened_json_helper(json, expected);-- let json = r#"- {- "key1": [- "value1",- {- "key2": "this\" is @ a # test"- }- ]- }- "#;- let expected = r#"key1="value1"-key1.key2="this\" is @ a # test""#;-- flattened_json_helper(json, expected);- }-- #[test]- fn bigram_tokenizer() {- assert!(tokenize_bigram("").is_empty());- assert!(tokenize_bigram("test").is_empty());-- assert_eq!(tokenize_bigram("this is"), vec!["thisis"]);- assert_eq!(tokenize_bigram("this is a"), vec!["thisis", "isa",]);- assert_eq!(- tokenize_bigram("this is a test"),- vec!["thisis", "isa", "atest",]- );-- // '.' is a stopword- assert_eq!(tokenize_bigram("this.is"), vec!["thisis"]);- }-- #[test]- fn trigram_tokenizer() {- assert!(tokenize_trigram("").is_empty());- assert!(tokenize_trigram("test").is_empty());- assert!(tokenize_trigram("this is").is_empty());-- assert_eq!(tokenize_trigram("this is a"), vec!["thisisa",]);- assert_eq!(- tokenize_trigram("this is a test"),- vec!["thisisa", "isatest",]- );- }-- #[test]- fn han() {- assert_eq!(- tokenize_simple("test 漢.com"),- vec!["test", "漢", ".", "com"]- );- }-- #[test]- fn hiragana() {- assert_eq!(- tokenize_simple("test あ.com"),- vec!["test", "あ", ".", "com"]- );- }-- #[test]- fn katakana() {- assert_eq!(- tokenize_simple("test ダ.com"),- vec!["test", "ダ", ".", "com"]- );- }-- #[test]- fn cyrillic() {- assert_eq!(tokenize_simple("test б.com"), vec!["test", "б", ".", "com"]);- }-- #[test]- fn arabic() {- assert_eq!(tokenize_simple("test ب.com"), vec!["test", "ب", ".", "com"]);- }-- #[test]- fn url() {- assert_eq!(- tokenize_url("https://www.example.com"),- vec!["www", ".", "example", ".", "com ",]- );-- assert_eq!(- tokenize_url("https://www.example.com/test"),- vec!["www", ".", "example", ".", "com ", "/", "test",]- );-- assert_eq!(tokenize_url("example.com"), vec!["example", ".", "com ",]);-- assert_eq!(- tokenize_url("example.com/another/path"),- vec!["example", ".", "com ", "/", "another", "/", "path",]- );-- assert_eq!(tokenize_url(".com"), vec![".", "com ",])- }-- #[test]- fn multiple_urls() {- assert_eq!(- tokenize_url("https://www.example.com\nhttps://www.example.com"),- vec!["www", ".", "example", ".", "com ", "\n", "www", ".", "example", ".", "com ",]- );-- assert_eq!(- tokenize_url("https://www.example.com/test\nhttps://www.abcd.com"),- vec![- "www", ".", "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".",- "com ",- ]- );-- assert_eq!(- tokenize_url("https://example.com/test\nhttps://www.abcd.com/test"),- vec![- "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".", "com ", "/",- "test",- ]- );- }-- #[test]- fn identity() {- assert_eq!(tokenize_identity("this is a test"), vec!["this is a test"]);- assert_eq!(tokenize_identity("a-b"), vec!["a-b"]);}}diff --git a/crates/core/src/tokenizer/script.rs b/crates/core/src/tokenizer/script.rsnew file mode 100644index 00000000..60f762b4--- /dev/null+++ b/crates/core/src/tokenizer/script.rs@@ -0,0 +1,44 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::script_tokenizer::ScriptTokenizer;++#[derive(Debug, PartialEq, Default, Clone, Copy)]+pub enum Script {+ Latin,++ #[default]+ Other,+}++impl From<char> for Script {+ fn from(c: char) -> Self {+ if c.is_ascii() {+ Script::Latin+ } else {+ Script::Other+ }+ }+}++impl Script {+ pub fn tokenizer(self) -> Box<dyn ScriptTokenizer> {+ match self {+ Script::Latin => Box::new(super::script_tokenizer::Latin),+ Script::Other => Box::new(super::script_tokenizer::Latin),+ }+ }+}diff --git a/crates/core/src/tokenizer/script_tokenizer.rs b/crates/core/src/tokenizer/script_tokenizer.rsnew file mode 100644index 00000000..00e2f756--- /dev/null+++ b/crates/core/src/tokenizer/script_tokenizer.rs@@ -0,0 +1,73 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::{+ split_preserve::StrSplitPreserveWithRange,+ split_whitespace_with_range::SplitWhitespaceWithRange, Token,+};++pub trait ScriptTokenizer {+ fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token> + 'a>;+}++pub struct Latin;++impl ScriptTokenizer for Latin {+ fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token> + 'a> {+ Box::new(+ text.split_whitespace_with_range()+ .flat_map(|(txt, span)| {+ let offset = span.start;+ txt.split_preserve_with_range(|c| !c.is_alphabetic() && !c.is_numeric())+ .map(move |(txt, span)| {+ let span = offset + span.start..offset + span.end;+ (txt, span)+ })+ })+ .map(|(txt, span)| Token::new(txt.to_string(), span)),+ )+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use proptest::prelude::*;++ #[test]+ fn test_latin() {+ let tokenizer = Latin;+ let txt = "Hello, world! 123";+ let tokens: Vec<_> = tokenizer.tokenize(txt).collect();+ assert_eq!(tokens.len(), 5);+ assert_eq!(tokens[0].text(), "Hello");+ assert_eq!(tokens[1].text(), ",");+ assert_eq!(tokens[2].text(), "world");+ assert_eq!(tokens[3].text(), "!");+ assert_eq!(tokens[4].text(), "123");+ }++ proptest! {+ #[test]+ fn prop_latin_correct_span(txt: String) {+ let tokenizer = Latin;+ let tokens: Vec<_> = tokenizer.tokenize(&txt).collect();+ for token in tokens {+ assert_eq!(&txt[token.span()], token.text());+ }+ }+ }+}diff --git a/crates/core/src/tokenizer/segmenter.rs b/crates/core/src/tokenizer/segmenter.rsnew file mode 100644index 00000000..c7a91eb3--- /dev/null+++ b/crates/core/src/tokenizer/segmenter.rs@@ -0,0 +1,156 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::{script::Script, Token};++/// A segment is a part of a text where the entire segment has the same script and langage.+#[derive(Clone)]+pub struct Segment<'a> {+ full_text: &'a str,+ span: std::ops::Range<usize>,+ script: Script,+}++impl<'a> Segment<'a> {+ pub fn script(&self) -> Script {+ self.script+ }++ pub fn text(&self) -> &'a str {+ &self.full_text[self.span.clone()]+ }++ pub fn span(&self) -> std::ops::Range<usize> {+ self.span.clone()+ }++ pub fn tokenize(&self) -> impl Iterator<Item = Token> + 'a {+ let offset = self.span.start;+ let script = self.script;++ script+ .tokenizer()+ .tokenize(self.text())+ .map(move |mut token| {+ token.offset(offset);+ token+ })+ }+}++pub trait Segmenter {+ fn segments(&self) -> SegmentIterator;+}++impl Segmenter for str {+ fn segments(&self) -> SegmentIterator<'_> {+ SegmentIterator::new(self)+ }+}++impl Segmenter for String {+ fn segments(&self) -> SegmentIterator<'_> {+ SegmentIterator::new(self)+ }+}++pub struct SegmentIterator<'a> {+ prev_end: usize,+ input: &'a str,+}++impl<'a> SegmentIterator<'a> {+ pub fn new(input: &'a str) -> Self {+ Self { input, prev_end: 0 }+ }+}++impl<'a> Iterator for SegmentIterator<'a> {+ type Item = Segment<'a>;++ fn next(&mut self) -> Option<Self::Item> {+ if self.prev_end >= self.input.len() {+ return None;+ }++ let start = self.prev_end;+ let mut end = start;+ let mut script = None;++ while end < self.input.len() {+ let c = self.input[end..].chars().next().unwrap();+ let next_script = Script::from(c);++ if let Some(script) = &script {+ if &next_script != script {+ break;+ }+ } else {+ script = Some(next_script);+ }++ end += c.len_utf8();+ }++ self.prev_end = end;++ Some(Segment {+ script: script.unwrap_or_default(),+ full_text: self.input,+ span: start..end,+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use proptest::prelude::*;++ #[test]+ fn test_segments() {+ let txt = "Hello, world! This is a test.";+ let segments: Vec<_> = txt.segments().collect();+ assert_eq!(segments.len(), 1);+ assert_eq!(segments[0].text(), txt);+ assert_eq!(segments[0].script, Script::Latin);++ let txt = "こんにちは、世界!";+ let segments: Vec<_> = txt.segments().collect();++ assert_eq!(segments.len(), 1);+ assert_eq!(segments[0].text(), txt);+ assert_eq!(segments[0].script, Script::Other);++ let txt = "Hello, こんにちは、世界!";+ let segments: Vec<_> = txt.segments().collect();++ assert_eq!(segments.len(), 2);+ assert_eq!(segments[0].text(), "Hello, ");+ assert_eq!(segments[0].script, Script::Latin);+ assert_eq!(segments[1].text(), "こんにちは、世界!");+ assert_eq!(segments[1].script, Script::Other);+ }++ proptest! {+ #[test]+ fn proptest_byte_offsets(txt in ".*") {+ for segment in txt.segments() {+ assert!(!segment.text().is_empty());+ }+ }+ }+}diff --git a/crates/core/src/tokenizer/split_preserve.rs b/crates/core/src/tokenizer/split_preserve.rsindex f0bb4afc..e6506610 100644--- a/crates/core/src/tokenizer/split_preserve.rs+++ b/crates/core/src/tokenizer/split_preserve.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub struct SplitPreserve<'a, P>+pub struct SplitPreserveWithRange<'a, P>whereP: Fn(char) -> bool,{@@ -24,7 +24,7 @@ wherelast_pred: Option<char>,}-impl<'a, P> SplitPreserve<'a, P>+impl<'a, P> SplitPreserveWithRange<'a, P>whereP: Fn(char) -> bool,{@@ -38,45 +38,81 @@ where}}-impl<'a, P> Iterator for SplitPreserve<'a, P>+impl<'a, P> Iterator for SplitPreserveWithRange<'a, P>whereP: Fn(char) -> bool,{- type Item = &'a str;+ type Item = (&'a str, std::ops::Range<usize>);fn next(&mut self) -> Option<Self::Item> {- if self.start >= self.s.len() {- return None;+ if let Some(c) = self.last_pred.take() {+ let range = self.start - c.len_utf8()..self.start;+ return Some((&self.s[range.clone()], range));}- if let Some(c) = self.last_pred.take() {- return Some(&self.s[self.start - c.len_utf8()..self.start]);+ if self.start >= self.s.len() {+ return None;}for (i, c) in self.s[self.start..].char_indices() {if (self.pred)(c) {- self.last_pred = Some(c);+ let range = self.start..self.start + i;+ let res = &self.s[range.clone()];+ self.start += i + c.len_utf8();- let res = &self.s[self.start..self.start + i];+ if i == 0 {+ let range = self.start - c.len_utf8()..self.start;+ return Some((&self.s[range.clone()], range));+ }- self.start += i + c.len_utf8();+ self.last_pred = Some(c);- return Some(res);+ return Some((res, range));}}if self.start < self.s.len() {- let res = &self.s[self.start..];+ let range = self.start..self.s.len();+ let res = &self.s[range.clone()];self.start = self.s.len();- Some(res)+ Some((res, range))} else {None}}}+pub struct SplitPreserve<'a, P>+where+ P: Fn(char) -> bool,+{+ inner: SplitPreserveWithRange<'a, P>,+}++impl<'a, P> SplitPreserve<'a, P>+where+ P: Fn(char) -> bool,+{+ fn new(s: &'a str, pred: P) -> Self {+ Self {+ inner: SplitPreserveWithRange::new(s, pred),+ }+ }+}++impl<'a, P> Iterator for SplitPreserve<'a, P>+where+ P: Fn(char) -> bool,+{+ type Item = &'a str;++ fn next(&mut self) -> Option<Self::Item> {+ self.inner.next().map(|(s, _)| s)+ }+}+pub trait StrSplitPreserve {fn split_preserve<F>(&self, pred: F) -> SplitPreserve<F>where@@ -101,6 +137,30 @@ impl StrSplitPreserve for String {}}+pub trait StrSplitPreserveWithRange {+ fn split_preserve_with_range<F>(&self, pred: F) -> SplitPreserveWithRange<F>+ where+ F: Fn(char) -> bool;+}++impl StrSplitPreserveWithRange for str {+ fn split_preserve_with_range<F>(&self, pred: F) -> SplitPreserveWithRange<F>+ where+ F: Fn(char) -> bool,+ {+ SplitPreserveWithRange::new(self, pred)+ }+}++impl StrSplitPreserveWithRange for String {+ fn split_preserve_with_range<F>(&self, pred: F) -> SplitPreserveWithRange<F>+ where+ F: Fn(char) -> bool,+ {+ SplitPreserveWithRange::new(self, pred)+ }+}+#[cfg(test)]mod tests {use super::*;@@ -115,4 +175,50 @@ mod tests {let res = "hello".split_preserve(|c| c == '.').collect::<Vec<_>>();assert_eq!(res, vec!["hello"]);}++ #[test]+ fn test_starts_with() {+ let res = ".hello.brave.new.world"+ .split_preserve(|c| c == '.')+ .collect::<Vec<_>>();+ assert_eq!(+ res,+ vec![".", "hello", ".", "brave", ".", "new", ".", "world"]+ );+ }++ #[test]+ fn test_ends_with() {+ let res = "hello.brave.new.world."+ .split_preserve(|c| c == '.')+ .collect::<Vec<_>>();+ assert_eq!(+ res,+ vec!["hello", ".", "brave", ".", "new", ".", "world", "."]+ );+ }++ #[test]+ fn test_empty() {+ let res = "".split_preserve(|c| c == '.').collect::<Vec<_>>();+ assert_eq!(res, vec![] as Vec<&str>);+ }++ #[test]+ fn test_no_split() {+ let res = "hello".split_preserve(|c| c == '.').collect::<Vec<_>>();+ assert_eq!(res, vec!["hello"]);+ }++ #[test]+ fn test_single_char() {+ let res = ".".split_preserve(|c| c == '.').collect::<Vec<_>>();+ assert_eq!(res, vec!["."]);+ }++ #[test]+ fn test_multi_char() {+ let res = "....".split_preserve(|c| c == '.').collect::<Vec<_>>();+ assert_eq!(res, vec![".", ".", ".", "."]);+ }}diff --git a/crates/core/src/tokenizer/split_whitespace_with_range.rs b/crates/core/src/tokenizer/split_whitespace_with_range.rsnew file mode 100644index 00000000..001aeafe--- /dev/null+++ b/crates/core/src/tokenizer/split_whitespace_with_range.rs@@ -0,0 +1,112 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub trait SplitWhitespaceWithRange {+ fn split_whitespace_with_range(&self) -> SplitWhitespaceWithRangeIter;+}++pub struct SplitWhitespaceWithRangeIter<'a> {+ s: &'a str,+ start: usize,+}++impl<'a> SplitWhitespaceWithRangeIter<'a> {+ fn new(s: &'a str) -> Self {+ Self { s, start: 0 }+ }+}++impl<'a> Iterator for SplitWhitespaceWithRangeIter<'a> {+ type Item = (&'a str, std::ops::Range<usize>);++ fn next(&mut self) -> Option<Self::Item> {+ for c in self.s[self.start..].chars() {+ if !c.is_whitespace() {+ break;+ }+ self.start += c.len_utf8();+ }++ if self.start >= self.s.len() {+ return None;+ }++ let start = self.s[self.start..].find(|c: char| !c.is_whitespace())?;+ let start = self.start + start;+ let end = self.s[start..]+ .find(char::is_whitespace)+ .map(|end| start + end)+ .unwrap_or(self.s.len());+ let range = start..end;+ self.start = end;+ Some((&self.s[range.clone()], range))+ }+}++impl SplitWhitespaceWithRange for str {+ fn split_whitespace_with_range(&self) -> SplitWhitespaceWithRangeIter {+ SplitWhitespaceWithRangeIter::new(self)+ }+}++impl SplitWhitespaceWithRange for String {+ fn split_whitespace_with_range(&self) -> SplitWhitespaceWithRangeIter {+ SplitWhitespaceWithRangeIter::new(self)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use proptest::prelude::*;++ #[test]+ fn test_split_whitespace_with_range() {+ let txt = "Hello, world! 123";+ let tokens: Vec<_> = txt.split_whitespace_with_range().collect();+ assert_eq!(tokens.len(), 3);+ assert_eq!(tokens[0], ("Hello,", 0..6));+ assert_eq!(tokens[1], ("world!", 7..13));+ assert_eq!(tokens[2], ("123", 14..17));+ }++ #[test]+ fn test_split_whitespace_with_range_empty() {+ let txt = "";+ let tokens: Vec<_> = txt.split_whitespace_with_range().collect();+ assert_eq!(tokens.len(), 0);+ }++ #[test]+ fn test_multi_whitespace() {+ let txt = "Hello, world! 123";+ let tokens: Vec<_> = txt.split_whitespace_with_range().collect();+ assert_eq!(tokens.len(), 3);+ assert_eq!(tokens[0], ("Hello,", 0..6));+ assert_eq!(tokens[1], ("world!", 9..15));+ assert_eq!(tokens[2], ("123", 16..19));+ }++ proptest! {+ #[test]+ fn prop_split_whitespace_with_range(s: String) {+ let tokens: Vec<_> = s.split_whitespace_with_range().collect();+ for (txt, range) in tokens {+ assert_eq!(&s[range.clone()], txt);+ }+ }+ }+}diff --git a/crates/core/src/tokenizer/stemmer.rs b/crates/core/src/tokenizer/stemmer.rsnew file mode 100644index 00000000..58608625--- /dev/null+++ b/crates/core/src/tokenizer/stemmer.rs@@ -0,0 +1,80 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use whatlang::Lang;++pub struct Stemmer(tantivy::tokenizer::Stemmer);++impl Stemmer {+ pub fn into_tantivy(self) -> tantivy::tokenizer::Stemmer {+ self.0+ }+}++impl From<Lang> for Stemmer {+ fn from(lang: Lang) -> Self {+ match lang {+ Lang::Dan => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Danish,+ )),+ Lang::Ara => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Arabic,+ )),+ Lang::Nld => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Dutch,+ )),+ Lang::Fin => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Finnish,+ )),+ Lang::Fra => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::French,+ )),+ Lang::Deu => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::German,+ )),+ Lang::Hun => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Hungarian,+ )),+ Lang::Ita => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Italian,+ )),+ Lang::Por => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Portuguese,+ )),+ Lang::Ron => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Romanian,+ )),+ Lang::Rus => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Russian,+ )),+ Lang::Spa => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Spanish,+ )),+ Lang::Swe => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Swedish,+ )),+ Lang::Tam => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Tamil,+ )),+ Lang::Tur => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::Turkish,+ )),+ _ => Stemmer(tantivy::tokenizer::Stemmer::new(+ tantivy::tokenizer::Language::English,+ )),+ }+ }+}diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rsindex 9f4f721f..71b09997 100644--- a/crates/core/src/warc.rs+++ b/crates/core/src/warc.rs@@ -30,7 +30,7 @@ use flate2::write::GzEncoder;use flate2::Compression;use fnv::FnvHashSet;#[cfg(test)]-use proptest_derive::Arbitrary;+use proptest::prelude::*;use tracing::{debug, trace};@@ -208,15 +208,31 @@ struct RawWarcRecord {}#[derive(Debug)]-#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]+#[cfg_attr(test, derive(Clone, PartialEq))]pub struct WarcRecord {pub request: Request,pub response: Response,pub metadata: Metadata,}+#[cfg(test)]+impl Arbitrary for WarcRecord {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: ()) -> Self::Strategy {+ (any::<Request>(), any::<Response>(), any::<Metadata>())+ .prop_map(|(request, response, metadata)| Self {+ request,+ response,+ metadata,+ })+ .boxed()+ }+}+#[derive(Debug)]-#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]+#[cfg_attr(test, derive(Clone, PartialEq))]pub struct Request {// WARC-Target-URIpub url: String,@@ -234,8 +250,17 @@ impl Request {}}+#[cfg(test)]+impl Arbitrary for Request {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: ()) -> Self::Strategy {+ ".+".prop_map(|url| Self { url }).boxed()+ }+}+#[derive(Debug, Clone, Copy, PartialEq, Eq)]-#[cfg_attr(test, derive(Arbitrary))]pub enum PayloadType {Html,Pdf,@@ -243,6 +268,22 @@ pub enum PayloadType {Atom,}+#[cfg(test)]+impl Arbitrary for PayloadType {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: ()) -> Self::Strategy {+ prop_oneof![+ Just(Self::Html),+ Just(Self::Pdf),+ Just(Self::Rss),+ Just(Self::Atom),+ ]+ .boxed()+ }+}+impl FromStr for PayloadType {type Err = Error;@@ -273,7 +314,7 @@ impl Display for PayloadType {}#[derive(Debug)]-#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]+#[cfg_attr(test, derive(Clone, PartialEq))]pub struct Response {pub body: String,pub payload_type: Option<PayloadType>,@@ -297,8 +338,20 @@ impl Response {}}+#[cfg(test)]+impl Arbitrary for Response {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: ()) -> Self::Strategy {+ (".+", any::<Option<PayloadType>>())+ .prop_map(|(body, payload_type)| Self { body, payload_type })+ .boxed()+ }+}+#[derive(Debug)]-#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]+#[cfg_attr(test, derive(Clone, PartialEq))]pub struct Metadata {// fetchTimeMspub fetch_time_ms: u64,@@ -325,6 +378,18 @@ impl Metadata {}}+#[cfg(test)]+impl Arbitrary for Metadata {+ type Parameters = ();+ type Strategy = BoxedStrategy<Self>;++ fn arbitrary_with(_args: ()) -> Self::Strategy {+ (0..10000u64)+ .prop_map(|fetch_time_ms| Self { fetch_time_ms })+ .boxed()+ }+}+pub struct RecordIterator<R: Read> {reader: BufReader<MultiGzDecoder<R>>,num_reads: usize,@@ -671,8 +736,6 @@ mod tests {use super::*;use core::panic;- use proptest::prelude::*;-#[test]fn it_works() {let raw = b"\diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex bcd4a0c9..f289a5af 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -74,7 +74,7 @@ impl Html {pub fn pretokenize_url_for_site_operator(&self) -> PreTokenizedString {self.pretokenize_string_with(self.url().to_string(),- tokenizer::Tokenizer::Url(tokenizer::UrlTokenizer),+ tokenizer::FieldTokenizer::Url(tokenizer::fields::UrlTokenizer),)}@@ -114,7 +114,7 @@ impl Html {fn pretokenize_string_with(&self,text: String,- tokenizer: tokenizer::Tokenizer,+ tokenizer: tokenizer::FieldTokenizer,) -> PreTokenizedString {let mut tokenizer = tokenizer;diff --git a/crates/core/src/webpage/schema_org/mod.rs b/crates/core/src/webpage/schema_org/mod.rsindex 73c6d635..7287300f 100644--- a/crates/core/src/webpage/schema_org/mod.rs+++ b/crates/core/src/webpage/schema_org/mod.rs@@ -18,7 +18,7 @@ use std::collections::HashMap;use kuchiki::NodeRef;-use crate::tokenizer::FlattenedJson;+use crate::tokenizer::fields::FlattenedJson;use crate::{OneOrMany, Result};mod json_ld;