web search done right
upgrade tantivy to 0.21.0
Mikkel Denkere576342175e8
diff --git a/Cargo.lock b/Cargo.lockindex 5e494217..88db2912 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -828,15 +828,6 @@ version = "1.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"-[[package]]-name = "combine"-version = "4.6.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"-dependencies = [- "memchr",-]-[[package]]name = "concurrent-queue"version = "2.2.0"@@ -1344,37 +1335,12 @@ dependencies = ["zune-inflate",]-[[package]]-name = "fail"-version = "0.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"-dependencies = [- "log",- "once_cell",- "rand 0.8.5",-]-[[package]]name = "fastdivide"version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04"-[[package]]-name = "fastfield_codecs"-version = "0.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "374a3a53c1bd5fb31b10084229290eafb0a05f260ec90f1f726afffda4877a8a"-dependencies = [- "fastdivide",- "itertools 0.10.5",- "log",- "ownedbytes",- "tantivy-bitpacker",- "tantivy-common",-]-[[package]]name = "fastrand"version = "2.0.0"@@ -1471,6 +1437,16 @@ dependencies = ["winapi",]+[[package]]+name = "fs4"+version = "0.6.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47"+dependencies = [+ "rustix",+ "windows-sys 0.48.0",+]+[[package]]name = "fst"version = "0.4.7"@@ -2076,6 +2052,15 @@ dependencies = ["either",]+[[package]]+name = "itertools"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "0.4.8"@@ -2301,11 +2286,11 @@ dependencies = [[[package]]name = "lru"-version = "0.7.8"+version = "0.11.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"+checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21"dependencies = [- "hashbrown 0.12.3",+ "hashbrown 0.14.0",][[package]]@@ -2333,6 +2318,12 @@ dependencies = ["twox-hash",]+[[package]]+name = "lz4_flex"+version = "0.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8"+[[package]]name = "mac"version = "0.1.1"@@ -2458,6 +2449,15 @@ dependencies = ["libc",]+[[package]]+name = "memmap2"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6"+dependencies = [+ "libc",+]+[[package]]name = "memoffset"version = "0.9.0"@@ -2548,12 +2548,9 @@ dependencies = [[[package]]name = "murmurhash32"-version = "0.2.0"+version = "0.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a"-dependencies = [- "byteorder",-]+checksum = "d9380db4c04d219ac5c51d14996bbf2c2e9a15229771b53f8671eb6c83cf44df"[[package]]name = "nanorand"@@ -2861,9 +2858,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"[[package]]name = "ownedbytes"-version = "0.4.0"+version = "0.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8e957eaa64a299f39755416e5b3128c505e9d63a91d0453771ad2ccd3907f8db"+checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7"dependencies = ["stable_deref_trait",]@@ -3866,7 +3863,7 @@ dependencies = ["futures","histogram","itertools 0.10.5",- "lz4_flex",+ "lz4_flex 0.9.5","num-bigint","num_enum","rand 0.8.5",@@ -3893,7 +3890,7 @@ dependencies = ["byteorder","bytes","chrono",- "lz4_flex",+ "lz4_flex 0.9.5","num-bigint","num_enum","scylla-macros",@@ -4120,6 +4117,15 @@ version = "0.3.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"+[[package]]+name = "sketches-ddsketch"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1"+dependencies = [+ "serde",+]+[[package]]name = "slab"version = "0.4.9"@@ -4251,7 +4257,7 @@ dependencies = ["maplit","md5","memmap",- "memmap2",+ "memmap2 0.5.10","mime","min-max-heap","num_cpus",@@ -4395,49 +4401,49 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"[[package]]name = "tantivy"-version = "0.19.2"+version = "0.21.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5bb26a6b22c84d8be41d99a14016d6f04d30d8d31a2ea411a8ab553af5cc490d"+checksum = "c1d4675fed6fe2218ce11445374e181e864a8ffd0f28e7e0591ccfc38cd000ae"dependencies = [- "aho-corasick 0.7.20",+ "aho-corasick 1.0.4","arc-swap","async-trait",- "base64 0.13.1",+ "base64 0.21.2","bitpacking","byteorder","census","crc32fast","crossbeam-channel","downcast-rs",- "fail","fastdivide",- "fastfield_codecs",- "fs2",+ "fs4","htmlescape",- "itertools 0.10.5",+ "itertools 0.11.0","levenshtein_automata","log","lru",- "lz4_flex",+ "lz4_flex 0.11.1","measure_time",- "memmap2",+ "memmap2 0.7.1","murmurhash32","num_cpus","once_cell","oneshot",- "ownedbytes","rayon","regex","rust-stemmers","rustc-hash","serde","serde_json",+ "sketches-ddsketch","smallvec",- "stable_deref_trait","tantivy-bitpacker",+ "tantivy-columnar","tantivy-common","tantivy-fst","tantivy-query-grammar",+ "tantivy-stacker",+ "tantivy-tokenizer-api","tempfile","thiserror","time 0.3.27",@@ -4447,18 +4453,40 @@ dependencies = [[[package]]name = "tantivy-bitpacker"-version = "0.3.0"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a"+dependencies = [+ "bitpacking",+]++[[package]]+name = "tantivy-columnar"+version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e71a0c95b82d4292b097a09b989a6380d28c3a86800c841a2d03bae1fc8b9fa6"+checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70"+dependencies = [+ "fastdivide",+ "fnv",+ "itertools 0.11.0",+ "serde",+ "tantivy-bitpacker",+ "tantivy-common",+ "tantivy-sstable",+ "tantivy-stacker",+][[package]]name = "tantivy-common"-version = "0.4.0"+version = "0.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "14fef4182bb60df9a4b92cd8ecab39ba2e50a05542934af17eef1f49660705cb"+checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1"dependencies = [+ "async-trait","byteorder","ownedbytes",+ "serde",+ "time 0.3.27",][[package]]@@ -4474,13 +4502,41 @@ dependencies = [[[package]]name = "tantivy-query-grammar"-version = "0.19.0"+version = "0.21.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "343e3ada4c1c480953f6960f8a21ce9c76611480ffdd4f4e230fdddce0fc5331"+checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77"dependencies = [- "combine",- "once_cell",- "regex",+ "nom",+]++[[package]]+name = "tantivy-sstable"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df"+dependencies = [+ "tantivy-common",+ "tantivy-fst",+ "zstd 0.12.4",+]++[[package]]+name = "tantivy-stacker"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c"+dependencies = [+ "murmurhash32",+ "tantivy-common",+]++[[package]]+name = "tantivy-tokenizer-api"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66"+dependencies = [+ "serde",][[package]]@@ -5571,7 +5627,7 @@ dependencies = ["pbkdf2","sha1","time 0.3.27",- "zstd",+ "zstd 0.11.2+zstd.1.5.2",][[package]]@@ -5580,7 +5636,16 @@ version = "0.11.2+zstd.1.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"dependencies = [- "zstd-safe",+ "zstd-safe 5.0.2+zstd.1.5.2",+]++[[package]]+name = "zstd"+version = "0.12.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"+dependencies = [+ "zstd-safe 6.0.6",][[package]]@@ -5593,6 +5658,16 @@ dependencies = ["zstd-sys",]+[[package]]+name = "zstd-safe"+version = "6.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"+dependencies = [+ "libc",+ "zstd-sys",+]+[[package]]name = "zstd-sys"version = "2.0.8+zstd.1.5.5"diff --git a/Cargo.toml b/Cargo.tomlindex 55be4625..982b38df 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -24,7 +24,7 @@ reqwest-eventsource = "0.4.0"reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }serde = { version = "1.0.137", features = ["rc", "derive"] }serde_json = "1.0.81"-tantivy = "0.19.2"+tantivy = "0.21.0"thiserror = "1.0.31"toml = "0.5.9"tracing = "0.1.34"diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rsindex 507fe190..13eb5716 100644--- a/core/src/entity_index/mod.rs+++ b/core/src/entity_index/mod.rs@@ -316,7 +316,8 @@ impl EntityIndex {let entity_abstract = self.schema.get_field("abstract").unwrap();let mut term_queries = Vec::new();- let mut stream = Normal::default().token_stream(query);+ let mut tokenizer = Normal::default();+ let mut stream = tokenizer.token_stream(query);while let Some(token) = stream.next() {if self.stopwords.contains(&token.text) {continue;diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rsindex 049f20d7..523cd9ef 100644--- a/core/src/fastfield_reader.rs+++ b/core/src/fastfield_reader.rs@@ -16,11 +16,11 @@use std::{collections::HashMap, sync::Arc};-use tantivy::{fastfield::Column, DocId, SegmentId};+use tantivy::{columnar::ColumnValues, DocId, SegmentId};use crate::{enum_map::EnumMap,- schema::{DataType, FastField, Field, ALL_FIELDS},+ schema::{DataType, FastField, ALL_FIELDS},};#[derive(Default, Clone)]@@ -43,24 +43,20 @@ impl FastFieldReader {pub fn new(tv_searcher: &tantivy::Searcher) -> Self {let mut segments = HashMap::new();- let schema = tv_searcher.schema();-for reader in tv_searcher.segment_readers() {let fastfield_readers = reader.fast_fields();let mut field_readers = Vec::new();for field in ALL_FIELDS.iter().filter_map(|field| field.as_fast()) {- let tv_field = schema.get_field(Field::Fast(field).name()).unwrap();-let field_reader = match field.data_type() {DataType::U64 => {- let reader = fastfield_readers.u64(tv_field).unwrap();- FieldReader::U64(reader)+ let reader = fastfield_readers.u64(field.name()).unwrap();+ FieldReader::U64(reader.values)}DataType::F64 => {- let reader = fastfield_readers.f64(tv_field).unwrap();- FieldReader::F64(reader)+ let reader = fastfield_readers.f64(field.name()).unwrap();+ FieldReader::F64(reader.values)}};@@ -108,8 +104,8 @@ impl From<FieldValue> for Option<u64> {}pub enum FieldReader {- U64(Arc<dyn Column<u64>>),- F64(Arc<dyn Column<f64>>),+ U64(Arc<dyn ColumnValues<u64>>),+ F64(Arc<dyn ColumnValues<f64>>),}impl FieldReader {diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex d1169ed4..969e9944 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -45,7 +45,9 @@ use crate::ranking::SignalAggregator;use crate::schema::{FastField, Field, TextField, ALL_FIELDS};use crate::search_ctx::Ctx;use crate::snippet::TextSnippet;-use crate::tokenizer::{BigramTokenizer, Identity, SiteOperatorUrlTokenizer, TrigramTokenizer};+use crate::tokenizer::{+ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer,+};use crate::webgraph::NodeID;use crate::webpage::region::Region;use crate::webpage::{schema_org, Webpage};@@ -159,6 +161,11 @@ impl InvertedIndex {.tokenizers().register(tokenizer.as_str(), tokenizer);+ let tokenizer = Tokenizer::Json(JsonField);+ tantivy_index+ .tokenizers()+ .register(tokenizer.as_str(), tokenizer);+let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;let merge_policy = NoMergePolicy;diff --git a/core/src/query/const_query.rs b/core/src/query/const_query.rsindex a1346e7c..32f218dc 100644--- a/core/src/query/const_query.rs+++ b/core/src/query/const_query.rs@@ -46,13 +46,22 @@ impl Query for ConstQuery {enable_scoring: tantivy::query::EnableScoring,) -> tantivy::Result<Box<dyn Weight>> {let schema = match enable_scoring {- tantivy::query::EnableScoring::Enabled(searcher) => searcher.schema(),- tantivy::query::EnableScoring::Disabled(s) => s,+ tantivy::query::EnableScoring::Enabled {+ searcher,+ statistics_provider: _,+ } => searcher.schema(),+ tantivy::query::EnableScoring::Disabled {+ schema,+ searcher_opt: _,+ } => schema,};Ok(Box::new(ConstWeight {subweight: self.subquery- .weight(tantivy::query::EnableScoring::Disabled(schema))?,+ .weight(tantivy::query::EnableScoring::Disabled {+ schema,+ searcher_opt: None,+ })?,score: self.score,}))}diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 25d2674b..47f0ace7 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -327,7 +327,8 @@ impl Term {Field::Fast(_) => vec![tantivy::Term::from_field_text(tantivy_field, term)],Field::Text(text_field) => {let mut terms: Vec<tantivy::Term> = Vec::new();- let mut token_stream = text_field.query_tokenizer().token_stream(term);+ let mut tokenizer = text_field.query_tokenizer();+ let mut token_stream = tokenizer.token_stream(term);token_stream.process(&mut |token| {let term = tantivy::Term::from_field_text(tantivy_field, &token.text);terms.push(term);diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex 88b3eefd..751b3051 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -106,11 +106,8 @@ impl PatternQuery {for pattern in &patterns {match pattern {PatternPart::Raw(text) => {- let mut stream = field- .as_text()- .unwrap()- .indexing_tokenizer()- .token_stream(text);+ let mut tokenizer = field.as_text().unwrap().indexing_tokenizer();+ let mut stream = tokenizer.token_stream(text);while let Some(token) = stream.next() {new_patterns.push(PatternPart::Raw(token.text.clone()));@@ -141,14 +138,17 @@ impl tantivy::query::Query for PatternQuery {scoring: tantivy::query::EnableScoring<'_>,) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {let bm25_weight = match scoring {- tantivy::query::EnableScoring::Enabled(searcher) => {+ tantivy::query::EnableScoring::Enabled {+ searcher,+ statistics_provider: _,+ } => {if self.raw_terms.is_empty() {None} else {Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)}}- tantivy::query::EnableScoring::Disabled(_) => None,+ tantivy::query::EnableScoring::Disabled { .. } => None,};if self.can_optimize_site_domain {diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 3c757982..d9a404d5 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -864,8 +864,8 @@ impl SignalAggregator {.collect::<String>();let mut terms = Vec::new();- let mut stream =- text_field.indexing_tokenizer().token_stream(&simple_query);+ let mut tokenizer = text_field.indexing_tokenizer();+ let mut stream = tokenizer.token_stream(&simple_query);while let Some(token) = stream.next() {let term = tantivy::Term::from_field_text(tv_field, &token.text);@@ -922,7 +922,10 @@ impl SignalAggregator {.map(|(_, rule)| RuleBoost {docset: rule.query- .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))+ .weight(tantivy::query::EnableScoring::Enabled {+ searcher: tv_searcher,+ statistics_provider: tv_searcher,+ }).unwrap().scorer(segment_reader, 0.0).unwrap(),@@ -957,7 +960,7 @@ impl SignalAggregator {let mut queries = Vec::new();for field in proximity_fields {let tv_field = schema.get_field(field.name()).unwrap();- let tokenizer = field.indexing_tokenizer();+ let mut tokenizer = field.indexing_tokenizer();let mut terms = Vec::with_capacity(query.simple_terms.len());for term in &query.simple_terms {@@ -979,7 +982,10 @@ impl SignalAggregator {let query = tantivy::query::BooleanQuery::new(queries);let docset = query- .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))+ .weight(tantivy::query::EnableScoring::Enabled {+ searcher: tv_searcher,+ statistics_provider: tv_searcher,+ }).unwrap().scorer(segment_reader, 1.0).unwrap();diff --git a/core/src/schema.rs b/core/src/schema.rsindex 7c2653b9..6998960f 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -14,9 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::schema::{- Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,-};+use tantivy::schema::{IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions};use crate::tokenizer::{BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,@@ -442,146 +440,120 @@ impl Field {Field::Text(TextField::SafetyClassification) => {IndexingOption::Text(self.default_text_options())}- Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::HostCentrality) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::PageCentrality) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::FetchTimeMs) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::TrackerScore) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),+ Field::Fast(FastField::IsHomepage) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::HostCentrality) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::PageCentrality) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::FetchTimeMs) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::TrackerScore) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }Field::Fast(FastField::LastUpdated) => IndexingOption::Integer(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_stored().set_indexed(),),Field::Fast(FastField::Region) => IndexingOption::Integer(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_stored().set_indexed(),),- Field::Fast(FastField::NumCleanBodyTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::NumDescriptionTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::NumTitleTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::NumMicroformatTagsTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::NumUrlTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::NumDomainTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::NumUrlForSiteOperatorTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::NumFlattenedSchemaTokens) => IndexingOption::Integer(- NumericOptions::default()- .set_fast(Cardinality::SingleValue)- .set_indexed(),- ),- Field::Fast(FastField::SiteHash1) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::SiteHash2) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::UrlWithoutQueryHash1) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::UrlWithoutQueryHash2) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::UrlHash1) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::UrlHash2) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::DomainHash1) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::DomainHash2) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::TitleHash1) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),- Field::Fast(FastField::TitleHash2) => IndexingOption::Integer(- NumericOptions::default().set_fast(Cardinality::SingleValue),- ),+ Field::Fast(FastField::NumCleanBodyTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::NumDescriptionTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::NumTitleTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::NumMicroformatTagsTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::NumUrlTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::NumDomainTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::NumFlattenedSchemaTokens) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }+ Field::Fast(FastField::SiteHash1) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::SiteHash2) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::UrlWithoutQueryHash1) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::UrlWithoutQueryHash2) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::UrlHash1) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::UrlHash2) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::DomainHash1) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::DomainHash2) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::TitleHash1) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::TitleHash2) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }Field::Fast(FastField::PreComputedScore) => IndexingOption::Float(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_indexed().set_stored(),),Field::Fast(FastField::HostNodeID1) => IndexingOption::Integer(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_indexed().set_stored(),),Field::Fast(FastField::HostNodeID2) => IndexingOption::Integer(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_indexed().set_stored(),),Field::Fast(FastField::SimHash) => IndexingOption::Integer(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_indexed().set_stored(),),Field::Fast(FastField::NumPathAndQuerySlashes) => IndexingOption::Integer(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_indexed().set_stored(),),Field::Fast(FastField::NumPathAndQueryDigits) => IndexingOption::Integer(NumericOptions::default()- .set_fast(Cardinality::SingleValue)+ .set_fast().set_indexed().set_stored(),),diff --git a/core/src/simhash.rs b/core/src/simhash.rsindex 0df0456e..08b275b4 100644--- a/core/src/simhash.rs+++ b/core/src/simhash.rs@@ -30,9 +30,9 @@ fn hash_token(token: &tantivy::tokenizer::Token) -> HashType {}pub fn hash(text: &str) -> HashType {- let tokenizer = Tokenizer::default();+ let mut tokenizer = Tokenizer::default();- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, text);+ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, text);let mut v = [0i64; HashType::BITS as usize];diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex 718e788e..b205d49a 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -79,13 +79,13 @@ struct SnippetBuilder {impl SnippetBuilder {fn highlight(&mut self, terms: &HashSet<String>, lang: whatlang::Lang) {- for tokenizer in [+ for mut tokenizer in [Tokenizer::Stemmed(Stemmed::with_forced_language(lang)),- Tokenizer::Bigram(BigramTokenizer {}),- Tokenizer::Trigram(TrigramTokenizer {}),+ Tokenizer::Bigram(BigramTokenizer::default()),+ Tokenizer::Trigram(TrigramTokenizer::default()),] {let mut stream =- tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, &self.fragment);+ tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &self.fragment);while let Some(tok) = stream.next() {if terms.contains(&tok.text) {self.highlights.push(tok.offset_from..tok.offset_to);@@ -137,12 +137,12 @@ fn snippet_string_builder(lang: whatlang::Lang,config: SnippetConfig,) -> SnippetBuilder {- let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));+ let mut tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));let terms: HashSet<String> = terms.iter().flat_map(|term| {- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, term);+ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);let mut res = Vec::new();while let Some(tok) = stream.next() {@@ -162,7 +162,8 @@ fn snippet_string_builder(let mut doc_terms = HashMap::new();{- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, &sentence);+ let mut stream =+ tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &sentence);while let Some(tok) = stream.next() {*doc_terms.entry(tok.text.clone()).or_insert(0) += 1;}diff --git a/core/src/tokenizer/mod.rs b/core/src/tokenizer/mod.rsindex 051430c9..77739be4 100644--- a/core/src/tokenizer/mod.rs+++ b/core/src/tokenizer/mod.rs@@ -55,7 +55,7 @@ impl From<Lang> for MyStemmer {}}-#[derive(Clone, Debug)]+#[derive(Clone)]pub enum Tokenizer {Normal(Normal),Identity(Identity),@@ -90,9 +90,10 @@ impl Default for Tokenizer {}}-#[derive(Clone, Default, Debug)]+#[derive(Clone, Default)]pub struct Normal {stopwords: Option<Vec<String>>,+ analyzer: Option<TextAnalyzer>,}impl Normal {@@ -103,12 +104,15 @@ impl Normal {pub fn with_stopwords(stopwords: Vec<String>) -> Self {Self {stopwords: Some(stopwords),+ analyzer: None,}}}-#[derive(Clone, Default, Debug)]-pub struct BigramTokenizer {}+#[derive(Clone, Default)]+pub struct BigramTokenizer {+ inner_tokenizer: Normal,+}impl BigramTokenizer {pub fn as_str() -> &'static str {@@ -116,8 +120,10 @@ impl BigramTokenizer {}}-#[derive(Clone, Default, Debug)]-pub struct TrigramTokenizer {}+#[derive(Clone, Default)]+pub struct TrigramTokenizer {+ inner_tokenizer: Normal,+}impl TrigramTokenizer {pub fn as_str() -> &'static str {@@ -125,9 +131,10 @@ impl TrigramTokenizer {}}-#[derive(Clone, Default, Debug)]+#[derive(Clone, Default)]pub struct Stemmed {force_language: Option<Lang>,+ analyzer: Option<TextAnalyzer>,}impl Stemmed {@@ -137,6 +144,7 @@ impl Stemmed {pub fn with_forced_language(lang: Lang) -> Self {Self {force_language: Some(lang),+ analyzer: None,}}}@@ -151,7 +159,9 @@ impl Identity {}impl tantivy::tokenizer::Tokenizer for Tokenizer {- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {match self {Tokenizer::Normal(tokenizer) => tokenizer.token_stream(text),Tokenizer::Stemmed(tokenizer) => tokenizer.token_stream(text),@@ -165,52 +175,70 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {}impl tantivy::tokenizer::Tokenizer for Normal {- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let mut analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);+ type TokenStream<'a> = BoxTokenStream<'a>;- if let Some(stopwords) = &self.stopwords {- analyzer = analyzer.filter(StopWordFilter::remove(stopwords.clone()));- }+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let builder = TextAnalyzer::builder(Simple).filter(LowerCaser);++ self.analyzer = if let Some(stopwords) = &self.stopwords {+ Some(+ builder+ .filter(StopWordFilter::remove(stopwords.clone()))+ .build(),+ )+ } else {+ Some(builder.build())+ };- analyzer.token_stream(text)+ self.analyzer.as_mut().unwrap().token_stream(text)}}impl tantivy::tokenizer::Tokenizer for Stemmed {- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let builder = TextAnalyzer::builder(Simple).filter(LowerCaser);let lang = match self.force_language {Some(lang) => Some(lang),None => whatlang::detect_lang(text),};- match lang {- Some(lang) => analyzer.filter(MyStemmer::from(lang).0).token_stream(text),- None => analyzer.token_stream(text),- }+ self.analyzer = match lang {+ Some(lang) => Some(builder.filter(MyStemmer::from(lang).0).build()),+ None => Some(builder.build()),+ };++ self.analyzer.as_mut().unwrap().token_stream(text)}}impl tantivy::tokenizer::Tokenizer for Identity {- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- BoxTokenStream::from(IdentityTokenStream::from(text.to_string()))+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {+ BoxTokenStream::new(IdentityTokenStream::from(text.to_string()))}}impl tantivy::tokenizer::Tokenizer for BigramTokenizer {- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let inner = Normal::default().token_stream(text);- let stream: NGramTokenStream<2> = NGramTokenStream::new(inner);- BoxTokenStream::from(stream)+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let inner_stream = self.inner_tokenizer.token_stream(text);+ let stream: NGramTokenStream<2> = NGramTokenStream::new(inner_stream);+ BoxTokenStream::new(stream)}}impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let inner = Normal::default().token_stream(text);+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let inner = self.inner_tokenizer.token_stream(text);let stream: NGramTokenStream<3> = NGramTokenStream::new(inner);- BoxTokenStream::from(stream)+ BoxTokenStream::new(stream)}}@@ -275,9 +303,11 @@ pub struct SimpleTokenStream<'a> {}impl tantivy::tokenizer::Tokenizer for Simple {- fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {let lexer = Token::lexer(text);- BoxTokenStream::from(SimpleTokenStream {+ BoxTokenStream::new(SimpleTokenStream {lexer,token: None,next_position: 0,@@ -389,6 +419,7 @@ impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'apub struct FlattenedJson {flattened_json: String,+ inner_tokenizer: JsonField,}struct IntermediateFlatValue {@@ -459,11 +490,14 @@ impl FlattenedJson {let flattened_json = itertools::intersperse(flatten(val), "\n".to_string()).collect();- Ok(Self { flattened_json })+ Ok(Self {+ flattened_json,+ inner_tokenizer: JsonField,+ })}- pub fn token_stream(&self) -> BoxTokenStream {- tantivy::tokenizer::Tokenizer::token_stream(&JsonField, &self.flattened_json)+ pub fn token_stream(&mut self) -> BoxTokenStream {+ tantivy::tokenizer::Tokenizer::token_stream(&mut self.inner_tokenizer, &self.flattened_json)}pub fn text(&self) -> &str {@@ -481,8 +515,10 @@ impl JsonField {}impl tantivy::tokenizer::Tokenizer for JsonField {- fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {- BoxTokenStream::from(JsonFieldTokenStream {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {+ BoxTokenStream::new(JsonFieldTokenStream {text,chars: text.char_indices(),token: tantivy::tokenizer::Token::default(),@@ -578,7 +614,9 @@ impl SiteOperatorUrlTokenizer {}impl tantivy::tokenizer::Tokenizer for SiteOperatorUrlTokenizer {- fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {let parsed_url = url::Url::parse(text).or_else(|_| url::Url::parse(&format!("http://{}", text))).map(|url| {@@ -617,7 +655,7 @@ impl tantivy::tokenizer::Tokenizer for SiteOperatorUrlTokenizer {}).unwrap_or_default();- BoxTokenStream::from(SiteOperatorUrlTokenStream {+ BoxTokenStream::new(SiteOperatorUrlTokenStream {url: parsed_url,token: tantivy::tokenizer::Token::default(),})@@ -694,7 +732,8 @@ mod tests {fn tokenize_simple(s: &str) -> Vec<String> {let mut res = Vec::new();- let mut stream = Normal::default().token_stream(s);+ let mut tokenizer = Normal::default();+ let mut stream = tokenizer.token_stream(s);while let Some(token) = stream.next() {res.push(token.text.clone());@@ -705,7 +744,8 @@ mod tests {fn tokenize_json(s: &str) -> Vec<String> {let mut res = Vec::new();- let mut stream = JsonField.token_stream(s);+ let mut tokenizer = JsonField;+ let mut stream = tokenizer.token_stream(s);while let Some(token) = stream.next() {res.push(token.text.clone());@@ -716,7 +756,8 @@ mod tests {fn tokenize_bigram(s: &str) -> Vec<String> {let mut res = Vec::new();- let mut stream = Tokenizer::Bigram(BigramTokenizer::default()).token_stream(s);+ let mut tokenizer = Tokenizer::Bigram(BigramTokenizer::default());+ let mut stream = tokenizer.token_stream(s);while let Some(token) = stream.next() {res.push(token.text.clone());@@ -727,7 +768,9 @@ mod tests {fn tokenize_trigram(s: &str) -> Vec<String> {let mut res = Vec::new();- let mut stream = Tokenizer::Trigram(TrigramTokenizer::default()).token_stream(s);++ let mut tokenizer = Tokenizer::Trigram(TrigramTokenizer::default());+ let mut stream = tokenizer.token_stream(s);while let Some(token) = stream.next() {res.push(token.text.clone());@@ -738,7 +781,8 @@ mod tests {fn tokenize_url(s: &str) -> Vec<String> {let mut res = Vec::new();- let mut stream = SiteOperatorUrlTokenizer.token_stream(s);+ let mut tokenizer = SiteOperatorUrlTokenizer;+ let mut stream = tokenizer.token_stream(s);while let Some(token) = stream.next() {res.push(token.text.clone());diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex b10c3306..a17be861 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -916,6 +916,8 @@ impl Html {text: String,tokenizer: tokenizer::Tokenizer,) -> PreTokenizedString {+ let mut tokenizer = tokenizer;+let mut tokens = Vec::new();{@@ -960,13 +962,15 @@ impl Html {let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();let pretokenized_schema_json = match schema_org::flattened_json(schemas) {- Ok(f) => {+ Ok(mut f) => {let mut tokens = Vec::new();- let mut stream = f.token_stream();+ {+ let mut stream = f.token_stream();- while let Some(token) = stream.next() {- tokens.push(token.clone());+ while let Some(token) = stream.next() {+ tokens.push(token.clone());+ }}PreTokenizedString {@@ -994,7 +998,7 @@ impl Html {for field in &ALL_FIELDS {let tantivy_field = schema.get_field(field.name())- .unwrap_or_else(|| panic!("Unknown field: {}", field.name()));+ .unwrap_or_else(|_| panic!("Unknown field: {}", field.name()));match field {Field::Text(TextField::Title) => {