Skip to content

web search done right

upgrade tantivy to 0.21.0

Mikkel Denkere576342175e8
diff --git a/Cargo.lock b/Cargo.lock
index 5e494217..88db2912 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -828,15 +828,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
-[[package]]
-name = "combine"
-version = "4.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
-dependencies = [
- "memchr",
-]
-
[[package]]
name = "concurrent-queue"
version = "2.2.0"
@@ -1344,37 +1335,12 @@ dependencies = [
"zune-inflate",
]
-[[package]]
-name = "fail"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"
-dependencies = [
- "log",
- "once_cell",
- "rand 0.8.5",
-]
-
[[package]]
name = "fastdivide"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04"
-[[package]]
-name = "fastfield_codecs"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "374a3a53c1bd5fb31b10084229290eafb0a05f260ec90f1f726afffda4877a8a"
-dependencies = [
- "fastdivide",
- "itertools 0.10.5",
- "log",
- "ownedbytes",
- "tantivy-bitpacker",
- "tantivy-common",
-]
-
[[package]]
name = "fastrand"
version = "2.0.0"
@@ -1471,6 +1437,16 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "fs4"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47"
+dependencies = [
+ "rustix",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "fst"
version = "0.4.7"
@@ -2076,6 +2052,15 @@ dependencies = [
"either",
]
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "0.4.8"
@@ -2301,11 +2286,11 @@ dependencies = [
[[package]]
name = "lru"
-version = "0.7.8"
+version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"
+checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21"
dependencies = [
- "hashbrown 0.12.3",
+ "hashbrown 0.14.0",
]
[[package]]
@@ -2333,6 +2318,12 @@ dependencies = [
"twox-hash",
]
+[[package]]
+name = "lz4_flex"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8"
+
[[package]]
name = "mac"
version = "0.1.1"
@@ -2458,6 +2449,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "memmap2"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "memoffset"
version = "0.9.0"
@@ -2548,12 +2548,9 @@ dependencies = [
[[package]]
name = "murmurhash32"
-version = "0.2.0"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a"
-dependencies = [
- "byteorder",
-]
+checksum = "d9380db4c04d219ac5c51d14996bbf2c2e9a15229771b53f8671eb6c83cf44df"
[[package]]
name = "nanorand"
@@ -2861,9 +2858,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "ownedbytes"
-version = "0.4.0"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e957eaa64a299f39755416e5b3128c505e9d63a91d0453771ad2ccd3907f8db"
+checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7"
dependencies = [
"stable_deref_trait",
]
@@ -3866,7 +3863,7 @@ dependencies = [
"futures",
"histogram",
"itertools 0.10.5",
- "lz4_flex",
+ "lz4_flex 0.9.5",
"num-bigint",
"num_enum",
"rand 0.8.5",
@@ -3893,7 +3890,7 @@ dependencies = [
"byteorder",
"bytes",
"chrono",
- "lz4_flex",
+ "lz4_flex 0.9.5",
"num-bigint",
"num_enum",
"scylla-macros",
@@ -4120,6 +4117,15 @@ version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
+[[package]]
+name = "sketches-ddsketch"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "slab"
version = "0.4.9"
@@ -4251,7 +4257,7 @@ dependencies = [
"maplit",
"md5",
"memmap",
- "memmap2",
+ "memmap2 0.5.10",
"mime",
"min-max-heap",
"num_cpus",
@@ -4395,49 +4401,49 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "tantivy"
-version = "0.19.2"
+version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bb26a6b22c84d8be41d99a14016d6f04d30d8d31a2ea411a8ab553af5cc490d"
+checksum = "c1d4675fed6fe2218ce11445374e181e864a8ffd0f28e7e0591ccfc38cd000ae"
dependencies = [
- "aho-corasick 0.7.20",
+ "aho-corasick 1.0.4",
"arc-swap",
"async-trait",
- "base64 0.13.1",
+ "base64 0.21.2",
"bitpacking",
"byteorder",
"census",
"crc32fast",
"crossbeam-channel",
"downcast-rs",
- "fail",
"fastdivide",
- "fastfield_codecs",
- "fs2",
+ "fs4",
"htmlescape",
- "itertools 0.10.5",
+ "itertools 0.11.0",
"levenshtein_automata",
"log",
"lru",
- "lz4_flex",
+ "lz4_flex 0.11.1",
"measure_time",
- "memmap2",
+ "memmap2 0.7.1",
"murmurhash32",
"num_cpus",
"once_cell",
"oneshot",
- "ownedbytes",
"rayon",
"regex",
"rust-stemmers",
"rustc-hash",
"serde",
"serde_json",
+ "sketches-ddsketch",
"smallvec",
- "stable_deref_trait",
"tantivy-bitpacker",
+ "tantivy-columnar",
"tantivy-common",
"tantivy-fst",
"tantivy-query-grammar",
+ "tantivy-stacker",
+ "tantivy-tokenizer-api",
"tempfile",
"thiserror",
"time 0.3.27",
@@ -4447,18 +4453,40 @@ dependencies = [
[[package]]
name = "tantivy-bitpacker"
-version = "0.3.0"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a"
+dependencies = [
+ "bitpacking",
+]
+
+[[package]]
+name = "tantivy-columnar"
+version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e71a0c95b82d4292b097a09b989a6380d28c3a86800c841a2d03bae1fc8b9fa6"
+checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70"
+dependencies = [
+ "fastdivide",
+ "fnv",
+ "itertools 0.11.0",
+ "serde",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-sstable",
+ "tantivy-stacker",
+]
[[package]]
name = "tantivy-common"
-version = "0.4.0"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14fef4182bb60df9a4b92cd8ecab39ba2e50a05542934af17eef1f49660705cb"
+checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1"
dependencies = [
+ "async-trait",
"byteorder",
"ownedbytes",
+ "serde",
+ "time 0.3.27",
]
[[package]]
@@ -4474,13 +4502,41 @@ dependencies = [
[[package]]
name = "tantivy-query-grammar"
-version = "0.19.0"
+version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343e3ada4c1c480953f6960f8a21ce9c76611480ffdd4f4e230fdddce0fc5331"
+checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77"
dependencies = [
- "combine",
- "once_cell",
- "regex",
+ "nom",
+]
+
+[[package]]
+name = "tantivy-sstable"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df"
+dependencies = [
+ "tantivy-common",
+ "tantivy-fst",
+ "zstd 0.12.4",
+]
+
+[[package]]
+name = "tantivy-stacker"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c"
+dependencies = [
+ "murmurhash32",
+ "tantivy-common",
+]
+
+[[package]]
+name = "tantivy-tokenizer-api"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66"
+dependencies = [
+ "serde",
]
[[package]]
@@ -5571,7 +5627,7 @@ dependencies = [
"pbkdf2",
"sha1",
"time 0.3.27",
- "zstd",
+ "zstd 0.11.2+zstd.1.5.2",
]
[[package]]
@@ -5580,7 +5636,16 @@ version = "0.11.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
dependencies = [
- "zstd-safe",
+ "zstd-safe 5.0.2+zstd.1.5.2",
+]
+
+[[package]]
+name = "zstd"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"
+dependencies = [
+ "zstd-safe 6.0.6",
]
[[package]]
@@ -5593,6 +5658,16 @@ dependencies = [
"zstd-sys",
]
+[[package]]
+name = "zstd-safe"
+version = "6.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"
+dependencies = [
+ "libc",
+ "zstd-sys",
+]
+
[[package]]
name = "zstd-sys"
version = "2.0.8+zstd.1.5.5"
diff --git a/Cargo.toml b/Cargo.toml
index 55be4625..982b38df 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,7 +24,7 @@ reqwest-eventsource = "0.4.0"
reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }
serde = { version = "1.0.137", features = ["rc", "derive"] }
serde_json = "1.0.81"
-tantivy = "0.19.2"
+tantivy = "0.21.0"
thiserror = "1.0.31"
toml = "0.5.9"
tracing = "0.1.34"
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index 507fe190..13eb5716 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -316,7 +316,8 @@ impl EntityIndex {
let entity_abstract = self.schema.get_field("abstract").unwrap();
let mut term_queries = Vec::new();
- let mut stream = Normal::default().token_stream(query);
+ let mut tokenizer = Normal::default();
+ let mut stream = tokenizer.token_stream(query);
while let Some(token) = stream.next() {
if self.stopwords.contains(&token.text) {
continue;
diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rs
index 049f20d7..523cd9ef 100644
--- a/core/src/fastfield_reader.rs
+++ b/core/src/fastfield_reader.rs
@@ -16,11 +16,11 @@
use std::{collections::HashMap, sync::Arc};
-use tantivy::{fastfield::Column, DocId, SegmentId};
+use tantivy::{columnar::ColumnValues, DocId, SegmentId};
use crate::{
enum_map::EnumMap,
- schema::{DataType, FastField, Field, ALL_FIELDS},
+ schema::{DataType, FastField, ALL_FIELDS},
};
#[derive(Default, Clone)]
@@ -43,24 +43,20 @@ impl FastFieldReader {
pub fn new(tv_searcher: &tantivy::Searcher) -> Self {
let mut segments = HashMap::new();
- let schema = tv_searcher.schema();
-
for reader in tv_searcher.segment_readers() {
let fastfield_readers = reader.fast_fields();
let mut field_readers = Vec::new();
for field in ALL_FIELDS.iter().filter_map(|field| field.as_fast()) {
- let tv_field = schema.get_field(Field::Fast(field).name()).unwrap();
-
let field_reader = match field.data_type() {
DataType::U64 => {
- let reader = fastfield_readers.u64(tv_field).unwrap();
- FieldReader::U64(reader)
+ let reader = fastfield_readers.u64(field.name()).unwrap();
+ FieldReader::U64(reader.values)
}
DataType::F64 => {
- let reader = fastfield_readers.f64(tv_field).unwrap();
- FieldReader::F64(reader)
+ let reader = fastfield_readers.f64(field.name()).unwrap();
+ FieldReader::F64(reader.values)
}
};
@@ -108,8 +104,8 @@ impl From<FieldValue> for Option<u64> {
}
pub enum FieldReader {
- U64(Arc<dyn Column<u64>>),
- F64(Arc<dyn Column<f64>>),
+ U64(Arc<dyn ColumnValues<u64>>),
+ F64(Arc<dyn ColumnValues<f64>>),
}
impl FieldReader {
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index d1169ed4..969e9944 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -45,7 +45,9 @@ use crate::ranking::SignalAggregator;
use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
use crate::search_ctx::Ctx;
use crate::snippet::TextSnippet;
-use crate::tokenizer::{BigramTokenizer, Identity, SiteOperatorUrlTokenizer, TrigramTokenizer};
+use crate::tokenizer::{
+ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer,
+};
use crate::webgraph::NodeID;
use crate::webpage::region::Region;
use crate::webpage::{schema_org, Webpage};
@@ -159,6 +161,11 @@ impl InvertedIndex {
.tokenizers()
.register(tokenizer.as_str(), tokenizer);
+ let tokenizer = Tokenizer::Json(JsonField);
+ tantivy_index
+ .tokenizers()
+ .register(tokenizer.as_str(), tokenizer);
+
let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;
let merge_policy = NoMergePolicy;
diff --git a/core/src/query/const_query.rs b/core/src/query/const_query.rs
index a1346e7c..32f218dc 100644
--- a/core/src/query/const_query.rs
+++ b/core/src/query/const_query.rs
@@ -46,13 +46,22 @@ impl Query for ConstQuery {
enable_scoring: tantivy::query::EnableScoring,
) -> tantivy::Result<Box<dyn Weight>> {
let schema = match enable_scoring {
- tantivy::query::EnableScoring::Enabled(searcher) => searcher.schema(),
- tantivy::query::EnableScoring::Disabled(s) => s,
+ tantivy::query::EnableScoring::Enabled {
+ searcher,
+ statistics_provider: _,
+ } => searcher.schema(),
+ tantivy::query::EnableScoring::Disabled {
+ schema,
+ searcher_opt: _,
+ } => schema,
};
Ok(Box::new(ConstWeight {
subweight: self
.subquery
- .weight(tantivy::query::EnableScoring::Disabled(schema))?,
+ .weight(tantivy::query::EnableScoring::Disabled {
+ schema,
+ searcher_opt: None,
+ })?,
score: self.score,
}))
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 25d2674b..47f0ace7 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -327,7 +327,8 @@ impl Term {
Field::Fast(_) => vec![tantivy::Term::from_field_text(tantivy_field, term)],
Field::Text(text_field) => {
let mut terms: Vec<tantivy::Term> = Vec::new();
- let mut token_stream = text_field.query_tokenizer().token_stream(term);
+ let mut tokenizer = text_field.query_tokenizer();
+ let mut token_stream = tokenizer.token_stream(term);
token_stream.process(&mut |token| {
let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
terms.push(term);
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 88b3eefd..751b3051 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -106,11 +106,8 @@ impl PatternQuery {
for pattern in &patterns {
match pattern {
PatternPart::Raw(text) => {
- let mut stream = field
- .as_text()
- .unwrap()
- .indexing_tokenizer()
- .token_stream(text);
+ let mut tokenizer = field.as_text().unwrap().indexing_tokenizer();
+ let mut stream = tokenizer.token_stream(text);
while let Some(token) = stream.next() {
new_patterns.push(PatternPart::Raw(token.text.clone()));
@@ -141,14 +138,17 @@ impl tantivy::query::Query for PatternQuery {
scoring: tantivy::query::EnableScoring<'_>,
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
let bm25_weight = match scoring {
- tantivy::query::EnableScoring::Enabled(searcher) => {
+ tantivy::query::EnableScoring::Enabled {
+ searcher,
+ statistics_provider: _,
+ } => {
if self.raw_terms.is_empty() {
None
} else {
Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)
}
}
- tantivy::query::EnableScoring::Disabled(_) => None,
+ tantivy::query::EnableScoring::Disabled { .. } => None,
};
if self.can_optimize_site_domain {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 3c757982..d9a404d5 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -864,8 +864,8 @@ impl SignalAggregator {
.collect::<String>();
let mut terms = Vec::new();
- let mut stream =
- text_field.indexing_tokenizer().token_stream(&simple_query);
+ let mut tokenizer = text_field.indexing_tokenizer();
+ let mut stream = tokenizer.token_stream(&simple_query);
while let Some(token) = stream.next() {
let term = tantivy::Term::from_field_text(tv_field, &token.text);
@@ -922,7 +922,10 @@ impl SignalAggregator {
.map(|(_, rule)| RuleBoost {
docset: rule
.query
- .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))
+ .weight(tantivy::query::EnableScoring::Enabled {
+ searcher: tv_searcher,
+ statistics_provider: tv_searcher,
+ })
.unwrap()
.scorer(segment_reader, 0.0)
.unwrap(),
@@ -957,7 +960,7 @@ impl SignalAggregator {
let mut queries = Vec::new();
for field in proximity_fields {
let tv_field = schema.get_field(field.name()).unwrap();
- let tokenizer = field.indexing_tokenizer();
+ let mut tokenizer = field.indexing_tokenizer();
let mut terms = Vec::with_capacity(query.simple_terms.len());
for term in &query.simple_terms {
@@ -979,7 +982,10 @@ impl SignalAggregator {
let query = tantivy::query::BooleanQuery::new(queries);
let docset = query
- .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))
+ .weight(tantivy::query::EnableScoring::Enabled {
+ searcher: tv_searcher,
+ statistics_provider: tv_searcher,
+ })
.unwrap()
.scorer(segment_reader, 1.0)
.unwrap();
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 7c2653b9..6998960f 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -14,9 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::schema::{
- Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
-};
+use tantivy::schema::{IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions};
use crate::tokenizer::{
BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,
@@ -442,146 +440,120 @@ impl Field {
Field::Text(TextField::SafetyClassification) => {
IndexingOption::Text(self.default_text_options())
}
- Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::HostCentrality) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::PageCentrality) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::FetchTimeMs) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::TrackerScore) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
+ Field::Fast(FastField::IsHomepage) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::HostCentrality) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::PageCentrality) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::FetchTimeMs) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::TrackerScore) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
Field::Fast(FastField::LastUpdated) => IndexingOption::Integer(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_stored()
.set_indexed(),
),
Field::Fast(FastField::Region) => IndexingOption::Integer(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_stored()
.set_indexed(),
),
- Field::Fast(FastField::NumCleanBodyTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::NumDescriptionTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::NumTitleTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::NumMicroformatTagsTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::NumUrlTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::NumDomainTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::NumUrlForSiteOperatorTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::NumFlattenedSchemaTokens) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
- .set_indexed(),
- ),
- Field::Fast(FastField::SiteHash1) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::SiteHash2) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::UrlWithoutQueryHash1) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::UrlWithoutQueryHash2) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::UrlHash1) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::UrlHash2) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::DomainHash1) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::DomainHash2) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::TitleHash1) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
- Field::Fast(FastField::TitleHash2) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
+ Field::Fast(FastField::NumCleanBodyTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::NumDescriptionTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::NumTitleTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::NumMicroformatTagsTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::NumUrlTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::NumDomainTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::NumFlattenedSchemaTokens) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
+ Field::Fast(FastField::SiteHash1) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::SiteHash2) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::UrlWithoutQueryHash1) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::UrlWithoutQueryHash2) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::UrlHash1) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::UrlHash2) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::DomainHash1) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::DomainHash2) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::TitleHash1) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::TitleHash2) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
Field::Fast(FastField::PreComputedScore) => IndexingOption::Float(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_indexed()
.set_stored(),
),
Field::Fast(FastField::HostNodeID1) => IndexingOption::Integer(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_indexed()
.set_stored(),
),
Field::Fast(FastField::HostNodeID2) => IndexingOption::Integer(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_indexed()
.set_stored(),
),
Field::Fast(FastField::SimHash) => IndexingOption::Integer(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_indexed()
.set_stored(),
),
Field::Fast(FastField::NumPathAndQuerySlashes) => IndexingOption::Integer(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_indexed()
.set_stored(),
),
Field::Fast(FastField::NumPathAndQueryDigits) => IndexingOption::Integer(
NumericOptions::default()
- .set_fast(Cardinality::SingleValue)
+ .set_fast()
.set_indexed()
.set_stored(),
),
diff --git a/core/src/simhash.rs b/core/src/simhash.rs
index 0df0456e..08b275b4 100644
--- a/core/src/simhash.rs
+++ b/core/src/simhash.rs
@@ -30,9 +30,9 @@ fn hash_token(token: &tantivy::tokenizer::Token) -> HashType {
}
pub fn hash(text: &str) -> HashType {
- let tokenizer = Tokenizer::default();
+ let mut tokenizer = Tokenizer::default();
- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, text);
+ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, text);
let mut v = [0i64; HashType::BITS as usize];
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 718e788e..b205d49a 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -79,13 +79,13 @@ struct SnippetBuilder {
impl SnippetBuilder {
fn highlight(&mut self, terms: &HashSet<String>, lang: whatlang::Lang) {
- for tokenizer in [
+ for mut tokenizer in [
Tokenizer::Stemmed(Stemmed::with_forced_language(lang)),
- Tokenizer::Bigram(BigramTokenizer {}),
- Tokenizer::Trigram(TrigramTokenizer {}),
+ Tokenizer::Bigram(BigramTokenizer::default()),
+ Tokenizer::Trigram(TrigramTokenizer::default()),
] {
let mut stream =
- tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, &self.fragment);
+ tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &self.fragment);
while let Some(tok) = stream.next() {
if terms.contains(&tok.text) {
self.highlights.push(tok.offset_from..tok.offset_to);
@@ -137,12 +137,12 @@ fn snippet_string_builder(
lang: whatlang::Lang,
config: SnippetConfig,
) -> SnippetBuilder {
- let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));
+ let mut tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));
let terms: HashSet<String> = terms
.iter()
.flat_map(|term| {
- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, term);
+ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);
let mut res = Vec::new();
while let Some(tok) = stream.next() {
@@ -162,7 +162,8 @@ fn snippet_string_builder(
let mut doc_terms = HashMap::new();
{
- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, &sentence);
+ let mut stream =
+ tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &sentence);
while let Some(tok) = stream.next() {
*doc_terms.entry(tok.text.clone()).or_insert(0) += 1;
}
diff --git a/core/src/tokenizer/mod.rs b/core/src/tokenizer/mod.rs
index 051430c9..77739be4 100644
--- a/core/src/tokenizer/mod.rs
+++ b/core/src/tokenizer/mod.rs
@@ -55,7 +55,7 @@ impl From<Lang> for MyStemmer {
}
}
-#[derive(Clone, Debug)]
+#[derive(Clone)]
pub enum Tokenizer {
Normal(Normal),
Identity(Identity),
@@ -90,9 +90,10 @@ impl Default for Tokenizer {
}
}
-#[derive(Clone, Default, Debug)]
+#[derive(Clone, Default)]
pub struct Normal {
stopwords: Option<Vec<String>>,
+ analyzer: Option<TextAnalyzer>,
}
impl Normal {
@@ -103,12 +104,15 @@ impl Normal {
pub fn with_stopwords(stopwords: Vec<String>) -> Self {
Self {
stopwords: Some(stopwords),
+ analyzer: None,
}
}
}
-#[derive(Clone, Default, Debug)]
-pub struct BigramTokenizer {}
+#[derive(Clone, Default)]
+pub struct BigramTokenizer {
+ inner_tokenizer: Normal,
+}
impl BigramTokenizer {
pub fn as_str() -> &'static str {
@@ -116,8 +120,10 @@ impl BigramTokenizer {
}
}
-#[derive(Clone, Default, Debug)]
-pub struct TrigramTokenizer {}
+#[derive(Clone, Default)]
+pub struct TrigramTokenizer {
+ inner_tokenizer: Normal,
+}
impl TrigramTokenizer {
pub fn as_str() -> &'static str {
@@ -125,9 +131,10 @@ impl TrigramTokenizer {
}
}
-#[derive(Clone, Default, Debug)]
+#[derive(Clone, Default)]
pub struct Stemmed {
force_language: Option<Lang>,
+ analyzer: Option<TextAnalyzer>,
}
impl Stemmed {
@@ -137,6 +144,7 @@ impl Stemmed {
pub fn with_forced_language(lang: Lang) -> Self {
Self {
force_language: Some(lang),
+ analyzer: None,
}
}
}
@@ -151,7 +159,9 @@ impl Identity {
}
impl tantivy::tokenizer::Tokenizer for Tokenizer {
- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
match self {
Tokenizer::Normal(tokenizer) => tokenizer.token_stream(text),
Tokenizer::Stemmed(tokenizer) => tokenizer.token_stream(text),
@@ -165,52 +175,70 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {
}
impl tantivy::tokenizer::Tokenizer for Normal {
- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let mut analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);
+ type TokenStream<'a> = BoxTokenStream<'a>;
- if let Some(stopwords) = &self.stopwords {
- analyzer = analyzer.filter(StopWordFilter::remove(stopwords.clone()));
- }
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+ let builder = TextAnalyzer::builder(Simple).filter(LowerCaser);
+
+ self.analyzer = if let Some(stopwords) = &self.stopwords {
+ Some(
+ builder
+ .filter(StopWordFilter::remove(stopwords.clone()))
+ .build(),
+ )
+ } else {
+ Some(builder.build())
+ };
- analyzer.token_stream(text)
+ self.analyzer.as_mut().unwrap().token_stream(text)
}
}
impl tantivy::tokenizer::Tokenizer for Stemmed {
- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+ let builder = TextAnalyzer::builder(Simple).filter(LowerCaser);
let lang = match self.force_language {
Some(lang) => Some(lang),
None => whatlang::detect_lang(text),
};
- match lang {
- Some(lang) => analyzer.filter(MyStemmer::from(lang).0).token_stream(text),
- None => analyzer.token_stream(text),
- }
+ self.analyzer = match lang {
+ Some(lang) => Some(builder.filter(MyStemmer::from(lang).0).build()),
+ None => Some(builder.build()),
+ };
+
+ self.analyzer.as_mut().unwrap().token_stream(text)
}
}
impl tantivy::tokenizer::Tokenizer for Identity {
- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- BoxTokenStream::from(IdentityTokenStream::from(text.to_string()))
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {
+ BoxTokenStream::new(IdentityTokenStream::from(text.to_string()))
}
}
impl tantivy::tokenizer::Tokenizer for BigramTokenizer {
- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let inner = Normal::default().token_stream(text);
- let stream: NGramTokenStream<2> = NGramTokenStream::new(inner);
- BoxTokenStream::from(stream)
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+ let inner_stream = self.inner_tokenizer.token_stream(text);
+ let stream: NGramTokenStream<2> = NGramTokenStream::new(inner_stream);
+ BoxTokenStream::new(stream)
}
}
impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {
- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let inner = Normal::default().token_stream(text);
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+ let inner = self.inner_tokenizer.token_stream(text);
let stream: NGramTokenStream<3> = NGramTokenStream::new(inner);
- BoxTokenStream::from(stream)
+ BoxTokenStream::new(stream)
}
}
@@ -275,9 +303,11 @@ pub struct SimpleTokenStream<'a> {
}
impl tantivy::tokenizer::Tokenizer for Simple {
- fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {
let lexer = Token::lexer(text);
- BoxTokenStream::from(SimpleTokenStream {
+ BoxTokenStream::new(SimpleTokenStream {
lexer,
token: None,
next_position: 0,
@@ -389,6 +419,7 @@ impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'a
pub struct FlattenedJson {
flattened_json: String,
+ inner_tokenizer: JsonField,
}
struct IntermediateFlatValue {
@@ -459,11 +490,14 @@ impl FlattenedJson {
let flattened_json = itertools::intersperse(flatten(val), "\n".to_string()).collect();
- Ok(Self { flattened_json })
+ Ok(Self {
+ flattened_json,
+ inner_tokenizer: JsonField,
+ })
}
- pub fn token_stream(&self) -> BoxTokenStream {
- tantivy::tokenizer::Tokenizer::token_stream(&JsonField, &self.flattened_json)
+ pub fn token_stream(&mut self) -> BoxTokenStream {
+ tantivy::tokenizer::Tokenizer::token_stream(&mut self.inner_tokenizer, &self.flattened_json)
}
pub fn text(&self) -> &str {
@@ -481,8 +515,10 @@ impl JsonField {
}
impl tantivy::tokenizer::Tokenizer for JsonField {
- fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
- BoxTokenStream::from(JsonFieldTokenStream {
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {
+ BoxTokenStream::new(JsonFieldTokenStream {
text,
chars: text.char_indices(),
token: tantivy::tokenizer::Token::default(),
@@ -578,7 +614,9 @@ impl SiteOperatorUrlTokenizer {
}
impl tantivy::tokenizer::Tokenizer for SiteOperatorUrlTokenizer {
- fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {
let parsed_url = url::Url::parse(text)
.or_else(|_| url::Url::parse(&format!("http://{}", text)))
.map(|url| {
@@ -617,7 +655,7 @@ impl tantivy::tokenizer::Tokenizer for SiteOperatorUrlTokenizer {
})
.unwrap_or_default();
- BoxTokenStream::from(SiteOperatorUrlTokenStream {
+ BoxTokenStream::new(SiteOperatorUrlTokenStream {
url: parsed_url,
token: tantivy::tokenizer::Token::default(),
})
@@ -694,7 +732,8 @@ mod tests {
fn tokenize_simple(s: &str) -> Vec<String> {
let mut res = Vec::new();
- let mut stream = Normal::default().token_stream(s);
+ let mut tokenizer = Normal::default();
+ let mut stream = tokenizer.token_stream(s);
while let Some(token) = stream.next() {
res.push(token.text.clone());
@@ -705,7 +744,8 @@ mod tests {
fn tokenize_json(s: &str) -> Vec<String> {
let mut res = Vec::new();
- let mut stream = JsonField.token_stream(s);
+ let mut tokenizer = JsonField;
+ let mut stream = tokenizer.token_stream(s);
while let Some(token) = stream.next() {
res.push(token.text.clone());
@@ -716,7 +756,8 @@ mod tests {
fn tokenize_bigram(s: &str) -> Vec<String> {
let mut res = Vec::new();
- let mut stream = Tokenizer::Bigram(BigramTokenizer::default()).token_stream(s);
+ let mut tokenizer = Tokenizer::Bigram(BigramTokenizer::default());
+ let mut stream = tokenizer.token_stream(s);
while let Some(token) = stream.next() {
res.push(token.text.clone());
@@ -727,7 +768,9 @@ mod tests {
fn tokenize_trigram(s: &str) -> Vec<String> {
let mut res = Vec::new();
- let mut stream = Tokenizer::Trigram(TrigramTokenizer::default()).token_stream(s);
+
+ let mut tokenizer = Tokenizer::Trigram(TrigramTokenizer::default());
+ let mut stream = tokenizer.token_stream(s);
while let Some(token) = stream.next() {
res.push(token.text.clone());
@@ -738,7 +781,8 @@ mod tests {
fn tokenize_url(s: &str) -> Vec<String> {
let mut res = Vec::new();
- let mut stream = SiteOperatorUrlTokenizer.token_stream(s);
+ let mut tokenizer = SiteOperatorUrlTokenizer;
+ let mut stream = tokenizer.token_stream(s);
while let Some(token) = stream.next() {
res.push(token.text.clone());
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index b10c3306..a17be861 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -916,6 +916,8 @@ impl Html {
text: String,
tokenizer: tokenizer::Tokenizer,
) -> PreTokenizedString {
+ let mut tokenizer = tokenizer;
+
let mut tokens = Vec::new();
{
@@ -960,13 +962,15 @@ impl Html {
let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
let pretokenized_schema_json = match schema_org::flattened_json(schemas) {
- Ok(f) => {
+ Ok(mut f) => {
let mut tokens = Vec::new();
- let mut stream = f.token_stream();
+ {
+ let mut stream = f.token_stream();
- while let Some(token) = stream.next() {
- tokens.push(token.clone());
+ while let Some(token) = stream.next() {
+ tokens.push(token.clone());
+ }
}
PreTokenizedString {
@@ -994,7 +998,7 @@ impl Html {
for field in &ALL_FIELDS {
let tantivy_field = schema
.get_field(field.name())
- .unwrap_or_else(|| panic!("Unknown field: {}", field.name()));
+ .unwrap_or_else(|_| panic!("Unknown field: {}", field.name()));
match field {
Field::Text(TextField::Title) => {