Skip to content

web search done right

upgrade tantivy

Mikkel Denkerfac468cc3c18
diff --git a/Cargo.lock b/Cargo.lock
index 13b05b96..e1eba30f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -710,16 +710,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "ctor"
-version = "0.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
-dependencies = [
- "quote",
- "syn",
-]
-
[[package]]
name = "cuely"
version = "0.1.0"
@@ -924,9 +914,12 @@ checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04"
[[package]]
name = "fastfield_codecs"
-version = "0.2.0"
-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"
+version = "0.3.0"
+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"
dependencies = [
+ "fastdivide",
+ "itertools",
+ "log",
"ownedbytes",
"tantivy-bitpacker",
"tantivy-common",
@@ -1554,7 +1547,7 @@ dependencies = [
"petgraph",
"pico-args",
"regex",
- "regex-syntax 0.6.27",
+ "regex-syntax",
"string_cache",
"term",
"tiny-keccak",
@@ -1674,7 +1667,7 @@ dependencies = [
"fnv",
"proc-macro2",
"quote",
- "regex-syntax 0.6.27",
+ "regex-syntax",
"syn",
]
@@ -1686,6 +1679,7 @@ checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"
dependencies = [
"cfg-if",
"generator",
+ "pin-utils",
"scoped-tls",
"tracing",
"tracing-subscriber",
@@ -1984,9 +1978,9 @@ checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"
[[package]]
name = "oneshot"
-version = "0.1.3"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3b8d98df258e762e901eb4349d66d5a5f5a7a994db8df82ff596011773be535"
+checksum = "fc22d22931513428ea6cc089e942d38600e3d00976eef8c86de6b8a3aadec6eb"
dependencies = [
"loom",
]
@@ -2066,19 +2060,10 @@ version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa"
-[[package]]
-name = "output_vt100"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"
-dependencies = [
- "winapi",
-]
-
[[package]]
name = "ownedbytes"
-version = "0.3.0"
-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"
+version = "0.4.0"
+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"
dependencies = [
"stable_deref_trait",
]
@@ -2303,18 +2288,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
-[[package]]
-name = "pretty_assertions"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563"
-dependencies = [
- "ansi_term",
- "ctor",
- "diff",
- "output_vt100",
-]
-
[[package]]
name = "proc-macro-error"
version = "1.0.4"
@@ -2511,7 +2484,7 @@ checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
dependencies = [
"aho-corasick",
"memchr",
- "regex-syntax 0.6.27",
+ "regex-syntax",
]
[[package]]
@@ -2520,15 +2493,9 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
- "regex-syntax 0.6.27",
+ "regex-syntax",
]
-[[package]]
-name = "regex-syntax"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"
-
[[package]]
name = "regex-syntax"
version = "0.6.27"
@@ -3060,9 +3027,10 @@ checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
[[package]]
name = "tantivy"
-version = "0.18.0"
-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"
+version = "0.19.0"
+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"
dependencies = [
+ "aho-corasick",
"arc-swap",
"async-trait",
"base64",
@@ -3075,7 +3043,6 @@ dependencies = [
"fail",
"fastdivide",
"fastfield_codecs",
- "fnv",
"fs2",
"htmlescape",
"itertools",
@@ -3090,10 +3057,10 @@ dependencies = [
"once_cell",
"oneshot",
"ownedbytes",
- "pretty_assertions",
"rayon",
"regex",
"rust-stemmers",
+ "rustc-hash",
"serde",
"serde_json",
"smallvec",
@@ -3111,13 +3078,13 @@ dependencies = [
[[package]]
name = "tantivy-bitpacker"
-version = "0.2.0"
-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"
+version = "0.3.0"
+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"
[[package]]
name = "tantivy-common"
-version = "0.3.0"
-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"
+version = "0.4.0"
+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"
dependencies = [
"byteorder",
"ownedbytes",
@@ -3125,19 +3092,19 @@ dependencies = [
[[package]]
name = "tantivy-fst"
-version = "0.3.0"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb20cdc0d83e9184560bdde9cd60142dbb4af2e0f770e88fce45770495224205"
+checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944"
dependencies = [
"byteorder",
- "regex-syntax 0.4.2",
+ "regex-syntax",
"utf8-ranges",
]
[[package]]
name = "tantivy-query-grammar"
-version = "0.18.0"
-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"
+version = "0.19.0"
+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"
dependencies = [
"combine",
"once_cell",
diff --git a/Cargo.toml b/Cargo.toml
index f4061bbb..90b6d062 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,7 @@ rayon = "1.5.3"
reqwest = { version = "0.11.10", features = ["blocking", "stream"] }
serde = { version = "1.0.137", features = ["rc", "derive"] }
serde_json = "1.0.81"
-tantivy = { git = "https://github.com/quickwit-oss/tantivy.git", rev = "a451f6d60dc4cb751457d11a3cc9d038d9b63713" }
+tantivy = { git = "https://github.com/quickwit-oss/tantivy.git", rev = "5d4535de83d214765c29501f0418f1ff38a607b1" }
thiserror = "1.0.31"
toml = "0.5.9"
tracing = "0.1.34"
diff --git a/core/src/fastfield_cache.rs b/core/src/fastfield_cache.rs
index de245512..edc4e575 100644
--- a/core/src/fastfield_cache.rs
+++ b/core/src/fastfield_cache.rs
@@ -19,7 +19,7 @@ use std::{
sync::{Arc, RwLock},
};
-use tantivy::{fastfield::Column, DocId, SegmentId};
+use tantivy::{DocId, SegmentId};
use crate::schema::{DataType, FastField, Field, ALL_FIELDS};
diff --git a/core/src/query/const_query.rs b/core/src/query/const_query.rs
index 3a519b23..a9bfd9f7 100644
--- a/core/src/query/const_query.rs
+++ b/core/src/query/const_query.rs
@@ -16,7 +16,7 @@
use tantivy::{
query::{Explanation, Query, Scorer, Weight},
- DocSet, Score, Searcher,
+ DocSet, Score,
};
#[derive(Debug)]
@@ -43,11 +43,10 @@ impl Clone for ConstQuery {
impl Query for ConstQuery {
fn weight(
&self,
- searcher: &Searcher,
- scoring_enabled: bool,
+ enable_scoring: tantivy::query::EnableScoring,
) -> tantivy::Result<Box<dyn Weight>> {
Ok(Box::new(ConstWeight {
- subweight: self.subquery.weight(searcher, scoring_enabled)?,
+ subweight: self.subquery.weight(enable_scoring)?,
score: self.score,
}))
}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 822cae6f..389b8286 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ index::Index,
ranking::SignalAggregator,
schema::{Field, TextField},
searcher::NUM_RESULTS_PER_PAGE,
@@ -159,10 +160,12 @@ impl Query {
})
}
- pub fn set_optic(&mut self, optic: &Optic, schema: &Schema) {
+ pub fn set_optic(&mut self, optic: &Optic, index: &Index) {
let mut subqueries = vec![(Occur::Must, self.tantivy_query.box_clone())];
- subqueries.append(&mut optic.as_multiple_tantivy(schema));
+ subqueries.append(
+ &mut optic.as_multiple_tantivy(&index.schema(), index.inverted_index.fastfield_cache()),
+ );
self.tantivy_query = Box::new(BooleanQuery::new(subqueries))
}
@@ -191,10 +194,9 @@ impl Query {
impl tantivy::query::Query for Query {
fn weight(
&self,
- searcher: &tantivy::Searcher,
- scoring_enabled: bool,
+ enable_scoring: tantivy::query::EnableScoring,
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
- self.tantivy_query.weight(searcher, scoring_enabled)
+ self.tantivy_query.weight(enable_scoring)
}
fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a tantivy::Term, bool)) {
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 68d02efe..67b512fb 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -14,6 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::sync::Arc;
+
use itertools::Itertools;
use optics::{Action, MatchLocation, Matching, Optic, Rule};
use tantivy::{
@@ -21,20 +23,32 @@ use tantivy::{
schema::Schema,
};
-use crate::{ranking::optics::SCALE, schema::TextField};
+use crate::{fastfield_cache::FastFieldCache, ranking::optics::SCALE, schema::TextField};
use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};
pub trait AsTantivyQuery {
- fn as_tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query>;
+ fn as_tantivy(
+ &self,
+ schema: &Schema,
+ fastfield_cache: Arc<FastFieldCache>,
+ ) -> Box<dyn tantivy::query::Query>;
}
pub trait AsMultipleTantivyQuery {
- fn as_multiple_tantivy(&self, schema: &Schema) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;
+ fn as_multiple_tantivy(
+ &self,
+ schema: &Schema,
+ fastfield_cache: Arc<FastFieldCache>,
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;
}
impl AsMultipleTantivyQuery for Optic {
- fn as_multiple_tantivy(&self, schema: &Schema) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
+ fn as_multiple_tantivy(
+ &self,
+ schema: &Schema,
+ fastfields: Arc<FastFieldCache>,
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
if self.discard_non_matching {
vec![(
Occur::Must,
@@ -43,7 +57,7 @@ impl AsMultipleTantivyQuery for Optic {
.iter()
.chain(self.site_rankings.rules().iter())
.filter_map(|rule| {
- let queries = rule.as_multiple_tantivy(schema);
+ let queries = rule.as_multiple_tantivy(schema, fastfields.clone());
if queries.is_empty() {
None
@@ -60,18 +74,27 @@ impl AsMultipleTantivyQuery for Optic {
self.rules
.iter()
.chain(self.site_rankings.rules().iter())
- .filter_map(|rule| rule.as_multiple_tantivy(schema).pop())
+ .filter_map(|rule| rule.as_multiple_tantivy(schema, fastfields.clone()).pop())
.collect()
}
}
}
impl AsMultipleTantivyQuery for Rule {
- fn as_multiple_tantivy(&self, schema: &Schema) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
+ fn as_multiple_tantivy(
+ &self,
+ schema: &Schema,
+ fastfield_cache: Arc<FastFieldCache>,
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
let mut subqueries: Vec<_> = self
.matches
.iter()
- .map(|matching| (Occur::Must, matching.as_tantivy(schema)))
+ .map(|matching| {
+ (
+ Occur::Must,
+ matching.as_tantivy(schema, fastfield_cache.clone()),
+ )
+ })
.collect();
if subqueries.is_empty() {
@@ -107,33 +130,69 @@ impl AsMultipleTantivyQuery for Rule {
}
impl AsTantivyQuery for Matching {
- fn as_tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query> {
+ fn as_tantivy(
+ &self,
+ schema: &Schema,
+ fastfield_cache: Arc<FastFieldCache>,
+ ) -> Box<dyn tantivy::query::Query> {
match &self.location {
- MatchLocation::Site => {
- PatternQuery::new(self.pattern.clone(), TextField::Site, schema).box_clone()
- }
- MatchLocation::Url => {
- PatternQuery::new(self.pattern.clone(), TextField::Url, schema).box_clone()
- }
- MatchLocation::Domain => {
- PatternQuery::new(self.pattern.clone(), TextField::Domain, schema).box_clone()
- }
- MatchLocation::Title => {
- PatternQuery::new(self.pattern.clone(), TextField::Title, schema).box_clone()
- }
+ MatchLocation::Site => PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Site,
+ schema,
+ fastfield_cache,
+ )
+ .box_clone(),
+ MatchLocation::Url => PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Url,
+ schema,
+ fastfield_cache,
+ )
+ .box_clone(),
+ MatchLocation::Domain => PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Domain,
+ schema,
+ fastfield_cache,
+ )
+ .box_clone(),
+ MatchLocation::Title => PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Title,
+ schema,
+ fastfield_cache,
+ )
+ .box_clone(),
MatchLocation::Description => UnionQuery::from(vec![
- PatternQuery::new(self.pattern.clone(), TextField::Description, schema).box_clone(),
- PatternQuery::new(self.pattern.clone(), TextField::DmozDescription, schema)
- .box_clone(),
+ PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Description,
+ schema,
+ fastfield_cache.clone(),
+ )
+ .box_clone(),
+ PatternQuery::new(
+ self.pattern.clone(),
+ TextField::DmozDescription,
+ schema,
+ fastfield_cache,
+ )
+ .box_clone(),
])
.box_clone(),
- MatchLocation::Content => {
- PatternQuery::new(self.pattern.clone(), TextField::CleanBody, schema).box_clone()
- }
+ MatchLocation::Content => PatternQuery::new(
+ self.pattern.clone(),
+ TextField::CleanBody,
+ schema,
+ fastfield_cache,
+ )
+ .box_clone(),
MatchLocation::Schema => PatternQuery::new(
self.pattern.clone(),
TextField::FlattenedSchemaOrgJson,
schema,
+ fastfield_cache,
)
.box_clone(),
}
@@ -146,13 +205,11 @@ mod tests {
gen_temp_path,
index::Index,
ranking::centrality_store::CentralityStore,
- schema::create_schema,
searcher::{LocalSearcher, SearchQuery},
webgraph::{Node, WebgraphBuilder},
webpage::{Html, Webpage},
};
- use super::*;
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
#[test]
@@ -291,13 +348,6 @@ mod tests {
assert_eq!(res[1].url, "https://www.b.com");
}
- #[test]
- fn quickstart_as_query() {
- optics::parse(include_str!("../../../optics/testcases/quickstart.optic"))
- .unwrap()
- .as_multiple_tantivy(&create_schema());
- }
-
#[test]
fn example_optics_dont_crash() {
let mut index = Index::temporary().expect("Unable to open index");
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index b5740617..52d2fc45 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -14,9 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::sync::Arc;
+
use optics::PatternPart;
use tantivy::{
- fastfield::{Column, DynamicFastFieldReader},
fieldnorm::FieldNormReader,
postings::SegmentPostings,
query::{EmptyScorer, Explanation, Scorer},
@@ -26,16 +27,28 @@ use tantivy::{
};
use crate::{
+ fastfield_cache::FastFieldCache,
query::intersection::Intersection,
ranking::bm25::Bm25Weight,
schema::{FastField, Field, TextField, ALL_FIELDS},
};
-#[derive(Debug, Clone)]
+#[derive(Clone)]
pub struct PatternQuery {
patterns: Vec<PatternPart>,
field: tantivy::schema::Field,
raw_terms: Vec<tantivy::Term>,
+ fastfield_cache: Arc<FastFieldCache>,
+}
+
+impl std::fmt::Debug for PatternQuery {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("PatternQuery")
+ .field("patterns", &self.patterns)
+ .field("field", &self.field)
+ .field("raw_terms", &self.raw_terms)
+ .finish()
+ }
}
impl PatternQuery {
@@ -43,6 +56,7 @@ impl PatternQuery {
patterns: Vec<PatternPart>,
field: TextField,
schema: &tantivy::schema::Schema,
+ fastfield_cache: Arc<FastFieldCache>,
) -> Self {
let mut raw_terms = Vec::new();
@@ -67,6 +81,7 @@ impl PatternQuery {
patterns,
field: tv_field,
raw_terms,
+ fastfield_cache,
}
}
}
@@ -74,17 +89,21 @@ impl PatternQuery {
impl tantivy::query::Query for PatternQuery {
fn weight(
&self,
- searcher: &tantivy::Searcher,
- scoring_enabled: bool,
+ scoring: tantivy::query::EnableScoring<'_>,
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
- let bm25_weight = Bm25Weight::for_terms(searcher, &self.raw_terms)?;
+ let bm25_weight = match scoring {
+ tantivy::query::EnableScoring::Enabled(searcher) => {
+ Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)
+ }
+ tantivy::query::EnableScoring::Disabled(_) => None,
+ };
Ok(Box::new(PatternWeight {
similarity_weight: bm25_weight,
- scoring_enabled,
raw_terms: self.raw_terms.clone(),
patterns: self.patterns.clone(),
field: self.field,
+ fastfield_cache: self.fastfield_cache.clone(),
}))
}
@@ -102,16 +121,16 @@ enum SmallPatternPart {
}
struct PatternWeight {
- similarity_weight: Bm25Weight,
- scoring_enabled: bool,
+ similarity_weight: Option<Bm25Weight>,
patterns: Vec<PatternPart>,
raw_terms: Vec<tantivy::Term>,
field: tantivy::schema::Field,
+ fastfield_cache: Arc<FastFieldCache>,
}
impl PatternWeight {
fn fieldnorm_reader(&self, reader: &SegmentReader) -> tantivy::Result<FieldNormReader> {
- if self.scoring_enabled {
+ if self.similarity_weight.is_some() {
if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {
return Ok(fieldnorm_reader);
}
@@ -124,7 +143,12 @@ impl PatternWeight {
reader: &SegmentReader,
boost: Score,
) -> tantivy::Result<Option<PatternScorer>> {
- let similarity_weight = self.similarity_weight.boost_by(boost);
+ let similarity_weight = if let Some(weight) = &self.similarity_weight {
+ weight.boost_by(boost)
+ } else {
+ return Ok(None);
+ };
+
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
let mut term_postings_list = Vec::new();
@@ -149,49 +173,16 @@ impl PatternWeight {
})
.collect();
- let num_tokens_reader = match &ALL_FIELDS[self.field.field_id() as usize] {
- Field::Text(TextField::Title) => reader.fast_fields().u64(
- reader
- .schema()
- .get_field(Field::Fast(FastField::NumTitleTokens).name())
- .unwrap(),
- ),
- Field::Text(TextField::CleanBody) => reader.fast_fields().u64(
- reader
- .schema()
- .get_field(Field::Fast(FastField::NumCleanBodyTokens).name())
- .unwrap(),
- ),
- Field::Text(TextField::Url) => reader.fast_fields().u64(
- reader
- .schema()
- .get_field(Field::Fast(FastField::NumUrlTokens).name())
- .unwrap(),
- ),
- Field::Text(TextField::Domain) => reader.fast_fields().u64(
- reader
- .schema()
- .get_field(Field::Fast(FastField::NumDomainTokens).name())
- .unwrap(),
- ),
- Field::Text(TextField::Site) => reader.fast_fields().u64(
- reader
- .schema()
- .get_field(Field::Fast(FastField::NumSiteTokens).name())
- .unwrap(),
- ),
- Field::Text(TextField::Description) => reader.fast_fields().u64(
- reader
- .schema()
- .get_field(Field::Fast(FastField::NumDescriptionTokens).name())
- .unwrap(),
- ),
- Field::Text(TextField::FlattenedSchemaOrgJson) => reader.fast_fields().u64(
- reader
- .schema()
- .get_field(Field::Fast(FastField::NumFlattenedSchemaTokens).name())
- .unwrap(),
- ),
+ let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {
+ Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),
+ Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),
+ Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),
+ Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
+ Field::Text(TextField::Site) => Ok(FastField::NumSiteTokens),
+ Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
+ Field::Text(TextField::FlattenedSchemaOrgJson) => {
+ Ok(FastField::NumFlattenedSchemaTokens)
+ }
field => Err(TantivyError::InvalidArgument(format!(
"{} is not supported in pattern query",
field.name()
@@ -203,7 +194,9 @@ impl PatternWeight {
term_postings_list,
fieldnorm_reader,
small_patterns,
- num_tokens_reader,
+ reader.segment_id(),
+ num_tokens_fastfield,
+ self.fastfield_cache.clone(),
)))
}
}
@@ -244,7 +237,12 @@ impl tantivy::query::Weight for PatternWeight {
let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
let phrase_count = scorer.phrase_count();
let mut explanation = Explanation::new("Pattern Scorer", scorer.score());
- explanation.add_detail(self.similarity_weight.explain(fieldnorm_id, phrase_count));
+ explanation.add_detail(
+ self.similarity_weight
+ .as_ref()
+ .unwrap()
+ .explain(fieldnorm_id, phrase_count),
+ );
Ok(explanation)
}
}
@@ -258,7 +256,9 @@ struct PatternScorer {
left: Vec<u32>,
right: Vec<u32>,
phrase_count: u32,
- num_tokens_reader: DynamicFastFieldReader<u64>,
+ segment: tantivy::SegmentId,
+ num_tokens_field: FastField,
+ fastfield_cache: Arc<FastFieldCache>,
}
impl PatternScorer {
@@ -267,7 +267,9 @@ impl PatternScorer {
term_postings_list: Vec<SegmentPostings>,
fieldnorm_reader: FieldNormReader,
pattern: Vec<SmallPatternPart>,
- num_tokens_reader: DynamicFastFieldReader<u64>,
+ segment: tantivy::SegmentId,
+ num_tokens_field: FastField,
+ fastfield_cache: Arc<FastFieldCache>,
) -> Self {
let num_query_terms = term_postings_list.len();
@@ -280,7 +282,9 @@ impl PatternScorer {
left: Vec::with_capacity(100),
right: Vec::with_capacity(100),
phrase_count: 0,
- num_tokens_reader,
+ segment,
+ num_tokens_field,
+ fastfield_cache,
}
}
fn phrase_count(&self) -> u32 {
@@ -305,7 +309,12 @@ impl PatternScorer {
let mut current_right_term = 1;
let mut slop = 1;
- let num_tokens_doc = self.num_tokens_reader.get_val(self.doc() as u64);
+ let num_tokens_doc = self
+ .fastfield_cache
+ .get_segment(&self.segment)
+ .get_doc_cache(&self.num_tokens_field)
+ .get_u64(&self.doc())
+ .unwrap();
for (i, pattern_part) in self.pattern.iter().enumerate().skip(1) {
match pattern_part {
diff --git a/core/src/query/union.rs b/core/src/query/union.rs
index 0323f79a..2327d24d 100644
--- a/core/src/query/union.rs
+++ b/core/src/query/union.rs
@@ -173,13 +173,12 @@ impl Clone for UnionQuery {
impl tantivy::query::Query for UnionQuery {
fn weight(
&self,
- searcher: &tantivy::Searcher,
- scoring_enabled: bool,
+ enable_scoring: tantivy::query::EnableScoring,
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
let mut weights = Vec::new();
for query in &self.subqueries {
- weights.push(query.weight(searcher, scoring_enabled)?);
+ weights.push(query.weight(enable_scoring)?);
}
Ok(Box::new(UnionWeight { weights }))
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 4bdf33e3..24ee3390 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -140,7 +140,7 @@ impl LocalSearcher {
.into_iter()
.fold(Optic::default(), |acc, elem| acc.merge(elem));
- parsed_query.set_optic(&optic, &self.index.schema());
+ parsed_query.set_optic(&optic, &self.index);
let mut ranker = Ranker::new(
self.index.region_count.clone(),