web search done right
upgrade tantivy
Mikkel Denkerfac468cc3c18
diff --git a/Cargo.lock b/Cargo.lockindex 13b05b96..e1eba30f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -710,16 +710,6 @@ dependencies = ["memchr",]-[[package]]-name = "ctor"-version = "0.1.22"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"-dependencies = [- "quote",- "syn",-]-[[package]]name = "cuely"version = "0.1.0"@@ -924,9 +914,12 @@ checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04"[[package]]name = "fastfield_codecs"-version = "0.2.0"-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"+version = "0.3.0"+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"dependencies = [+ "fastdivide",+ "itertools",+ "log","ownedbytes","tantivy-bitpacker","tantivy-common",@@ -1554,7 +1547,7 @@ dependencies = ["petgraph","pico-args","regex",- "regex-syntax 0.6.27",+ "regex-syntax","string_cache","term","tiny-keccak",@@ -1674,7 +1667,7 @@ dependencies = ["fnv","proc-macro2","quote",- "regex-syntax 0.6.27",+ "regex-syntax","syn",]@@ -1686,6 +1679,7 @@ checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"dependencies = ["cfg-if","generator",+ "pin-utils","scoped-tls","tracing","tracing-subscriber",@@ -1984,9 +1978,9 @@ checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"[[package]]name = "oneshot"-version = "0.1.3"+version = "0.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f3b8d98df258e762e901eb4349d66d5a5f5a7a994db8df82ff596011773be535"+checksum = "fc22d22931513428ea6cc089e942d38600e3d00976eef8c86de6b8a3aadec6eb"dependencies = ["loom",]@@ -2066,19 +2060,10 @@ version = "6.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa"-[[package]]-name = "output_vt100"-version = "0.1.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"-dependencies = [- "winapi",-]-[[package]]name = "ownedbytes"-version = "0.3.0"-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"+version = "0.4.0"+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"dependencies = ["stable_deref_trait",]@@ -2303,18 +2288,6 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"-[[package]]-name = "pretty_assertions"-version = "1.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563"-dependencies = [- "ansi_term",- "ctor",- "diff",- "output_vt100",-]-[[package]]name = "proc-macro-error"version = "1.0.4"@@ -2511,7 +2484,7 @@ checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"dependencies = ["aho-corasick","memchr",- "regex-syntax 0.6.27",+ "regex-syntax",][[package]]@@ -2520,15 +2493,9 @@ version = "0.1.10"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"dependencies = [- "regex-syntax 0.6.27",+ "regex-syntax",]-[[package]]-name = "regex-syntax"-version = "0.4.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"-[[package]]name = "regex-syntax"version = "0.6.27"@@ -3060,9 +3027,10 @@ checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"[[package]]name = "tantivy"-version = "0.18.0"-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"+version = "0.19.0"+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"dependencies = [+ "aho-corasick","arc-swap","async-trait","base64",@@ -3075,7 +3043,6 @@ dependencies = ["fail","fastdivide","fastfield_codecs",- "fnv","fs2","htmlescape","itertools",@@ -3090,10 +3057,10 @@ dependencies = ["once_cell","oneshot","ownedbytes",- "pretty_assertions","rayon","regex","rust-stemmers",+ "rustc-hash","serde","serde_json","smallvec",@@ -3111,13 +3078,13 @@ dependencies = [[[package]]name = "tantivy-bitpacker"-version = "0.2.0"-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"+version = "0.3.0"+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"[[package]]name = "tantivy-common"-version = "0.3.0"-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"+version = "0.4.0"+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"dependencies = ["byteorder","ownedbytes",@@ -3125,19 +3092,19 @@ dependencies = [[[package]]name = "tantivy-fst"-version = "0.3.0"+version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cb20cdc0d83e9184560bdde9cd60142dbb4af2e0f770e88fce45770495224205"+checksum = "fc3c506b1a8443a3a65352df6382a1fb6a7afe1a02e871cee0d25e2c3d5f3944"dependencies = ["byteorder",- "regex-syntax 0.4.2",+ "regex-syntax","utf8-ranges",][[package]]name = "tantivy-query-grammar"-version = "0.18.0"-source = "git+https://github.com/quickwit-oss/tantivy.git?rev=a451f6d60dc4cb751457d11a3cc9d038d9b63713#a451f6d60dc4cb751457d11a3cc9d038d9b63713"+version = "0.19.0"+source = "git+https://github.com/quickwit-oss/tantivy.git?rev=5d4535de83d214765c29501f0418f1ff38a607b1#5d4535de83d214765c29501f0418f1ff38a607b1"dependencies = ["combine","once_cell",diff --git a/Cargo.toml b/Cargo.tomlindex f4061bbb..90b6d062 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -18,7 +18,7 @@ rayon = "1.5.3"reqwest = { version = "0.11.10", features = ["blocking", "stream"] }serde = { version = "1.0.137", features = ["rc", "derive"] }serde_json = "1.0.81"-tantivy = { git = "https://github.com/quickwit-oss/tantivy.git", rev = "a451f6d60dc4cb751457d11a3cc9d038d9b63713" }+tantivy = { git = "https://github.com/quickwit-oss/tantivy.git", rev = "5d4535de83d214765c29501f0418f1ff38a607b1" }thiserror = "1.0.31"toml = "0.5.9"tracing = "0.1.34"diff --git a/core/src/fastfield_cache.rs b/core/src/fastfield_cache.rsindex de245512..edc4e575 100644--- a/core/src/fastfield_cache.rs+++ b/core/src/fastfield_cache.rs@@ -19,7 +19,7 @@ use std::{sync::{Arc, RwLock},};-use tantivy::{fastfield::Column, DocId, SegmentId};+use tantivy::{DocId, SegmentId};use crate::schema::{DataType, FastField, Field, ALL_FIELDS};diff --git a/core/src/query/const_query.rs b/core/src/query/const_query.rsindex 3a519b23..a9bfd9f7 100644--- a/core/src/query/const_query.rs+++ b/core/src/query/const_query.rs@@ -16,7 +16,7 @@use tantivy::{query::{Explanation, Query, Scorer, Weight},- DocSet, Score, Searcher,+ DocSet, Score,};#[derive(Debug)]@@ -43,11 +43,10 @@ impl Clone for ConstQuery {impl Query for ConstQuery {fn weight(&self,- searcher: &Searcher,- scoring_enabled: bool,+ enable_scoring: tantivy::query::EnableScoring,) -> tantivy::Result<Box<dyn Weight>> {Ok(Box::new(ConstWeight {- subweight: self.subquery.weight(searcher, scoring_enabled)?,+ subweight: self.subquery.weight(enable_scoring)?,score: self.score,}))}diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 822cae6f..389b8286 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ index::Index,ranking::SignalAggregator,schema::{Field, TextField},searcher::NUM_RESULTS_PER_PAGE,@@ -159,10 +160,12 @@ impl Query {})}- pub fn set_optic(&mut self, optic: &Optic, schema: &Schema) {+ pub fn set_optic(&mut self, optic: &Optic, index: &Index) {let mut subqueries = vec![(Occur::Must, self.tantivy_query.box_clone())];- subqueries.append(&mut optic.as_multiple_tantivy(schema));+ subqueries.append(+ &mut optic.as_multiple_tantivy(&index.schema(), index.inverted_index.fastfield_cache()),+ );self.tantivy_query = Box::new(BooleanQuery::new(subqueries))}@@ -191,10 +194,9 @@ impl Query {impl tantivy::query::Query for Query {fn weight(&self,- searcher: &tantivy::Searcher,- scoring_enabled: bool,+ enable_scoring: tantivy::query::EnableScoring,) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {- self.tantivy_query.weight(searcher, scoring_enabled)+ self.tantivy_query.weight(enable_scoring)}fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a tantivy::Term, bool)) {diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 68d02efe..67b512fb 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -14,6 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::sync::Arc;+use itertools::Itertools;use optics::{Action, MatchLocation, Matching, Optic, Rule};use tantivy::{@@ -21,20 +23,32 @@ use tantivy::{schema::Schema,};-use crate::{ranking::optics::SCALE, schema::TextField};+use crate::{fastfield_cache::FastFieldCache, ranking::optics::SCALE, schema::TextField};use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};pub trait AsTantivyQuery {- fn as_tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query>;+ fn as_tantivy(+ &self,+ schema: &Schema,+ fastfield_cache: Arc<FastFieldCache>,+ ) -> Box<dyn tantivy::query::Query>;}pub trait AsMultipleTantivyQuery {- fn as_multiple_tantivy(&self, schema: &Schema) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;+ fn as_multiple_tantivy(+ &self,+ schema: &Schema,+ fastfield_cache: Arc<FastFieldCache>,+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;}impl AsMultipleTantivyQuery for Optic {- fn as_multiple_tantivy(&self, schema: &Schema) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {+ fn as_multiple_tantivy(+ &self,+ schema: &Schema,+ fastfields: Arc<FastFieldCache>,+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {if self.discard_non_matching {vec![(Occur::Must,@@ -43,7 +57,7 @@ impl AsMultipleTantivyQuery for Optic {.iter().chain(self.site_rankings.rules().iter()).filter_map(|rule| {- let queries = rule.as_multiple_tantivy(schema);+ let queries = rule.as_multiple_tantivy(schema, fastfields.clone());if queries.is_empty() {None@@ -60,18 +74,27 @@ impl AsMultipleTantivyQuery for Optic {self.rules.iter().chain(self.site_rankings.rules().iter())- .filter_map(|rule| rule.as_multiple_tantivy(schema).pop())+ .filter_map(|rule| rule.as_multiple_tantivy(schema, fastfields.clone()).pop()).collect()}}}impl AsMultipleTantivyQuery for Rule {- fn as_multiple_tantivy(&self, schema: &Schema) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {+ fn as_multiple_tantivy(+ &self,+ schema: &Schema,+ fastfield_cache: Arc<FastFieldCache>,+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {let mut subqueries: Vec<_> = self.matches.iter()- .map(|matching| (Occur::Must, matching.as_tantivy(schema)))+ .map(|matching| {+ (+ Occur::Must,+ matching.as_tantivy(schema, fastfield_cache.clone()),+ )+ }).collect();if subqueries.is_empty() {@@ -107,33 +130,69 @@ impl AsMultipleTantivyQuery for Rule {}impl AsTantivyQuery for Matching {- fn as_tantivy(&self, schema: &Schema) -> Box<dyn tantivy::query::Query> {+ fn as_tantivy(+ &self,+ schema: &Schema,+ fastfield_cache: Arc<FastFieldCache>,+ ) -> Box<dyn tantivy::query::Query> {match &self.location {- MatchLocation::Site => {- PatternQuery::new(self.pattern.clone(), TextField::Site, schema).box_clone()- }- MatchLocation::Url => {- PatternQuery::new(self.pattern.clone(), TextField::Url, schema).box_clone()- }- MatchLocation::Domain => {- PatternQuery::new(self.pattern.clone(), TextField::Domain, schema).box_clone()- }- MatchLocation::Title => {- PatternQuery::new(self.pattern.clone(), TextField::Title, schema).box_clone()- }+ MatchLocation::Site => PatternQuery::new(+ self.pattern.clone(),+ TextField::Site,+ schema,+ fastfield_cache,+ )+ .box_clone(),+ MatchLocation::Url => PatternQuery::new(+ self.pattern.clone(),+ TextField::Url,+ schema,+ fastfield_cache,+ )+ .box_clone(),+ MatchLocation::Domain => PatternQuery::new(+ self.pattern.clone(),+ TextField::Domain,+ schema,+ fastfield_cache,+ )+ .box_clone(),+ MatchLocation::Title => PatternQuery::new(+ self.pattern.clone(),+ TextField::Title,+ schema,+ fastfield_cache,+ )+ .box_clone(),MatchLocation::Description => UnionQuery::from(vec![- PatternQuery::new(self.pattern.clone(), TextField::Description, schema).box_clone(),- PatternQuery::new(self.pattern.clone(), TextField::DmozDescription, schema)- .box_clone(),+ PatternQuery::new(+ self.pattern.clone(),+ TextField::Description,+ schema,+ fastfield_cache.clone(),+ )+ .box_clone(),+ PatternQuery::new(+ self.pattern.clone(),+ TextField::DmozDescription,+ schema,+ fastfield_cache,+ )+ .box_clone(),]).box_clone(),- MatchLocation::Content => {- PatternQuery::new(self.pattern.clone(), TextField::CleanBody, schema).box_clone()- }+ MatchLocation::Content => PatternQuery::new(+ self.pattern.clone(),+ TextField::CleanBody,+ schema,+ fastfield_cache,+ )+ .box_clone(),MatchLocation::Schema => PatternQuery::new(self.pattern.clone(),TextField::FlattenedSchemaOrgJson,schema,+ fastfield_cache,).box_clone(),}@@ -146,13 +205,11 @@ mod tests {gen_temp_path,index::Index,ranking::centrality_store::CentralityStore,- schema::create_schema,searcher::{LocalSearcher, SearchQuery},webgraph::{Node, WebgraphBuilder},webpage::{Html, Webpage},};- use super::*;const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";#[test]@@ -291,13 +348,6 @@ mod tests {assert_eq!(res[1].url, "https://www.b.com");}- #[test]- fn quickstart_as_query() {- optics::parse(include_str!("../../../optics/testcases/quickstart.optic"))- .unwrap()- .as_multiple_tantivy(&create_schema());- }-#[test]fn example_optics_dont_crash() {let mut index = Index::temporary().expect("Unable to open index");diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex b5740617..52d2fc45 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -14,9 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::sync::Arc;+use optics::PatternPart;use tantivy::{- fastfield::{Column, DynamicFastFieldReader},fieldnorm::FieldNormReader,postings::SegmentPostings,query::{EmptyScorer, Explanation, Scorer},@@ -26,16 +27,28 @@ use tantivy::{};use crate::{+ fastfield_cache::FastFieldCache,query::intersection::Intersection,ranking::bm25::Bm25Weight,schema::{FastField, Field, TextField, ALL_FIELDS},};-#[derive(Debug, Clone)]+#[derive(Clone)]pub struct PatternQuery {patterns: Vec<PatternPart>,field: tantivy::schema::Field,raw_terms: Vec<tantivy::Term>,+ fastfield_cache: Arc<FastFieldCache>,+}++impl std::fmt::Debug for PatternQuery {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("PatternQuery")+ .field("patterns", &self.patterns)+ .field("field", &self.field)+ .field("raw_terms", &self.raw_terms)+ .finish()+ }}impl PatternQuery {@@ -43,6 +56,7 @@ impl PatternQuery {patterns: Vec<PatternPart>,field: TextField,schema: &tantivy::schema::Schema,+ fastfield_cache: Arc<FastFieldCache>,) -> Self {let mut raw_terms = Vec::new();@@ -67,6 +81,7 @@ impl PatternQuery {patterns,field: tv_field,raw_terms,+ fastfield_cache,}}}@@ -74,17 +89,21 @@ impl PatternQuery {impl tantivy::query::Query for PatternQuery {fn weight(&self,- searcher: &tantivy::Searcher,- scoring_enabled: bool,+ scoring: tantivy::query::EnableScoring<'_>,) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {- let bm25_weight = Bm25Weight::for_terms(searcher, &self.raw_terms)?;+ let bm25_weight = match scoring {+ tantivy::query::EnableScoring::Enabled(searcher) => {+ Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)+ }+ tantivy::query::EnableScoring::Disabled(_) => None,+ };Ok(Box::new(PatternWeight {similarity_weight: bm25_weight,- scoring_enabled,raw_terms: self.raw_terms.clone(),patterns: self.patterns.clone(),field: self.field,+ fastfield_cache: self.fastfield_cache.clone(),}))}@@ -102,16 +121,16 @@ enum SmallPatternPart {}struct PatternWeight {- similarity_weight: Bm25Weight,- scoring_enabled: bool,+ similarity_weight: Option<Bm25Weight>,patterns: Vec<PatternPart>,raw_terms: Vec<tantivy::Term>,field: tantivy::schema::Field,+ fastfield_cache: Arc<FastFieldCache>,}impl PatternWeight {fn fieldnorm_reader(&self, reader: &SegmentReader) -> tantivy::Result<FieldNormReader> {- if self.scoring_enabled {+ if self.similarity_weight.is_some() {if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {return Ok(fieldnorm_reader);}@@ -124,7 +143,12 @@ impl PatternWeight {reader: &SegmentReader,boost: Score,) -> tantivy::Result<Option<PatternScorer>> {- let similarity_weight = self.similarity_weight.boost_by(boost);+ let similarity_weight = if let Some(weight) = &self.similarity_weight {+ weight.boost_by(boost)+ } else {+ return Ok(None);+ };+let fieldnorm_reader = self.fieldnorm_reader(reader)?;let mut term_postings_list = Vec::new();@@ -149,49 +173,16 @@ impl PatternWeight {}).collect();- let num_tokens_reader = match &ALL_FIELDS[self.field.field_id() as usize] {- Field::Text(TextField::Title) => reader.fast_fields().u64(- reader- .schema()- .get_field(Field::Fast(FastField::NumTitleTokens).name())- .unwrap(),- ),- Field::Text(TextField::CleanBody) => reader.fast_fields().u64(- reader- .schema()- .get_field(Field::Fast(FastField::NumCleanBodyTokens).name())- .unwrap(),- ),- Field::Text(TextField::Url) => reader.fast_fields().u64(- reader- .schema()- .get_field(Field::Fast(FastField::NumUrlTokens).name())- .unwrap(),- ),- Field::Text(TextField::Domain) => reader.fast_fields().u64(- reader- .schema()- .get_field(Field::Fast(FastField::NumDomainTokens).name())- .unwrap(),- ),- Field::Text(TextField::Site) => reader.fast_fields().u64(- reader- .schema()- .get_field(Field::Fast(FastField::NumSiteTokens).name())- .unwrap(),- ),- Field::Text(TextField::Description) => reader.fast_fields().u64(- reader- .schema()- .get_field(Field::Fast(FastField::NumDescriptionTokens).name())- .unwrap(),- ),- Field::Text(TextField::FlattenedSchemaOrgJson) => reader.fast_fields().u64(- reader- .schema()- .get_field(Field::Fast(FastField::NumFlattenedSchemaTokens).name())- .unwrap(),- ),+ let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {+ Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),+ Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),+ Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),+ Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),+ Field::Text(TextField::Site) => Ok(FastField::NumSiteTokens),+ Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),+ Field::Text(TextField::FlattenedSchemaOrgJson) => {+ Ok(FastField::NumFlattenedSchemaTokens)+ }field => Err(TantivyError::InvalidArgument(format!("{} is not supported in pattern query",field.name()@@ -203,7 +194,9 @@ impl PatternWeight {term_postings_list,fieldnorm_reader,small_patterns,- num_tokens_reader,+ reader.segment_id(),+ num_tokens_fastfield,+ self.fastfield_cache.clone(),)))}}@@ -244,7 +237,12 @@ impl tantivy::query::Weight for PatternWeight {let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);let phrase_count = scorer.phrase_count();let mut explanation = Explanation::new("Pattern Scorer", scorer.score());- explanation.add_detail(self.similarity_weight.explain(fieldnorm_id, phrase_count));+ explanation.add_detail(+ self.similarity_weight+ .as_ref()+ .unwrap()+ .explain(fieldnorm_id, phrase_count),+ );Ok(explanation)}}@@ -258,7 +256,9 @@ struct PatternScorer {left: Vec<u32>,right: Vec<u32>,phrase_count: u32,- num_tokens_reader: DynamicFastFieldReader<u64>,+ segment: tantivy::SegmentId,+ num_tokens_field: FastField,+ fastfield_cache: Arc<FastFieldCache>,}impl PatternScorer {@@ -267,7 +267,9 @@ impl PatternScorer {term_postings_list: Vec<SegmentPostings>,fieldnorm_reader: FieldNormReader,pattern: Vec<SmallPatternPart>,- num_tokens_reader: DynamicFastFieldReader<u64>,+ segment: tantivy::SegmentId,+ num_tokens_field: FastField,+ fastfield_cache: Arc<FastFieldCache>,) -> Self {let num_query_terms = term_postings_list.len();@@ -280,7 +282,9 @@ impl PatternScorer {left: Vec::with_capacity(100),right: Vec::with_capacity(100),phrase_count: 0,- num_tokens_reader,+ segment,+ num_tokens_field,+ fastfield_cache,}}fn phrase_count(&self) -> u32 {@@ -305,7 +309,12 @@ impl PatternScorer {let mut current_right_term = 1;let mut slop = 1;- let num_tokens_doc = self.num_tokens_reader.get_val(self.doc() as u64);+ let num_tokens_doc = self+ .fastfield_cache+ .get_segment(&self.segment)+ .get_doc_cache(&self.num_tokens_field)+ .get_u64(&self.doc())+ .unwrap();for (i, pattern_part) in self.pattern.iter().enumerate().skip(1) {match pattern_part {diff --git a/core/src/query/union.rs b/core/src/query/union.rsindex 0323f79a..2327d24d 100644--- a/core/src/query/union.rs+++ b/core/src/query/union.rs@@ -173,13 +173,12 @@ impl Clone for UnionQuery {impl tantivy::query::Query for UnionQuery {fn weight(&self,- searcher: &tantivy::Searcher,- scoring_enabled: bool,+ enable_scoring: tantivy::query::EnableScoring,) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {let mut weights = Vec::new();for query in &self.subqueries {- weights.push(query.weight(searcher, scoring_enabled)?);+ weights.push(query.weight(enable_scoring)?);}Ok(Box::new(UnionWeight { weights }))diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 4bdf33e3..24ee3390 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -140,7 +140,7 @@ impl LocalSearcher {.into_iter().fold(Optic::default(), |acc, elem| acc.merge(elem));- parsed_query.set_optic(&optic, &self.index.schema());+ parsed_query.set_optic(&optic, &self.index);let mut ranker = Ranker::new(self.index.region_count.clone(),