web search done right
row order numerical fields used for ranking
Mikkel Denker76d7323524ac
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rsindex ee46d4f5..886ab995 100644--- a/crates/core/src/collector/top_docs.rs+++ b/crates/core/src/collector/top_docs.rs@@ -25,9 +25,9 @@ use tantivy::{};use crate::{- columnfield_reader,config::CollectorConfig,inverted_index::{DocAddress, WebpagePointer},+ numericalfield_reader,prehashed::Prehashed,ranking::initial::{InitialScoreTweaker, Score},schema::{numerical_field, NumericalFieldEnum},@@ -40,7 +40,7 @@ pub struct TopDocs {top_n: usize,offset: usize,max_docs: Option<MaxDocsConsidered>,- columnfield_reader: columnfield_reader::ColumnFieldReader,+ columnfield_reader: numericalfield_reader::NumericalFieldReader,de_rank_similar: bool,collector_config: CollectorConfig,}@@ -48,7 +48,7 @@ pub struct TopDocs {impl TopDocs {pub fn with_limit(top_n: usize,- columnfield_reader: columnfield_reader::ColumnFieldReader,+ columnfield_reader: numericalfield_reader::NumericalFieldReader,) -> Self {Self {top_n,@@ -114,7 +114,7 @@ impl TopDocs {}pub struct TopSegmentCollector {- columnfield_segment_reader: Arc<columnfield_reader::SegmentReader>,+ columnfield_segment_reader: Arc<numericalfield_reader::SegmentReader>,max_docs: Option<usize>,num_docs_taken: usize,segment_ord: SegmentOrdinal,diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rsindex ed347100..410d7cf5 100644--- a/crates/core/src/inverted_index/indexing.rs+++ b/crates/core/src/inverted_index/indexing.rs@@ -18,7 +18,7 @@ use tantivy::merge_policy::NoMergePolicy;use tantivy::{IndexWriter, SegmentMeta};-use crate::columnfield_reader::ColumnFieldReader;+use crate::numericalfield_reader::NumericalFieldReader;use crate::webpage::Webpage;use crate::Result;@@ -126,7 +126,7 @@ impl InvertedIndex {.expect("writer has not been prepared").commit()?;self.reader.reload()?;- self.columnfield_reader = ColumnFieldReader::new(&self.reader.searcher());+ self.columnfield_reader = NumericalFieldReader::new(&self.reader.searcher());Ok(())}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 4f8043a7..e810702d 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -40,8 +40,8 @@ use tantivy::tokenizer::TokenizerManager;use tantivy::{IndexReader, IndexWriter, TantivyDocument};use crate::collector::{approx_count, Hashes};-use crate::columnfield_reader::ColumnFieldReader;use crate::config::SnippetConfig;+use crate::numericalfield_reader::NumericalFieldReader;use crate::ranking::initial::Score;@@ -138,7 +138,7 @@ pub struct InvertedIndex {reader: IndexReader,schema: Arc<Schema>,snippet_config: SnippetConfig,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,}impl InvertedIndex {@@ -170,7 +170,7 @@ impl InvertedIndex {let reader: IndexReader = tantivy_index.reader_builder().try_into()?;- let columnfield_reader = ColumnFieldReader::new(&reader.searcher());+ let columnfield_reader = NumericalFieldReader::new(&reader.searcher());Ok(InvertedIndex {writer: None,@@ -183,7 +183,7 @@ impl InvertedIndex {})}- pub fn columnfield_reader(&self) -> ColumnFieldReader {+ pub fn columnfield_reader(&self) -> NumericalFieldReader {self.columnfield_reader.clone()}diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex f5ec6dfc..875e0cf0 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -26,8 +26,8 @@ use url::Url;use crate::collector::approx_count::ApproxCount;use crate::collector::{approx_count, MainCollector};-use crate::columnfield_reader::ColumnFieldReader;use crate::highlighted::HighlightedFragment;+use crate::numericalfield_reader::NumericalFieldReader;use crate::query::shortcircuit::ShortCircuitQuery;use crate::query::Query;use crate::ranking::pipeline::LocalRecallRankingWebpage;@@ -101,7 +101,7 @@ impl InvertedIndex {ctx: &Ctx,pointers: Vec<WebpagePointer>,mut computer: SignalComputer,- columnfield_reader: &ColumnFieldReader,+ columnfield_reader: &NumericalFieldReader,) -> Result<Vec<LocalRecallRankingWebpage>> {let mut top_websites = Vec::new();diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 5160a249..5d336a4d 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -47,7 +47,6 @@ pub mod bangs;mod bincode_utils;pub mod canon_index;mod collector;-mod columnfield_reader;pub mod config;pub mod crawler;pub mod distributed;@@ -68,6 +67,7 @@ mod live_index;mod metrics;mod models;pub mod naive_bayes;+mod numericalfield_reader;pub mod prehashed;pub mod query;mod rake;diff --git a/crates/core/src/columnfield_reader.rs b/crates/core/src/numericalfield_reader.rssimilarity index 62%rename from crates/core/src/columnfield_reader.rsrename to crates/core/src/numericalfield_reader.rsindex d69d6119..aecf36a6 100644--- a/crates/core/src/columnfield_reader.rs+++ b/crates/core/src/numericalfield_reader.rs@@ -20,20 +20,23 @@ use tantivy::{columnar::ColumnValues, index::SegmentId, DocId};use crate::{enum_map::EnumMap,- schema::{numerical_field::NumericalField, DataType, Field, NumericalFieldEnum},+ schema::{+ numerical_field::{NumericalField, Orientation},+ DataType, Field, NumericalFieldEnum,+ },};-#[derive(Default, Clone)]-struct InnerColumnFieldReader {+#[derive(Default)]+struct InnerNumericalFieldReader {segments: HashMap<SegmentId, Arc<SegmentReader>>,}#[derive(Default, Clone)]-pub struct ColumnFieldReader {- inner: Arc<InnerColumnFieldReader>,+pub struct NumericalFieldReader {+ inner: Arc<InnerNumericalFieldReader>,}-impl ColumnFieldReader {+impl NumericalFieldReader {pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentReader> {Arc::clone(self.inner.segments.get(segment).unwrap())}@@ -43,11 +46,12 @@ impl ColumnFieldReader {}}-impl ColumnFieldReader {+impl NumericalFieldReader {pub fn new(tv_searcher: &tantivy::Searcher) -> Self {let mut segments = HashMap::new();for reader in tv_searcher.segment_readers() {+ let mut field_ids = EnumMap::new();let columnfield_readers = reader.column_fields();let mut u64s = EnumMap::new();@@ -78,28 +82,35 @@ impl ColumnFieldReader {}}};++ field_ids.insert(+ field,+ reader.schema().get_field(field.name()).unwrap().field_id(),+ );}segments.insert(reader.segment_id(),Arc::new(SegmentReader {- field_readers: AllReaders {+ row_reader: reader.row_fields().clone(),+ columnar_readers: ColumnarReaders {u64s,bytes,bools,f64s,},+ field_ids,}),);}Self {- inner: Arc::new(InnerColumnFieldReader { segments }),+ inner: Arc::new(InnerNumericalFieldReader { segments }),}}}-struct AllReaders {+struct ColumnarReaders {u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,bools: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<bool>>>,@@ -195,48 +206,87 @@ impl From<Value> for Option<bool> {}pub struct FieldReader<'a> {- readers: &'a AllReaders,+ columnar_readers: &'a ColumnarReaders,+ row: Option<tantivy::roworder::Row<'a>>,+ field_ids: &'a EnumMap<NumericalFieldEnum, u32>,doc: DocId,}impl<'a> FieldReader<'a> {pub fn get(&self, field: NumericalFieldEnum) -> Option<Value> {- match field.data_type() {- DataType::U64 => Some(self.readers.u64s.get(field)?.get_val(self.doc).into()),-- DataType::F64 => Some(self.readers.f64s.get(field)?.get_val(self.doc).into()),-- DataType::Bool => Some(self.readers.bools.get(field)?.get_val(self.doc).into()),-- DataType::Bytes => {- let reader = self.readers.bytes.get(field)?;- let ord = reader.ords().values.get_val(self.doc);-- if ord > reader.num_terms() as u64 || reader.num_terms() == 0 {- return None;- }+ if field.orientation().contains(Orientation::ROW) {+ let field_id = self.field_ids.get(field)?;++ self.row.as_ref().and_then(|row| match field.data_type() {+ DataType::U64 => row.get_u64(field_id).map(Value::U64),+ DataType::F64 => row.get_f64(field_id).map(Value::F64),+ DataType::Bool => row.get_bool(field_id).map(Value::Bool),+ DataType::Bytes => unimplemented!("bytes fields cannot be row oriented"),+ })+ } else if field.orientation().contains(Orientation::COLUMNAR) {+ match field.data_type() {+ DataType::U64 => Some(+ self.columnar_readers+ .u64s+ .get(field)?+ .get_val(self.doc)+ .into(),+ ),++ DataType::F64 => Some(+ self.columnar_readers+ .f64s+ .get(field)?+ .get_val(self.doc)+ .into(),+ ),++ DataType::Bool => Some(+ self.columnar_readers+ .bools+ .get(field)?+ .get_val(self.doc)+ .into(),+ ),++ DataType::Bytes => {+ let reader = self.columnar_readers.bytes.get(field)?;+ let ord = reader.ords().values.get_val(self.doc);++ if ord > reader.num_terms() as u64 || reader.num_terms() == 0 {+ return None;+ }- let mut bytes = Vec::new();- reader.ord_to_bytes(ord, &mut bytes).unwrap();+ let mut bytes = Vec::new();+ reader.ord_to_bytes(ord, &mut bytes).unwrap();- if bytes.is_empty() {- None- } else {- Some(bytes.into())+ if bytes.is_empty() {+ None+ } else {+ Some(bytes.into())+ }}}+ } else {+ None}}}pub struct SegmentReader {- field_readers: AllReaders,+ columnar_readers: ColumnarReaders,+ row_reader: tantivy::roworder::readers::RowFieldReaders,+ field_ids: EnumMap<NumericalFieldEnum, u32>,}impl SegmentReader {pub fn get_field_reader(&self, doc: DocId) -> FieldReader<'_> {+ let row = self.row_reader.row_index().get_row(doc as usize);+FieldReader {- readers: &self.field_readers,+ columnar_readers: &self.columnar_readers,+ field_ids: &self.field_ids,+ row,doc,}}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 623c1216..03f1dd39 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -22,7 +22,7 @@ use tantivy::{schema::Schema,};-use crate::{columnfield_reader::ColumnFieldReader, schema::text_field, webpage::schema_org};+use crate::{numericalfield_reader::NumericalFieldReader, schema::text_field, webpage::schema_org};use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};@@ -30,7 +30,7 @@ pub trait AsTantivyQuery {fn as_tantivy(&self,schema: &Schema,- columnfield_reader: &ColumnFieldReader,+ columnfield_reader: &NumericalFieldReader,) -> Box<dyn tantivy::query::Query>;}@@ -38,7 +38,7 @@ pub trait AsMultipleTantivyQuery {fn as_multiple_tantivy(&self,schema: &Schema,- columnfield_reader: &ColumnFieldReader,+ columnfield_reader: &NumericalFieldReader,) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;}@@ -46,7 +46,7 @@ impl AsMultipleTantivyQuery for Optic {fn as_multiple_tantivy(&self,schema: &Schema,- columnfields: &ColumnFieldReader,+ columnfields: &NumericalFieldReader,) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {if self.discard_non_matching {let block = (@@ -92,7 +92,7 @@ pub trait AsSearchableRule {fn as_searchable_rule(&self,schema: &Schema,- columnfield_reader: &ColumnFieldReader,+ columnfield_reader: &NumericalFieldReader,) -> Option<(Occur, SearchableRule)>;}@@ -100,7 +100,7 @@ impl AsSearchableRule for Rule {fn as_searchable_rule(&self,schema: &Schema,- columnfield_reader: &ColumnFieldReader,+ columnfield_reader: &NumericalFieldReader,) -> Option<(Occur, SearchableRule)> {let mut subqueries: Vec<_> = self.matches@@ -167,7 +167,7 @@ impl AsTantivyQuery for Matching {fn as_tantivy(&self,schema: &Schema,- columnfield_reader: &ColumnFieldReader,+ columnfield_reader: &NumericalFieldReader,) -> Box<dyn tantivy::query::Query> {match &self.location {MatchLocation::Site => ConstQuery::new(diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rsindex 22537582..d9139b01 100644--- a/crates/core/src/query/pattern_query/mod.rs+++ b/crates/core/src/query/pattern_query/mod.rs@@ -22,7 +22,7 @@ use optics::PatternPart;use tantivy::tokenizer::Tokenizer;use crate::{- columnfield_reader::ColumnFieldReader,+ numericalfield_reader::NumericalFieldReader,schema::{text_field::TextField, Field, TextFieldEnum},};@@ -34,7 +34,7 @@ pub struct PatternQuery {can_optimize_site_domain: bool,field: tantivy::schema::Field,raw_terms: Vec<tantivy::Term>,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,}impl std::fmt::Debug for PatternQuery {@@ -52,7 +52,7 @@ impl PatternQuery {patterns: Vec<PatternPart>,field: TextFieldEnum,schema: &tantivy::schema::Schema,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,) -> Self {let field = Field::Text(field);let tv_field = schema.get_field(field.name()).unwrap();diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rsindex 3f58d9a7..edc745fb 100644--- a/crates/core/src/query/pattern_query/scorer.rs+++ b/crates/core/src/query/pattern_query/scorer.rs@@ -23,7 +23,7 @@ use tantivy::{};use crate::{- columnfield_reader::{self, ColumnFieldReader},+ numericalfield_reader::{self, NumericalFieldReader},query::intersection::Intersection,schema::NumericalFieldEnum,};@@ -126,7 +126,7 @@ impl Scorer for AllScorer {}pub struct EmptyFieldScorer {- pub segment_reader: Arc<columnfield_reader::SegmentReader>,+ pub segment_reader: Arc<numericalfield_reader::SegmentReader>,pub num_tokens_columnfield: NumericalFieldEnum,pub all_scorer: AllScorer,}@@ -211,7 +211,7 @@ pub struct NormalPatternScorer {right: Vec<u32>,phrase_count: u32,num_tokens_field: NumericalFieldEnum,- segment_reader: Arc<columnfield_reader::SegmentReader>,+ segment_reader: Arc<numericalfield_reader::SegmentReader>,}impl NormalPatternScorer {@@ -220,7 +220,7 @@ impl NormalPatternScorer {pattern: Vec<SmallPatternPart>,segment: tantivy::index::SegmentId,num_tokens_field: NumericalFieldEnum,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,) -> Self {let num_query_terms = term_postings_list.len();let segment_reader = columnfield_reader.get_segment(&segment);diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rsindex 8d1cbc05..920f967d 100644--- a/crates/core/src/query/pattern_query/weight.rs+++ b/crates/core/src/query/pattern_query/weight.rs@@ -22,7 +22,7 @@ use tantivy::{};use crate::{- columnfield_reader::ColumnFieldReader,+ numericalfield_reader::NumericalFieldReader,schema::{numerical_field,text_field::{self, TextField},@@ -114,7 +114,7 @@ pub struct PatternWeight {pub patterns: Vec<PatternPart>,pub raw_terms: Vec<tantivy::Term>,pub field: tantivy::schema::Field,- pub columnfield_reader: ColumnFieldReader,+ pub columnfield_reader: NumericalFieldReader,}impl PatternWeight {diff --git a/crates/core/src/ranking/initial.rs b/crates/core/src/ranking/initial.rsindex 2f4264cb..e0aca051 100644--- a/crates/core/src/ranking/initial.rs+++ b/crates/core/src/ranking/initial.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::columnfield_reader::ColumnFieldReader;+use crate::numericalfield_reader::NumericalFieldReader;use chrono::Utc;use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};@@ -24,7 +24,7 @@ use super::SignalComputer;pub struct InitialScoreTweaker {tv_searcher: tantivy::Searcher,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,computer: SignalComputer,}@@ -38,7 +38,7 @@ impl InitialScoreTweaker {pub fn new(tv_searcher: tantivy::Searcher,computer: SignalComputer,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,) -> Self {Self {tv_searcher,diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 702f39b8..a590511f 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -28,8 +28,8 @@ use initial::InitialScoreTweaker;use crate::{collector::{MainCollector, MaxDocsConsidered, TopDocs},- columnfield_reader::ColumnFieldReader,config::CollectorConfig,+ numericalfield_reader::NumericalFieldReader,search_ctx::Ctx,searcher::NUM_RESULTS_PER_PAGE,};@@ -41,7 +41,7 @@ pub struct Ranker {max_docs: Option<MaxDocsConsidered>,offset: Option<usize>,computer: SignalComputer,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,de_rank_similar: bool,num_results: Option<usize>,collector_config: CollectorConfig,@@ -50,7 +50,7 @@ pub struct Ranker {impl Ranker {pub fn new(computer: SignalComputer,- columnfield_reader: ColumnFieldReader,+ columnfield_reader: NumericalFieldReader,collector_config: CollectorConfig,) -> Self {Ranker {diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex d3672eb4..fb2d2fb0 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -17,11 +17,12 @@use std::sync::Arc;use crate::{- collector, columnfield_reader,+ collector,config::CollectorConfig,enum_map::EnumMap,inverted_index::WebpagePointer,models::dual_encoder::DualEncoder,+ numericalfield_reader,ranking::{bitvec_similarity, inbound_similarity,models::lambdamart::LambdaMART,@@ -154,7 +155,7 @@ impl LocalRecallRankingWebpage {/// the index to calculate bm25.pub fn new(pointer: WebpagePointer,- columnfield_reader: &columnfield_reader::SegmentReader,+ columnfield_reader: &numericalfield_reader::SegmentReader,computer: &mut SignalComputer,) -> Self {let columnfields = columnfield_reader.get_field_reader(pointer.address.doc_id);diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex 68ddc197..97928ffd 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -19,7 +19,7 @@ use crate::query::{Query, MAX_TERMS_FOR_NGRAM_LOOKUPS};use crate::ranking::bm25f::MultiBm25FWeight;use crate::schema::text_field::TextField;use crate::Result;-use crate::{columnfield_reader, enum_map::EnumMap, schema::TextFieldEnum, webpage::Webpage};+use crate::{enum_map::EnumMap, numericalfield_reader, schema::TextFieldEnum, webpage::Webpage};use std::cell::RefCell;@@ -121,7 +121,7 @@ pub struct OpticBoosts {pub struct SegmentReader {text_fields: EnumMap<TextFieldEnum, TextFieldData>,optic_boosts: OpticBoosts,- columnfield_reader: Arc<columnfield_reader::SegmentReader>,+ numericalfield_reader: Arc<numericalfield_reader::SegmentReader>,}impl SegmentReader {@@ -129,8 +129,8 @@ impl SegmentReader {&mut self.text_fields}- pub fn columnfield_reader(&self) -> &columnfield_reader::SegmentReader {- &self.columnfield_reader+ pub fn numericalfield_reader(&self) -> &numericalfield_reader::SegmentReader {+ &self.numericalfield_reader}}@@ -322,7 +322,7 @@ impl SignalComputer {&self,tv_searcher: &tantivy::Searcher,segment_reader: &tantivy::SegmentReader,- columnfield_reader: &columnfield_reader::ColumnFieldReader,+ numericalfield_reader: &numericalfield_reader::NumericalFieldReader,) -> Vec<RuleBoost> {let mut optic_rule_boosts = Vec::new();@@ -331,7 +331,7 @@ impl SignalComputer {.optic_rules.iter().filter_map(|rule| {- rule.as_searchable_rule(tv_searcher.schema(), columnfield_reader)+ rule.as_searchable_rule(tv_searcher.schema(), numericalfield_reader)}).map(|(_, rule)| RuleBoost {docset: rule@@ -355,16 +355,17 @@ impl SignalComputer {&mut self,tv_searcher: &tantivy::Searcher,segment_reader: &tantivy::SegmentReader,- columnfield_reader: &columnfield_reader::ColumnFieldReader,+ numericalfield_reader: &numericalfield_reader::NumericalFieldReader,) -> Result<()> {- let columnfield_segment_reader =- columnfield_reader.get_segment(&segment_reader.segment_id());+ let numericalfield_segment_reader =+ numericalfield_reader.get_segment(&segment_reader.segment_id());let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;- let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, columnfield_reader);+ let optic_rule_boosts =+ self.prepare_optic(tv_searcher, segment_reader, numericalfield_reader);self.segment_reader = Some(RefCell::new(SegmentReader {text_fields,- columnfield_reader: columnfield_segment_reader,+ numericalfield_reader: numericalfield_segment_reader,optic_boosts: OpticBoosts {rules: optic_rule_boosts,},diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex 84167f8e..cb66611f 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -116,10 +116,10 @@ impl Signal for HostCentrality {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_f64()).unwrap();Some(val)@@ -156,10 +156,10 @@ impl Signal for HostCentralityRank {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_rank(val as f64))@@ -196,10 +196,10 @@ impl Signal for PageCentrality {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_f64()).unwrap();Some(val)@@ -236,10 +236,10 @@ impl Signal for PageCentralityRank {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_rank(val as f64))@@ -274,10 +274,10 @@ impl Signal for IsHomepage {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_bool()).unwrap();@@ -324,10 +324,10 @@ impl Signal for FetchTimeMs {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let fetch_time_ms = columnfield_reader- .get(self.as_columnfield().unwrap())+ let fetch_time_ms = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap() as usize;@@ -375,10 +375,10 @@ impl Signal for UpdateTimestamp {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap() as usize;@@ -417,10 +417,10 @@ impl Signal for TrackerScore {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_trackers(val as f64))@@ -457,10 +457,10 @@ impl Signal for Region {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();let region = crate::webpage::Region::from_id(val);@@ -595,10 +595,10 @@ impl Signal for UrlDigits {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_digits(val as f64))@@ -642,10 +642,10 @@ impl Signal for UrlSlashes {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_slashes(val as f64))@@ -683,10 +683,10 @@ impl Signal for LinkDensity {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);- let val = columnfield_reader- .get(self.as_columnfield().unwrap())+ let val = numericalfield_reader+ .get(self.as_numericalfield().unwrap()).and_then(|v| v.as_f64()).unwrap();Some(score_link_density(val))diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rsindex 9de46936..8c375afb 100644--- a/crates/core/src/ranking/signal/prelude.rs+++ b/crates/core/src/ranking/signal/prelude.rs@@ -46,7 +46,7 @@ pub trait Signal:self.as_field().and_then(|field| field.as_text())}- fn as_columnfield(&self) -> Option<NumericalFieldEnum> {+ fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {self.as_field().and_then(|field| field.as_numerical())}}diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rsindex 87bff767..c7e62c30 100644--- a/crates/core/src/schema/numerical_field.rs+++ b/crates/core/src/schema/numerical_field.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>+use bitflags::bitflags;use enum_dispatch::enum_dispatch;use strum::{EnumDiscriminants, VariantArray};use tantivy::{@@ -31,6 +32,16 @@ use crate::{use super::IndexingOption;+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode, PartialEq, Eq, Hash)]+pub struct Orientation(u8);++bitflags! {+ impl Orientation: u8 {+ const ROW = 1 << 0;+ const COLUMNAR = 1 << 1;+ }+}+#[enum_dispatch]pub trait NumericalField:Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash@@ -66,9 +77,18 @@ pub trait NumericalField:}fn indexing_option(&self) -> IndexingOption {+ let orientation = self.orientation();match self.data_type() {DataType::U64 | DataType::Bool | DataType::F64 => {- let mut opt = NumericOptions::default().set_columnar();+ let mut opt = NumericOptions::default();++ if orientation.contains(Orientation::COLUMNAR) {+ opt = opt.set_columnar();+ }++ if orientation.contains(Orientation::ROW) {+ opt = opt.set_row_order();+ }if self.is_stored() {opt = opt.set_stored();@@ -81,7 +101,11 @@ pub trait NumericalField:IndexingOption::Integer(opt)}DataType::Bytes => {- let mut opt = BytesOptions::default().set_columnar().set_indexed();+ let mut opt = BytesOptions::default().set_indexed();++ if orientation.contains(Orientation::COLUMNAR) {+ opt = opt.set_columnar();+ }if self.is_stored() {opt = opt.set_stored();@@ -97,6 +121,10 @@ pub trait NumericalField:.get_field(self.name()).unwrap_or_else(|_| unreachable!("Unknown field: {}", self.name()))}++ fn orientation(&self) -> Orientation {+ Orientation::COLUMNAR+ }}#[enum_dispatch(NumericalField)]@@ -275,6 +303,10 @@ impl NumericalField for HostCentrality {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -304,6 +336,10 @@ impl NumericalField for HostCentralityRank {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -336,6 +372,10 @@ impl NumericalField for PageCentrality {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -364,6 +404,10 @@ impl NumericalField for PageCentralityRank {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -392,6 +436,10 @@ impl NumericalField for FetchTimeMs {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -420,6 +468,10 @@ impl NumericalField for LastUpdated {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -440,6 +492,10 @@ impl NumericalField for TrackerScore {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -481,6 +537,10 @@ impl NumericalField for Region {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -968,6 +1028,10 @@ impl NumericalField for HostNodeID {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -1052,6 +1116,10 @@ impl NumericalField for NumPathAndQuerySlashes {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -1090,6 +1158,10 @@ impl NumericalField for NumPathAndQueryDigits {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -1118,6 +1190,10 @@ impl NumericalField for LikelyHasAds {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -1146,6 +1222,10 @@ impl NumericalField for LikelyHasPaywall {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -1174,6 +1254,10 @@ impl NumericalField for LinkDensity {Ok(())}++ fn orientation(&self) -> Orientation {+ Orientation::ROW+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]diff --git a/crates/core/src/search_ctx.rs b/crates/core/src/search_ctx.rsindex d0f768ae..db45efe1 100644--- a/crates/core/src/search_ctx.rs+++ b/crates/core/src/search_ctx.rs@@ -14,10 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::columnfield_reader::ColumnFieldReader;+use crate::numericalfield_reader::NumericalFieldReader;#[derive(Clone)]pub struct Ctx {pub tv_searcher: tantivy::Searcher,- pub columnfield_reader: ColumnFieldReader,+ pub columnfield_reader: NumericalFieldReader,}diff --git a/crates/tantivy/src/roworder/mod.rs b/crates/tantivy/src/roworder/mod.rsindex a14f66a1..31ef3b24 100644--- a/crates/tantivy/src/roworder/mod.rs+++ b/crates/tantivy/src/roworder/mod.rs@@ -282,6 +282,12 @@ impl AsFieldId for crate::schema::Field {}}+impl AsFieldId for u32 {+ fn field_id(&self) -> u32 {+ *self+ }+}+impl<'a> Row<'a> {pub fn get<F: AsFieldId>(&self, field: &F) -> Option<&RowValue> {self.get_by_field_id(field.field_id())diff --git a/crates/tantivy/src/schema/bytes_options.rs b/crates/tantivy/src/schema/bytes_options.rsindex 142067be..5fb1222a 100644--- a/crates/tantivy/src/schema/bytes_options.rs+++ b/crates/tantivy/src/schema/bytes_options.rs@@ -228,7 +228,7 @@ mod tests {indexed: true,fieldnorms: true,columnar: false,- stored: false+ stored: false,});}@@ -247,7 +247,7 @@ mod tests {indexed: false,fieldnorms: false,columnar: false,- stored: false+ stored: false,});}@@ -267,7 +267,7 @@ mod tests {indexed: true,fieldnorms: false,columnar: false,- stored: false+ stored: false,});}@@ -288,7 +288,7 @@ mod tests {indexed: false,fieldnorms: true,columnar: false,- stored: false+ stored: false,});}