Skip to content

web search done right

row order numerical fields used for ranking

Mikkel Denker76d7323524ac
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rs
index ee46d4f5..886ab995 100644
--- a/crates/core/src/collector/top_docs.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -25,9 +25,9 @@ use tantivy::{
};
use crate::{
- columnfield_reader,
config::CollectorConfig,
inverted_index::{DocAddress, WebpagePointer},
+ numericalfield_reader,
prehashed::Prehashed,
ranking::initial::{InitialScoreTweaker, Score},
schema::{numerical_field, NumericalFieldEnum},
@@ -40,7 +40,7 @@ pub struct TopDocs {
top_n: usize,
offset: usize,
max_docs: Option<MaxDocsConsidered>,
- columnfield_reader: columnfield_reader::ColumnFieldReader,
+ columnfield_reader: numericalfield_reader::NumericalFieldReader,
de_rank_similar: bool,
collector_config: CollectorConfig,
}
@@ -48,7 +48,7 @@ pub struct TopDocs {
impl TopDocs {
pub fn with_limit(
top_n: usize,
- columnfield_reader: columnfield_reader::ColumnFieldReader,
+ columnfield_reader: numericalfield_reader::NumericalFieldReader,
) -> Self {
Self {
top_n,
@@ -114,7 +114,7 @@ impl TopDocs {
}
pub struct TopSegmentCollector {
- columnfield_segment_reader: Arc<columnfield_reader::SegmentReader>,
+ columnfield_segment_reader: Arc<numericalfield_reader::SegmentReader>,
max_docs: Option<usize>,
num_docs_taken: usize,
segment_ord: SegmentOrdinal,
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index ed347100..410d7cf5 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -18,7 +18,7 @@ use tantivy::merge_policy::NoMergePolicy;
use tantivy::{IndexWriter, SegmentMeta};
-use crate::columnfield_reader::ColumnFieldReader;
+use crate::numericalfield_reader::NumericalFieldReader;
use crate::webpage::Webpage;
use crate::Result;
@@ -126,7 +126,7 @@ impl InvertedIndex {
.expect("writer has not been prepared")
.commit()?;
self.reader.reload()?;
- self.columnfield_reader = ColumnFieldReader::new(&self.reader.searcher());
+ self.columnfield_reader = NumericalFieldReader::new(&self.reader.searcher());
Ok(())
}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 4f8043a7..e810702d 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -40,8 +40,8 @@ use tantivy::tokenizer::TokenizerManager;
use tantivy::{IndexReader, IndexWriter, TantivyDocument};
use crate::collector::{approx_count, Hashes};
-use crate::columnfield_reader::ColumnFieldReader;
use crate::config::SnippetConfig;
+use crate::numericalfield_reader::NumericalFieldReader;
use crate::ranking::initial::Score;
@@ -138,7 +138,7 @@ pub struct InvertedIndex {
reader: IndexReader,
schema: Arc<Schema>,
snippet_config: SnippetConfig,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
}
impl InvertedIndex {
@@ -170,7 +170,7 @@ impl InvertedIndex {
let reader: IndexReader = tantivy_index.reader_builder().try_into()?;
- let columnfield_reader = ColumnFieldReader::new(&reader.searcher());
+ let columnfield_reader = NumericalFieldReader::new(&reader.searcher());
Ok(InvertedIndex {
writer: None,
@@ -183,7 +183,7 @@ impl InvertedIndex {
})
}
- pub fn columnfield_reader(&self) -> ColumnFieldReader {
+ pub fn columnfield_reader(&self) -> NumericalFieldReader {
self.columnfield_reader.clone()
}
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index f5ec6dfc..875e0cf0 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -26,8 +26,8 @@ use url::Url;
use crate::collector::approx_count::ApproxCount;
use crate::collector::{approx_count, MainCollector};
-use crate::columnfield_reader::ColumnFieldReader;
use crate::highlighted::HighlightedFragment;
+use crate::numericalfield_reader::NumericalFieldReader;
use crate::query::shortcircuit::ShortCircuitQuery;
use crate::query::Query;
use crate::ranking::pipeline::LocalRecallRankingWebpage;
@@ -101,7 +101,7 @@ impl InvertedIndex {
ctx: &Ctx,
pointers: Vec<WebpagePointer>,
mut computer: SignalComputer,
- columnfield_reader: &ColumnFieldReader,
+ columnfield_reader: &NumericalFieldReader,
) -> Result<Vec<LocalRecallRankingWebpage>> {
let mut top_websites = Vec::new();
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 5160a249..5d336a4d 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -47,7 +47,6 @@ pub mod bangs;
mod bincode_utils;
pub mod canon_index;
mod collector;
-mod columnfield_reader;
pub mod config;
pub mod crawler;
pub mod distributed;
@@ -68,6 +67,7 @@ mod live_index;
mod metrics;
mod models;
pub mod naive_bayes;
+mod numericalfield_reader;
pub mod prehashed;
pub mod query;
mod rake;
diff --git a/crates/core/src/columnfield_reader.rs b/crates/core/src/numericalfield_reader.rs
similarity index 62%
rename from crates/core/src/columnfield_reader.rs
rename to crates/core/src/numericalfield_reader.rs
index d69d6119..aecf36a6 100644
--- a/crates/core/src/columnfield_reader.rs
+++ b/crates/core/src/numericalfield_reader.rs
@@ -20,20 +20,23 @@ use tantivy::{columnar::ColumnValues, index::SegmentId, DocId};
use crate::{
enum_map::EnumMap,
- schema::{numerical_field::NumericalField, DataType, Field, NumericalFieldEnum},
+ schema::{
+ numerical_field::{NumericalField, Orientation},
+ DataType, Field, NumericalFieldEnum,
+ },
};
-#[derive(Default, Clone)]
-struct InnerColumnFieldReader {
+#[derive(Default)]
+struct InnerNumericalFieldReader {
segments: HashMap<SegmentId, Arc<SegmentReader>>,
}
#[derive(Default, Clone)]
-pub struct ColumnFieldReader {
- inner: Arc<InnerColumnFieldReader>,
+pub struct NumericalFieldReader {
+ inner: Arc<InnerNumericalFieldReader>,
}
-impl ColumnFieldReader {
+impl NumericalFieldReader {
pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentReader> {
Arc::clone(self.inner.segments.get(segment).unwrap())
}
@@ -43,11 +46,12 @@ impl ColumnFieldReader {
}
}
-impl ColumnFieldReader {
+impl NumericalFieldReader {
pub fn new(tv_searcher: &tantivy::Searcher) -> Self {
let mut segments = HashMap::new();
for reader in tv_searcher.segment_readers() {
+ let mut field_ids = EnumMap::new();
let columnfield_readers = reader.column_fields();
let mut u64s = EnumMap::new();
@@ -78,28 +82,35 @@ impl ColumnFieldReader {
}
}
};
+
+ field_ids.insert(
+ field,
+ reader.schema().get_field(field.name()).unwrap().field_id(),
+ );
}
segments.insert(
reader.segment_id(),
Arc::new(SegmentReader {
- field_readers: AllReaders {
+ row_reader: reader.row_fields().clone(),
+ columnar_readers: ColumnarReaders {
u64s,
bytes,
bools,
f64s,
},
+ field_ids,
}),
);
}
Self {
- inner: Arc::new(InnerColumnFieldReader { segments }),
+ inner: Arc::new(InnerNumericalFieldReader { segments }),
}
}
}
-struct AllReaders {
+struct ColumnarReaders {
u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,
f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,
bools: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<bool>>>,
@@ -195,48 +206,87 @@ impl From<Value> for Option<bool> {
}
pub struct FieldReader<'a> {
- readers: &'a AllReaders,
+ columnar_readers: &'a ColumnarReaders,
+ row: Option<tantivy::roworder::Row<'a>>,
+ field_ids: &'a EnumMap<NumericalFieldEnum, u32>,
doc: DocId,
}
impl<'a> FieldReader<'a> {
pub fn get(&self, field: NumericalFieldEnum) -> Option<Value> {
- match field.data_type() {
- DataType::U64 => Some(self.readers.u64s.get(field)?.get_val(self.doc).into()),
-
- DataType::F64 => Some(self.readers.f64s.get(field)?.get_val(self.doc).into()),
-
- DataType::Bool => Some(self.readers.bools.get(field)?.get_val(self.doc).into()),
-
- DataType::Bytes => {
- let reader = self.readers.bytes.get(field)?;
- let ord = reader.ords().values.get_val(self.doc);
-
- if ord > reader.num_terms() as u64 || reader.num_terms() == 0 {
- return None;
- }
+ if field.orientation().contains(Orientation::ROW) {
+ let field_id = self.field_ids.get(field)?;
+
+ self.row.as_ref().and_then(|row| match field.data_type() {
+ DataType::U64 => row.get_u64(field_id).map(Value::U64),
+ DataType::F64 => row.get_f64(field_id).map(Value::F64),
+ DataType::Bool => row.get_bool(field_id).map(Value::Bool),
+ DataType::Bytes => unimplemented!("bytes fields cannot be row oriented"),
+ })
+ } else if field.orientation().contains(Orientation::COLUMNAR) {
+ match field.data_type() {
+ DataType::U64 => Some(
+ self.columnar_readers
+ .u64s
+ .get(field)?
+ .get_val(self.doc)
+ .into(),
+ ),
+
+ DataType::F64 => Some(
+ self.columnar_readers
+ .f64s
+ .get(field)?
+ .get_val(self.doc)
+ .into(),
+ ),
+
+ DataType::Bool => Some(
+ self.columnar_readers
+ .bools
+ .get(field)?
+ .get_val(self.doc)
+ .into(),
+ ),
+
+ DataType::Bytes => {
+ let reader = self.columnar_readers.bytes.get(field)?;
+ let ord = reader.ords().values.get_val(self.doc);
+
+ if ord > reader.num_terms() as u64 || reader.num_terms() == 0 {
+ return None;
+ }
- let mut bytes = Vec::new();
- reader.ord_to_bytes(ord, &mut bytes).unwrap();
+ let mut bytes = Vec::new();
+ reader.ord_to_bytes(ord, &mut bytes).unwrap();
- if bytes.is_empty() {
- None
- } else {
- Some(bytes.into())
+ if bytes.is_empty() {
+ None
+ } else {
+ Some(bytes.into())
+ }
}
}
+ } else {
+ None
}
}
}
pub struct SegmentReader {
- field_readers: AllReaders,
+ columnar_readers: ColumnarReaders,
+ row_reader: tantivy::roworder::readers::RowFieldReaders,
+ field_ids: EnumMap<NumericalFieldEnum, u32>,
}
impl SegmentReader {
pub fn get_field_reader(&self, doc: DocId) -> FieldReader<'_> {
+ let row = self.row_reader.row_index().get_row(doc as usize);
+
FieldReader {
- readers: &self.field_readers,
+ columnar_readers: &self.columnar_readers,
+ field_ids: &self.field_ids,
+ row,
doc,
}
}
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 623c1216..03f1dd39 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -22,7 +22,7 @@ use tantivy::{
schema::Schema,
};
-use crate::{columnfield_reader::ColumnFieldReader, schema::text_field, webpage::schema_org};
+use crate::{numericalfield_reader::NumericalFieldReader, schema::text_field, webpage::schema_org};
use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};
@@ -30,7 +30,7 @@ pub trait AsTantivyQuery {
fn as_tantivy(
&self,
schema: &Schema,
- columnfield_reader: &ColumnFieldReader,
+ columnfield_reader: &NumericalFieldReader,
) -> Box<dyn tantivy::query::Query>;
}
@@ -38,7 +38,7 @@ pub trait AsMultipleTantivyQuery {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- columnfield_reader: &ColumnFieldReader,
+ columnfield_reader: &NumericalFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;
}
@@ -46,7 +46,7 @@ impl AsMultipleTantivyQuery for Optic {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- columnfields: &ColumnFieldReader,
+ columnfields: &NumericalFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
if self.discard_non_matching {
let block = (
@@ -92,7 +92,7 @@ pub trait AsSearchableRule {
fn as_searchable_rule(
&self,
schema: &Schema,
- columnfield_reader: &ColumnFieldReader,
+ columnfield_reader: &NumericalFieldReader,
) -> Option<(Occur, SearchableRule)>;
}
@@ -100,7 +100,7 @@ impl AsSearchableRule for Rule {
fn as_searchable_rule(
&self,
schema: &Schema,
- columnfield_reader: &ColumnFieldReader,
+ columnfield_reader: &NumericalFieldReader,
) -> Option<(Occur, SearchableRule)> {
let mut subqueries: Vec<_> = self
.matches
@@ -167,7 +167,7 @@ impl AsTantivyQuery for Matching {
fn as_tantivy(
&self,
schema: &Schema,
- columnfield_reader: &ColumnFieldReader,
+ columnfield_reader: &NumericalFieldReader,
) -> Box<dyn tantivy::query::Query> {
match &self.location {
MatchLocation::Site => ConstQuery::new(
diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rs
index 22537582..d9139b01 100644
--- a/crates/core/src/query/pattern_query/mod.rs
+++ b/crates/core/src/query/pattern_query/mod.rs
@@ -22,7 +22,7 @@ use optics::PatternPart;
use tantivy::tokenizer::Tokenizer;
use crate::{
- columnfield_reader::ColumnFieldReader,
+ numericalfield_reader::NumericalFieldReader,
schema::{text_field::TextField, Field, TextFieldEnum},
};
@@ -34,7 +34,7 @@ pub struct PatternQuery {
can_optimize_site_domain: bool,
field: tantivy::schema::Field,
raw_terms: Vec<tantivy::Term>,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
}
impl std::fmt::Debug for PatternQuery {
@@ -52,7 +52,7 @@ impl PatternQuery {
patterns: Vec<PatternPart>,
field: TextFieldEnum,
schema: &tantivy::schema::Schema,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
) -> Self {
let field = Field::Text(field);
let tv_field = schema.get_field(field.name()).unwrap();
diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rs
index 3f58d9a7..edc745fb 100644
--- a/crates/core/src/query/pattern_query/scorer.rs
+++ b/crates/core/src/query/pattern_query/scorer.rs
@@ -23,7 +23,7 @@ use tantivy::{
};
use crate::{
- columnfield_reader::{self, ColumnFieldReader},
+ numericalfield_reader::{self, NumericalFieldReader},
query::intersection::Intersection,
schema::NumericalFieldEnum,
};
@@ -126,7 +126,7 @@ impl Scorer for AllScorer {
}
pub struct EmptyFieldScorer {
- pub segment_reader: Arc<columnfield_reader::SegmentReader>,
+ pub segment_reader: Arc<numericalfield_reader::SegmentReader>,
pub num_tokens_columnfield: NumericalFieldEnum,
pub all_scorer: AllScorer,
}
@@ -211,7 +211,7 @@ pub struct NormalPatternScorer {
right: Vec<u32>,
phrase_count: u32,
num_tokens_field: NumericalFieldEnum,
- segment_reader: Arc<columnfield_reader::SegmentReader>,
+ segment_reader: Arc<numericalfield_reader::SegmentReader>,
}
impl NormalPatternScorer {
@@ -220,7 +220,7 @@ impl NormalPatternScorer {
pattern: Vec<SmallPatternPart>,
segment: tantivy::index::SegmentId,
num_tokens_field: NumericalFieldEnum,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
) -> Self {
let num_query_terms = term_postings_list.len();
let segment_reader = columnfield_reader.get_segment(&segment);
diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rs
index 8d1cbc05..920f967d 100644
--- a/crates/core/src/query/pattern_query/weight.rs
+++ b/crates/core/src/query/pattern_query/weight.rs
@@ -22,7 +22,7 @@ use tantivy::{
};
use crate::{
- columnfield_reader::ColumnFieldReader,
+ numericalfield_reader::NumericalFieldReader,
schema::{
numerical_field,
text_field::{self, TextField},
@@ -114,7 +114,7 @@ pub struct PatternWeight {
pub patterns: Vec<PatternPart>,
pub raw_terms: Vec<tantivy::Term>,
pub field: tantivy::schema::Field,
- pub columnfield_reader: ColumnFieldReader,
+ pub columnfield_reader: NumericalFieldReader,
}
impl PatternWeight {
diff --git a/crates/core/src/ranking/initial.rs b/crates/core/src/ranking/initial.rs
index 2f4264cb..e0aca051 100644
--- a/crates/core/src/ranking/initial.rs
+++ b/crates/core/src/ranking/initial.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::columnfield_reader::ColumnFieldReader;
+use crate::numericalfield_reader::NumericalFieldReader;
use chrono::Utc;
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
@@ -24,7 +24,7 @@ use super::SignalComputer;
pub struct InitialScoreTweaker {
tv_searcher: tantivy::Searcher,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
computer: SignalComputer,
}
@@ -38,7 +38,7 @@ impl InitialScoreTweaker {
pub fn new(
tv_searcher: tantivy::Searcher,
computer: SignalComputer,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
) -> Self {
Self {
tv_searcher,
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index 702f39b8..a590511f 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -28,8 +28,8 @@ use initial::InitialScoreTweaker;
use crate::{
collector::{MainCollector, MaxDocsConsidered, TopDocs},
- columnfield_reader::ColumnFieldReader,
config::CollectorConfig,
+ numericalfield_reader::NumericalFieldReader,
search_ctx::Ctx,
searcher::NUM_RESULTS_PER_PAGE,
};
@@ -41,7 +41,7 @@ pub struct Ranker {
max_docs: Option<MaxDocsConsidered>,
offset: Option<usize>,
computer: SignalComputer,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
de_rank_similar: bool,
num_results: Option<usize>,
collector_config: CollectorConfig,
@@ -50,7 +50,7 @@ pub struct Ranker {
impl Ranker {
pub fn new(
computer: SignalComputer,
- columnfield_reader: ColumnFieldReader,
+ columnfield_reader: NumericalFieldReader,
collector_config: CollectorConfig,
) -> Self {
Ranker {
diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rs
index d3672eb4..fb2d2fb0 100644
--- a/crates/core/src/ranking/pipeline/stages/recall.rs
+++ b/crates/core/src/ranking/pipeline/stages/recall.rs
@@ -17,11 +17,12 @@
use std::sync::Arc;
use crate::{
- collector, columnfield_reader,
+ collector,
config::CollectorConfig,
enum_map::EnumMap,
inverted_index::WebpagePointer,
models::dual_encoder::DualEncoder,
+ numericalfield_reader,
ranking::{
bitvec_similarity, inbound_similarity,
models::lambdamart::LambdaMART,
@@ -154,7 +155,7 @@ impl LocalRecallRankingWebpage {
/// the index to calculate bm25.
pub fn new(
pointer: WebpagePointer,
- columnfield_reader: &columnfield_reader::SegmentReader,
+ columnfield_reader: &numericalfield_reader::SegmentReader,
computer: &mut SignalComputer,
) -> Self {
let columnfields = columnfield_reader.get_field_reader(pointer.address.doc_id);
diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rs
index 68ddc197..97928ffd 100644
--- a/crates/core/src/ranking/signal/computer/mod.rs
+++ b/crates/core/src/ranking/signal/computer/mod.rs
@@ -19,7 +19,7 @@ use crate::query::{Query, MAX_TERMS_FOR_NGRAM_LOOKUPS};
use crate::ranking::bm25f::MultiBm25FWeight;
use crate::schema::text_field::TextField;
use crate::Result;
-use crate::{columnfield_reader, enum_map::EnumMap, schema::TextFieldEnum, webpage::Webpage};
+use crate::{enum_map::EnumMap, numericalfield_reader, schema::TextFieldEnum, webpage::Webpage};
use std::cell::RefCell;
@@ -121,7 +121,7 @@ pub struct OpticBoosts {
pub struct SegmentReader {
text_fields: EnumMap<TextFieldEnum, TextFieldData>,
optic_boosts: OpticBoosts,
- columnfield_reader: Arc<columnfield_reader::SegmentReader>,
+ numericalfield_reader: Arc<numericalfield_reader::SegmentReader>,
}
impl SegmentReader {
@@ -129,8 +129,8 @@ impl SegmentReader {
&mut self.text_fields
}
- pub fn columnfield_reader(&self) -> &columnfield_reader::SegmentReader {
- &self.columnfield_reader
+ pub fn numericalfield_reader(&self) -> &numericalfield_reader::SegmentReader {
+ &self.numericalfield_reader
}
}
@@ -322,7 +322,7 @@ impl SignalComputer {
&self,
tv_searcher: &tantivy::Searcher,
segment_reader: &tantivy::SegmentReader,
- columnfield_reader: &columnfield_reader::ColumnFieldReader,
+ numericalfield_reader: &numericalfield_reader::NumericalFieldReader,
) -> Vec<RuleBoost> {
let mut optic_rule_boosts = Vec::new();
@@ -331,7 +331,7 @@ impl SignalComputer {
.optic_rules
.iter()
.filter_map(|rule| {
- rule.as_searchable_rule(tv_searcher.schema(), columnfield_reader)
+ rule.as_searchable_rule(tv_searcher.schema(), numericalfield_reader)
})
.map(|(_, rule)| RuleBoost {
docset: rule
@@ -355,16 +355,17 @@ impl SignalComputer {
&mut self,
tv_searcher: &tantivy::Searcher,
segment_reader: &tantivy::SegmentReader,
- columnfield_reader: &columnfield_reader::ColumnFieldReader,
+ numericalfield_reader: &numericalfield_reader::NumericalFieldReader,
) -> Result<()> {
- let columnfield_segment_reader =
- columnfield_reader.get_segment(&segment_reader.segment_id());
+ let numericalfield_segment_reader =
+ numericalfield_reader.get_segment(&segment_reader.segment_id());
let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;
- let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, columnfield_reader);
+ let optic_rule_boosts =
+ self.prepare_optic(tv_searcher, segment_reader, numericalfield_reader);
self.segment_reader = Some(RefCell::new(SegmentReader {
text_fields,
- columnfield_reader: columnfield_segment_reader,
+ numericalfield_reader: numericalfield_segment_reader,
optic_boosts: OpticBoosts {
rules: optic_rule_boosts,
},
diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rs
index 84167f8e..cb66611f 100644
--- a/crates/core/src/ranking/signal/non_text.rs
+++ b/crates/core/src/ranking/signal/non_text.rs
@@ -116,10 +116,10 @@ impl Signal for HostCentrality {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_f64())
.unwrap();
Some(val)
@@ -156,10 +156,10 @@ impl Signal for HostCentralityRank {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
Some(score_rank(val as f64))
@@ -196,10 +196,10 @@ impl Signal for PageCentrality {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_f64())
.unwrap();
Some(val)
@@ -236,10 +236,10 @@ impl Signal for PageCentralityRank {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
Some(score_rank(val as f64))
@@ -274,10 +274,10 @@ impl Signal for IsHomepage {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_bool())
.unwrap();
@@ -324,10 +324,10 @@ impl Signal for FetchTimeMs {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let fetch_time_ms = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let fetch_time_ms = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap() as usize;
@@ -375,10 +375,10 @@ impl Signal for UpdateTimestamp {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap() as usize;
@@ -417,10 +417,10 @@ impl Signal for TrackerScore {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
Some(score_trackers(val as f64))
@@ -457,10 +457,10 @@ impl Signal for Region {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
let region = crate::webpage::Region::from_id(val);
@@ -595,10 +595,10 @@ impl Signal for UrlDigits {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
Some(score_digits(val as f64))
@@ -642,10 +642,10 @@ impl Signal for UrlSlashes {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
Some(score_slashes(val as f64))
@@ -683,10 +683,10 @@ impl Signal for LinkDensity {
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
- let val = columnfield_reader
- .get(self.as_columnfield().unwrap())
+ let val = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_f64())
.unwrap();
Some(score_link_density(val))
diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rs
index 9de46936..8c375afb 100644
--- a/crates/core/src/ranking/signal/prelude.rs
+++ b/crates/core/src/ranking/signal/prelude.rs
@@ -46,7 +46,7 @@ pub trait Signal:
self.as_field().and_then(|field| field.as_text())
}
- fn as_columnfield(&self) -> Option<NumericalFieldEnum> {
+ fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {
self.as_field().and_then(|field| field.as_numerical())
}
}
diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rs
index 87bff767..c7e62c30 100644
--- a/crates/core/src/schema/numerical_field.rs
+++ b/crates/core/src/schema/numerical_field.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
+use bitflags::bitflags;
use enum_dispatch::enum_dispatch;
use strum::{EnumDiscriminants, VariantArray};
use tantivy::{
@@ -31,6 +32,16 @@ use crate::{
use super::IndexingOption;
+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode, PartialEq, Eq, Hash)]
+pub struct Orientation(u8);
+
+bitflags! {
+ impl Orientation: u8 {
+ const ROW = 1 << 0;
+ const COLUMNAR = 1 << 1;
+ }
+}
+
#[enum_dispatch]
pub trait NumericalField:
Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash
@@ -66,9 +77,18 @@ pub trait NumericalField:
}
fn indexing_option(&self) -> IndexingOption {
+ let orientation = self.orientation();
match self.data_type() {
DataType::U64 | DataType::Bool | DataType::F64 => {
- let mut opt = NumericOptions::default().set_columnar();
+ let mut opt = NumericOptions::default();
+
+ if orientation.contains(Orientation::COLUMNAR) {
+ opt = opt.set_columnar();
+ }
+
+ if orientation.contains(Orientation::ROW) {
+ opt = opt.set_row_order();
+ }
if self.is_stored() {
opt = opt.set_stored();
@@ -81,7 +101,11 @@ pub trait NumericalField:
IndexingOption::Integer(opt)
}
DataType::Bytes => {
- let mut opt = BytesOptions::default().set_columnar().set_indexed();
+ let mut opt = BytesOptions::default().set_indexed();
+
+ if orientation.contains(Orientation::COLUMNAR) {
+ opt = opt.set_columnar();
+ }
if self.is_stored() {
opt = opt.set_stored();
@@ -97,6 +121,10 @@ pub trait NumericalField:
.get_field(self.name())
.unwrap_or_else(|_| unreachable!("Unknown field: {}", self.name()))
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::COLUMNAR
+ }
}
#[enum_dispatch(NumericalField)]
@@ -275,6 +303,10 @@ impl NumericalField for HostCentrality {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -304,6 +336,10 @@ impl NumericalField for HostCentralityRank {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -336,6 +372,10 @@ impl NumericalField for PageCentrality {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -364,6 +404,10 @@ impl NumericalField for PageCentralityRank {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -392,6 +436,10 @@ impl NumericalField for FetchTimeMs {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -420,6 +468,10 @@ impl NumericalField for LastUpdated {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -440,6 +492,10 @@ impl NumericalField for TrackerScore {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -481,6 +537,10 @@ impl NumericalField for Region {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -968,6 +1028,10 @@ impl NumericalField for HostNodeID {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -1052,6 +1116,10 @@ impl NumericalField for NumPathAndQuerySlashes {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -1090,6 +1158,10 @@ impl NumericalField for NumPathAndQueryDigits {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -1118,6 +1190,10 @@ impl NumericalField for LikelyHasAds {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -1146,6 +1222,10 @@ impl NumericalField for LikelyHasPaywall {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -1174,6 +1254,10 @@ impl NumericalField for LinkDensity {
Ok(())
}
+
+ fn orientation(&self) -> Orientation {
+ Orientation::ROW
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
diff --git a/crates/core/src/search_ctx.rs b/crates/core/src/search_ctx.rs
index d0f768ae..db45efe1 100644
--- a/crates/core/src/search_ctx.rs
+++ b/crates/core/src/search_ctx.rs
@@ -14,10 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::columnfield_reader::ColumnFieldReader;
+use crate::numericalfield_reader::NumericalFieldReader;
#[derive(Clone)]
pub struct Ctx {
pub tv_searcher: tantivy::Searcher,
- pub columnfield_reader: ColumnFieldReader,
+ pub columnfield_reader: NumericalFieldReader,
}
diff --git a/crates/tantivy/src/roworder/mod.rs b/crates/tantivy/src/roworder/mod.rs
index a14f66a1..31ef3b24 100644
--- a/crates/tantivy/src/roworder/mod.rs
+++ b/crates/tantivy/src/roworder/mod.rs
@@ -282,6 +282,12 @@ impl AsFieldId for crate::schema::Field {
}
}
+impl AsFieldId for u32 {
+ fn field_id(&self) -> u32 {
+ *self
+ }
+}
+
impl<'a> Row<'a> {
pub fn get<F: AsFieldId>(&self, field: &F) -> Option<&RowValue> {
self.get_by_field_id(field.field_id())
diff --git a/crates/tantivy/src/schema/bytes_options.rs b/crates/tantivy/src/schema/bytes_options.rs
index 142067be..5fb1222a 100644
--- a/crates/tantivy/src/schema/bytes_options.rs
+++ b/crates/tantivy/src/schema/bytes_options.rs
@@ -228,7 +228,7 @@ mod tests {
indexed: true,
fieldnorms: true,
columnar: false,
- stored: false
+ stored: false,
}
);
}
@@ -247,7 +247,7 @@ mod tests {
indexed: false,
fieldnorms: false,
columnar: false,
- stored: false
+ stored: false,
}
);
}
@@ -267,7 +267,7 @@ mod tests {
indexed: true,
fieldnorms: false,
columnar: false,
- stored: false
+ stored: false,
}
);
}
@@ -288,7 +288,7 @@ mod tests {
indexed: false,
fieldnorms: true,
columnar: false,
- stored: false
+ stored: false,
}
);
}