Skip to content

web search done right

re-use row field reader for document across signals

Mikkel Denkerab6ed35bb80b
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rs
index 7e957465..eedfb903 100644
--- a/crates/core/src/collector/top_docs.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
-use std::{collections::HashMap, sync::Arc};
+use std::collections::HashMap;
use bloom::combine_u64s;
use min_max_heap::MinMaxHeap;
@@ -101,7 +101,10 @@ impl TopDocs {
.map(|max_docs| max_docs.total_docs / max_docs.segments);
Ok(TopSegmentCollector {
- columnfield_segment_reader: self.columnfield_reader.get_segment(&segment.segment_id()),
+ columnfield_segment_reader: self
+ .columnfield_reader
+ .borrow_segment(&segment.segment_id())
+ .clone(),
max_docs,
num_docs_taken: 0,
segment_ord: segment_local_id,
@@ -114,7 +117,7 @@ impl TopDocs {
}
pub struct TopSegmentCollector {
- columnfield_segment_reader: Arc<numericalfield_reader::SegmentReader>,
+ columnfield_segment_reader: numericalfield_reader::SegmentReader,
max_docs: Option<usize>,
num_docs_taken: usize,
segment_ord: SegmentOrdinal,
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index f0bdc896..6293727d 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -127,7 +127,12 @@ impl InvertedIndex {
.then_with(|| a.1.address.doc_id.cmp(&b.1.address.doc_id))
});
+ if pointers.is_empty() {
+ return Ok(vec![]);
+ }
+
let mut prev_segment = None;
+ let mut numeric_segment_reader = None;
for (orig_index, pointer) in pointers {
let update_segment = match prev_segment {
Some(prev_segment) if prev_segment != pointer.address.segment => true,
@@ -138,6 +143,11 @@ impl InvertedIndex {
let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);
if update_segment {
computer.register_segment(&ctx.tv_searcher, segment_reader, columnfield_reader)?;
+ numeric_segment_reader = Some(
+ columnfield_reader
+ .borrow_segment(&segment_reader.segment_id())
+ .clone(),
+ );
}
prev_segment = Some(pointer.address.segment);
@@ -146,7 +156,7 @@ impl InvertedIndex {
orig_index,
LocalRecallRankingWebpage::new(
pointer,
- columnfield_reader.borrow_segment(&segment_reader.segment_id()),
+ numeric_segment_reader.as_mut().unwrap(),
&mut computer,
),
));
diff --git a/crates/core/src/numericalfield_reader.rs b/crates/core/src/numericalfield_reader.rs
index aecf36a6..ace5e941 100644
--- a/crates/core/src/numericalfield_reader.rs
+++ b/crates/core/src/numericalfield_reader.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -28,7 +28,7 @@ use crate::{
#[derive(Default)]
struct InnerNumericalFieldReader {
- segments: HashMap<SegmentId, Arc<SegmentReader>>,
+ segments: HashMap<SegmentId, SegmentReader>,
}
#[derive(Default, Clone)]
@@ -37,12 +37,8 @@ pub struct NumericalFieldReader {
}
impl NumericalFieldReader {
- pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentReader> {
- Arc::clone(self.inner.segments.get(segment).unwrap())
- }
-
pub fn borrow_segment(&self, segment: &SegmentId) -> &SegmentReader {
- self.inner.segments.get(segment).unwrap()
+ self.inner.segments.get(segment).as_ref().unwrap()
}
}
@@ -91,7 +87,7 @@ impl NumericalFieldReader {
segments.insert(
reader.segment_id(),
- Arc::new(SegmentReader {
+ SegmentReader {
row_reader: reader.row_fields().clone(),
columnar_readers: ColumnarReaders {
u64s,
@@ -100,7 +96,8 @@ impl NumericalFieldReader {
f64s,
},
field_ids,
- }),
+ row: None,
+ },
);
}
@@ -110,6 +107,7 @@ impl NumericalFieldReader {
}
}
+#[derive(Clone)]
struct ColumnarReaders {
u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,
f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,
@@ -207,7 +205,7 @@ impl From<Value> for Option<bool> {
pub struct FieldReader<'a> {
columnar_readers: &'a ColumnarReaders,
- row: Option<tantivy::roworder::Row<'a>>,
+ row: Option<&'a tantivy::roworder::Row>,
field_ids: &'a EnumMap<NumericalFieldEnum, u32>,
doc: DocId,
}
@@ -273,20 +271,47 @@ impl<'a> FieldReader<'a> {
}
}
+struct Row {
+ doc: DocId,
+ row: tantivy::roworder::Row,
+}
+
pub struct SegmentReader {
columnar_readers: ColumnarReaders,
row_reader: tantivy::roworder::readers::RowFieldReaders,
field_ids: EnumMap<NumericalFieldEnum, u32>,
+ row: Option<Row>,
+}
+
+impl Clone for SegmentReader {
+ fn clone(&self) -> Self {
+ Self {
+ columnar_readers: self.columnar_readers.clone(),
+ row_reader: self.row_reader.clone(),
+ field_ids: self.field_ids.clone(),
+ row: None,
+ }
+ }
}
impl SegmentReader {
+ pub fn prepare_row_for_doc(&mut self, doc: DocId) {
+ self.row = Some(Row {
+ doc,
+ row: self.row_reader.row_index().get_row(doc as usize).unwrap(),
+ });
+ }
+
pub fn get_field_reader(&self, doc: DocId) -> FieldReader<'_> {
- let row = self.row_reader.row_index().get_row(doc as usize);
+ debug_assert!(
+ self.row.as_ref().map(|r| r.doc == doc).unwrap_or(true),
+ "Row prepared for wrong doc"
+ );
FieldReader {
columnar_readers: &self.columnar_readers,
field_ids: &self.field_ids,
- row,
+ row: self.row.as_ref().map(|r| &r.row),
doc,
}
}
diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rs
index edc745fb..4818ff4d 100644
--- a/crates/core/src/query/pattern_query/scorer.rs
+++ b/crates/core/src/query/pattern_query/scorer.rs
@@ -14,8 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::sync::Arc;
-
use tantivy::{
postings::{Postings, SegmentPostings},
query::Scorer,
@@ -126,7 +124,7 @@ impl Scorer for AllScorer {
}
pub struct EmptyFieldScorer {
- pub segment_reader: Arc<numericalfield_reader::SegmentReader>,
+ pub segment_reader: numericalfield_reader::SegmentReader,
pub num_tokens_columnfield: NumericalFieldEnum,
pub all_scorer: AllScorer,
}
@@ -211,7 +209,7 @@ pub struct NormalPatternScorer {
right: Vec<u32>,
phrase_count: u32,
num_tokens_field: NumericalFieldEnum,
- segment_reader: Arc<numericalfield_reader::SegmentReader>,
+ segment_reader: numericalfield_reader::SegmentReader,
}
impl NormalPatternScorer {
@@ -223,7 +221,7 @@ impl NormalPatternScorer {
columnfield_reader: NumericalFieldReader,
) -> Self {
let num_query_terms = term_postings_list.len();
- let segment_reader = columnfield_reader.get_segment(&segment);
+ let segment_reader = columnfield_reader.borrow_segment(&segment).clone();
let mut s = Self {
pattern_all_simple: pattern.iter().all(|p| matches!(p, SmallPatternPart::Term)),
diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rs
index 920f967d..65824943 100644
--- a/crates/core/src/query/pattern_query/weight.rs
+++ b/crates/core/src/query/pattern_query/weight.rs
@@ -182,7 +182,10 @@ impl PatternWeight {
{
return Ok(Some(PatternScorer::EmptyField(EmptyFieldScorer {
num_tokens_columnfield,
- segment_reader: self.columnfield_reader.get_segment(&reader.segment_id()),
+ segment_reader: self
+ .columnfield_reader
+ .borrow_segment(&reader.segment_id())
+ .clone(),
all_scorer: AllScorer {
doc: 0,
max_doc: reader.max_doc(),
diff --git a/crates/core/src/ranking/computer/mod.rs b/crates/core/src/ranking/computer/mod.rs
index b660be5f..86e2edf4 100644
--- a/crates/core/src/ranking/computer/mod.rs
+++ b/crates/core/src/ranking/computer/mod.rs
@@ -160,7 +160,7 @@ pub struct OpticBoosts {
pub struct SegmentReader {
text_fields: EnumMap<TextFieldEnum, TextFieldData>,
optic_boosts: OpticBoosts,
- numericalfield_reader: Arc<numericalfield_reader::SegmentReader>,
+ numericalfield_reader: numericalfield_reader::SegmentReader,
}
impl SegmentReader {
@@ -171,6 +171,10 @@ impl SegmentReader {
pub fn numericalfield_reader(&self) -> &numericalfield_reader::SegmentReader {
&self.numericalfield_reader
}
+
+ pub fn prepare_for_doc(&mut self, doc: DocId) {
+ self.numericalfield_reader.prepare_row_for_doc(doc);
+ }
}
#[derive(Clone)]
@@ -410,8 +414,9 @@ impl SignalComputer {
segment_reader: &tantivy::SegmentReader,
numericalfield_reader: &numericalfield_reader::NumericalFieldReader,
) -> Result<()> {
- let numericalfield_segment_reader =
- numericalfield_reader.get_segment(&segment_reader.segment_id());
+ let numericalfield_segment_reader = numericalfield_reader
+ .borrow_segment(&segment_reader.segment_id())
+ .clone();
let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;
let optic_rule_boosts =
self.prepare_optic(tv_searcher, segment_reader, numericalfield_reader);
diff --git a/crates/core/src/ranking/computer/order.rs b/crates/core/src/ranking/computer/order.rs
index 69df4170..b14caaa9 100644
--- a/crates/core/src/ranking/computer/order.rs
+++ b/crates/core/src/ranking/computer/order.rs
@@ -65,6 +65,12 @@ impl SignalComputeOrder {
doc: DocId,
signal_computer: &'a SignalComputer,
) -> impl Iterator<Item = ComputedSignal> + 'a {
+ signal_computer
+ .segment_reader
+ .as_ref()
+ .unwrap()
+ .borrow_mut()
+ .prepare_for_doc(doc);
self.text_signals
.values()
.flat_map(move |ngram| ngram.compute(doc, signal_computer))
diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rs
index 3122e5ca..a1d5118e 100644
--- a/crates/core/src/ranking/pipeline/stages/recall.rs
+++ b/crates/core/src/ranking/pipeline/stages/recall.rs
@@ -162,9 +162,10 @@ impl LocalRecallRankingWebpage {
/// the index to calculate bm25.
pub fn new(
pointer: WebpagePointer,
- columnfield_reader: &numericalfield_reader::SegmentReader,
+ columnfield_reader: &mut numericalfield_reader::SegmentReader,
computer: &mut SignalComputer,
) -> Self {
+ columnfield_reader.prepare_row_for_doc(pointer.address.doc_id);
let columnfields = columnfield_reader.get_field_reader(pointer.address.doc_id);
let title_embedding: Option<Vec<u8>> = columnfields
diff --git a/crates/tantivy/src/roworder/mod.rs b/crates/tantivy/src/roworder/mod.rs
index 31ef3b24..479d052c 100644
--- a/crates/tantivy/src/roworder/mod.rs
+++ b/crates/tantivy/src/roworder/mod.rs
@@ -255,12 +255,12 @@ impl RowValue {
}
}
-pub struct Row<'a> {
- index: &'a [Option<usize>],
+pub struct Row {
+ index: Vec<Option<usize>>,
values: Vec<RowValue>,
}
-impl<'a> fmt::Debug for Row<'a> {
+impl fmt::Debug for Row {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Row").field("values", &self.values).finish()
}
@@ -288,7 +288,7 @@ impl AsFieldId for u32 {
}
}
-impl<'a> Row<'a> {
+impl Row {
pub fn get<F: AsFieldId>(&self, field: &F) -> Option<&RowValue> {
self.get_by_field_id(field.field_id())
}
@@ -427,7 +427,7 @@ impl RowIndex {
})
}
- pub fn get_row(&self, idx: usize) -> Option<Row<'_>> {
+ pub fn get_row(&self, idx: usize) -> Option<Row> {
if self.header.field_order().is_empty() {
return None;
}
@@ -456,12 +456,12 @@ impl RowIndex {
}
Some(Row {
- index: &self.index,
+ index: self.index.clone(),
values,
})
}
- pub fn iter(&self) -> impl Iterator<Item = Row<'_>> + '_ {
+ pub fn iter(&self) -> impl Iterator<Item = Row> + '_ {
(0..self.num_rows()).filter_map(move |idx| self.get_row(idx))
}