web search done right
re-use row field reader for document across signals
Mikkel Denkerab6ed35bb80b
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rsindex 7e957465..eedfb903 100644--- a/crates/core/src/collector/top_docs.rs+++ b/crates/core/src/collector/top_docs.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-use std::{collections::HashMap, sync::Arc};+use std::collections::HashMap;use bloom::combine_u64s;use min_max_heap::MinMaxHeap;@@ -101,7 +101,10 @@ impl TopDocs {.map(|max_docs| max_docs.total_docs / max_docs.segments);Ok(TopSegmentCollector {- columnfield_segment_reader: self.columnfield_reader.get_segment(&segment.segment_id()),+ columnfield_segment_reader: self+ .columnfield_reader+ .borrow_segment(&segment.segment_id())+ .clone(),max_docs,num_docs_taken: 0,segment_ord: segment_local_id,@@ -114,7 +117,7 @@ impl TopDocs {}pub struct TopSegmentCollector {- columnfield_segment_reader: Arc<numericalfield_reader::SegmentReader>,+ columnfield_segment_reader: numericalfield_reader::SegmentReader,max_docs: Option<usize>,num_docs_taken: usize,segment_ord: SegmentOrdinal,diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex f0bdc896..6293727d 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -127,7 +127,12 @@ impl InvertedIndex {.then_with(|| a.1.address.doc_id.cmp(&b.1.address.doc_id))});+ if pointers.is_empty() {+ return Ok(vec![]);+ }+let mut prev_segment = None;+ let mut numeric_segment_reader = None;for (orig_index, pointer) in pointers {let update_segment = match prev_segment {Some(prev_segment) if prev_segment != pointer.address.segment => true,@@ -138,6 +143,11 @@ impl InvertedIndex {let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);if update_segment {computer.register_segment(&ctx.tv_searcher, segment_reader, columnfield_reader)?;+ numeric_segment_reader = Some(+ columnfield_reader+ .borrow_segment(&segment_reader.segment_id())+ .clone(),+ );}prev_segment = Some(pointer.address.segment);@@ -146,7 +156,7 @@ impl InvertedIndex {orig_index,LocalRecallRankingWebpage::new(pointer,- columnfield_reader.borrow_segment(&segment_reader.segment_id()),+ numeric_segment_reader.as_mut().unwrap(),&mut computer,),));diff --git a/crates/core/src/numericalfield_reader.rs b/crates/core/src/numericalfield_reader.rsindex aecf36a6..ace5e941 100644--- a/crates/core/src/numericalfield_reader.rs+++ b/crates/core/src/numericalfield_reader.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -28,7 +28,7 @@ use crate::{#[derive(Default)]struct InnerNumericalFieldReader {- segments: HashMap<SegmentId, Arc<SegmentReader>>,+ segments: HashMap<SegmentId, SegmentReader>,}#[derive(Default, Clone)]@@ -37,12 +37,8 @@ pub struct NumericalFieldReader {}impl NumericalFieldReader {- pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentReader> {- Arc::clone(self.inner.segments.get(segment).unwrap())- }-pub fn borrow_segment(&self, segment: &SegmentId) -> &SegmentReader {- self.inner.segments.get(segment).unwrap()+ self.inner.segments.get(segment).as_ref().unwrap()}}@@ -91,7 +87,7 @@ impl NumericalFieldReader {segments.insert(reader.segment_id(),- Arc::new(SegmentReader {+ SegmentReader {row_reader: reader.row_fields().clone(),columnar_readers: ColumnarReaders {u64s,@@ -100,7 +96,8 @@ impl NumericalFieldReader {f64s,},field_ids,- }),+ row: None,+ },);}@@ -110,6 +107,7 @@ impl NumericalFieldReader {}}+#[derive(Clone)]struct ColumnarReaders {u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,@@ -207,7 +205,7 @@ impl From<Value> for Option<bool> {pub struct FieldReader<'a> {columnar_readers: &'a ColumnarReaders,- row: Option<tantivy::roworder::Row<'a>>,+ row: Option<&'a tantivy::roworder::Row>,field_ids: &'a EnumMap<NumericalFieldEnum, u32>,doc: DocId,}@@ -273,20 +271,47 @@ impl<'a> FieldReader<'a> {}}+struct Row {+ doc: DocId,+ row: tantivy::roworder::Row,+}+pub struct SegmentReader {columnar_readers: ColumnarReaders,row_reader: tantivy::roworder::readers::RowFieldReaders,field_ids: EnumMap<NumericalFieldEnum, u32>,+ row: Option<Row>,+}++impl Clone for SegmentReader {+ fn clone(&self) -> Self {+ Self {+ columnar_readers: self.columnar_readers.clone(),+ row_reader: self.row_reader.clone(),+ field_ids: self.field_ids.clone(),+ row: None,+ }+ }}impl SegmentReader {+ pub fn prepare_row_for_doc(&mut self, doc: DocId) {+ self.row = Some(Row {+ doc,+ row: self.row_reader.row_index().get_row(doc as usize).unwrap(),+ });+ }+pub fn get_field_reader(&self, doc: DocId) -> FieldReader<'_> {- let row = self.row_reader.row_index().get_row(doc as usize);+ debug_assert!(+ self.row.as_ref().map(|r| r.doc == doc).unwrap_or(true),+ "Row prepared for wrong doc"+ );FieldReader {columnar_readers: &self.columnar_readers,field_ids: &self.field_ids,- row,+ row: self.row.as_ref().map(|r| &r.row),doc,}}diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rsindex edc745fb..4818ff4d 100644--- a/crates/core/src/query/pattern_query/scorer.rs+++ b/crates/core/src/query/pattern_query/scorer.rs@@ -14,8 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::sync::Arc;-use tantivy::{postings::{Postings, SegmentPostings},query::Scorer,@@ -126,7 +124,7 @@ impl Scorer for AllScorer {}pub struct EmptyFieldScorer {- pub segment_reader: Arc<numericalfield_reader::SegmentReader>,+ pub segment_reader: numericalfield_reader::SegmentReader,pub num_tokens_columnfield: NumericalFieldEnum,pub all_scorer: AllScorer,}@@ -211,7 +209,7 @@ pub struct NormalPatternScorer {right: Vec<u32>,phrase_count: u32,num_tokens_field: NumericalFieldEnum,- segment_reader: Arc<numericalfield_reader::SegmentReader>,+ segment_reader: numericalfield_reader::SegmentReader,}impl NormalPatternScorer {@@ -223,7 +221,7 @@ impl NormalPatternScorer {columnfield_reader: NumericalFieldReader,) -> Self {let num_query_terms = term_postings_list.len();- let segment_reader = columnfield_reader.get_segment(&segment);+ let segment_reader = columnfield_reader.borrow_segment(&segment).clone();let mut s = Self {pattern_all_simple: pattern.iter().all(|p| matches!(p, SmallPatternPart::Term)),diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rsindex 920f967d..65824943 100644--- a/crates/core/src/query/pattern_query/weight.rs+++ b/crates/core/src/query/pattern_query/weight.rs@@ -182,7 +182,10 @@ impl PatternWeight {{return Ok(Some(PatternScorer::EmptyField(EmptyFieldScorer {num_tokens_columnfield,- segment_reader: self.columnfield_reader.get_segment(&reader.segment_id()),+ segment_reader: self+ .columnfield_reader+ .borrow_segment(&reader.segment_id())+ .clone(),all_scorer: AllScorer {doc: 0,max_doc: reader.max_doc(),diff --git a/crates/core/src/ranking/computer/mod.rs b/crates/core/src/ranking/computer/mod.rsindex b660be5f..86e2edf4 100644--- a/crates/core/src/ranking/computer/mod.rs+++ b/crates/core/src/ranking/computer/mod.rs@@ -160,7 +160,7 @@ pub struct OpticBoosts {pub struct SegmentReader {text_fields: EnumMap<TextFieldEnum, TextFieldData>,optic_boosts: OpticBoosts,- numericalfield_reader: Arc<numericalfield_reader::SegmentReader>,+ numericalfield_reader: numericalfield_reader::SegmentReader,}impl SegmentReader {@@ -171,6 +171,10 @@ impl SegmentReader {pub fn numericalfield_reader(&self) -> &numericalfield_reader::SegmentReader {&self.numericalfield_reader}++ pub fn prepare_for_doc(&mut self, doc: DocId) {+ self.numericalfield_reader.prepare_row_for_doc(doc);+ }}#[derive(Clone)]@@ -410,8 +414,9 @@ impl SignalComputer {segment_reader: &tantivy::SegmentReader,numericalfield_reader: &numericalfield_reader::NumericalFieldReader,) -> Result<()> {- let numericalfield_segment_reader =- numericalfield_reader.get_segment(&segment_reader.segment_id());+ let numericalfield_segment_reader = numericalfield_reader+ .borrow_segment(&segment_reader.segment_id())+ .clone();let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;let optic_rule_boosts =self.prepare_optic(tv_searcher, segment_reader, numericalfield_reader);diff --git a/crates/core/src/ranking/computer/order.rs b/crates/core/src/ranking/computer/order.rsindex 69df4170..b14caaa9 100644--- a/crates/core/src/ranking/computer/order.rs+++ b/crates/core/src/ranking/computer/order.rs@@ -65,6 +65,12 @@ impl SignalComputeOrder {doc: DocId,signal_computer: &'a SignalComputer,) -> impl Iterator<Item = ComputedSignal> + 'a {+ signal_computer+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut()+ .prepare_for_doc(doc);self.text_signals.values().flat_map(move |ngram| ngram.compute(doc, signal_computer))diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex 3122e5ca..a1d5118e 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -162,9 +162,10 @@ impl LocalRecallRankingWebpage {/// the index to calculate bm25.pub fn new(pointer: WebpagePointer,- columnfield_reader: &numericalfield_reader::SegmentReader,+ columnfield_reader: &mut numericalfield_reader::SegmentReader,computer: &mut SignalComputer,) -> Self {+ columnfield_reader.prepare_row_for_doc(pointer.address.doc_id);let columnfields = columnfield_reader.get_field_reader(pointer.address.doc_id);let title_embedding: Option<Vec<u8>> = columnfieldsdiff --git a/crates/tantivy/src/roworder/mod.rs b/crates/tantivy/src/roworder/mod.rsindex 31ef3b24..479d052c 100644--- a/crates/tantivy/src/roworder/mod.rs+++ b/crates/tantivy/src/roworder/mod.rs@@ -255,12 +255,12 @@ impl RowValue {}}-pub struct Row<'a> {- index: &'a [Option<usize>],+pub struct Row {+ index: Vec<Option<usize>>,values: Vec<RowValue>,}-impl<'a> fmt::Debug for Row<'a> {+impl fmt::Debug for Row {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {f.debug_struct("Row").field("values", &self.values).finish()}@@ -288,7 +288,7 @@ impl AsFieldId for u32 {}}-impl<'a> Row<'a> {+impl Row {pub fn get<F: AsFieldId>(&self, field: &F) -> Option<&RowValue> {self.get_by_field_id(field.field_id())}@@ -427,7 +427,7 @@ impl RowIndex {})}- pub fn get_row(&self, idx: usize) -> Option<Row<'_>> {+ pub fn get_row(&self, idx: usize) -> Option<Row> {if self.header.field_order().is_empty() {return None;}@@ -456,12 +456,12 @@ impl RowIndex {}Some(Row {- index: &self.index,+ index: self.index.clone(),values,})}- pub fn iter(&self) -> impl Iterator<Item = Row<'_>> + '_ {+ pub fn iter(&self) -> impl Iterator<Item = Row> + '_ {(0..self.num_rows()).filter_map(move |idx| self.get_row(idx))}