Skip to content

web search done right

spans for term proximity ranking

Mikkel Denker976cc70d34ac
diff --git a/src/query/field_union.rs b/src/query/field_union.rs
new file mode 100644
index 00000000..f8a4b0a3
--- /dev/null
+++ b/src/query/field_union.rs
@@ -0,0 +1,73 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use tantivy::query::Scorer;
+use tantivy::{DocId, DocSet, Score, TERMINATED};
+
+use super::term_scorer::TermScorerForField;
+
+/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.
+pub struct FieldUnion {
+ pub docsets: Vec<TermScorerForField>,
+}
+
+impl From<Vec<TermScorerForField>> for FieldUnion {
+ fn from(docsets: Vec<TermScorerForField>) -> FieldUnion {
+ FieldUnion { docsets }
+ }
+}
+
+impl DocSet for FieldUnion {
+ fn advance(&mut self) -> DocId {
+ let current_doc = self.doc();
+
+ for scorer in self
+ .docsets
+ .iter_mut()
+ .filter(|scorer| scorer.doc() == current_doc)
+ {
+ scorer.advance();
+ }
+
+ self.doc()
+ }
+
+ fn doc(&self) -> DocId {
+ self.docsets
+ .iter()
+ .map(|scorer| scorer.doc())
+ .min()
+ .unwrap_or(TERMINATED)
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.docsets
+ .iter()
+ .map(|docset| docset.size_hint())
+ .max()
+ .unwrap_or(0u32)
+ }
+}
+
+impl Scorer for FieldUnion {
+ fn score(&mut self) -> Score {
+ let current_doc = self.doc();
+ self.docsets
+ .iter_mut()
+ .filter(|scorer| scorer.doc() == current_doc)
+ .map(|scorer| scorer.score())
+ .sum()
+ }
+}
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 021fe684..2d80e406 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -32,8 +32,10 @@ use tantivy::{
use self::bm25::Bm25Weight;
mod bm25;
-mod scorer;
-mod union;
+mod field_union;
+mod term_intersection;
+mod term_scorer;
+mod vec_docset;
mod weight;
/// A combinator that takes a parser `inner` and produces a parser that also consumes leading
@@ -50,6 +52,7 @@ where
fn term(term: &str) -> IResult<&str, &str> {
ws(take_while(|c: char| !c.is_whitespace()))(term)
}
+
pub struct FieldData {
tantivy: tantivy::schema::Field,
scoring: Bm25Weight,
@@ -57,6 +60,13 @@ pub struct FieldData {
boost: Option<f32>,
analyzer: Option<TextAnalyzer>,
}
+impl std::fmt::Debug for FieldData {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("FieldData")
+ .field("tanitvy", &self.tantivy)
+ .finish()
+ }
+}
#[derive(Clone, Debug)]
pub struct Query {
diff --git a/src/query/term_intersection.rs b/src/query/term_intersection.rs
new file mode 100644
index 00000000..92d03ec1
--- /dev/null
+++ b/src/query/term_intersection.rs
@@ -0,0 +1,476 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use itertools::Itertools;
+use tantivy::query::{EmptyScorer, Scorer};
+use tantivy::{DocId, DocSet, Postings, Score, TERMINATED};
+
+use super::field_union::FieldUnion;
+use super::term_scorer;
+
+const MAX_WIDTH: u32 = 10;
+
+pub fn intersect_terms(mut scorers: Vec<FieldUnion>) -> Box<dyn Scorer> {
+ if scorers.is_empty() {
+ return Box::new(EmptyScorer);
+ }
+ scorers.sort_by_key(|scorer| scorer.size_hint());
+ let doc = go_to_first_doc(&mut scorers[..]);
+ if doc == TERMINATED {
+ return Box::new(EmptyScorer);
+ }
+ Box::new(TermIntersection { docsets: scorers })
+}
+
+pub struct TermIntersection {
+ docsets: Vec<FieldUnion>,
+}
+
+fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {
+ assert!(!docsets.is_empty());
+ let mut candidate = docsets.iter().map(TDocSet::doc).max().unwrap();
+ 'outer: loop {
+ for docset in docsets.iter_mut() {
+ let seek_doc = docset.seek(candidate);
+ if seek_doc > candidate {
+ candidate = docset.doc();
+ continue 'outer;
+ }
+ }
+ return candidate;
+ }
+}
+
+impl DocSet for TermIntersection {
+ fn advance(&mut self) -> DocId {
+ let mut candidate = self.docsets[0].advance();
+
+ 'outer: loop {
+ for docset in self.docsets.iter_mut() {
+ let seek_doc = docset.seek(candidate);
+ if seek_doc > candidate {
+ candidate = self.docsets[0].seek(seek_doc);
+ continue 'outer;
+ }
+ }
+ debug_assert!(self.docsets.iter().all(|docset| docset.doc() == candidate));
+ return candidate;
+ }
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ self.docsets[0].seek(target);
+ let doc = go_to_first_doc(&mut self.docsets);
+ debug_assert!(self.docsets.iter().all(|docset| docset.doc() == doc));
+ debug_assert!(doc >= target);
+ doc
+ }
+
+ fn doc(&self) -> DocId {
+ self.docsets[0].doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.docsets[0].size_hint()
+ }
+}
+
+/// modified bm25 score based on term proximity.
+/// more details in the papers "Viewing Term Proximity from a Different Perspective"
+/// and "A Short Note on Proximity-based Scoring of Documents with Multiple Fields"
+impl Scorer for TermIntersection {
+ fn score(&mut self) -> Score {
+ if self.doc() == TERMINATED {
+ return 0.0;
+ }
+
+ let num_docsets = self.docsets.len();
+
+ let postings = self
+ .docsets
+ .iter_mut()
+ .map(|docset| {
+ docset
+ .docsets
+ .iter_mut()
+ .map(|docset| &mut docset.postings)
+ .collect_vec()
+ })
+ .collect_vec();
+
+ debug_assert_eq!(postings.len(), num_docsets);
+
+ let mut fields = Vec::new();
+ for term in postings {
+ for (i, field) in term.into_iter().enumerate() {
+ if i >= fields.len() {
+ fields.push(Vec::new());
+ }
+ fields[i].push(field);
+ }
+ }
+
+ debug_assert!(fields.iter().all(|terms| terms.len() == num_docsets));
+
+ let mut positions: Vec<Vec<u32>> = Vec::new();
+ for _ in 0..fields.get(0).map(|t| t.len()).unwrap_or(0) {
+ positions.push(Vec::new());
+ }
+
+ fields
+ .into_iter()
+ .map(|terms| {
+ for (i, term) in terms.into_iter().enumerate() {
+ if term.doc() == TERMINATED {
+ for pos in &mut positions[i..] {
+ if !pos.is_empty() {
+ pos.clear();
+ }
+ }
+ break;
+ }
+
+ term.positions(positions.get_mut(i).unwrap());
+ }
+
+ if positions[0].is_empty() {
+ // field has no matches in any/all of the terms.
+ return 0.0;
+ }
+
+ 1.0
+ })
+ .sum::<Score>();
+
+ self.docsets.iter_mut().map(Scorer::score).sum::<Score>()
+ }
+}
+
+struct SpansIterator {
+ num_terms: usize,
+ hits: HitsIterator,
+ current_hit: Option<Hit>,
+ spillover_hit: Option<Hit>,
+}
+
+impl SpansIterator {
+ pub fn new(positions: Vec<Vec<u32>>) -> Self {
+ let num_terms = positions.len();
+ let hits = HitsIterator::new(positions);
+
+ Self {
+ hits,
+ num_terms,
+ current_hit: None,
+ spillover_hit: None,
+ }
+ }
+}
+
+impl Iterator for SpansIterator {
+ type Item = Span;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let mut current_span = Span::default();
+ let mut term_seen_before = vec![None; self.num_terms];
+
+ if let Some(spillover) = self.spillover_hit.take() {
+ term_seen_before[spillover.term_id as usize] = Some(spillover.position);
+ current_span.terms.push(SpanTermInfo {
+ term_id: spillover.term_id,
+ position: spillover.position,
+ });
+ }
+
+ loop {
+ match (self.current_hit.take(), self.hits.next()) {
+ (None, Some(next_hit)) => self.current_hit = Some(next_hit),
+ (Some(current_hit), None) => {
+ term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);
+
+ current_span.terms.push(SpanTermInfo {
+ term_id: current_hit.term_id,
+ position: current_hit.position,
+ });
+
+ self.current_hit = None;
+
+ return Some(current_span);
+ }
+ (Some(current_hit), Some(next_hit)) => {
+ term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);
+
+ let dist = next_hit.position - current_hit.position + 1;
+ if dist > MAX_WIDTH || current_hit.term_id == next_hit.term_id {
+ // case 1 and 2 from paper
+ current_span.terms.push(SpanTermInfo {
+ term_id: current_hit.term_id,
+ position: current_hit.position,
+ });
+
+ self.current_hit = Some(next_hit);
+
+ return Some(current_span);
+ } else if let Some(prev_hit) = term_seen_before[next_hit.term_id as usize] {
+ // case 3 from paper
+ let dist_prev = current_hit.position - prev_hit + 1;
+ let dist_next = next_hit.position - current_hit.position + 1;
+
+ if dist_prev < dist_next {
+ current_span.terms.push(SpanTermInfo {
+ term_id: current_hit.term_id,
+ position: current_hit.position,
+ });
+ } else {
+ self.spillover_hit = Some(current_hit);
+ }
+ self.current_hit = Some(next_hit);
+ return Some(current_span);
+ } else {
+ // case 4
+ current_span.terms.push(SpanTermInfo {
+ term_id: current_hit.term_id,
+ position: current_hit.position,
+ });
+ self.current_hit = Some(next_hit);
+ }
+ }
+ (None, None) => return None,
+ }
+ }
+ }
+}
+
+#[derive(Debug)]
+struct SpanTermInfo {
+ term_id: u32,
+ position: u32,
+}
+
+#[derive(Default, Debug)]
+struct Span {
+ terms: Vec<SpanTermInfo>,
+}
+
+impl Span {
+ fn width(&self) -> u32 {
+ match (self.terms.first(), self.terms.last()) {
+ (Some(first), Some(last)) if first.term_id != last.term_id => {
+ (last.position - first.position + 1).min(MAX_WIDTH)
+ }
+ _ => MAX_WIDTH,
+ }
+ }
+
+ fn num_terms(&self) -> u32 {
+ self.terms.len() as u32
+ }
+
+ fn has_term(&self, term_id: u32) -> bool {
+ self.terms.iter().any(|term| term.term_id == term_id)
+ }
+}
+
+#[derive(PartialEq, Debug)]
+struct Hit {
+ position: u32,
+ term_id: u32,
+}
+
+struct HitsIterator {
+ positions: Vec<Vec<u32>>,
+ cursors: Vec<usize>,
+}
+
+impl HitsIterator {
+ fn new(positions: Vec<Vec<u32>>) -> Self {
+ let cursors = positions.iter().map(|_| 0).collect();
+
+ Self { positions, cursors }
+ }
+}
+
+impl Iterator for HitsIterator {
+ type Item = Hit;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let mut smallest_idx = None;
+
+ for idx in 0..self.positions.len() {
+ let cursor = self.cursors[idx];
+ if cursor >= self.positions[idx].len() {
+ continue;
+ }
+
+ smallest_idx = match smallest_idx {
+ Some(s_idx) => {
+ let tmp: &Vec<u32> = &self.positions[s_idx];
+ if self.positions[idx][cursor] < tmp[self.cursors[s_idx]] {
+ Some(idx)
+ } else {
+ smallest_idx
+ }
+ }
+ None => Some(idx),
+ };
+ }
+
+ smallest_idx.map(|idx| {
+ let res = self.positions[idx][self.cursors[idx]];
+ self.cursors[idx] += 1;
+
+ Hit {
+ position: res,
+ term_id: idx as u32,
+ }
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn hits_iterator() {
+ let positions = vec![vec![0, 2, 6], vec![1, 4], vec![3, 5]];
+
+ let mut iter = HitsIterator::new(positions);
+
+ assert_eq!(
+ iter.next(),
+ Some(Hit {
+ position: 0,
+ term_id: 0,
+ })
+ );
+ assert_eq!(
+ iter.next(),
+ Some(Hit {
+ position: 1,
+ term_id: 1,
+ })
+ );
+ assert_eq!(
+ iter.next(),
+ Some(Hit {
+ position: 2,
+ term_id: 0,
+ })
+ );
+ assert_eq!(
+ iter.next(),
+ Some(Hit {
+ position: 3,
+ term_id: 2,
+ })
+ );
+ assert_eq!(
+ iter.next(),
+ Some(Hit {
+ position: 4,
+ term_id: 1,
+ })
+ );
+ assert_eq!(
+ iter.next(),
+ Some(Hit {
+ position: 5,
+ term_id: 2,
+ })
+ );
+ assert_eq!(
+ iter.next(),
+ Some(Hit {
+ position: 6,
+ term_id: 0,
+ })
+ );
+ assert_eq!(iter.next(), None);
+ }
+
+ #[test]
+ fn span_info() {
+ let span = Span {
+ terms: vec![
+ SpanTermInfo {
+ term_id: 1,
+ position: 0,
+ },
+ SpanTermInfo {
+ term_id: 2,
+ position: 1,
+ },
+ SpanTermInfo {
+ term_id: 3,
+ position: 3,
+ },
+ ],
+ };
+
+ assert_eq!(span.width(), 4);
+ assert_eq!(span.num_terms(), 3);
+ assert!(span.has_term(1));
+ assert!(span.has_term(2));
+ assert!(span.has_term(3));
+ assert!(!span.has_term(4));
+
+ let span = Span {
+ terms: vec![SpanTermInfo {
+ term_id: 1,
+ position: 0,
+ }],
+ };
+
+ assert_eq!(span.width(), MAX_WIDTH);
+
+ let span = Span {
+ terms: vec![
+ SpanTermInfo {
+ term_id: 1,
+ position: 0,
+ },
+ SpanTermInfo {
+ term_id: 1,
+ position: MAX_WIDTH + 10,
+ },
+ ],
+ };
+
+ assert_eq!(span.width(), MAX_WIDTH);
+ }
+
+ #[test]
+ fn hits_to_span() {
+ // example from paper
+ // positions:
+ // sea: 5, 29
+ // thousand: 7, 10
+ // years: 8, 11
+
+ let positions = vec![vec![5, 29], vec![7, 10], vec![8, 11]];
+
+ let spans: Vec<Span> = SpansIterator::new(positions).collect();
+
+ assert_eq!(spans.len(), 3);
+ assert_eq!(spans[0].width(), 4);
+ assert_eq!(spans[1].width(), 2);
+ assert_eq!(spans[2].width(), 10);
+
+ assert_eq!(spans[0].num_terms(), 3);
+ assert_eq!(spans[1].num_terms(), 2);
+ assert_eq!(spans[2].num_terms(), 1);
+ }
+}
diff --git a/src/query/scorer.rs b/src/query/term_scorer.rs
similarity index 88%
rename from src/query/scorer.rs
rename to src/query/term_scorer.rs
index fe60a8c7..e54ad09a 100644
--- a/src/query/scorer.rs
+++ b/src/query/term_scorer.rs
@@ -21,19 +21,19 @@ use tantivy::DocSet;
use tantivy::{DocId, Score};
#[derive(Clone)]
-pub struct Scorer {
- postings: SegmentPostings,
+pub struct TermScorerForField {
+ pub postings: SegmentPostings,
fieldnorm_reader: FieldNormReader,
similarity_weight: Bm25Weight,
}
-impl Scorer {
+impl TermScorerForField {
pub fn new(
postings: SegmentPostings,
fieldnorm_reader: FieldNormReader,
similarity_weight: Bm25Weight,
- ) -> Scorer {
- Scorer {
+ ) -> TermScorerForField {
+ TermScorerForField {
postings,
fieldnorm_reader,
similarity_weight,
@@ -49,7 +49,7 @@ impl Scorer {
}
}
-impl DocSet for Scorer {
+impl DocSet for TermScorerForField {
fn advance(&mut self) -> DocId {
self.postings.advance()
}
@@ -67,7 +67,7 @@ impl DocSet for Scorer {
}
}
-impl tantivy::query::Scorer for Scorer {
+impl tantivy::query::Scorer for TermScorerForField {
fn score(&mut self) -> Score {
let fieldnorm_id = self.fieldnorm_id();
let term_freq = self.term_freq();
diff --git a/src/query/union.rs b/src/query/union.rs
deleted file mode 100644
index 3db21791..00000000
--- a/src/query/union.rs
+++ /dev/null
@@ -1,99 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::query::Scorer;
-use tantivy::{DocId, DocSet, Score, TERMINATED};
-
-/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.
-pub struct Union<TScorer> {
- docsets: Vec<TScorer>,
- score: Score,
- doc: DocId,
-}
-
-impl<TScorer> From<Vec<TScorer>> for Union<TScorer>
-where
- TScorer: Scorer,
-{
- fn from(docsets: Vec<TScorer>) -> Union<TScorer> {
- let non_empty_docsets: Vec<TScorer> = docsets
- .into_iter()
- .filter(|docset| docset.doc() != TERMINATED)
- .collect();
- let mut union = Union {
- docsets: non_empty_docsets,
- doc: 0,
- score: 0.0,
- };
-
- union.advance();
- union
- }
-}
-
-impl<TScorer> DocSet for Union<TScorer>
-where
- TScorer: Scorer,
-{
- fn advance(&mut self) -> DocId {
- if let Some(min_doc) = self.docsets.iter().map(|scorer| scorer.doc()).min() {
- self.doc = min_doc;
-
- if self.doc == TERMINATED {
- return self.doc;
- }
-
- self.score = self
- .docsets
- .iter_mut()
- .filter(|scorer| scorer.doc() == self.doc)
- .map(|scorer| scorer.score())
- .sum();
-
- for scorer in self
- .docsets
- .iter_mut()
- .filter(|scorer| scorer.doc() == min_doc)
- {
- scorer.advance();
- }
- } else {
- self.doc = TERMINATED;
- }
-
- self.doc
- }
-
- fn doc(&self) -> DocId {
- self.doc
- }
-
- fn size_hint(&self) -> u32 {
- self.docsets
- .iter()
- .map(|docset| docset.size_hint())
- .max()
- .unwrap_or(0u32)
- }
-}
-
-impl<TScorer> Scorer for Union<TScorer>
-where
- TScorer: Scorer,
-{
- fn score(&mut self) -> Score {
- self.score
- }
-}
diff --git a/src/query/vec_docset.rs b/src/query/vec_docset.rs
new file mode 100644
index 00000000..418c136d
--- /dev/null
+++ b/src/query/vec_docset.rs
@@ -0,0 +1,79 @@
+// taken from tantivy source for tests.
+// source: https://github.com/quickwit-oss/tantivy/blob/main/src/query/vec_docset.rs
+#![allow(dead_code)]
+
+use tantivy::{DocId, DocSet, TERMINATED};
+
+/// Simulate a `Postings` objects from a `VecPostings`.
+/// `VecPostings` only exist for testing purposes.
+///
+/// Term frequencies always return 1.
+/// No positions are returned.
+pub struct VecDocSet {
+ doc_ids: Vec<DocId>,
+ cursor: usize,
+}
+
+impl From<Vec<DocId>> for VecDocSet {
+ fn from(doc_ids: Vec<DocId>) -> VecDocSet {
+ VecDocSet { doc_ids, cursor: 0 }
+ }
+}
+
+impl DocSet for VecDocSet {
+ fn advance(&mut self) -> DocId {
+ self.cursor += 1;
+ if self.cursor >= self.doc_ids.len() {
+ self.cursor = self.doc_ids.len();
+ return TERMINATED;
+ }
+ self.doc()
+ }
+
+ fn doc(&self) -> DocId {
+ if self.cursor == self.doc_ids.len() {
+ return TERMINATED;
+ }
+ self.doc_ids[self.cursor]
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.doc_ids.len() as u32
+ }
+}
+
+#[cfg(test)]
+pub mod tests {
+
+ use super::*;
+
+ #[test]
+ pub fn test_vec_postings() {
+ let doc_ids: Vec<DocId> = (0u32..1024u32).map(|e| e * 3).collect();
+ let mut postings = VecDocSet::from(doc_ids);
+ assert_eq!(postings.doc(), 0u32);
+ assert_eq!(postings.advance(), 3u32);
+ assert_eq!(postings.doc(), 3u32);
+ assert_eq!(postings.seek(14u32), 15u32);
+ assert_eq!(postings.doc(), 15u32);
+ assert_eq!(postings.seek(300u32), 300u32);
+ assert_eq!(postings.doc(), 300u32);
+ assert_eq!(postings.seek(6000u32), TERMINATED);
+ }
+
+ #[test]
+ pub fn test_fill_buffer() {
+ let doc_ids: Vec<DocId> = (1u32..210u32).collect();
+ let mut postings = VecDocSet::from(doc_ids);
+ let mut buffer = vec![1000u32; 100];
+ assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);
+ for i in 0u32..100u32 {
+ assert_eq!(buffer[i as usize], i + 1);
+ }
+ assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);
+ for i in 0u32..100u32 {
+ assert_eq!(buffer[i as usize], i + 101);
+ }
+ assert_eq!(postings.fill_buffer(&mut buffer[..]), 9);
+ }
+}
diff --git a/src/query/weight.rs b/src/query/weight.rs
index ded3bff5..5165b491 100644
--- a/src/query/weight.rs
+++ b/src/query/weight.rs
@@ -13,12 +13,14 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use super::scorer::Scorer;
-use super::union::Union;
+
+use super::field_union::FieldUnion;
+use super::term_intersection::intersect_terms;
+use super::term_scorer::TermScorerForField;
use super::FieldData;
use tantivy::fieldnorm::FieldNormReader;
use tantivy::postings::SegmentPostings;
-use tantivy::query::{intersect_scorers, Explanation};
+use tantivy::query::Explanation;
use tantivy::{DocId, Score, SegmentReader};
pub struct Weight {
@@ -33,10 +35,12 @@ impl tantivy::query::Weight for Weight {
reader: &SegmentReader,
boost: Score,
) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
- let mut term_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
+ // each term has a union over the fields
+ // e.g. if the term both occurs in the title and body
+ let mut term_scorers: Vec<FieldUnion> = Vec::new();
for term_text in &self.terms {
- let mut field_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
+ let mut field_scorers: Vec<TermScorerForField> = Vec::new();
for field_data in &self.fields {
let inverted_index = reader.inverted_index(field_data.tantivy)?;
@@ -80,18 +84,18 @@ impl tantivy::query::Weight for Weight {
let posting = postings_opt.unwrap_or_else(SegmentPostings::empty);
- field_scorers.push(Box::new(Scorer::new(
+ field_scorers.push(TermScorerForField::new(
posting,
fieldnorm_reader,
similarity_weight,
- )));
+ ));
}
}
- term_scorers.push(Box::new(Union::from(field_scorers)));
+ term_scorers.push(FieldUnion::from(field_scorers));
}
- Ok(intersect_scorers(term_scorers))
+ Ok(intersect_terms(term_scorers))
}
fn explain(&self, _reader: &SegmentReader, _doc: DocId) -> tantivy::Result<Explanation> {