web search done right
spans for term proximity ranking
Mikkel Denker976cc70d34ac
diff --git a/src/query/field_union.rs b/src/query/field_union.rsnew file mode 100644index 00000000..f8a4b0a3--- /dev/null+++ b/src/query/field_union.rs@@ -0,0 +1,73 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use tantivy::query::Scorer;+use tantivy::{DocId, DocSet, Score, TERMINATED};++use super::term_scorer::TermScorerForField;++/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.+pub struct FieldUnion {+ pub docsets: Vec<TermScorerForField>,+}++impl From<Vec<TermScorerForField>> for FieldUnion {+ fn from(docsets: Vec<TermScorerForField>) -> FieldUnion {+ FieldUnion { docsets }+ }+}++impl DocSet for FieldUnion {+ fn advance(&mut self) -> DocId {+ let current_doc = self.doc();++ for scorer in self+ .docsets+ .iter_mut()+ .filter(|scorer| scorer.doc() == current_doc)+ {+ scorer.advance();+ }++ self.doc()+ }++ fn doc(&self) -> DocId {+ self.docsets+ .iter()+ .map(|scorer| scorer.doc())+ .min()+ .unwrap_or(TERMINATED)+ }++ fn size_hint(&self) -> u32 {+ self.docsets+ .iter()+ .map(|docset| docset.size_hint())+ .max()+ .unwrap_or(0u32)+ }+}++impl Scorer for FieldUnion {+ fn score(&mut self) -> Score {+ let current_doc = self.doc();+ self.docsets+ .iter_mut()+ .filter(|scorer| scorer.doc() == current_doc)+ .map(|scorer| scorer.score())+ .sum()+ }+}diff --git a/src/query/mod.rs b/src/query/mod.rsindex 021fe684..2d80e406 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -32,8 +32,10 @@ use tantivy::{use self::bm25::Bm25Weight;mod bm25;-mod scorer;-mod union;+mod field_union;+mod term_intersection;+mod term_scorer;+mod vec_docset;mod weight;/// A combinator that takes a parser `inner` and produces a parser that also consumes leading@@ -50,6 +52,7 @@ wherefn term(term: &str) -> IResult<&str, &str> {ws(take_while(|c: char| !c.is_whitespace()))(term)}+pub struct FieldData {tantivy: tantivy::schema::Field,scoring: Bm25Weight,@@ -57,6 +60,13 @@ pub struct FieldData {boost: Option<f32>,analyzer: Option<TextAnalyzer>,}+impl std::fmt::Debug for FieldData {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("FieldData")+ .field("tanitvy", &self.tantivy)+ .finish()+ }+}#[derive(Clone, Debug)]pub struct Query {diff --git a/src/query/term_intersection.rs b/src/query/term_intersection.rsnew file mode 100644index 00000000..92d03ec1--- /dev/null+++ b/src/query/term_intersection.rs@@ -0,0 +1,476 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use itertools::Itertools;+use tantivy::query::{EmptyScorer, Scorer};+use tantivy::{DocId, DocSet, Postings, Score, TERMINATED};++use super::field_union::FieldUnion;+use super::term_scorer;++const MAX_WIDTH: u32 = 10;++pub fn intersect_terms(mut scorers: Vec<FieldUnion>) -> Box<dyn Scorer> {+ if scorers.is_empty() {+ return Box::new(EmptyScorer);+ }+ scorers.sort_by_key(|scorer| scorer.size_hint());+ let doc = go_to_first_doc(&mut scorers[..]);+ if doc == TERMINATED {+ return Box::new(EmptyScorer);+ }+ Box::new(TermIntersection { docsets: scorers })+}++pub struct TermIntersection {+ docsets: Vec<FieldUnion>,+}++fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {+ assert!(!docsets.is_empty());+ let mut candidate = docsets.iter().map(TDocSet::doc).max().unwrap();+ 'outer: loop {+ for docset in docsets.iter_mut() {+ let seek_doc = docset.seek(candidate);+ if seek_doc > candidate {+ candidate = docset.doc();+ continue 'outer;+ }+ }+ return candidate;+ }+}++impl DocSet for TermIntersection {+ fn advance(&mut self) -> DocId {+ let mut candidate = self.docsets[0].advance();++ 'outer: loop {+ for docset in self.docsets.iter_mut() {+ let seek_doc = docset.seek(candidate);+ if seek_doc > candidate {+ candidate = self.docsets[0].seek(seek_doc);+ continue 'outer;+ }+ }+ debug_assert!(self.docsets.iter().all(|docset| docset.doc() == candidate));+ return candidate;+ }+ }++ fn seek(&mut self, target: DocId) -> DocId {+ self.docsets[0].seek(target);+ let doc = go_to_first_doc(&mut self.docsets);+ debug_assert!(self.docsets.iter().all(|docset| docset.doc() == doc));+ debug_assert!(doc >= target);+ doc+ }++ fn doc(&self) -> DocId {+ self.docsets[0].doc()+ }++ fn size_hint(&self) -> u32 {+ self.docsets[0].size_hint()+ }+}++/// modified bm25 score based on term proximity.+/// more details in the papers "Viewing Term Proximity from a Different Perspective"+/// and "A Short Note on Proximity-based Scoring of Documents with Multiple Fields"+impl Scorer for TermIntersection {+ fn score(&mut self) -> Score {+ if self.doc() == TERMINATED {+ return 0.0;+ }++ let num_docsets = self.docsets.len();++ let postings = self+ .docsets+ .iter_mut()+ .map(|docset| {+ docset+ .docsets+ .iter_mut()+ .map(|docset| &mut docset.postings)+ .collect_vec()+ })+ .collect_vec();++ debug_assert_eq!(postings.len(), num_docsets);++ let mut fields = Vec::new();+ for term in postings {+ for (i, field) in term.into_iter().enumerate() {+ if i >= fields.len() {+ fields.push(Vec::new());+ }+ fields[i].push(field);+ }+ }++ debug_assert!(fields.iter().all(|terms| terms.len() == num_docsets));++ let mut positions: Vec<Vec<u32>> = Vec::new();+ for _ in 0..fields.get(0).map(|t| t.len()).unwrap_or(0) {+ positions.push(Vec::new());+ }++ fields+ .into_iter()+ .map(|terms| {+ for (i, term) in terms.into_iter().enumerate() {+ if term.doc() == TERMINATED {+ for pos in &mut positions[i..] {+ if !pos.is_empty() {+ pos.clear();+ }+ }+ break;+ }++ term.positions(positions.get_mut(i).unwrap());+ }++ if positions[0].is_empty() {+ // field has no matches in any/all of the terms.+ return 0.0;+ }++ 1.0+ })+ .sum::<Score>();++ self.docsets.iter_mut().map(Scorer::score).sum::<Score>()+ }+}++struct SpansIterator {+ num_terms: usize,+ hits: HitsIterator,+ current_hit: Option<Hit>,+ spillover_hit: Option<Hit>,+}++impl SpansIterator {+ pub fn new(positions: Vec<Vec<u32>>) -> Self {+ let num_terms = positions.len();+ let hits = HitsIterator::new(positions);++ Self {+ hits,+ num_terms,+ current_hit: None,+ spillover_hit: None,+ }+ }+}++impl Iterator for SpansIterator {+ type Item = Span;++ fn next(&mut self) -> Option<Self::Item> {+ let mut current_span = Span::default();+ let mut term_seen_before = vec![None; self.num_terms];++ if let Some(spillover) = self.spillover_hit.take() {+ term_seen_before[spillover.term_id as usize] = Some(spillover.position);+ current_span.terms.push(SpanTermInfo {+ term_id: spillover.term_id,+ position: spillover.position,+ });+ }++ loop {+ match (self.current_hit.take(), self.hits.next()) {+ (None, Some(next_hit)) => self.current_hit = Some(next_hit),+ (Some(current_hit), None) => {+ term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);++ current_span.terms.push(SpanTermInfo {+ term_id: current_hit.term_id,+ position: current_hit.position,+ });++ self.current_hit = None;++ return Some(current_span);+ }+ (Some(current_hit), Some(next_hit)) => {+ term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);++ let dist = next_hit.position - current_hit.position + 1;+ if dist > MAX_WIDTH || current_hit.term_id == next_hit.term_id {+ // case 1 and 2 from paper+ current_span.terms.push(SpanTermInfo {+ term_id: current_hit.term_id,+ position: current_hit.position,+ });++ self.current_hit = Some(next_hit);++ return Some(current_span);+ } else if let Some(prev_hit) = term_seen_before[next_hit.term_id as usize] {+ // case 3 from paper+ let dist_prev = current_hit.position - prev_hit + 1;+ let dist_next = next_hit.position - current_hit.position + 1;++ if dist_prev < dist_next {+ current_span.terms.push(SpanTermInfo {+ term_id: current_hit.term_id,+ position: current_hit.position,+ });+ } else {+ self.spillover_hit = Some(current_hit);+ }+ self.current_hit = Some(next_hit);+ return Some(current_span);+ } else {+ // case 4+ current_span.terms.push(SpanTermInfo {+ term_id: current_hit.term_id,+ position: current_hit.position,+ });+ self.current_hit = Some(next_hit);+ }+ }+ (None, None) => return None,+ }+ }+ }+}++#[derive(Debug)]+struct SpanTermInfo {+ term_id: u32,+ position: u32,+}++#[derive(Default, Debug)]+struct Span {+ terms: Vec<SpanTermInfo>,+}++impl Span {+ fn width(&self) -> u32 {+ match (self.terms.first(), self.terms.last()) {+ (Some(first), Some(last)) if first.term_id != last.term_id => {+ (last.position - first.position + 1).min(MAX_WIDTH)+ }+ _ => MAX_WIDTH,+ }+ }++ fn num_terms(&self) -> u32 {+ self.terms.len() as u32+ }++ fn has_term(&self, term_id: u32) -> bool {+ self.terms.iter().any(|term| term.term_id == term_id)+ }+}++#[derive(PartialEq, Debug)]+struct Hit {+ position: u32,+ term_id: u32,+}++struct HitsIterator {+ positions: Vec<Vec<u32>>,+ cursors: Vec<usize>,+}++impl HitsIterator {+ fn new(positions: Vec<Vec<u32>>) -> Self {+ let cursors = positions.iter().map(|_| 0).collect();++ Self { positions, cursors }+ }+}++impl Iterator for HitsIterator {+ type Item = Hit;++ fn next(&mut self) -> Option<Self::Item> {+ let mut smallest_idx = None;++ for idx in 0..self.positions.len() {+ let cursor = self.cursors[idx];+ if cursor >= self.positions[idx].len() {+ continue;+ }++ smallest_idx = match smallest_idx {+ Some(s_idx) => {+ let tmp: &Vec<u32> = &self.positions[s_idx];+ if self.positions[idx][cursor] < tmp[self.cursors[s_idx]] {+ Some(idx)+ } else {+ smallest_idx+ }+ }+ None => Some(idx),+ };+ }++ smallest_idx.map(|idx| {+ let res = self.positions[idx][self.cursors[idx]];+ self.cursors[idx] += 1;++ Hit {+ position: res,+ term_id: idx as u32,+ }+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn hits_iterator() {+ let positions = vec![vec![0, 2, 6], vec![1, 4], vec![3, 5]];++ let mut iter = HitsIterator::new(positions);++ assert_eq!(+ iter.next(),+ Some(Hit {+ position: 0,+ term_id: 0,+ })+ );+ assert_eq!(+ iter.next(),+ Some(Hit {+ position: 1,+ term_id: 1,+ })+ );+ assert_eq!(+ iter.next(),+ Some(Hit {+ position: 2,+ term_id: 0,+ })+ );+ assert_eq!(+ iter.next(),+ Some(Hit {+ position: 3,+ term_id: 2,+ })+ );+ assert_eq!(+ iter.next(),+ Some(Hit {+ position: 4,+ term_id: 1,+ })+ );+ assert_eq!(+ iter.next(),+ Some(Hit {+ position: 5,+ term_id: 2,+ })+ );+ assert_eq!(+ iter.next(),+ Some(Hit {+ position: 6,+ term_id: 0,+ })+ );+ assert_eq!(iter.next(), None);+ }++ #[test]+ fn span_info() {+ let span = Span {+ terms: vec![+ SpanTermInfo {+ term_id: 1,+ position: 0,+ },+ SpanTermInfo {+ term_id: 2,+ position: 1,+ },+ SpanTermInfo {+ term_id: 3,+ position: 3,+ },+ ],+ };++ assert_eq!(span.width(), 4);+ assert_eq!(span.num_terms(), 3);+ assert!(span.has_term(1));+ assert!(span.has_term(2));+ assert!(span.has_term(3));+ assert!(!span.has_term(4));++ let span = Span {+ terms: vec![SpanTermInfo {+ term_id: 1,+ position: 0,+ }],+ };++ assert_eq!(span.width(), MAX_WIDTH);++ let span = Span {+ terms: vec![+ SpanTermInfo {+ term_id: 1,+ position: 0,+ },+ SpanTermInfo {+ term_id: 1,+ position: MAX_WIDTH + 10,+ },+ ],+ };++ assert_eq!(span.width(), MAX_WIDTH);+ }++ #[test]+ fn hits_to_span() {+ // example from paper+ // positions:+ // sea: 5, 29+ // thousand: 7, 10+ // years: 8, 11++ let positions = vec![vec![5, 29], vec![7, 10], vec![8, 11]];++ let spans: Vec<Span> = SpansIterator::new(positions).collect();++ assert_eq!(spans.len(), 3);+ assert_eq!(spans[0].width(), 4);+ assert_eq!(spans[1].width(), 2);+ assert_eq!(spans[2].width(), 10);++ assert_eq!(spans[0].num_terms(), 3);+ assert_eq!(spans[1].num_terms(), 2);+ assert_eq!(spans[2].num_terms(), 1);+ }+}diff --git a/src/query/scorer.rs b/src/query/term_scorer.rssimilarity index 88%rename from src/query/scorer.rsrename to src/query/term_scorer.rsindex fe60a8c7..e54ad09a 100644--- a/src/query/scorer.rs+++ b/src/query/term_scorer.rs@@ -21,19 +21,19 @@ use tantivy::DocSet;use tantivy::{DocId, Score};#[derive(Clone)]-pub struct Scorer {- postings: SegmentPostings,+pub struct TermScorerForField {+ pub postings: SegmentPostings,fieldnorm_reader: FieldNormReader,similarity_weight: Bm25Weight,}-impl Scorer {+impl TermScorerForField {pub fn new(postings: SegmentPostings,fieldnorm_reader: FieldNormReader,similarity_weight: Bm25Weight,- ) -> Scorer {- Scorer {+ ) -> TermScorerForField {+ TermScorerForField {postings,fieldnorm_reader,similarity_weight,@@ -49,7 +49,7 @@ impl Scorer {}}-impl DocSet for Scorer {+impl DocSet for TermScorerForField {fn advance(&mut self) -> DocId {self.postings.advance()}@@ -67,7 +67,7 @@ impl DocSet for Scorer {}}-impl tantivy::query::Scorer for Scorer {+impl tantivy::query::Scorer for TermScorerForField {fn score(&mut self) -> Score {let fieldnorm_id = self.fieldnorm_id();let term_freq = self.term_freq();diff --git a/src/query/union.rs b/src/query/union.rsdeleted file mode 100644index 3db21791..00000000--- a/src/query/union.rs+++ /dev/null@@ -1,99 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::query::Scorer;-use tantivy::{DocId, DocSet, Score, TERMINATED};--/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.-pub struct Union<TScorer> {- docsets: Vec<TScorer>,- score: Score,- doc: DocId,-}--impl<TScorer> From<Vec<TScorer>> for Union<TScorer>-where- TScorer: Scorer,-{- fn from(docsets: Vec<TScorer>) -> Union<TScorer> {- let non_empty_docsets: Vec<TScorer> = docsets- .into_iter()- .filter(|docset| docset.doc() != TERMINATED)- .collect();- let mut union = Union {- docsets: non_empty_docsets,- doc: 0,- score: 0.0,- };-- union.advance();- union- }-}--impl<TScorer> DocSet for Union<TScorer>-where- TScorer: Scorer,-{- fn advance(&mut self) -> DocId {- if let Some(min_doc) = self.docsets.iter().map(|scorer| scorer.doc()).min() {- self.doc = min_doc;-- if self.doc == TERMINATED {- return self.doc;- }-- self.score = self- .docsets- .iter_mut()- .filter(|scorer| scorer.doc() == self.doc)- .map(|scorer| scorer.score())- .sum();-- for scorer in self- .docsets- .iter_mut()- .filter(|scorer| scorer.doc() == min_doc)- {- scorer.advance();- }- } else {- self.doc = TERMINATED;- }-- self.doc- }-- fn doc(&self) -> DocId {- self.doc- }-- fn size_hint(&self) -> u32 {- self.docsets- .iter()- .map(|docset| docset.size_hint())- .max()- .unwrap_or(0u32)- }-}--impl<TScorer> Scorer for Union<TScorer>-where- TScorer: Scorer,-{- fn score(&mut self) -> Score {- self.score- }-}diff --git a/src/query/vec_docset.rs b/src/query/vec_docset.rsnew file mode 100644index 00000000..418c136d--- /dev/null+++ b/src/query/vec_docset.rs@@ -0,0 +1,79 @@+// taken from tantivy source for tests.+// source: https://github.com/quickwit-oss/tantivy/blob/main/src/query/vec_docset.rs+#![allow(dead_code)]++use tantivy::{DocId, DocSet, TERMINATED};++/// Simulate a `Postings` objects from a `VecPostings`.+/// `VecPostings` only exist for testing purposes.+///+/// Term frequencies always return 1.+/// No positions are returned.+pub struct VecDocSet {+ doc_ids: Vec<DocId>,+ cursor: usize,+}++impl From<Vec<DocId>> for VecDocSet {+ fn from(doc_ids: Vec<DocId>) -> VecDocSet {+ VecDocSet { doc_ids, cursor: 0 }+ }+}++impl DocSet for VecDocSet {+ fn advance(&mut self) -> DocId {+ self.cursor += 1;+ if self.cursor >= self.doc_ids.len() {+ self.cursor = self.doc_ids.len();+ return TERMINATED;+ }+ self.doc()+ }++ fn doc(&self) -> DocId {+ if self.cursor == self.doc_ids.len() {+ return TERMINATED;+ }+ self.doc_ids[self.cursor]+ }++ fn size_hint(&self) -> u32 {+ self.doc_ids.len() as u32+ }+}++#[cfg(test)]+pub mod tests {++ use super::*;++ #[test]+ pub fn test_vec_postings() {+ let doc_ids: Vec<DocId> = (0u32..1024u32).map(|e| e * 3).collect();+ let mut postings = VecDocSet::from(doc_ids);+ assert_eq!(postings.doc(), 0u32);+ assert_eq!(postings.advance(), 3u32);+ assert_eq!(postings.doc(), 3u32);+ assert_eq!(postings.seek(14u32), 15u32);+ assert_eq!(postings.doc(), 15u32);+ assert_eq!(postings.seek(300u32), 300u32);+ assert_eq!(postings.doc(), 300u32);+ assert_eq!(postings.seek(6000u32), TERMINATED);+ }++ #[test]+ pub fn test_fill_buffer() {+ let doc_ids: Vec<DocId> = (1u32..210u32).collect();+ let mut postings = VecDocSet::from(doc_ids);+ let mut buffer = vec![1000u32; 100];+ assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);+ for i in 0u32..100u32 {+ assert_eq!(buffer[i as usize], i + 1);+ }+ assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);+ for i in 0u32..100u32 {+ assert_eq!(buffer[i as usize], i + 101);+ }+ assert_eq!(postings.fill_buffer(&mut buffer[..]), 9);+ }+}diff --git a/src/query/weight.rs b/src/query/weight.rsindex ded3bff5..5165b491 100644--- a/src/query/weight.rs+++ b/src/query/weight.rs@@ -13,12 +13,14 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use super::scorer::Scorer;-use super::union::Union;++use super::field_union::FieldUnion;+use super::term_intersection::intersect_terms;+use super::term_scorer::TermScorerForField;use super::FieldData;use tantivy::fieldnorm::FieldNormReader;use tantivy::postings::SegmentPostings;-use tantivy::query::{intersect_scorers, Explanation};+use tantivy::query::Explanation;use tantivy::{DocId, Score, SegmentReader};pub struct Weight {@@ -33,10 +35,12 @@ impl tantivy::query::Weight for Weight {reader: &SegmentReader,boost: Score,) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {- let mut term_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();+ // each term has a union over the fields+ // e.g. if the term both occurs in the title and body+ let mut term_scorers: Vec<FieldUnion> = Vec::new();for term_text in &self.terms {- let mut field_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();+ let mut field_scorers: Vec<TermScorerForField> = Vec::new();for field_data in &self.fields {let inverted_index = reader.inverted_index(field_data.tantivy)?;@@ -80,18 +84,18 @@ impl tantivy::query::Weight for Weight {let posting = postings_opt.unwrap_or_else(SegmentPostings::empty);- field_scorers.push(Box::new(Scorer::new(+ field_scorers.push(TermScorerForField::new(posting,fieldnorm_reader,similarity_weight,- )));+ ));}}- term_scorers.push(Box::new(Union::from(field_scorers)));+ term_scorers.push(FieldUnion::from(field_scorers));}- Ok(intersect_scorers(term_scorers))+ Ok(intersect_terms(term_scorers))}fn explain(&self, _reader: &SegmentReader, _doc: DocId) -> tantivy::Result<Explanation> {