Skip to content

web search done right

term proximity ranking

Mikkel Denker822213c10e33
diff --git a/src/index.rs b/src/index.rs
index 1ed894f1..3a08e12a 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -710,4 +710,79 @@ mod tests {
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
+
+ #[test]
+ fn proximity_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("termA termB", index.schema()).expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} termA termB d d d d d d d d d
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.first.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} termA d d d d d d d d d termB
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.third.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} termA d d d d termB d d d d d
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.second.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 3);
+ assert_eq!(result.documents.len(), 3);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+ assert_eq!(result.documents[1].url, "https://www.second.com");
+ assert_eq!(result.documents[2].url, "https://www.third.com");
+ }
}
diff --git a/src/query/bm25.rs b/src/query/bm25.rs
index d880b776..a88ab939 100644
--- a/src/query/bm25.rs
+++ b/src/query/bm25.rs
@@ -130,13 +130,12 @@ impl Bm25Weight {
}
#[inline]
- pub fn score(&self, fieldnorm_id: u8, term_freq: u32) -> Score {
+ pub fn score(&self, fieldnorm_id: u8, term_freq: Score) -> Score {
self.weight * self.tf_factor(fieldnorm_id, term_freq)
}
#[inline]
- pub(crate) fn tf_factor(&self, fieldnorm_id: u8, term_freq: u32) -> Score {
- let term_freq = term_freq as Score;
+ pub(crate) fn tf_factor(&self, fieldnorm_id: u8, term_freq: Score) -> Score {
let norm = self.cache[fieldnorm_id as usize];
term_freq / (term_freq + norm)
}
diff --git a/src/query/term_intersection.rs b/src/query/term_intersection.rs
index 92d03ec1..b312535c 100644
--- a/src/query/term_intersection.rs
+++ b/src/query/term_intersection.rs
@@ -13,14 +13,16 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
use itertools::Itertools;
use tantivy::query::{EmptyScorer, Scorer};
use tantivy::{DocId, DocSet, Postings, Score, TERMINATED};
use super::field_union::FieldUnion;
-use super::term_scorer;
-const MAX_WIDTH: u32 = 10;
+const MAX_WIDTH: u32 = 45;
+const LAMBDA: f32 = 0.55;
+const GAMMA: f32 = 0.25;
pub fn intersect_terms(mut scorers: Vec<FieldUnion>) -> Box<dyn Scorer> {
if scorers.is_empty() {
@@ -105,7 +107,7 @@ impl Scorer for TermIntersection {
docset
.docsets
.iter_mut()
- .map(|docset| &mut docset.postings)
+ .map(|docset| (&mut docset.postings, &docset.similarity_weight))
.collect_vec()
})
.collect_vec();
@@ -131,8 +133,11 @@ impl Scorer for TermIntersection {
fields
.into_iter()
- .map(|terms| {
- for (i, term) in terms.into_iter().enumerate() {
+ .enumerate()
+ .map(|(field_id, terms)| {
+ let mut weights = Vec::new();
+ for (i, (term, weight)) in terms.into_iter().enumerate() {
+ weights.push(weight);
if term.doc() == TERMINATED {
for pos in &mut positions[i..] {
if !pos.is_empty() {
@@ -150,29 +155,48 @@ impl Scorer for TermIntersection {
return 0.0;
}
- 1.0
+ let spans: Vec<Span> = SpansIterator::new(positions.clone()).collect();
+
+ positions
+ .iter()
+ .zip_eq(weights.iter())
+ .enumerate()
+ .map(|(term_id, (_pos, weight))| {
+ let rc = spans
+ .iter()
+ .filter(|span| span.has_term(term_id as u32))
+ .map(|span| span.relevance_contribution())
+ .sum::<f32>();
+
+ weight.score(field_id.try_into().unwrap(), rc)
+ })
+ .sum::<Score>()
})
- .sum::<Score>();
-
- self.docsets.iter_mut().map(Scorer::score).sum::<Score>()
+ .sum::<Score>()
}
}
struct SpansIterator {
num_terms: usize,
hits: HitsIterator,
+ max_width: u32,
current_hit: Option<Hit>,
spillover_hit: Option<Hit>,
}
impl SpansIterator {
pub fn new(positions: Vec<Vec<u32>>) -> Self {
+ Self::new_with_max_width(positions, MAX_WIDTH)
+ }
+
+ fn new_with_max_width(positions: Vec<Vec<u32>>, max_width: u32) -> Self {
let num_terms = positions.len();
let hits = HitsIterator::new(positions);
Self {
hits,
num_terms,
+ max_width,
current_hit: None,
spillover_hit: None,
}
@@ -183,7 +207,10 @@ impl Iterator for SpansIterator {
type Item = Span;
fn next(&mut self) -> Option<Self::Item> {
- let mut current_span = Span::default();
+ let mut current_span = Span {
+ max_width: self.max_width,
+ ..Default::default()
+ };
let mut term_seen_before = vec![None; self.num_terms];
if let Some(spillover) = self.spillover_hit.take() {
@@ -213,7 +240,7 @@ impl Iterator for SpansIterator {
term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);
let dist = next_hit.position - current_hit.position + 1;
- if dist > MAX_WIDTH || current_hit.term_id == next_hit.term_id {
+ if dist > self.max_width || current_hit.term_id == next_hit.term_id {
// case 1 and 2 from paper
current_span.terms.push(SpanTermInfo {
term_id: current_hit.term_id,
@@ -259,18 +286,28 @@ struct SpanTermInfo {
position: u32,
}
-#[derive(Default, Debug)]
+#[derive(Debug)]
struct Span {
+ max_width: u32,
terms: Vec<SpanTermInfo>,
}
+impl Default for Span {
+ fn default() -> Self {
+ Self {
+ max_width: MAX_WIDTH,
+ terms: Default::default(),
+ }
+ }
+}
+
impl Span {
fn width(&self) -> u32 {
match (self.terms.first(), self.terms.last()) {
(Some(first), Some(last)) if first.term_id != last.term_id => {
- (last.position - first.position + 1).min(MAX_WIDTH)
+ (last.position - first.position + 1).min(self.max_width)
}
- _ => MAX_WIDTH,
+ _ => self.max_width,
}
}
@@ -281,6 +318,10 @@ impl Span {
fn has_term(&self, term_id: u32) -> bool {
self.terms.iter().any(|term| term.term_id == term_id)
}
+
+ fn relevance_contribution(&self) -> f32 {
+ (self.num_terms() as f32).powf(LAMBDA) / (self.width() as f32).powf(GAMMA)
+ }
}
#[derive(PartialEq, Debug)]
@@ -418,6 +459,7 @@ mod tests {
position: 3,
},
],
+ ..Default::default()
};
assert_eq!(span.width(), 4);
@@ -432,6 +474,7 @@ mod tests {
term_id: 1,
position: 0,
}],
+ ..Default::default()
};
assert_eq!(span.width(), MAX_WIDTH);
@@ -447,6 +490,7 @@ mod tests {
position: MAX_WIDTH + 10,
},
],
+ ..Default::default()
};
assert_eq!(span.width(), MAX_WIDTH);
@@ -462,7 +506,7 @@ mod tests {
let positions = vec![vec![5, 29], vec![7, 10], vec![8, 11]];
- let spans: Vec<Span> = SpansIterator::new(positions).collect();
+ let spans: Vec<Span> = SpansIterator::new_with_max_width(positions, 10).collect();
assert_eq!(spans.len(), 3);
assert_eq!(spans[0].width(), 4);
diff --git a/src/query/term_scorer.rs b/src/query/term_scorer.rs
index e54ad09a..a2e2dd44 100644
--- a/src/query/term_scorer.rs
+++ b/src/query/term_scorer.rs
@@ -24,7 +24,7 @@ use tantivy::{DocId, Score};
pub struct TermScorerForField {
pub postings: SegmentPostings,
fieldnorm_reader: FieldNormReader,
- similarity_weight: Bm25Weight,
+ pub similarity_weight: Bm25Weight,
}
impl TermScorerForField {
@@ -71,6 +71,7 @@ impl tantivy::query::Scorer for TermScorerForField {
fn score(&mut self) -> Score {
let fieldnorm_id = self.fieldnorm_id();
let term_freq = self.term_freq();
- self.similarity_weight.score(fieldnorm_id, term_freq)
+ self.similarity_weight
+ .score(fieldnorm_id, term_freq as Score)
}
}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 6793beea..83359216 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -62,6 +62,6 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;
let is_homepage = self.is_homepage_reader.get(doc) as f64;
- score + 1_000.0 * centrality + 5.0 * is_homepage
+ score + 400.0 * centrality + 5.0 * is_homepage
}
}