Skip to content

web search done right

make bm25 constants configurable for each field

Mikkel Denker3844e29bd11a
diff --git a/crates/core/src/ranking/bm25.rs b/crates/core/src/ranking/bm25.rs
index 7b40caef..853b4329 100644
--- a/crates/core/src/ranking/bm25.rs
+++ b/crates/core/src/ranking/bm25.rs
@@ -5,8 +5,20 @@ use itertools::Itertools;
use tantivy::fieldnorm::FieldNormReader;
use tantivy::{Score, Searcher, Term};
-pub const K1: Score = 1.2;
-pub const B: Score = 0.75;
+const K1: Score = 1.2;
+const B: Score = 0.75;
+
+#[derive(Clone, Copy)]
+pub struct Bm25Constants {
+ pub k1: Score,
+ pub b: Score,
+}
+
+impl Default for Bm25Constants {
+ fn default() -> Self {
+ Bm25Constants { k1: K1, b: B }
+ }
+}
pub fn idf(doc_freq: u64, doc_count: u64) -> Score {
assert!(doc_count >= doc_freq, "{doc_count} >= {doc_freq}");
@@ -14,15 +26,19 @@ pub fn idf(doc_freq: u64, doc_count: u64) -> Score {
(1.0 + x).ln()
}
-fn cached_tf_component(fieldnorm: u32, average_fieldnorm: Score) -> Score {
- K1 * (1.0 - B + B * fieldnorm as Score / average_fieldnorm)
+fn cached_tf_component(
+ fieldnorm: u32,
+ average_fieldnorm: Score,
+ constants: Bm25Constants,
+) -> Score {
+ constants.k1 * (1.0 - constants.b + constants.b * fieldnorm as Score / average_fieldnorm)
}
-pub fn compute_tf_cache(average_fieldnorm: Score) -> [Score; 256] {
+pub fn compute_tf_cache(average_fieldnorm: Score, constants: Bm25Constants) -> [Score; 256] {
let mut cache: [Score; 256] = [0.0; 256];
for (fieldnorm_id, cache_mut) in cache.iter_mut().enumerate() {
let fieldnorm = FieldNormReader::id_to_fieldnorm(fieldnorm_id as u8);
- *cache_mut = cached_tf_component(fieldnorm, average_fieldnorm);
+ *cache_mut = cached_tf_component(fieldnorm, average_fieldnorm, constants);
}
cache
}
@@ -33,7 +49,11 @@ pub struct MultiBm25Weight {
}
impl MultiBm25Weight {
- pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> tantivy::Result<Self> {
+ pub fn for_terms(
+ searcher: &Searcher,
+ terms: &[Term],
+ constants: Bm25Constants,
+ ) -> tantivy::Result<Self> {
if terms.is_empty() {
return Ok(Self {
weights: Vec::new(),
@@ -66,6 +86,7 @@ impl MultiBm25Weight {
term_doc_freq,
total_num_docs,
average_fieldnorm,
+ constants,
));
}
@@ -88,6 +109,7 @@ impl MultiBm25Weight {
#[derive(Clone)]
pub struct Bm25Weight {
weight: Score,
+ constants: Bm25Constants,
cache: [Score; 256],
}
@@ -96,15 +118,17 @@ impl Bm25Weight {
term_doc_freq: u64,
total_num_docs: u64,
avg_fieldnorm: Score,
+ constants: Bm25Constants,
) -> Bm25Weight {
let idf = idf(term_doc_freq, total_num_docs);
- Bm25Weight::new(idf, avg_fieldnorm)
+ Bm25Weight::new(idf, avg_fieldnorm, constants)
}
- pub fn new(weight: Score, average_fieldnorm: Score) -> Bm25Weight {
+ pub fn new(weight: Score, average_fieldnorm: Score, constants: Bm25Constants) -> Bm25Weight {
Bm25Weight {
weight,
- cache: compute_tf_cache(average_fieldnorm),
+ cache: compute_tf_cache(average_fieldnorm, constants),
+ constants,
}
}
@@ -117,7 +141,7 @@ impl Bm25Weight {
pub fn tf_factor(&self, fieldnorm_id: u8, term_freq: u32) -> Score {
let term_freq = term_freq as Score;
let norm = self.cache[fieldnorm_id as usize];
- (term_freq * (K1 + 1.0)) / (term_freq + norm)
+ (term_freq * (self.constants.k1 + 1.0)) / (term_freq + norm)
}
}
@@ -130,10 +154,11 @@ mod tests {
// assume the query is something like 'the end'
// 'the' appears in almost all docs (98)
// 'end' appears in a smalle subset (20)
+ let constants = Bm25Constants::default();
let weight = MultiBm25Weight {
weights: vec![
- Bm25Weight::for_one_term(98, 100, 1.0),
- Bm25Weight::for_one_term(20, 100, 1.0),
+ Bm25Weight::for_one_term(98, 100, 1.0, constants),
+ Bm25Weight::for_one_term(20, 100, 1.0, constants),
],
};
diff --git a/crates/core/src/ranking/bm25f.rs b/crates/core/src/ranking/bm25f.rs
index c9095062..ba5b549b 100644
--- a/crates/core/src/ranking/bm25f.rs
+++ b/crates/core/src/ranking/bm25f.rs
@@ -17,7 +17,7 @@
use itertools::Itertools;
use tantivy::{Score, Searcher, Term};
-use super::bm25::{compute_tf_cache, idf, K1};
+use super::bm25::{compute_tf_cache, idf, Bm25Constants};
/// A BM25F weight that uses the same IDF weight for all fields.
/// The idea is that the term 'the' might not appear very frequently e.g. in the title field,
@@ -39,7 +39,7 @@ pub struct MultiBm25FWeight {
}
impl MultiBm25FWeight {
- pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> Self {
+ pub fn for_terms(searcher: &Searcher, terms: &[Term], constants: Bm25Constants) -> Self {
if terms.is_empty() {
return Self {
weights: Vec::new(),
@@ -84,6 +84,7 @@ impl MultiBm25FWeight {
term_doc_freq,
total_num_docs,
average_fieldnorm,
+ constants,
));
}
@@ -105,6 +106,7 @@ impl MultiBm25FWeight {
pub struct Bm25FWeight {
weight: Score,
cache: [Score; 256],
+ constants: Bm25Constants,
}
impl Bm25FWeight {
@@ -112,15 +114,17 @@ impl Bm25FWeight {
term_doc_freq: u64,
total_num_docs: u64,
avg_fieldnorm: Score,
+ constants: Bm25Constants,
) -> Bm25FWeight {
let idf = idf(term_doc_freq, total_num_docs);
- Bm25FWeight::new(idf, avg_fieldnorm)
+ Bm25FWeight::new(idf, avg_fieldnorm, constants)
}
- pub fn new(weight: Score, average_fieldnorm: Score) -> Bm25FWeight {
+ pub fn new(weight: Score, average_fieldnorm: Score, constants: Bm25Constants) -> Bm25FWeight {
Bm25FWeight {
weight,
- cache: compute_tf_cache(average_fieldnorm),
+ cache: compute_tf_cache(average_fieldnorm, constants),
+ constants,
}
}
@@ -133,6 +137,6 @@ impl Bm25FWeight {
pub fn tf_factor(&self, coefficient: Score, fieldnorm_id: u8, term_freq: u32) -> Score {
let term_freq = term_freq as Score * coefficient;
let norm = self.cache[fieldnorm_id as usize];
- (term_freq * (K1 + 1.0)) / (term_freq + norm)
+ (term_freq * (self.constants.k1 + 1.0)) / (term_freq + norm)
}
}
diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rs
index 1f2f06e1..406a1dfd 100644
--- a/crates/core/src/ranking/signal/computer/mod.rs
+++ b/crates/core/src/ranking/signal/computer/mod.rs
@@ -289,8 +289,16 @@ impl SignalComputer {
matching_terms.push(term.clone());
}
}
- let bm25 = MultiBm25Weight::for_terms(tv_searcher, &matching_terms)?;
- let bm25f = MultiBm25FWeight::for_terms(tv_searcher, &matching_terms);
+ let bm25 = MultiBm25Weight::for_terms(
+ tv_searcher,
+ &matching_terms,
+ text_field.bm25_constants(),
+ )?;
+ let bm25f = MultiBm25FWeight::for_terms(
+ tv_searcher,
+ &matching_terms,
+ text_field.bm25_constants(),
+ );
text_fields.insert(
text_field,
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 7f20abc7..9ccb61ec 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -28,6 +28,7 @@ use whatlang::Lang;
use crate::{
enum_dispatch_from_discriminant,
enum_map::InsertEnumMapKey,
+ ranking::bm25::Bm25Constants,
tokenizer::{
self, BigramTokenizer, Identity, JsonField, Tokenizer, TrigramTokenizer, UrlTokenizer,
},
@@ -140,6 +141,10 @@ pub trait TextField:
fn tantivy_field(&self, schema: &tantivy::schema::Schema) -> Option<tantivy::schema::Field> {
schema.get_field(self.name()).ok()
}
+
+ fn bm25_constants(&self) -> Bm25Constants {
+ Bm25Constants::default()
+ }
}
#[enum_dispatch(TextField)]