web search done right
bm25+ to not over penalize long documents
Mikkel Denker2fd6db3cfa8f
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 43b4a734..9bf44796 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -332,6 +332,10 @@ impl<S: DatumStream> JobExecutor<S> {200 => {self.save_datum(datum.clone()).await;+ if datum.url != url {+ // todo: we might want to save the redirected url+ }+match Html::parse(&datum.body, datum.url.as_str()) {Ok(html) => {let new_urls = self.new_urls(&html);diff --git a/crates/core/src/ranking/bm25.rs b/crates/core/src/ranking/bm25.rsindex 26e967ee..f1cede25 100644--- a/crates/core/src/ranking/bm25.rs+++ b/crates/core/src/ranking/bm25.rs@@ -8,6 +8,7 @@ use tantivy::{Score, Searcher, Term};const K1: Score = 1.2;const B: Score = 0.75;+const DELTA: Score = 1.0;pub(crate) fn idf(doc_freq: u64, doc_count: u64) -> Score {assert!(doc_count >= doc_freq, "{doc_count} >= {doc_freq}");@@ -135,7 +136,7 @@ impl Bm25Weight {}pub(crate) fn new(idf_explain: Explanation, average_fieldnorm: Score) -> Bm25Weight {- let weight = idf_explain.value() * (1.0 + K1);+ let weight = idf_explain.value();Bm25Weight {idf_explain,weight,@@ -146,14 +147,14 @@ impl Bm25Weight {#[inline]pub fn score(&self, fieldnorm_id: u8, term_freq: u32) -> Score {- self.weight * self.tf_factor(fieldnorm_id, term_freq)+ self.weight * (self.tf_factor(fieldnorm_id, term_freq) + DELTA)}#[inline]pub(crate) fn tf_factor(&self, fieldnorm_id: u8, term_freq: u32) -> Score {let term_freq = term_freq as Score;let norm = self.cache[fieldnorm_id as usize];- term_freq / (term_freq + norm)+ (term_freq * (K1 + 1.0)) / (term_freq + norm)}pub fn explain(&self, fieldnorm_id: u8, term_freq: u32) -> Explanation {