web search done right
better keyphrases
Mikkel Denker7e5d3aafd212
diff --git a/configs/api.toml b/configs/api.tomlindex 6d066302..dd6f3c00 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -1,11 +1,11 @@cluster_id = "dev_api"# crossencoder_model_path = "data/cross_encoder"+bangs_path = "data/bangs.json"gossip_addr = "0.0.0.0:3005"gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]host = "0.0.0.0:3000"-management_host = "0.0.0.0:3011"# lambda_model_path = "data/lambdamart.txt"-bangs_path = "data/bangs.json"+management_host = "0.0.0.0:3011"# dual_encoder_model_path = "data/dual_encoder"prometheus_host = "0.0.0.0:3001"top_phrases_for_autosuggest = 1_000diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rsindex afab83ee..2726ab3a 100644--- a/crates/core/src/inverted_index/key_phrase.rs+++ b/crates/core/src/inverted_index/key_phrase.rs@@ -30,7 +30,6 @@ const NON_ALPHABETIC_CHAR_THRESHOLD: f64 = 0.25;struct Scorer {word_freq: HashMap<String, u64>,word_docs: HashMap<String, u64>,- total_doc_freq: f64,num_docs: u64,word_freq_threshold: u64,}@@ -39,7 +38,6 @@ impl Scorer {fn new(searcher: &tantivy::Searcher, field: tantivy::schema::Field) -> Self {let mut word_freq = HashMap::new();let mut word_docs = HashMap::new();- let mut total_doc_freq = 0.0;let mut num_docs = 0;for seg_reader in searcher.segment_readers() {@@ -63,7 +61,6 @@ impl Scorer {}}- total_doc_freq += seg_reader.num_docs() as f64;num_docs += inv_index.terms().num_terms() as u64;}@@ -77,7 +74,6 @@ impl Scorer {Self {word_freq,word_docs,- total_doc_freq,num_docs,word_freq_threshold,}@@ -93,11 +89,6 @@ impl Scorer {&self.word_docs}- #[inline]- fn total_doc_freq(&self) -> f64 {- self.total_doc_freq- }-#[inline]fn num_docs(&self) -> u64 {self.num_docs@@ -108,7 +99,7 @@ impl Scorer {self.word_freq_threshold}- fn score(&self, words: &[&str], doc_freq: u32) -> f64 {+ fn score(&self, words: &[&str]) -> f64 {let word_freq_threshold = self.word_freq_threshold();let mut score = 0.0;let num_words = words.len();@@ -133,9 +124,7 @@ impl Scorer {score += tf * idf;}- let cf = doc_freq as f64 / (self.total_doc_freq() + 1.0);-- score * (1.0 - cf)+ score}}@@ -164,7 +153,7 @@ impl KeyPhrase {for seg_reader in searcher.segment_readers() {let inv_index = seg_reader.inverted_index(field).unwrap();let mut stream = inv_index.terms().stream().unwrap();- while let Some((term, info)) = stream.next() {+ while let Some((term, _)) = stream.next() {let term_str = str::from_utf8(term).unwrap().to_string();let num_chars = term_str.chars().count();@@ -182,7 +171,12 @@ impl KeyPhrase {}let words = term_str.split_whitespace().collect::<Vec<_>>();- let score = scorer.score(&words, info.doc_freq);++ if words.is_empty() {+ continue;+ }++ let score = words.len().min(5) as f64 * scorer.score(&words);if score.is_normal() {let term_str = words.join(" ");diff --git a/crates/core/src/rake.rs b/crates/core/src/rake.rsindex 588a5aec..654df801 100644--- a/crates/core/src/rake.rs+++ b/crates/core/src/rake.rs@@ -31,7 +31,7 @@ use crate::stopwords;fn is_sent_split(c: char) -> bool {matches!(c,- '.' | '!' | '?' | '\n' | '\r' | '\t' | '\u{2026}' | '\u{2025}' | '\u{2024}'+ ',' | '.' | '!' | '?' | '\n' | '\r' | '\t' | '\u{2026}' | '\u{2025}' | '\u{2024}')}