Skip to content

web search done right

better keyphrases

Mikkel Denker7e5d3aafd212
diff --git a/configs/api.toml b/configs/api.toml
index 6d066302..dd6f3c00 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -1,11 +1,11 @@
cluster_id = "dev_api"
# crossencoder_model_path = "data/cross_encoder"
+bangs_path = "data/bangs.json"
gossip_addr = "0.0.0.0:3005"
gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]
host = "0.0.0.0:3000"
-management_host = "0.0.0.0:3011"
# lambda_model_path = "data/lambdamart.txt"
-bangs_path = "data/bangs.json"
+management_host = "0.0.0.0:3011"
# dual_encoder_model_path = "data/dual_encoder"
prometheus_host = "0.0.0.0:3001"
top_phrases_for_autosuggest = 1_000
diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rs
index afab83ee..2726ab3a 100644
--- a/crates/core/src/inverted_index/key_phrase.rs
+++ b/crates/core/src/inverted_index/key_phrase.rs
@@ -30,7 +30,6 @@ const NON_ALPHABETIC_CHAR_THRESHOLD: f64 = 0.25;
struct Scorer {
word_freq: HashMap<String, u64>,
word_docs: HashMap<String, u64>,
- total_doc_freq: f64,
num_docs: u64,
word_freq_threshold: u64,
}
@@ -39,7 +38,6 @@ impl Scorer {
fn new(searcher: &tantivy::Searcher, field: tantivy::schema::Field) -> Self {
let mut word_freq = HashMap::new();
let mut word_docs = HashMap::new();
- let mut total_doc_freq = 0.0;
let mut num_docs = 0;
for seg_reader in searcher.segment_readers() {
@@ -63,7 +61,6 @@ impl Scorer {
}
}
- total_doc_freq += seg_reader.num_docs() as f64;
num_docs += inv_index.terms().num_terms() as u64;
}
@@ -77,7 +74,6 @@ impl Scorer {
Self {
word_freq,
word_docs,
- total_doc_freq,
num_docs,
word_freq_threshold,
}
@@ -93,11 +89,6 @@ impl Scorer {
&self.word_docs
}
- #[inline]
- fn total_doc_freq(&self) -> f64 {
- self.total_doc_freq
- }
-
#[inline]
fn num_docs(&self) -> u64 {
self.num_docs
@@ -108,7 +99,7 @@ impl Scorer {
self.word_freq_threshold
}
- fn score(&self, words: &[&str], doc_freq: u32) -> f64 {
+ fn score(&self, words: &[&str]) -> f64 {
let word_freq_threshold = self.word_freq_threshold();
let mut score = 0.0;
let num_words = words.len();
@@ -133,9 +124,7 @@ impl Scorer {
score += tf * idf;
}
- let cf = doc_freq as f64 / (self.total_doc_freq() + 1.0);
-
- score * (1.0 - cf)
+ score
}
}
@@ -164,7 +153,7 @@ impl KeyPhrase {
for seg_reader in searcher.segment_readers() {
let inv_index = seg_reader.inverted_index(field).unwrap();
let mut stream = inv_index.terms().stream().unwrap();
- while let Some((term, info)) = stream.next() {
+ while let Some((term, _)) = stream.next() {
let term_str = str::from_utf8(term).unwrap().to_string();
let num_chars = term_str.chars().count();
@@ -182,7 +171,12 @@ impl KeyPhrase {
}
let words = term_str.split_whitespace().collect::<Vec<_>>();
- let score = scorer.score(&words, info.doc_freq);
+
+ if words.is_empty() {
+ continue;
+ }
+
+ let score = words.len().min(5) as f64 * scorer.score(&words);
if score.is_normal() {
let term_str = words.join(" ");
diff --git a/crates/core/src/rake.rs b/crates/core/src/rake.rs
index 588a5aec..654df801 100644
--- a/crates/core/src/rake.rs
+++ b/crates/core/src/rake.rs
@@ -31,7 +31,7 @@ use crate::stopwords;
fn is_sent_split(c: char) -> bool {
matches!(
c,
- '.' | '!' | '?' | '\n' | '\r' | '\t' | '\u{2026}' | '\u{2025}' | '\u{2024}'
+ ',' | '.' | '!' | '?' | '\n' | '\r' | '\t' | '\u{2026}' | '\u{2025}' | '\u{2024}'
)
}