web search done right
[web-spell] correctly calculate probabilities in cases where the ngram wasn't seen during training
Mikkel Denker5ebac6fdd472
diff --git a/configs/api.toml b/configs/api.tomlindex a87c5bf0..d7432de2 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -17,3 +17,7 @@ stackoverflow = 0.0[widgets]thesaurus_paths = ["data/english-wordnet-2022-subset.ttl"]++[correction_config]+correction_threshold = 3.0+lm_prob_weight = 1.0diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 774391f7..3f07c6d0 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -211,5 +211,40 @@ mod tests {assert_eq!(result.webpages.len(), 2);assert_eq!(result.webpages[0].url, "https://www.third.com/");assert_eq!(result.webpages[1].url, "https://www.second.com/");++ let result = searcher+ .search(&SearchQuery {+ query: "test".to_string(),+ host_rankings: Some(HostRankings {+ liked: vec![],+ disliked: vec![],+ blocked: vec!["www.first.com".to_string()],+ }),+ return_ranking_signals: true,+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.third.com/");+ assert_eq!(result.webpages[1].url, "https://www.second.com/");++ let result = searcher+ .search(&SearchQuery {+ query: "test".to_string(),+ host_rankings: Some(HostRankings {+ liked: vec!["first.com".to_string()],+ disliked: vec![],+ blocked: vec!["abc.first.com".to_string()],+ }),+ return_ranking_signals: true,+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 3);+ assert_eq!(result.webpages[0].url, "https://www.first.com/");+ assert_eq!(result.webpages[1].url, "https://www.second.com/");+ assert_eq!(result.webpages[2].url, "https://www.third.com/");}}diff --git a/core/src/web_spell/error_model.rs b/core/src/web_spell/error_model.rsindex af3f6276..ad869ddc 100644--- a/core/src/web_spell/error_model.rs+++ b/core/src/web_spell/error_model.rs@@ -204,7 +204,7 @@ impl ErrorModel {pub fn log_prob(&self, error: &ErrorSequence) -> f64 {match self.errors.get(error) {Some(count) => (*count as f64).log2() - (self.total as f64).log2(),- None => 0.0,+ None => 0.0 - (self.total as f64).log2(),}}}diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rsindex 2e601f53..ec031054 100644--- a/core/src/web_spell/spell_checker.rs+++ b/core/src/web_spell/spell_checker.rs@@ -97,7 +97,7 @@ impl LangSpellChecker {let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;- let error_log_prob = if candidate.as_str() == term {+ let error_log_prob = if candidate.as_str() != term {match error_model::possible_errors(term, candidate) {Some(error_seq) => {(1.0 - self.config.misspelled_prob).log2()@@ -106,8 +106,9 @@ impl LangSpellChecker {None => 0.0,}} else {- self.config.misspelled_prob+ self.config.misspelled_prob.log2()};+ tracing::trace!(?candidate, ?scaled_lm_log_prob, ?error_log_prob);let score = scaled_lm_log_prob + error_log_prob;@@ -153,14 +154,15 @@ impl LangSpellChecker {let this_term_context_idx = this_term_context_idx.unwrap();let term_log_prob = self.lm_logprob(this_term_context_idx, &context);- let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob;+ let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob+ + ((1.0 - self.config.misspelled_prob).log2());tracing::debug!(?term, ?scaled_term_log_prob);if let Some((best_term, score)) =self.score_candidates(term, &candidates, context, this_term_context_idx){- let diff = score.abs() - scaled_term_log_prob.abs();+ let diff = score - scaled_term_log_prob;tracing::debug!(?best_term, ?score, ?diff);if diff > self.config.correction_threshold {corrections.push((i, best_term.clone()));diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rsindex 346fd8b2..5caf57c4 100644--- a/core/src/web_spell/stupid_backoff.rs+++ b/core/src/web_spell/stupid_backoff.rs@@ -291,13 +291,17 @@ impl StupidBackoff {pub fn log_prob<S: NextWordsStrategy>(&self, words: &[String], strat: S) -> f64 {if words.len() >= self.ngrams.len() || words.is_empty() {- return 0.0;+ return -(self.n_counts[0] as f64).log2();}+ let mut strat = strat;if let Some(freq) = self.freq(words) {- (freq as f64 / self.n_counts[words.len() - 1] as f64).log2()+ if let Some(next_freq) = self.freq(strat.inverse().next_words(words)) {+ (freq as f64).log2() - (next_freq as f64).log2()+ } else {+ (freq as f64).log2() - (self.n_counts[words.len() - 1] as f64).log2()+ }} else {- let mut strat = strat;DISCOUNT.log2() + self.log_prob(strat.next_words(words), strat)}}@@ -326,43 +330,66 @@ impl StupidBackoff {}}-pub trait NextWordsStrategy {+pub trait NextWordsStrategy: Sized {+ type Inv: NextWordsStrategy;+fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String];+ fn inverse(self) -> Self::Inv;}pub struct LeftToRight;impl NextWordsStrategy for LeftToRight {+ type Inv = RightToLeft;+fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {&words[1..]}++ fn inverse(self) -> Self::Inv {+ RightToLeft+ }}pub struct RightToLeft;impl NextWordsStrategy for RightToLeft {+ type Inv = LeftToRight;+fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {&words[..words.len() - 1]}++ fn inverse(self) -> Self::Inv {+ LeftToRight+ }}#[derive(Default)]pub struct IntoMiddle {- last_right: bool,+ last_left: bool,}impl NextWordsStrategy for IntoMiddle {+ type Inv = IntoMiddle;+fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {- let res = if self.last_right {- &words[..words.len() - 1]- } else {+ let res = if self.last_left {&words[1..]+ } else {+ &words[..words.len() - 1]};- self.last_right = !self.last_right;+ self.last_left = !self.last_left;res}++ fn inverse(self) -> Self::Inv {+ // TODO: this is not entirely correct.+ // to score b in [a, b, c] we should score [a, c] in the rotated ngrams.+ self+ }}#[cfg(test)]diff --git a/core/src/web_spell/term_freqs.rs b/core/src/web_spell/term_freqs.rsindex ed6b6562..a76e22e6 100644--- a/core/src/web_spell/term_freqs.rs+++ b/core/src/web_spell/term_freqs.rs@@ -242,6 +242,33 @@ impl TermDict {self.save_meta()?;self.stored.push(stored);+ self.gc()?;++ Ok(())+ }++ fn gc(&self) -> Result<()> {+ let all_dicts = self+ .path+ .read_dir()?+ .filter_map(|e| e.ok())+ .filter(|e| e.path().extension().unwrap_or_default() == "dict")+ .map(|e| e.path())+ .collect::<Vec<_>>();++ for dict in all_dicts {+ if !self.metadata.dicts.contains(+ &dict+ .file_stem()+ .expect("dict files should have a filename")+ .to_str()+ .expect("dict filenames are created from uuid `.to_string()`, so they should be valid utf8")+ .parse()+ .expect("dict filenames are created from uuid `.to_string()`, so they should be valid uuids"),+ ) {+ std::fs::remove_file(dict)?;+ }+ }Ok(())}@@ -250,6 +277,7 @@ impl TermDict {let file = OpenOptions::new().create(true).write(true)+ .truncate(true).open(self.path.join("meta.json"))?;serde_json::to_writer_pretty(file, &self.metadata)?;