Skip to content

web search done right

[web-spell] correctly calculate probabilities in cases where the ngram wasn't seen during training

Mikkel Denker5ebac6fdd472
diff --git a/configs/api.toml b/configs/api.toml
index a87c5bf0..d7432de2 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -17,3 +17,7 @@ stackoverflow = 0.0
[widgets]
thesaurus_paths = ["data/english-wordnet-2022-subset.ttl"]
+
+[correction_config]
+correction_threshold = 3.0
+lm_prob_weight = 1.0
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 774391f7..3f07c6d0 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -211,5 +211,40 @@ mod tests {
assert_eq!(result.webpages.len(), 2);
assert_eq!(result.webpages[0].url, "https://www.third.com/");
assert_eq!(result.webpages[1].url, "https://www.second.com/");
+
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test".to_string(),
+ host_rankings: Some(HostRankings {
+ liked: vec![],
+ disliked: vec![],
+ blocked: vec!["www.first.com".to_string()],
+ }),
+ return_ranking_signals: true,
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.third.com/");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/");
+
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test".to_string(),
+ host_rankings: Some(HostRankings {
+ liked: vec!["first.com".to_string()],
+ disliked: vec![],
+ blocked: vec!["abc.first.com".to_string()],
+ }),
+ return_ranking_signals: true,
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages.len(), 3);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/");
+ assert_eq!(result.webpages[2].url, "https://www.third.com/");
}
}
diff --git a/core/src/web_spell/error_model.rs b/core/src/web_spell/error_model.rs
index af3f6276..ad869ddc 100644
--- a/core/src/web_spell/error_model.rs
+++ b/core/src/web_spell/error_model.rs
@@ -204,7 +204,7 @@ impl ErrorModel {
pub fn log_prob(&self, error: &ErrorSequence) -> f64 {
match self.errors.get(error) {
Some(count) => (*count as f64).log2() - (self.total as f64).log2(),
- None => 0.0,
+ None => 0.0 - (self.total as f64).log2(),
}
}
}
diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rs
index 2e601f53..ec031054 100644
--- a/core/src/web_spell/spell_checker.rs
+++ b/core/src/web_spell/spell_checker.rs
@@ -97,7 +97,7 @@ impl LangSpellChecker {
let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;
- let error_log_prob = if candidate.as_str() == term {
+ let error_log_prob = if candidate.as_str() != term {
match error_model::possible_errors(term, candidate) {
Some(error_seq) => {
(1.0 - self.config.misspelled_prob).log2()
@@ -106,8 +106,9 @@ impl LangSpellChecker {
None => 0.0,
}
} else {
- self.config.misspelled_prob
+ self.config.misspelled_prob.log2()
};
+ tracing::trace!(?candidate, ?scaled_lm_log_prob, ?error_log_prob);
let score = scaled_lm_log_prob + error_log_prob;
@@ -153,14 +154,15 @@ impl LangSpellChecker {
let this_term_context_idx = this_term_context_idx.unwrap();
let term_log_prob = self.lm_logprob(this_term_context_idx, &context);
- let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob;
+ let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob
+ + ((1.0 - self.config.misspelled_prob).log2());
tracing::debug!(?term, ?scaled_term_log_prob);
if let Some((best_term, score)) =
self.score_candidates(term, &candidates, context, this_term_context_idx)
{
- let diff = score.abs() - scaled_term_log_prob.abs();
+ let diff = score - scaled_term_log_prob;
tracing::debug!(?best_term, ?score, ?diff);
if diff > self.config.correction_threshold {
corrections.push((i, best_term.clone()));
diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rs
index 346fd8b2..5caf57c4 100644
--- a/core/src/web_spell/stupid_backoff.rs
+++ b/core/src/web_spell/stupid_backoff.rs
@@ -291,13 +291,17 @@ impl StupidBackoff {
pub fn log_prob<S: NextWordsStrategy>(&self, words: &[String], strat: S) -> f64 {
if words.len() >= self.ngrams.len() || words.is_empty() {
- return 0.0;
+ return -(self.n_counts[0] as f64).log2();
}
+ let mut strat = strat;
if let Some(freq) = self.freq(words) {
- (freq as f64 / self.n_counts[words.len() - 1] as f64).log2()
+ if let Some(next_freq) = self.freq(strat.inverse().next_words(words)) {
+ (freq as f64).log2() - (next_freq as f64).log2()
+ } else {
+ (freq as f64).log2() - (self.n_counts[words.len() - 1] as f64).log2()
+ }
} else {
- let mut strat = strat;
DISCOUNT.log2() + self.log_prob(strat.next_words(words), strat)
}
}
@@ -326,43 +330,66 @@ impl StupidBackoff {
}
}
-pub trait NextWordsStrategy {
+pub trait NextWordsStrategy: Sized {
+ type Inv: NextWordsStrategy;
+
fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String];
+ fn inverse(self) -> Self::Inv;
}
pub struct LeftToRight;
impl NextWordsStrategy for LeftToRight {
+ type Inv = RightToLeft;
+
fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {
&words[1..]
}
+
+ fn inverse(self) -> Self::Inv {
+ RightToLeft
+ }
}
pub struct RightToLeft;
impl NextWordsStrategy for RightToLeft {
+ type Inv = LeftToRight;
+
fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {
&words[..words.len() - 1]
}
+
+ fn inverse(self) -> Self::Inv {
+ LeftToRight
+ }
}
#[derive(Default)]
pub struct IntoMiddle {
- last_right: bool,
+ last_left: bool,
}
impl NextWordsStrategy for IntoMiddle {
+ type Inv = IntoMiddle;
+
fn next_words<'a>(&mut self, words: &'a [String]) -> &'a [String] {
- let res = if self.last_right {
- &words[..words.len() - 1]
- } else {
+ let res = if self.last_left {
&words[1..]
+ } else {
+ &words[..words.len() - 1]
};
- self.last_right = !self.last_right;
+ self.last_left = !self.last_left;
res
}
+
+ fn inverse(self) -> Self::Inv {
+ // TODO: this is not entirely correct.
+ // to score b in [a, b, c] we should score [a, c] in the rotated ngrams.
+ self
+ }
}
#[cfg(test)]
diff --git a/core/src/web_spell/term_freqs.rs b/core/src/web_spell/term_freqs.rs
index ed6b6562..a76e22e6 100644
--- a/core/src/web_spell/term_freqs.rs
+++ b/core/src/web_spell/term_freqs.rs
@@ -242,6 +242,33 @@ impl TermDict {
self.save_meta()?;
self.stored.push(stored);
+ self.gc()?;
+
+ Ok(())
+ }
+
+ fn gc(&self) -> Result<()> {
+ let all_dicts = self
+ .path
+ .read_dir()?
+ .filter_map(|e| e.ok())
+ .filter(|e| e.path().extension().unwrap_or_default() == "dict")
+ .map(|e| e.path())
+ .collect::<Vec<_>>();
+
+ for dict in all_dicts {
+ if !self.metadata.dicts.contains(
+ &dict
+ .file_stem()
+ .expect("dict files should have a filename")
+ .to_str()
+ .expect("dict filenames are created from uuid `.to_string()`, so they should be valid utf8")
+ .parse()
+ .expect("dict filenames are created from uuid `.to_string()`, so they should be valid uuids"),
+ ) {
+ std::fs::remove_file(dict)?;
+ }
+ }
Ok(())
}
@@ -250,6 +277,7 @@ impl TermDict {
let file = OpenOptions::new()
.create(true)
.write(true)
+ .truncate(true)
.open(self.path.join("meta.json"))?;
serde_json::to_writer_pretty(file, &self.metadata)?;