Skip to content

web search done right

better correction scales

Mikkel Denkere82f755a94bc
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index 4471aac8..57c14eb4 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -154,7 +154,7 @@ impl Correction {
}
pub fn correction_threshold() -> f64 {
- 100.0 // logprob difference
+ 50.0 // logprob difference
}
pub fn lm_prob_weight() -> f64 {
diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rs
index 47de1f8d..7eb3c615 100644
--- a/core/src/web_spell/spell_checker.rs
+++ b/core/src/web_spell/spell_checker.rs
@@ -106,8 +106,9 @@ impl LangSpellChecker {
let strat = IntoMiddle::default();
self.language_model.log_prob(&context, strat)
};
+ let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob;
- tracing::debug!(?term, ?term_log_prob);
+ tracing::debug!(?term, ?scaled_term_log_prob);
let mut best_term: Option<(String, f64)> = None;
@@ -151,7 +152,7 @@ impl LangSpellChecker {
}
if let Some((best_term, score)) = best_term {
- let diff = score.abs() - term_log_prob.abs();
+ let diff = score.abs() - scaled_term_log_prob.abs();
tracing::debug!(?best_term, ?score, ?diff);
if diff > self.config.correction_threshold {
corrections.push((i, best_term.clone()));