web search done right
better correction scales
Mikkel Denkere82f755a94bc
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex 4471aac8..57c14eb4 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -154,7 +154,7 @@ impl Correction {}pub fn correction_threshold() -> f64 {- 100.0 // logprob difference+ 50.0 // logprob difference}pub fn lm_prob_weight() -> f64 {diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rsindex 47de1f8d..7eb3c615 100644--- a/core/src/web_spell/spell_checker.rs+++ b/core/src/web_spell/spell_checker.rs@@ -106,8 +106,9 @@ impl LangSpellChecker {let strat = IntoMiddle::default();self.language_model.log_prob(&context, strat)};+ let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob;- tracing::debug!(?term, ?term_log_prob);+ tracing::debug!(?term, ?scaled_term_log_prob);let mut best_term: Option<(String, f64)> = None;@@ -151,7 +152,7 @@ impl LangSpellChecker {}if let Some((best_term, score)) = best_term {- let diff = score.abs() - term_log_prob.abs();+ let diff = score.abs() - scaled_term_log_prob.abs();tracing::debug!(?best_term, ?score, ?diff);if diff > self.config.correction_threshold {corrections.push((i, best_term.clone()));