web search done right
fixed bug in stupid_backoff model that caused last n_gram count to always be 0
Mikkel Denker1a6d8ff6bed9
diff --git a/configs/api.toml b/configs/api.tomlindex 5ebd9acd..27905bf7 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -4,10 +4,10 @@ gossip_addr = "0.0.0.0:3005"gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]host = "0.0.0.0:3000"# lambda_model_path = "data/lambdamart.txt"+bangs_path = "data/bangs.json"dual_encoder_model_path = "data/summarizer/dual_encoder"prometheus_host = "0.0.0.0:3001"queries_csv_path = "data/queries_us.csv"-bangs_path = "data/bangs.json"summarizer_path = "data/summarizer"[spell_check]@@ -17,7 +17,6 @@ path = "data/web_spell/checker"correction_threshold = 3.0lm_prob_weight = 1.0-[thresholds]entity_sidebar = 0.0stackoverflow = 0.0diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 0a00e15c..fa88e73e 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -20,7 +20,6 @@ use std::fs;use std::path::Path;use stract::config;use stract::entrypoint::autosuggest_scrape::{self, Gl};-use tracing::level_filters::LevelFilter;#[cfg(feature = "dev")]use stract::entrypoint::configure;@@ -262,9 +261,8 @@ fn main() -> Result<()> {.with_max_level(tracing::Level::INFO).with_env_filter(tracing_subscriber::EnvFilter::builder()- .with_default_directive(LevelFilter::OFF.into())- .from_env_lossy()- .add_directive("stract=info".parse().unwrap()),+ .with_default_directive("stract=info".parse().unwrap())+ .from_env_lossy(),).without_time().with_target(false)diff --git a/crates/core/src/web_spell/spell_checker.rs b/crates/core/src/web_spell/spell_checker.rsindex b63289dd..e7ac0b32 100644--- a/crates/core/src/web_spell/spell_checker.rs+++ b/crates/core/src/web_spell/spell_checker.rs@@ -157,14 +157,14 @@ impl LangSpellChecker {let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob+ ((1.0 - self.config.misspelled_prob).log2());- tracing::debug!(?term, ?scaled_term_log_prob);+ tracing::debug!(?term, ?term_log_prob, ?scaled_term_log_prob);if let Some((best_term, score)) =self.score_candidates(term, &candidates, context, this_term_context_idx){let diff = score - scaled_term_log_prob;tracing::debug!(?best_term, ?score, ?diff);- if diff > self.config.correction_threshold {+ if diff.is_finite() && diff > self.config.correction_threshold {corrections.push((i, best_term.clone()));terms[i] = best_term; // make sure the next terms use the corrected context}diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rsindex 998e2f46..f50bf459 100644--- a/crates/core/src/web_spell/stupid_backoff.rs+++ b/crates/core/src/web_spell/stupid_backoff.rs@@ -79,7 +79,7 @@ impl StupidBackoffTrainer {pub fn train(&mut self, tokens: &[String]) {for window in tokens.windows(self.max_ngram_size) {- for i in 1..window.len() {+ for i in 1..=window.len() {let ngram = Ngram {terms: window[..i].to_vec(),};@@ -433,5 +433,43 @@ mod tests {model.contexts("b"),vec![(vec!["a".to_string(), "b".to_string(), "c".to_string()], 1)]);++ assert_eq!(model.n_counts, vec![24, 24, 24]);+ }++ #[test]+ fn test_merge() {+ let mut a = StupidBackoffTrainer::new(3);++ a.train(&tokenize(+ "a b c d e f g h i j k l m n o p q r s t u v w x y z",+ ));++ let path = gen_temp_path();++ a.build(&path).unwrap();++ let a = StupidBackoff::open(&path).unwrap();++ let mut b = StupidBackoffTrainer::new(3);++ b.train(&tokenize(+ "a b c d e f g h i j k l m n o p q r s t u v w x y z",+ ));++ let path = gen_temp_path();++ b.build(&path).unwrap();++ let b = StupidBackoff::open(&path).unwrap();++ let path = gen_temp_path();+ let model = StupidBackoff::merge(vec![a, b], &path).unwrap();++ assert_eq!(model.n_counts, vec![48, 48, 48]);++ let model = StupidBackoff::open(path).unwrap();++ assert_eq!(model.n_counts, vec![48, 48, 48]);}}diff --git a/justfile b/justfileindex 9a8aedb7..4ae55f46 100644--- a/justfile+++ b/justfile@@ -2,7 +2,7 @@cd frontend && npm install./scripts/run_dev.py {{ARGS}}-export RUST_LOG := env_var_or_default("RUST_LOG", "info,stract=debug")+export RUST_LOG := env_var_or_default("RUST_LOG", "stract=debug")export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")