Skip to content

web search done right

fixed bug in stupid_backoff model that caused last n_gram count to always be 0

Mikkel Denker1a6d8ff6bed9
diff --git a/configs/api.toml b/configs/api.toml
index 5ebd9acd..27905bf7 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -4,10 +4,10 @@ gossip_addr = "0.0.0.0:3005"
gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]
host = "0.0.0.0:3000"
# lambda_model_path = "data/lambdamart.txt"
+bangs_path = "data/bangs.json"
dual_encoder_model_path = "data/summarizer/dual_encoder"
prometheus_host = "0.0.0.0:3001"
queries_csv_path = "data/queries_us.csv"
-bangs_path = "data/bangs.json"
summarizer_path = "data/summarizer"
[spell_check]
@@ -17,7 +17,6 @@ path = "data/web_spell/checker"
correction_threshold = 3.0
lm_prob_weight = 1.0
-
[thresholds]
entity_sidebar = 0.0
stackoverflow = 0.0
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 0a00e15c..fa88e73e 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -20,7 +20,6 @@ use std::fs;
use std::path::Path;
use stract::config;
use stract::entrypoint::autosuggest_scrape::{self, Gl};
-use tracing::level_filters::LevelFilter;
#[cfg(feature = "dev")]
use stract::entrypoint::configure;
@@ -262,9 +261,8 @@ fn main() -> Result<()> {
.with_max_level(tracing::Level::INFO)
.with_env_filter(
tracing_subscriber::EnvFilter::builder()
- .with_default_directive(LevelFilter::OFF.into())
- .from_env_lossy()
- .add_directive("stract=info".parse().unwrap()),
+ .with_default_directive("stract=info".parse().unwrap())
+ .from_env_lossy(),
)
.without_time()
.with_target(false)
diff --git a/crates/core/src/web_spell/spell_checker.rs b/crates/core/src/web_spell/spell_checker.rs
index b63289dd..e7ac0b32 100644
--- a/crates/core/src/web_spell/spell_checker.rs
+++ b/crates/core/src/web_spell/spell_checker.rs
@@ -157,14 +157,14 @@ impl LangSpellChecker {
let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob
+ ((1.0 - self.config.misspelled_prob).log2());
- tracing::debug!(?term, ?scaled_term_log_prob);
+ tracing::debug!(?term, ?term_log_prob, ?scaled_term_log_prob);
if let Some((best_term, score)) =
self.score_candidates(term, &candidates, context, this_term_context_idx)
{
let diff = score - scaled_term_log_prob;
tracing::debug!(?best_term, ?score, ?diff);
- if diff > self.config.correction_threshold {
+ if diff.is_finite() && diff > self.config.correction_threshold {
corrections.push((i, best_term.clone()));
terms[i] = best_term; // make sure the next terms use the corrected context
}
diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rs
index 998e2f46..f50bf459 100644
--- a/crates/core/src/web_spell/stupid_backoff.rs
+++ b/crates/core/src/web_spell/stupid_backoff.rs
@@ -79,7 +79,7 @@ impl StupidBackoffTrainer {
pub fn train(&mut self, tokens: &[String]) {
for window in tokens.windows(self.max_ngram_size) {
- for i in 1..window.len() {
+ for i in 1..=window.len() {
let ngram = Ngram {
terms: window[..i].to_vec(),
};
@@ -433,5 +433,43 @@ mod tests {
model.contexts("b"),
vec![(vec!["a".to_string(), "b".to_string(), "c".to_string()], 1)]
);
+
+ assert_eq!(model.n_counts, vec![24, 24, 24]);
+ }
+
+ #[test]
+ fn test_merge() {
+ let mut a = StupidBackoffTrainer::new(3);
+
+ a.train(&tokenize(
+ "a b c d e f g h i j k l m n o p q r s t u v w x y z",
+ ));
+
+ let path = gen_temp_path();
+
+ a.build(&path).unwrap();
+
+ let a = StupidBackoff::open(&path).unwrap();
+
+ let mut b = StupidBackoffTrainer::new(3);
+
+ b.train(&tokenize(
+ "a b c d e f g h i j k l m n o p q r s t u v w x y z",
+ ));
+
+ let path = gen_temp_path();
+
+ b.build(&path).unwrap();
+
+ let b = StupidBackoff::open(&path).unwrap();
+
+ let path = gen_temp_path();
+ let model = StupidBackoff::merge(vec![a, b], &path).unwrap();
+
+ assert_eq!(model.n_counts, vec![48, 48, 48]);
+
+ let model = StupidBackoff::open(path).unwrap();
+
+ assert_eq!(model.n_counts, vec![48, 48, 48]);
}
}
diff --git a/justfile b/justfile
index 9a8aedb7..4ae55f46 100644
--- a/justfile
+++ b/justfile
@@ -2,7 +2,7 @@
cd frontend && npm install
./scripts/run_dev.py {{ARGS}}
-export RUST_LOG := env_var_or_default("RUST_LOG", "info,stract=debug")
+export RUST_LOG := env_var_or_default("RUST_LOG", "stract=debug")
export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")