Skip to content

web search done right

new spell correction algorithm (web_spell)

Mikkel Denkere3dda0f528ff
diff --git a/configs/api.toml b/configs/api.toml
index 90eeb0ff..74c83910 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -6,6 +6,7 @@ host = "0.0.0.0:3000"
lambda_model_path = "data/lambdamart.txt"
prometheus_host = "0.0.0.0:3001"
queries_csv_path = "data/queries_us.csv"
+spell_corrector_path = "data/web_spell/corrector"
# qa_model_path = "data/qa_model"
bangs_path = "data/bangs.json"
summarizer_path = "data/summarizer"
diff --git a/configs/search_server.toml b/configs/search_server.toml
index ed4d70f7..c2a3898b 100644
--- a/configs/search_server.toml
+++ b/configs/search_server.toml
@@ -7,7 +7,6 @@ host_centrality_store_path = "data/centrality"
index_path = "data/index"
shard_id = 0
# linear_model_path = "data/linear_model.json"
-build_spell_dictionary = false
lambda_model_path = "data/lambdamart.txt"
[snippet]
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index 35722849..4471aac8 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -145,3 +145,19 @@ impl SearchQuery {
false
}
}
+
+pub struct Correction;
+
+impl Correction {
+ pub fn misspelled_prob() -> f64 {
+ 0.1
+ }
+
+ pub fn correction_threshold() -> f64 {
+ 100.0 // logprob difference
+ }
+
+ pub fn lm_prob_weight() -> f64 {
+ 5.77
+ }
+}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 1f23e112..4e4e2200 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -197,6 +197,7 @@ pub struct ApiConfig {
pub crossencoder_model_path: Option<String>,
pub lambda_model_path: Option<String>,
pub qa_model_path: Option<String>,
+ pub spell_corrector_path: Option<String>,
pub bangs_path: String,
pub summarizer_path: String,
pub query_store_db_host: Option<String>,
@@ -211,6 +212,9 @@ pub struct ApiConfig {
pub thresholds: ApiThresholds,
pub widgets: WidgetsConfig,
+
+ #[serde(default)]
+ pub correction_config: CorrectionConfig,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -258,9 +262,6 @@ pub struct SearchServerConfig {
#[serde(default)]
pub snippet: SnippetConfig,
-
- #[serde(default = "defaults::SearchServer::build_spell_dictionary")]
- pub build_spell_dictionary: bool,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -457,3 +458,30 @@ pub struct WebSpellConfig {
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
}
+
+#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize)]
+pub struct CorrectionConfig {
+ /// The probability that a word is misspelled
+ #[serde(default = "defaults::Correction::misspelled_prob")]
+ pub misspelled_prob: f64,
+
+ /// Lambda in eq. 2 (http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf)
+ #[serde(default = "defaults::Correction::lm_prob_weight")]
+ pub lm_prob_weight: f64,
+
+ /// The threshold that the difference between the log probability of the best
+ /// correction and the observed word must be above for the word to be
+ /// corrected
+ #[serde(default = "defaults::Correction::correction_threshold")]
+ pub correction_threshold: f64,
+}
+
+impl Default for CorrectionConfig {
+ fn default() -> Self {
+ Self {
+ misspelled_prob: defaults::Correction::misspelled_prob(),
+ lm_prob_weight: defaults::Correction::lm_prob_weight(),
+ correction_threshold: defaults::Correction::correction_threshold(),
+ }
+ }
+}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index abf06fc2..15860eeb 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -19,6 +19,7 @@ use tokio::io;
use tokio_stream::StreamExt;
use tracing::{debug, info};
+use crate::config::{LocalConfig, WebSpellConfig};
use crate::entrypoint::indexer::JobSettings;
use crate::entrypoint::{dmoz_parser, indexer};
use crate::Result;
@@ -69,6 +70,30 @@ fn download_files() {
});
}
+fn build_spellchecker() -> Result<()> {
+ debug!("Building spellchecker");
+ let spellchecker_path = Path::new(DATA_PATH).join("web_spell");
+
+ if !spellchecker_path.exists() {
+ crate::entrypoint::web_spell::run(WebSpellConfig {
+ languages: vec![whatlang::Lang::Eng],
+ output_path: spellchecker_path.to_str().unwrap().to_string(),
+ warc_source: crate::config::WarcSource::Local(LocalConfig {
+ folder: ".".to_string(),
+ names: vec![Path::new(DATA_PATH)
+ .join("sample.warc.gz")
+ .to_str()
+ .unwrap()
+ .to_string()],
+ }),
+ limit_warc_files: None,
+ skip_warc_files: None,
+ })?;
+ }
+
+ Ok(())
+}
+
fn create_webgraph() -> Result<()> {
debug!("Creating webgraph");
let out_path_host = Path::new(DATA_PATH).join("webgraph_host");
@@ -204,6 +229,7 @@ fn index_files() -> Result<()> {
parse_topics()?;
create_inverted_index()?;
create_entity_index()?;
+ build_spellchecker()?;
Ok(())
}
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 254c19cd..be99bb2a 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -85,10 +85,6 @@ impl SearchService {
local_searcher.set_lambda_model(LambdaMART::open(model_path)?);
}
- if config.build_spell_dictionary {
- local_searcher.build_spell_dict();
- }
-
local_searcher.set_collector_config(config.collector);
local_searcher.set_snippet_config(config.snippet);
diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rs
index 00d53374..77546f07 100644
--- a/core/src/entrypoint/web_spell.rs
+++ b/core/src/entrypoint/web_spell.rs
@@ -156,7 +156,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {
results,
Path::new(&config.output_path)
.join("corrector")
- .join(lang.to_string()),
+ .join(lang.code()),
)?;
debug!("second trainer created");
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
index 5dc83060..499d5e6f 100644
--- a/core/src/searcher/api.rs
+++ b/core/src/searcher/api.rs
@@ -36,6 +36,7 @@ use crate::search_prettifier::{
create_stackoverflow_sidebar, DisplayedAnswer, DisplayedEntity, DisplayedSidebar,
DisplayedWebpage, HighlightedSpellCorrection,
};
+use crate::web_spell::SpellChecker;
use crate::widgets::{Widget, Widgets};
use crate::{
bangs::Bangs,
@@ -83,6 +84,7 @@ pub struct ApiSearcher {
collector_config: CollectorConfig,
thresholds: ApiThresholds,
widgets: Widgets,
+ spell_checker: Option<SpellChecker>,
}
#[cfg(not(feature = "libtorch"))]
@@ -94,6 +96,7 @@ pub struct ApiSearcher {
collector_config: CollectorConfig,
thresholds: ApiThresholds,
widgets: Widgets,
+ spell_checker: Option<SpellChecker>,
}
impl ApiSearcher {
@@ -116,6 +119,9 @@ impl ApiSearcher {
collector_config: config.collector,
thresholds: config.thresholds,
widgets: Widgets::new(config.widgets).unwrap(),
+ spell_checker: config
+ .spell_corrector_path
+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
}
}
@@ -134,6 +140,9 @@ impl ApiSearcher {
collector_config: config.collector,
thresholds: config.thresholds,
widgets: Widgets::new(config.widgets).unwrap(),
+ spell_checker: config
+ .spell_corrector_path
+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
}
}
@@ -215,6 +224,7 @@ impl ApiSearcher {
results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));
if let Some((shard, website)) = results.pop() {
+ tracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");
if website.score > self.thresholds.stackoverflow {
let scored_websites =
vec![(0, distributed::ScoredWebsitePointer { website, shard })];
@@ -449,9 +459,9 @@ impl ApiSearcher {
let sidebar = self.sidebar(&initial_results, stackoverflow);
let spell_corrected_query = if widget.is_none() {
- initial_results
- .first()
- .and_then(|result| result.local_result.spell_corrected_query.clone())
+ self.spell_checker
+ .as_ref()
+ .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))
.map(HighlightedSpellCorrection::from)
} else {
None
diff --git a/core/src/web_spell/error_model.rs b/core/src/web_spell/error_model.rs
index a44e68c3..af3f6276 100644
--- a/core/src/web_spell/error_model.rs
+++ b/core/src/web_spell/error_model.rs
@@ -35,7 +35,7 @@ pub enum ErrorType {
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ErrorSequence(Vec<ErrorType>);
-fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {
+pub fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {
if a == b {
return None;
}
@@ -110,6 +110,42 @@ fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {
}
#[derive(Debug, Serialize, Deserialize)]
+struct StoredErrorModel {
+ errors: HashMap<String, u64>,
+ total: u64,
+}
+
+impl From<ErrorModel> for StoredErrorModel {
+ fn from(value: ErrorModel) -> Self {
+ let stored_errors = value
+ .errors
+ .into_iter()
+ .map(|(error_seq, count)| (serde_json::to_string(&error_seq).unwrap(), count))
+ .collect();
+
+ Self {
+ errors: stored_errors,
+ total: value.total,
+ }
+ }
+}
+
+impl From<StoredErrorModel> for ErrorModel {
+ fn from(value: StoredErrorModel) -> Self {
+ let errors = value
+ .errors
+ .into_iter()
+ .map(|(error_seq, count)| (serde_json::from_str(&error_seq).unwrap(), count))
+ .collect();
+
+ Self {
+ errors,
+ total: value.total,
+ }
+ }
+}
+
+#[derive(Debug)]
pub struct ErrorModel {
errors: HashMap<ErrorSequence, u64>,
total: u64,
@@ -138,7 +174,7 @@ impl ErrorModel {
let wrt = BufWriter::new(file);
- serde_json::to_writer_pretty(wrt, &self)?;
+ serde_json::to_writer_pretty(wrt, &StoredErrorModel::from(self))?;
Ok(())
}
@@ -148,12 +184,15 @@ impl ErrorModel {
let rdr = BufReader::new(file);
- Ok(serde_json::from_reader(rdr)?)
+ let stored: StoredErrorModel = serde_json::from_reader(rdr)?;
+
+ Ok(stored.into())
}
pub fn add(&mut self, a: &str, b: &str) {
if let Some(errors) = possible_errors(a, b) {
*self.errors.entry(errors).or_insert(0) += 1;
+ self.total += 1;
}
}
@@ -161,6 +200,13 @@ impl ErrorModel {
let count = self.errors.get(error).unwrap_or(&0);
*count as f64 / self.total as f64
}
+
+ pub fn log_prob(&self, error: &ErrorSequence) -> f64 {
+ match self.errors.get(error) {
+ Some(count) => (*count as f64).log2() - (self.total as f64).log2(),
+ None => 0.0,
+ }
+ }
}
#[cfg(test)]
diff --git a/core/src/web_spell/mod.rs b/core/src/web_spell/mod.rs
index b0a1e246..0df274e1 100644
--- a/core/src/web_spell/mod.rs
+++ b/core/src/web_spell/mod.rs
@@ -24,6 +24,7 @@ mod term_freqs;
mod trainer;
pub use error_model::ErrorModel;
+pub use spell_checker::SpellChecker;
pub use stupid_backoff::StupidBackoff;
pub use term_freqs::TermDict;
pub use trainer::FirstTrainer;
@@ -50,6 +51,9 @@ pub enum Error {
#[error("Bincode error: {0}")]
Bincode(#[from] bincode::Error),
+
+ #[error("Corrector not found")]
+ CorrectorNotFound,
}
pub type Result<T, E = Error> = std::result::Result<T, E>;
diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rs
index bfa7dc6d..47de1f8d 100644
--- a/core/src/web_spell/spell_checker.rs
+++ b/core/src/web_spell/spell_checker.rs
@@ -13,3 +13,223 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::Result;
+use std::{path::Path, str::FromStr};
+
+use fnv::FnvHashMap;
+use whatlang::Lang;
+
+use crate::{
+ config::CorrectionConfig,
+ web_spell::stupid_backoff::{IntoMiddle, LeftToRight, RightToLeft},
+};
+
+use super::{error_model, Correction, CorrectionTerm, Error, ErrorModel, StupidBackoff, TermDict};
+
+struct LangSpellChecker {
+ term_dict: TermDict,
+ language_model: StupidBackoff,
+ error_model: ErrorModel,
+ config: CorrectionConfig,
+}
+
+impl LangSpellChecker {
+ fn open<P: AsRef<Path>>(path: P, config: CorrectionConfig) -> Result<Self> {
+ let term_dict = TermDict::open(path.as_ref().join("term_dict"))?;
+ let language_model = StupidBackoff::open(path.as_ref().join("stupid_backoff"))?;
+ let error_model = ErrorModel::open(path.as_ref().join("error_model.json"))?;
+
+ Ok(Self {
+ term_dict,
+ language_model,
+ error_model,
+ config,
+ })
+ }
+
+ fn candidates(&self, term: &str) -> Vec<String> {
+ // one edit for words of
+ // up to four characters, two edits for up to twelve
+ // characters, and three for longer
+ let max_edit_distance = if term.len() <= 4 {
+ 1
+ } else if term.len() <= 12 {
+ 2
+ } else {
+ 3
+ };
+
+ self.term_dict.search(term, max_edit_distance)
+ }
+
+ fn correct_once(&self, text: &str) -> Option<Correction> {
+ let orig_terms = super::tokenize(text);
+ let mut terms = orig_terms.clone();
+
+ let mut corrections = Vec::new();
+
+ let num_terms = terms.len();
+ for i in 0..num_terms {
+ let term = &terms[i];
+ let candidates = self.candidates(term);
+
+ if candidates.is_empty() {
+ tracing::debug!("no candidates for {}", term);
+ continue;
+ }
+
+ // context around term
+ // if term is first or last, use two next/previous terms if they exist
+ // otherwise use one next/previous term (if they exist)
+ let mut context = Vec::new();
+ let mut j = i.saturating_sub(2);
+ let mut this_term_context_idx = None;
+ let limit = std::cmp::min(i + 3, terms.len());
+
+ while j < limit {
+ context.push(terms[j].clone());
+ if i == j {
+ this_term_context_idx = Some(context.len() - 1);
+ }
+ j += 1;
+ }
+
+ let this_term_context_idx = this_term_context_idx.unwrap();
+ let term_log_prob = if this_term_context_idx == 0 {
+ let strat = RightToLeft;
+ self.language_model.log_prob(&context, strat)
+ } else if this_term_context_idx == context.len() - 1 {
+ let strat = LeftToRight;
+ self.language_model.log_prob(&context, strat)
+ } else {
+ let strat = IntoMiddle::default();
+ self.language_model.log_prob(&context, strat)
+ };
+
+ tracing::debug!(?term, ?term_log_prob);
+
+ let mut best_term: Option<(String, f64)> = None;
+
+ for candidate in candidates {
+ if candidate == *term {
+ continue;
+ }
+
+ context[this_term_context_idx] = candidate.clone();
+
+ let log_prob = if this_term_context_idx == 0 {
+ let strat = RightToLeft;
+ self.language_model.log_prob(&context, strat)
+ } else if this_term_context_idx == context.len() - 1 {
+ let strat = LeftToRight;
+ self.language_model.log_prob(&context, strat)
+ } else {
+ let strat = IntoMiddle::default();
+ self.language_model.log_prob(&context, strat)
+ };
+
+ let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;
+
+ let error_log_prob = if &candidate == term {
+ match error_model::possible_errors(term, &candidate) {
+ Some(error_seq) => {
+ (1.0 - self.config.misspelled_prob).log2()
+ + self.error_model.log_prob(&error_seq)
+ }
+ None => 0.0,
+ }
+ } else {
+ self.config.misspelled_prob
+ };
+
+ let score = scaled_lm_log_prob + error_log_prob;
+
+ if best_term.is_none() || score > best_term.as_ref().unwrap().1 {
+ best_term = Some((candidate, score));
+ }
+ }
+
+ if let Some((best_term, score)) = best_term {
+ let diff = score.abs() - term_log_prob.abs();
+ tracing::debug!(?best_term, ?score, ?diff);
+ if diff > self.config.correction_threshold {
+ corrections.push((i, best_term.clone()));
+ terms[i] = best_term; // make sure the next terms use the corrected context
+ }
+ }
+ }
+
+ if corrections.is_empty() {
+ return None;
+ }
+
+ let mut res = Correction::empty(text.to_string());
+
+ for (orig, possible_correction) in orig_terms.into_iter().zip(terms.into_iter()) {
+ if orig == possible_correction {
+ res.push(CorrectionTerm::NotCorrected(orig));
+ } else {
+ res.push(CorrectionTerm::Corrected(possible_correction));
+ }
+ }
+
+ Some(res)
+ }
+
+ fn correct(&self, text: &str) -> Option<Correction> {
+ self.correct_once(text.to_lowercase().as_str())
+ }
+}
+
+pub struct SpellChecker {
+ lang_spell_checkers: FnvHashMap<Lang, LangSpellChecker>,
+}
+
+impl SpellChecker {
+ pub fn open<P: AsRef<Path>>(path: P, config: CorrectionConfig) -> Result<Self> {
+ if !path.as_ref().exists() {
+ return Err(Error::CorrectorNotFound);
+ }
+
+ if !path.as_ref().is_dir() {
+ return Err(Error::CorrectorNotFound);
+ }
+
+ let mut lang_spell_checkers = FnvHashMap::default();
+
+ for entry in std::fs::read_dir(path)? {
+ let entry = entry?;
+ let path = entry.path();
+
+ if !path.is_dir() {
+ continue;
+ }
+
+ let lang = match path.file_name().and_then(|s| s.to_str()) {
+ Some(lang) => lang,
+ None => continue,
+ };
+
+ let lang = match Lang::from_str(lang) {
+ Ok(lang) => lang,
+ Err(_) => {
+ tracing::warn!("Invalid language: {}", lang);
+ continue;
+ }
+ };
+
+ let lang_spell_checker = LangSpellChecker::open(path, config)?;
+ lang_spell_checkers.insert(lang, lang_spell_checker);
+ }
+
+ Ok(Self {
+ lang_spell_checkers,
+ })
+ }
+ pub fn correct(&self, text: &str, lang: &Lang) -> Option<Correction> {
+ self.lang_spell_checkers
+ .get(lang)
+ .and_then(|s| s.correct(text))
+ }
+}
diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rs
index ecb07c82..346fd8b2 100644
--- a/core/src/web_spell/stupid_backoff.rs
+++ b/core/src/web_spell/stupid_backoff.rs
@@ -233,6 +233,13 @@ impl StupidBackoff {
acc
});
+ let file = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .open(path.as_ref().join("n_counts.bin"))?;
+
+ bincode::serialize_into(file, &n_counts)?;
+
let file = OpenOptions::new()
.create(true)
.write(true)
@@ -295,6 +302,10 @@ impl StupidBackoff {
}
}
+ pub fn prob<S: NextWordsStrategy>(&self, words: &[String], strat: S) -> f64 {
+ self.log_prob(words, strat).exp2()
+ }
+
pub fn contexts(&self, word: &str) -> Vec<(Vec<String>, u64)> {
let q = word.to_string() + " ";
let automaton = fst::automaton::Str::new(&q).starts_with();
diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.py
index 3097f2db..3e605a8e 100644
--- a/ltr/auto_annotate.py
+++ b/ltr/auto_annotate.py
@@ -7,7 +7,7 @@ import json
import time
import requests
-NUM_RESULTS_PER_QUERY = 50
+NUM_RESULTS_PER_QUERY = 100
NUM_ENSEMBLE_PREDS = 1
PROMPT = """<|im_start|>system
Perform the task to the best of your ability.<|im_end|>
@@ -30,6 +30,9 @@ llm = Llama(
with open("data/queries_us.csv") as f:
all_queries = [line.strip() for line in f.readlines()]
+# shuffle queries
+np.random.shuffle(all_queries)
+
def setup_db():
db = sqlite3.connect("data/auto-ranking-annotation.sqlite")
@@ -72,6 +75,10 @@ for query in all_queries:
if sum([c.isalnum() for c in query]) / len(query) < 0.5:
continue
+ # only consider queries with at least two words
+ if len(query.split()) < 2:
+ continue
+
if len(query) > 100:
continue