web search done right
new spell correction algorithm (web_spell)
Mikkel Denkere3dda0f528ff
diff --git a/configs/api.toml b/configs/api.tomlindex 90eeb0ff..74c83910 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -6,6 +6,7 @@ host = "0.0.0.0:3000"lambda_model_path = "data/lambdamart.txt"prometheus_host = "0.0.0.0:3001"queries_csv_path = "data/queries_us.csv"+spell_corrector_path = "data/web_spell/corrector"# qa_model_path = "data/qa_model"bangs_path = "data/bangs.json"summarizer_path = "data/summarizer"diff --git a/configs/search_server.toml b/configs/search_server.tomlindex ed4d70f7..c2a3898b 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -7,7 +7,6 @@ host_centrality_store_path = "data/centrality"index_path = "data/index"shard_id = 0# linear_model_path = "data/linear_model.json"-build_spell_dictionary = falselambda_model_path = "data/lambdamart.txt"[snippet]diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex 35722849..4471aac8 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -145,3 +145,19 @@ impl SearchQuery {false}}++pub struct Correction;++impl Correction {+ pub fn misspelled_prob() -> f64 {+ 0.1+ }++ pub fn correction_threshold() -> f64 {+ 100.0 // logprob difference+ }++ pub fn lm_prob_weight() -> f64 {+ 5.77+ }+}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 1f23e112..4e4e2200 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -197,6 +197,7 @@ pub struct ApiConfig {pub crossencoder_model_path: Option<String>,pub lambda_model_path: Option<String>,pub qa_model_path: Option<String>,+ pub spell_corrector_path: Option<String>,pub bangs_path: String,pub summarizer_path: String,pub query_store_db_host: Option<String>,@@ -211,6 +212,9 @@ pub struct ApiConfig {pub thresholds: ApiThresholds,pub widgets: WidgetsConfig,++ #[serde(default)]+ pub correction_config: CorrectionConfig,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -258,9 +262,6 @@ pub struct SearchServerConfig {#[serde(default)]pub snippet: SnippetConfig,-- #[serde(default = "defaults::SearchServer::build_spell_dictionary")]- pub build_spell_dictionary: bool,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -457,3 +458,30 @@ pub struct WebSpellConfig {pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,}++#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize)]+pub struct CorrectionConfig {+ /// The probability that a word is misspelled+ #[serde(default = "defaults::Correction::misspelled_prob")]+ pub misspelled_prob: f64,++ /// Lambda in eq. 2 (http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf)+ #[serde(default = "defaults::Correction::lm_prob_weight")]+ pub lm_prob_weight: f64,++ /// The threshold that the difference between the log probability of the best+ /// correction and the observed word must be above for the word to be+ /// corrected+ #[serde(default = "defaults::Correction::correction_threshold")]+ pub correction_threshold: f64,+}++impl Default for CorrectionConfig {+ fn default() -> Self {+ Self {+ misspelled_prob: defaults::Correction::misspelled_prob(),+ lm_prob_weight: defaults::Correction::lm_prob_weight(),+ correction_threshold: defaults::Correction::correction_threshold(),+ }+ }+}diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex abf06fc2..15860eeb 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -19,6 +19,7 @@ use tokio::io;use tokio_stream::StreamExt;use tracing::{debug, info};+use crate::config::{LocalConfig, WebSpellConfig};use crate::entrypoint::indexer::JobSettings;use crate::entrypoint::{dmoz_parser, indexer};use crate::Result;@@ -69,6 +70,30 @@ fn download_files() {});}+fn build_spellchecker() -> Result<()> {+ debug!("Building spellchecker");+ let spellchecker_path = Path::new(DATA_PATH).join("web_spell");++ if !spellchecker_path.exists() {+ crate::entrypoint::web_spell::run(WebSpellConfig {+ languages: vec![whatlang::Lang::Eng],+ output_path: spellchecker_path.to_str().unwrap().to_string(),+ warc_source: crate::config::WarcSource::Local(LocalConfig {+ folder: ".".to_string(),+ names: vec![Path::new(DATA_PATH)+ .join("sample.warc.gz")+ .to_str()+ .unwrap()+ .to_string()],+ }),+ limit_warc_files: None,+ skip_warc_files: None,+ })?;+ }++ Ok(())+}+fn create_webgraph() -> Result<()> {debug!("Creating webgraph");let out_path_host = Path::new(DATA_PATH).join("webgraph_host");@@ -204,6 +229,7 @@ fn index_files() -> Result<()> {parse_topics()?;create_inverted_index()?;create_entity_index()?;+ build_spellchecker()?;Ok(())}diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 254c19cd..be99bb2a 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -85,10 +85,6 @@ impl SearchService {local_searcher.set_lambda_model(LambdaMART::open(model_path)?);}- if config.build_spell_dictionary {- local_searcher.build_spell_dict();- }-local_searcher.set_collector_config(config.collector);local_searcher.set_snippet_config(config.snippet);diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rsindex 00d53374..77546f07 100644--- a/core/src/entrypoint/web_spell.rs+++ b/core/src/entrypoint/web_spell.rs@@ -156,7 +156,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {results,Path::new(&config.output_path).join("corrector")- .join(lang.to_string()),+ .join(lang.code()),)?;debug!("second trainer created");diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex 5dc83060..499d5e6f 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -36,6 +36,7 @@ use crate::search_prettifier::{create_stackoverflow_sidebar, DisplayedAnswer, DisplayedEntity, DisplayedSidebar,DisplayedWebpage, HighlightedSpellCorrection,};+use crate::web_spell::SpellChecker;use crate::widgets::{Widget, Widgets};use crate::{bangs::Bangs,@@ -83,6 +84,7 @@ pub struct ApiSearcher {collector_config: CollectorConfig,thresholds: ApiThresholds,widgets: Widgets,+ spell_checker: Option<SpellChecker>,}#[cfg(not(feature = "libtorch"))]@@ -94,6 +96,7 @@ pub struct ApiSearcher {collector_config: CollectorConfig,thresholds: ApiThresholds,widgets: Widgets,+ spell_checker: Option<SpellChecker>,}impl ApiSearcher {@@ -116,6 +119,9 @@ impl ApiSearcher {collector_config: config.collector,thresholds: config.thresholds,widgets: Widgets::new(config.widgets).unwrap(),+ spell_checker: config+ .spell_corrector_path+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),}}@@ -134,6 +140,9 @@ impl ApiSearcher {collector_config: config.collector,thresholds: config.thresholds,widgets: Widgets::new(config.widgets).unwrap(),+ spell_checker: config+ .spell_corrector_path+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),}}@@ -215,6 +224,7 @@ impl ApiSearcher {results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));if let Some((shard, website)) = results.pop() {+ tracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");if website.score > self.thresholds.stackoverflow {let scored_websites =vec![(0, distributed::ScoredWebsitePointer { website, shard })];@@ -449,9 +459,9 @@ impl ApiSearcher {let sidebar = self.sidebar(&initial_results, stackoverflow);let spell_corrected_query = if widget.is_none() {- initial_results- .first()- .and_then(|result| result.local_result.spell_corrected_query.clone())+ self.spell_checker+ .as_ref()+ .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng)).map(HighlightedSpellCorrection::from)} else {Nonediff --git a/core/src/web_spell/error_model.rs b/core/src/web_spell/error_model.rsindex a44e68c3..af3f6276 100644--- a/core/src/web_spell/error_model.rs+++ b/core/src/web_spell/error_model.rs@@ -35,7 +35,7 @@ pub enum ErrorType {#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]pub struct ErrorSequence(Vec<ErrorType>);-fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {+pub fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {if a == b {return None;}@@ -110,6 +110,42 @@ fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {}#[derive(Debug, Serialize, Deserialize)]+struct StoredErrorModel {+ errors: HashMap<String, u64>,+ total: u64,+}++impl From<ErrorModel> for StoredErrorModel {+ fn from(value: ErrorModel) -> Self {+ let stored_errors = value+ .errors+ .into_iter()+ .map(|(error_seq, count)| (serde_json::to_string(&error_seq).unwrap(), count))+ .collect();++ Self {+ errors: stored_errors,+ total: value.total,+ }+ }+}++impl From<StoredErrorModel> for ErrorModel {+ fn from(value: StoredErrorModel) -> Self {+ let errors = value+ .errors+ .into_iter()+ .map(|(error_seq, count)| (serde_json::from_str(&error_seq).unwrap(), count))+ .collect();++ Self {+ errors,+ total: value.total,+ }+ }+}++#[derive(Debug)]pub struct ErrorModel {errors: HashMap<ErrorSequence, u64>,total: u64,@@ -138,7 +174,7 @@ impl ErrorModel {let wrt = BufWriter::new(file);- serde_json::to_writer_pretty(wrt, &self)?;+ serde_json::to_writer_pretty(wrt, &StoredErrorModel::from(self))?;Ok(())}@@ -148,12 +184,15 @@ impl ErrorModel {let rdr = BufReader::new(file);- Ok(serde_json::from_reader(rdr)?)+ let stored: StoredErrorModel = serde_json::from_reader(rdr)?;++ Ok(stored.into())}pub fn add(&mut self, a: &str, b: &str) {if let Some(errors) = possible_errors(a, b) {*self.errors.entry(errors).or_insert(0) += 1;+ self.total += 1;}}@@ -161,6 +200,13 @@ impl ErrorModel {let count = self.errors.get(error).unwrap_or(&0);*count as f64 / self.total as f64}++ pub fn log_prob(&self, error: &ErrorSequence) -> f64 {+ match self.errors.get(error) {+ Some(count) => (*count as f64).log2() - (self.total as f64).log2(),+ None => 0.0,+ }+ }}#[cfg(test)]diff --git a/core/src/web_spell/mod.rs b/core/src/web_spell/mod.rsindex b0a1e246..0df274e1 100644--- a/core/src/web_spell/mod.rs+++ b/core/src/web_spell/mod.rs@@ -24,6 +24,7 @@ mod term_freqs;mod trainer;pub use error_model::ErrorModel;+pub use spell_checker::SpellChecker;pub use stupid_backoff::StupidBackoff;pub use term_freqs::TermDict;pub use trainer::FirstTrainer;@@ -50,6 +51,9 @@ pub enum Error {#[error("Bincode error: {0}")]Bincode(#[from] bincode::Error),++ #[error("Corrector not found")]+ CorrectorNotFound,}pub type Result<T, E = Error> = std::result::Result<T, E>;diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rsindex bfa7dc6d..47de1f8d 100644--- a/core/src/web_spell/spell_checker.rs+++ b/core/src/web_spell/spell_checker.rs@@ -13,3 +13,223 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::Result;+use std::{path::Path, str::FromStr};++use fnv::FnvHashMap;+use whatlang::Lang;++use crate::{+ config::CorrectionConfig,+ web_spell::stupid_backoff::{IntoMiddle, LeftToRight, RightToLeft},+};++use super::{error_model, Correction, CorrectionTerm, Error, ErrorModel, StupidBackoff, TermDict};++struct LangSpellChecker {+ term_dict: TermDict,+ language_model: StupidBackoff,+ error_model: ErrorModel,+ config: CorrectionConfig,+}++impl LangSpellChecker {+ fn open<P: AsRef<Path>>(path: P, config: CorrectionConfig) -> Result<Self> {+ let term_dict = TermDict::open(path.as_ref().join("term_dict"))?;+ let language_model = StupidBackoff::open(path.as_ref().join("stupid_backoff"))?;+ let error_model = ErrorModel::open(path.as_ref().join("error_model.json"))?;++ Ok(Self {+ term_dict,+ language_model,+ error_model,+ config,+ })+ }++ fn candidates(&self, term: &str) -> Vec<String> {+ // one edit for words of+ // up to four characters, two edits for up to twelve+ // characters, and three for longer+ let max_edit_distance = if term.len() <= 4 {+ 1+ } else if term.len() <= 12 {+ 2+ } else {+ 3+ };++ self.term_dict.search(term, max_edit_distance)+ }++ fn correct_once(&self, text: &str) -> Option<Correction> {+ let orig_terms = super::tokenize(text);+ let mut terms = orig_terms.clone();++ let mut corrections = Vec::new();++ let num_terms = terms.len();+ for i in 0..num_terms {+ let term = &terms[i];+ let candidates = self.candidates(term);++ if candidates.is_empty() {+ tracing::debug!("no candidates for {}", term);+ continue;+ }++ // context around term+ // if term is first or last, use two next/previous terms if they exist+ // otherwise use one next/previous term (if they exist)+ let mut context = Vec::new();+ let mut j = i.saturating_sub(2);+ let mut this_term_context_idx = None;+ let limit = std::cmp::min(i + 3, terms.len());++ while j < limit {+ context.push(terms[j].clone());+ if i == j {+ this_term_context_idx = Some(context.len() - 1);+ }+ j += 1;+ }++ let this_term_context_idx = this_term_context_idx.unwrap();+ let term_log_prob = if this_term_context_idx == 0 {+ let strat = RightToLeft;+ self.language_model.log_prob(&context, strat)+ } else if this_term_context_idx == context.len() - 1 {+ let strat = LeftToRight;+ self.language_model.log_prob(&context, strat)+ } else {+ let strat = IntoMiddle::default();+ self.language_model.log_prob(&context, strat)+ };++ tracing::debug!(?term, ?term_log_prob);++ let mut best_term: Option<(String, f64)> = None;++ for candidate in candidates {+ if candidate == *term {+ continue;+ }++ context[this_term_context_idx] = candidate.clone();++ let log_prob = if this_term_context_idx == 0 {+ let strat = RightToLeft;+ self.language_model.log_prob(&context, strat)+ } else if this_term_context_idx == context.len() - 1 {+ let strat = LeftToRight;+ self.language_model.log_prob(&context, strat)+ } else {+ let strat = IntoMiddle::default();+ self.language_model.log_prob(&context, strat)+ };++ let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;++ let error_log_prob = if &candidate == term {+ match error_model::possible_errors(term, &candidate) {+ Some(error_seq) => {+ (1.0 - self.config.misspelled_prob).log2()+ + self.error_model.log_prob(&error_seq)+ }+ None => 0.0,+ }+ } else {+ self.config.misspelled_prob+ };++ let score = scaled_lm_log_prob + error_log_prob;++ if best_term.is_none() || score > best_term.as_ref().unwrap().1 {+ best_term = Some((candidate, score));+ }+ }++ if let Some((best_term, score)) = best_term {+ let diff = score.abs() - term_log_prob.abs();+ tracing::debug!(?best_term, ?score, ?diff);+ if diff > self.config.correction_threshold {+ corrections.push((i, best_term.clone()));+ terms[i] = best_term; // make sure the next terms use the corrected context+ }+ }+ }++ if corrections.is_empty() {+ return None;+ }++ let mut res = Correction::empty(text.to_string());++ for (orig, possible_correction) in orig_terms.into_iter().zip(terms.into_iter()) {+ if orig == possible_correction {+ res.push(CorrectionTerm::NotCorrected(orig));+ } else {+ res.push(CorrectionTerm::Corrected(possible_correction));+ }+ }++ Some(res)+ }++ fn correct(&self, text: &str) -> Option<Correction> {+ self.correct_once(text.to_lowercase().as_str())+ }+}++pub struct SpellChecker {+ lang_spell_checkers: FnvHashMap<Lang, LangSpellChecker>,+}++impl SpellChecker {+ pub fn open<P: AsRef<Path>>(path: P, config: CorrectionConfig) -> Result<Self> {+ if !path.as_ref().exists() {+ return Err(Error::CorrectorNotFound);+ }++ if !path.as_ref().is_dir() {+ return Err(Error::CorrectorNotFound);+ }++ let mut lang_spell_checkers = FnvHashMap::default();++ for entry in std::fs::read_dir(path)? {+ let entry = entry?;+ let path = entry.path();++ if !path.is_dir() {+ continue;+ }++ let lang = match path.file_name().and_then(|s| s.to_str()) {+ Some(lang) => lang,+ None => continue,+ };++ let lang = match Lang::from_str(lang) {+ Ok(lang) => lang,+ Err(_) => {+ tracing::warn!("Invalid language: {}", lang);+ continue;+ }+ };++ let lang_spell_checker = LangSpellChecker::open(path, config)?;+ lang_spell_checkers.insert(lang, lang_spell_checker);+ }++ Ok(Self {+ lang_spell_checkers,+ })+ }+ pub fn correct(&self, text: &str, lang: &Lang) -> Option<Correction> {+ self.lang_spell_checkers+ .get(lang)+ .and_then(|s| s.correct(text))+ }+}diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rsindex ecb07c82..346fd8b2 100644--- a/core/src/web_spell/stupid_backoff.rs+++ b/core/src/web_spell/stupid_backoff.rs@@ -233,6 +233,13 @@ impl StupidBackoff {acc});+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref().join("n_counts.bin"))?;++ bincode::serialize_into(file, &n_counts)?;+let file = OpenOptions::new().create(true).write(true)@@ -295,6 +302,10 @@ impl StupidBackoff {}}+ pub fn prob<S: NextWordsStrategy>(&self, words: &[String], strat: S) -> f64 {+ self.log_prob(words, strat).exp2()+ }+pub fn contexts(&self, word: &str) -> Vec<(Vec<String>, u64)> {let q = word.to_string() + " ";let automaton = fst::automaton::Str::new(&q).starts_with();diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex 3097f2db..3e605a8e 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -7,7 +7,7 @@ import jsonimport timeimport requests-NUM_RESULTS_PER_QUERY = 50+NUM_RESULTS_PER_QUERY = 100NUM_ENSEMBLE_PREDS = 1PROMPT = """<|im_start|>systemPerform the task to the best of your ability.<|im_end|>@@ -30,6 +30,9 @@ llm = Llama(with open("data/queries_us.csv") as f:all_queries = [line.strip() for line in f.readlines()]+# shuffle queries+np.random.shuffle(all_queries)+def setup_db():db = sqlite3.connect("data/auto-ranking-annotation.sqlite")@@ -72,6 +75,10 @@ for query in all_queries:if sum([c.isalnum() for c in query]) / len(query) < 0.5:continue+ # only consider queries with at least two words+ if len(query.split()) < 2:+ continue+if len(query) > 100:continue