web search done right
fix misunderstandings in the new spell corrector
Mikkel Denkerf77fa32b6489
diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rsindex 30b920fc..00d53374 100644--- a/core/src/entrypoint/web_spell.rs+++ b/core/src/entrypoint/web_spell.rs@@ -20,7 +20,7 @@ use anyhow::Result;use fnv::FnvHashMap;use itertools::Itertools;use serde::{Deserialize, Serialize};-use tracing::info;+use tracing::{debug, info};use whatlang::Lang;use crate::{@@ -90,12 +90,15 @@ impl SpellWorker {}fn next_training_step(self) -> Result<FnvHashMap<Lang, FirstTrainerResult>> {+ debug!("next training step");let mut result = FnvHashMap::default();for (lang, trainer) in self.trainer {result.insert(lang, trainer.next_training_step()?);}+ debug!("next training step created");+Ok(result)}}@@ -120,7 +123,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {}).collect_vec();- let num_workers = num_cpus::get();+ let num_workers = (num_cpus::get()).min(jobs.len());let mut handlers = Vec::new();for i in 0..num_workers {@@ -148,12 +151,14 @@ pub fn run(config: WebSpellConfig) -> Result<()> {}for (lang, results) in combined {+ info!("creating second trainer for {}", lang);let second_trainer = SecondTrainer::new(results,Path::new(&config.output_path).join("corrector").join(lang.to_string()),)?;+ debug!("second trainer created");second_trainer.train()?;}diff --git a/core/src/web_spell/mod.rs b/core/src/web_spell/mod.rsindex a50356c5..b0a1e246 100644--- a/core/src/web_spell/mod.rs+++ b/core/src/web_spell/mod.rs@@ -127,7 +127,7 @@ pub fn tokenize(text: &str) -> Vec<String> {}pub struct MergePointer<'a> {pub term: String,- pub freq: u64,+ pub value: u64,pub stream: fst::map::Stream<'a>,pub is_finished: bool,}@@ -137,9 +137,9 @@ impl<'a> MergePointer<'a> {self.is_finished = self.stream.next()- .map(|(term, freq)| {+ .map(|(term, value)| {self.term = std::str::from_utf8(term).unwrap().to_string();- self.freq = freq;+ self.value = value;}).is_none();diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rsindex 518301cf..f65df34d 100644--- a/core/src/web_spell/stupid_backoff.rs+++ b/core/src/web_spell/stupid_backoff.rs@@ -22,7 +22,7 @@ use std::{path::Path,};-use fst::IntoStreamer;+use fst::{Automaton, IntoStreamer, Streamer};use serde::{Deserialize, Serialize};const DISCOUNT: f64 = 0.4;@@ -53,6 +53,7 @@ impl AsRef<[u8]> for StoredNgram {pub struct StupidBackoffTrainer {max_ngram_size: usize,ngrams: BTreeMap<Ngram, u64>,+ rotated_ngrams: BTreeMap<Ngram, u64>,n_counts: Vec<u64>,}@@ -61,6 +62,7 @@ impl StupidBackoffTrainer {Self {max_ngram_size,ngrams: BTreeMap::new(),+ rotated_ngrams: BTreeMap::new(),n_counts: vec![0; max_ngram_size],}}@@ -79,6 +81,15 @@ impl StupidBackoffTrainer {self.n_counts[i - 1] += 1;}++ let mut ngram = Ngram {+ terms: window.to_vec(),+ };+ ngram.terms.rotate_left(1);+ self.rotated_ngrams+ .entry(ngram)+ .and_modify(|e| *e += 1)+ .or_insert(1);}}@@ -101,6 +112,21 @@ impl StupidBackoffTrainer {builder.finish()?;+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref().join("rotated_ngrams.bin"))?;++ let wtr = BufWriter::new(file);++ let mut builder = fst::MapBuilder::new(wtr)?;++ for (ngram, freq) in self.rotated_ngrams {+ builder.insert(StoredNgram::from(ngram), freq)?;+ }++ builder.finish()?;+bincode::serialize_into(File::create(path.as_ref().join("n_counts.bin"))?,&self.n_counts,@@ -110,8 +136,68 @@ impl StupidBackoffTrainer {}}+fn merge_streams(+ mut builder: fst::MapBuilder<BufWriter<File>>,+ streams: Vec<fst::map::Stream<'_, fst::automaton::AlwaysMatch>>,+) -> Result<()> {+ let mut pointers: Vec<_> = streams+ .into_iter()+ .map(|stream| MergePointer {+ term: String::new(),+ value: 0,+ stream,+ is_finished: false,+ })+ .collect();++ for pointer in pointers.iter_mut() {+ pointer.advance();+ }++ while pointers.iter().any(|p| !p.is_finished) {+ let mut min_pointer: Option<&MergePointer<'_>> = None;++ for pointer in pointers.iter() {+ if pointer.is_finished {+ continue;+ }++ if let Some(min) = min_pointer {+ if pointer.term < min.term {+ min_pointer = Some(pointer);+ }+ } else {+ min_pointer = Some(pointer);+ }+ }++ if let Some(min_pointer) = min_pointer {+ let term = min_pointer.term.clone();+ let mut freq = 0;++ for pointer in pointers.iter_mut() {+ if pointer.is_finished {+ continue;+ }++ if pointer.term == term {+ freq += pointer.value;+ pointer.advance();+ }+ }++ builder.insert(term, freq)?;+ }+ }++ builder.finish()?;++ Ok(())+}+pub struct StupidBackoff {ngrams: fst::Map<memmap::Mmap>,+ rotated_ngrams: fst::Map<memmap::Mmap>,n_counts: Vec<u64>,}@@ -120,21 +206,23 @@ impl StupidBackoff {let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };let ngrams = fst::Map::new(mmap)?;+ let mmap =+ unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };+ let rotated_ngrams = fst::Map::new(mmap)?;+let n_counts = bincode::deserialize_from(File::open(path.as_ref().join("n_counts.bin"))?)?;- Ok(Self { ngrams, n_counts })+ Ok(Self {+ ngrams,+ rotated_ngrams,+ n_counts,+ })}pub fn merge<P: AsRef<Path>>(models: Vec<Self>, path: P) -> Result<Self> {if !path.as_ref().exists() {std::fs::create_dir_all(path.as_ref())?;}-- let file = OpenOptions::new()- .create(true)- .write(true)- .open(path.as_ref().join("ngrams.bin"))?;-let n_counts = models.iter().fold(vec![0; models[0].n_counts.len()], |mut acc, m| {@@ -145,65 +233,39 @@ impl StupidBackoff {acc});- let wtr = BufWriter::new(file);- let mut builder = fst::MapBuilder::new(wtr)?;+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref().join("ngrams.bin"))?;- let mut pointers: Vec<_> = models- .iter()- .map(|d| MergePointer {- term: String::new(),- freq: 0,- stream: d.ngrams.stream(),- is_finished: false,- })- .collect();-- for pointer in pointers.iter_mut() {- pointer.advance();- }+ let wtr = BufWriter::new(file);+ let builder = fst::MapBuilder::new(wtr)?;- while pointers.iter().any(|p| !p.is_finished) {- let mut min_pointer: Option<&MergePointer<'_>> = None;+ let streams: Vec<_> = models.iter().map(|d| d.ngrams.stream()).collect();- for pointer in pointers.iter() {- if pointer.is_finished {- continue;- }+ merge_streams(builder, streams)?;- if let Some(min) = min_pointer {- if pointer.term < min.term {- min_pointer = Some(pointer);- }- } else {- min_pointer = Some(pointer);- }- }-- if let Some(min_pointer) = min_pointer {- let term = min_pointer.term.clone();- let mut freq = 0;+ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let ngrams = fst::Map::new(mmap)?;- for pointer in pointers.iter_mut() {- if pointer.is_finished {- continue;- }+ let file = OpenOptions::new()+ .create(true)+ .write(true)+ .open(path.as_ref().join("rotated_ngrams.bin"))?;- if pointer.term == term {- freq += pointer.freq;- pointer.advance();- }- }+ let wtr = BufWriter::new(file);+ let builder = fst::MapBuilder::new(wtr)?;- builder.insert(term, freq)?;- }- }+ let streams: Vec<_> = models.iter().map(|d| d.ngrams.stream()).collect();- builder.finish()?;+ merge_streams(builder, streams)?;- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };+ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let rotated_ngrams = fst::Map::new(mmap)?;Ok(Self {- ngrams: fst::Map::new(mmap)?,+ ngrams,+ rotated_ngrams,n_counts,})}@@ -233,18 +295,23 @@ impl StupidBackoff {}}- pub fn contexts(&self, word: &str) -> Vec<Vec<String>> {- let q = " ".to_string() + word + " ";- let automaton = fst::automaton::Subsequence::new(q.as_str());-- self.ngrams- .search(automaton)- .into_stream()- .into_str_keys()- .unwrap_or_default()- .into_iter()- .map(|s| tokenize(&s))- .collect()+ pub fn contexts(&self, word: &str) -> Vec<(Vec<String>, u64)> {+ let q = word.to_string() + " ";+ let automaton = fst::automaton::Str::new(&q).starts_with();++ let mut stream = self.rotated_ngrams.search(automaton).into_stream();++ let mut contexts = Vec::new();++ while let Some((ngram, freq)) = stream.next() {+ if let Ok(ngram) = std::str::from_utf8(ngram) {+ let mut ngram = tokenize(ngram);+ ngram.rotate_right(1);+ contexts.push((ngram, freq));+ }+ }++ contexts}}diff --git a/core/src/web_spell/term_freqs.rs b/core/src/web_spell/term_freqs.rsindex 0e420bb4..ed6b6562 100644--- a/core/src/web_spell/term_freqs.rs+++ b/core/src/web_spell/term_freqs.rs@@ -97,7 +97,7 @@ impl StoredDict {.iter().map(|d| MergePointer {term: String::new(),- freq: 0,+ value: 0,stream: d.map.stream(),is_finished: false,})@@ -134,7 +134,7 @@ impl StoredDict {}if pointer.term == term {- freq += pointer.freq;+ freq += pointer.value;pointer.advance();}}diff --git a/core/src/web_spell/trainer.rs b/core/src/web_spell/trainer.rsindex 46a27118..26b18f43 100644--- a/core/src/web_spell/trainer.rs+++ b/core/src/web_spell/trainer.rs@@ -14,9 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use indicatif::ParallelProgressIterator;use rayon::prelude::*;-use crate::web_spell::{error_model::ErrorModel, stupid_backoff::IntoMiddle};+use crate::web_spell::error_model::ErrorModel;use super::{stupid_backoff::StupidBackoffTrainer, tokenize, Result};use std::{@@ -112,10 +113,12 @@ impl SecondTrainer {}pub fn train(self) -> Result<()> {+ tracing::info!("training error model");let terms = self.term_dict.terms();let errors: Vec<_> = terms.into_par_iter()+ .progress().map(|term| {// one edit for words of// up to four characters, two edits for up to twelve@@ -128,37 +131,46 @@ impl SecondTrainer {3};- let possible_errors = self+ let possible_corrections = self.term_dict.search(&term, max_edit_distance).into_iter()- .filter(|incorrect| {- incorrect != &term- && 10 * self.term_dict.freq(incorrect).unwrap_or_default()- > self.term_dict.freq(&term).unwrap_or_default()+ .filter(|correction| {+ correction != &term+ && 10 * self.term_dict.freq(&term).unwrap_or_default()+ > self.term_dict.freq(correction).unwrap_or_default()}).collect::<Vec<_>>();- (term, possible_errors)+ (term, possible_corrections)}).filter(|(_, errors)| !errors.is_empty())- .map(|(term, possible_errors)| {+ .map(|(term, possible_corrections)| {let contexts = self.lm_model.contexts(&term);let best_terms = contexts.into_iter()+ .filter_map(+ |(context, count)| {+ if count < 10 {+ None+ } else {+ Some(context)+ }+ },+ ).filter(|c| c.len() != 3).map(|context| {- let most_probable = possible_errors+ let most_probable = possible_corrections.iter().chain(vec![&term]).map(|t| {let mut words = context.clone();words[1] = t.clone();- (self.lm_model.log_prob(&words, IntoMiddle::default()), t)+ (self.lm_model.freq(&words), t)})- .max_by(|(a, _), (b, _)| a.total_cmp(b))+ .max_by(|(a, _), (b, _)| a.cmp(b)).unwrap().1.clone();@@ -180,17 +192,16 @@ impl SecondTrainer {let mut error_model = ErrorModel::new();- for (term, correction) in errors+ for (term, possible_correction) in errors.into_iter()- .flat_map(|(term, correction)| {- correction- .keys()- .map(|other| (term.clone(), other.clone()))- .collect::<Vec<_>>()+ .flat_map(|(term, possible_corrections)| {+ possible_corrections+ .into_keys()+ .map(move |other| (term.clone(), other.clone()))}).filter(|(a, b)| a != b){- error_model.add(&term, &correction);+ error_model.add(&term, &possible_correction);}error_model.save(self.path.join("error_model.json"))?;diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex fef78a75..510c0c63 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -1,24 +1,29 @@-from openai import OpenAI-from os import environ+import numpy as npimport re+from llama_cpp import Llamaimport sqlite3-import requestsfrom tqdm import tqdmimport jsonimport time+import requests-NUM_RESULTS_PER_QUERY = 10-PROMPT = """You are an expert data annotator employed at Google. Your task is to evaluate search results on an integer scale of 0-4, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source or niche blog and answer the users query.-First briefly provide the reasoning for the evaluation and then give the relevancy on the form "Relevancy: {{}}" on a new line.+NUM_RESULTS_PER_QUERY = 20+NUM_ENSEMBLE_PREDS = 1+PROMPT = """<|im_start|>system+Perform the task to the best of your ability.<|im_end|>+<|im_start|>user+Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-4, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.Query: "{}"-URL: "{}"+Url: "{}"Title: "{}"-Snippet: "{}"-Explanation:"""--client = OpenAI(- api_key=environ.get("OPENAI_API_KEY"),+Snippet: "{}"<|im_end|>+<|im_start|>assistant+"""+llm = Llama(+ n_gpu_layers=-1,+ n_ctx=8000,+ model_path="data/neuralhermes-2.5-mistral-7b.Q5_K_M.gguf",)@@ -60,6 +65,16 @@ db = setup_db()cur = db.cursor()for query in all_queries:+ if len(query) < 3:+ continue++ # check if query has large percentage of non-alphanumeric characters+ if sum([c.isalnum() for c in query]) / len(query) < 0.5:+ continue++ if len(query) > 100:+ continue+cur.execute("""INSERT OR IGNORE INTO queries (query)@@ -169,23 +184,30 @@ for qid, query in tqdm(unannotated_queries.items()):for url, orig_rank, webpage_json in tqdm(unnanotated_results):webpage = json.loads(webpage_json)prompt = get_prompt(query, url, webpage["title"], webpage["snippet"])- res = (- client.chat.completions.create(- messages=[- {- "role": "system",- "content": prompt,- }- ],- model="gpt-4-1106-preview",+ res = 0+ n = 0+ for _ in range(NUM_ENSEMBLE_PREDS):+ output = llm.create_completion(+ prompt,+ max_tokens=-1,+ echo=False,+ temperature=0.4,+ stop=["<|im_start|>", "<|im_end|>"],)- .choices[0]- .message.content- )+ output = output["choices"][0]["text"]++ relevancy = get_relevancy(output)+ if relevancy is None:+ continue++ n += 1+ res += relevancy- relevancy = get_relevancy(res)- if relevancy is None:+ if n == 0:+ print("No relevancy annotation for", query, url)continue+ relevancy = np.round(res / n).astype(int)+ relevancy = max(0, min(4, int(relevancy)))cur.execute("""diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 3c856b28..6b2bdcc6 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -37,7 +37,7 @@ for qid in queries:if w["label"] is not None]- urls = sorted(urls, key=lambda x: x[1], reverse=True)+ urls = sorted(urls, key=lambda x: int(x[1]), reverse=True)queries[qid]["urls"] = urls@@ -105,9 +105,9 @@ model = lgb.LGBMRanker(objective="lambdarank",metric="ndcg",importance_type="gain",- num_leaves=20,+ num_leaves=15,n_estimators=n_estimators,- max_depth=10,+ max_depth=3,learning_rate=0.1,label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],)@@ -144,8 +144,8 @@ for i in range(len(X_test)):assert model.predict(X_test[i : i + 1]) == saved_model.predict(X_test[i : i + 1])# print an example-print("Example:")-t = X_test[0]-print("Features:")-pprint({id2feature[i]: v for i, v in enumerate(t)})-print("Score:", model.predict(t.reshape(1, -1))[0])+# print("Example:")+# t = X_test[0]+# print("Features:")+# pprint({id2feature[i]: v for i, v in enumerate(t)})+# print("Score:", model.predict(t.reshape(1, -1))[0])