Skip to content

web search done right

fix misunderstandings in the new spell corrector

Mikkel Denkerf77fa32b6489
diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rs
index 30b920fc..00d53374 100644
--- a/core/src/entrypoint/web_spell.rs
+++ b/core/src/entrypoint/web_spell.rs
@@ -20,7 +20,7 @@ use anyhow::Result;
use fnv::FnvHashMap;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
-use tracing::info;
+use tracing::{debug, info};
use whatlang::Lang;
use crate::{
@@ -90,12 +90,15 @@ impl SpellWorker {
}
fn next_training_step(self) -> Result<FnvHashMap<Lang, FirstTrainerResult>> {
+ debug!("next training step");
let mut result = FnvHashMap::default();
for (lang, trainer) in self.trainer {
result.insert(lang, trainer.next_training_step()?);
}
+ debug!("next training step created");
+
Ok(result)
}
}
@@ -120,7 +123,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {
})
.collect_vec();
- let num_workers = num_cpus::get();
+ let num_workers = (num_cpus::get()).min(jobs.len());
let mut handlers = Vec::new();
for i in 0..num_workers {
@@ -148,12 +151,14 @@ pub fn run(config: WebSpellConfig) -> Result<()> {
}
for (lang, results) in combined {
+ info!("creating second trainer for {}", lang);
let second_trainer = SecondTrainer::new(
results,
Path::new(&config.output_path)
.join("corrector")
.join(lang.to_string()),
)?;
+ debug!("second trainer created");
second_trainer.train()?;
}
diff --git a/core/src/web_spell/mod.rs b/core/src/web_spell/mod.rs
index a50356c5..b0a1e246 100644
--- a/core/src/web_spell/mod.rs
+++ b/core/src/web_spell/mod.rs
@@ -127,7 +127,7 @@ pub fn tokenize(text: &str) -> Vec<String> {
}
pub struct MergePointer<'a> {
pub term: String,
- pub freq: u64,
+ pub value: u64,
pub stream: fst::map::Stream<'a>,
pub is_finished: bool,
}
@@ -137,9 +137,9 @@ impl<'a> MergePointer<'a> {
self.is_finished = self
.stream
.next()
- .map(|(term, freq)| {
+ .map(|(term, value)| {
self.term = std::str::from_utf8(term).unwrap().to_string();
- self.freq = freq;
+ self.value = value;
})
.is_none();
diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rs
index 518301cf..f65df34d 100644
--- a/core/src/web_spell/stupid_backoff.rs
+++ b/core/src/web_spell/stupid_backoff.rs
@@ -22,7 +22,7 @@ use std::{
path::Path,
};
-use fst::IntoStreamer;
+use fst::{Automaton, IntoStreamer, Streamer};
use serde::{Deserialize, Serialize};
const DISCOUNT: f64 = 0.4;
@@ -53,6 +53,7 @@ impl AsRef<[u8]> for StoredNgram {
pub struct StupidBackoffTrainer {
max_ngram_size: usize,
ngrams: BTreeMap<Ngram, u64>,
+ rotated_ngrams: BTreeMap<Ngram, u64>,
n_counts: Vec<u64>,
}
@@ -61,6 +62,7 @@ impl StupidBackoffTrainer {
Self {
max_ngram_size,
ngrams: BTreeMap::new(),
+ rotated_ngrams: BTreeMap::new(),
n_counts: vec![0; max_ngram_size],
}
}
@@ -79,6 +81,15 @@ impl StupidBackoffTrainer {
self.n_counts[i - 1] += 1;
}
+
+ let mut ngram = Ngram {
+ terms: window.to_vec(),
+ };
+ ngram.terms.rotate_left(1);
+ self.rotated_ngrams
+ .entry(ngram)
+ .and_modify(|e| *e += 1)
+ .or_insert(1);
}
}
@@ -101,6 +112,21 @@ impl StupidBackoffTrainer {
builder.finish()?;
+ let file = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .open(path.as_ref().join("rotated_ngrams.bin"))?;
+
+ let wtr = BufWriter::new(file);
+
+ let mut builder = fst::MapBuilder::new(wtr)?;
+
+ for (ngram, freq) in self.rotated_ngrams {
+ builder.insert(StoredNgram::from(ngram), freq)?;
+ }
+
+ builder.finish()?;
+
bincode::serialize_into(
File::create(path.as_ref().join("n_counts.bin"))?,
&self.n_counts,
@@ -110,8 +136,68 @@ impl StupidBackoffTrainer {
}
}
+fn merge_streams(
+ mut builder: fst::MapBuilder<BufWriter<File>>,
+ streams: Vec<fst::map::Stream<'_, fst::automaton::AlwaysMatch>>,
+) -> Result<()> {
+ let mut pointers: Vec<_> = streams
+ .into_iter()
+ .map(|stream| MergePointer {
+ term: String::new(),
+ value: 0,
+ stream,
+ is_finished: false,
+ })
+ .collect();
+
+ for pointer in pointers.iter_mut() {
+ pointer.advance();
+ }
+
+ while pointers.iter().any(|p| !p.is_finished) {
+ let mut min_pointer: Option<&MergePointer<'_>> = None;
+
+ for pointer in pointers.iter() {
+ if pointer.is_finished {
+ continue;
+ }
+
+ if let Some(min) = min_pointer {
+ if pointer.term < min.term {
+ min_pointer = Some(pointer);
+ }
+ } else {
+ min_pointer = Some(pointer);
+ }
+ }
+
+ if let Some(min_pointer) = min_pointer {
+ let term = min_pointer.term.clone();
+ let mut freq = 0;
+
+ for pointer in pointers.iter_mut() {
+ if pointer.is_finished {
+ continue;
+ }
+
+ if pointer.term == term {
+ freq += pointer.value;
+ pointer.advance();
+ }
+ }
+
+ builder.insert(term, freq)?;
+ }
+ }
+
+ builder.finish()?;
+
+ Ok(())
+}
+
pub struct StupidBackoff {
ngrams: fst::Map<memmap::Mmap>,
+ rotated_ngrams: fst::Map<memmap::Mmap>,
n_counts: Vec<u64>,
}
@@ -120,21 +206,23 @@ impl StupidBackoff {
let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
let ngrams = fst::Map::new(mmap)?;
+ let mmap =
+ unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };
+ let rotated_ngrams = fst::Map::new(mmap)?;
+
let n_counts = bincode::deserialize_from(File::open(path.as_ref().join("n_counts.bin"))?)?;
- Ok(Self { ngrams, n_counts })
+ Ok(Self {
+ ngrams,
+ rotated_ngrams,
+ n_counts,
+ })
}
pub fn merge<P: AsRef<Path>>(models: Vec<Self>, path: P) -> Result<Self> {
if !path.as_ref().exists() {
std::fs::create_dir_all(path.as_ref())?;
}
-
- let file = OpenOptions::new()
- .create(true)
- .write(true)
- .open(path.as_ref().join("ngrams.bin"))?;
-
let n_counts = models
.iter()
.fold(vec![0; models[0].n_counts.len()], |mut acc, m| {
@@ -145,65 +233,39 @@ impl StupidBackoff {
acc
});
- let wtr = BufWriter::new(file);
- let mut builder = fst::MapBuilder::new(wtr)?;
+ let file = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .open(path.as_ref().join("ngrams.bin"))?;
- let mut pointers: Vec<_> = models
- .iter()
- .map(|d| MergePointer {
- term: String::new(),
- freq: 0,
- stream: d.ngrams.stream(),
- is_finished: false,
- })
- .collect();
-
- for pointer in pointers.iter_mut() {
- pointer.advance();
- }
+ let wtr = BufWriter::new(file);
+ let builder = fst::MapBuilder::new(wtr)?;
- while pointers.iter().any(|p| !p.is_finished) {
- let mut min_pointer: Option<&MergePointer<'_>> = None;
+ let streams: Vec<_> = models.iter().map(|d| d.ngrams.stream()).collect();
- for pointer in pointers.iter() {
- if pointer.is_finished {
- continue;
- }
+ merge_streams(builder, streams)?;
- if let Some(min) = min_pointer {
- if pointer.term < min.term {
- min_pointer = Some(pointer);
- }
- } else {
- min_pointer = Some(pointer);
- }
- }
-
- if let Some(min_pointer) = min_pointer {
- let term = min_pointer.term.clone();
- let mut freq = 0;
+ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ let ngrams = fst::Map::new(mmap)?;
- for pointer in pointers.iter_mut() {
- if pointer.is_finished {
- continue;
- }
+ let file = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .open(path.as_ref().join("rotated_ngrams.bin"))?;
- if pointer.term == term {
- freq += pointer.freq;
- pointer.advance();
- }
- }
+ let wtr = BufWriter::new(file);
+ let builder = fst::MapBuilder::new(wtr)?;
- builder.insert(term, freq)?;
- }
- }
+ let streams: Vec<_> = models.iter().map(|d| d.ngrams.stream()).collect();
- builder.finish()?;
+ merge_streams(builder, streams)?;
- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };
+ let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ let rotated_ngrams = fst::Map::new(mmap)?;
Ok(Self {
- ngrams: fst::Map::new(mmap)?,
+ ngrams,
+ rotated_ngrams,
n_counts,
})
}
@@ -233,18 +295,23 @@ impl StupidBackoff {
}
}
- pub fn contexts(&self, word: &str) -> Vec<Vec<String>> {
- let q = " ".to_string() + word + " ";
- let automaton = fst::automaton::Subsequence::new(q.as_str());
-
- self.ngrams
- .search(automaton)
- .into_stream()
- .into_str_keys()
- .unwrap_or_default()
- .into_iter()
- .map(|s| tokenize(&s))
- .collect()
+ pub fn contexts(&self, word: &str) -> Vec<(Vec<String>, u64)> {
+ let q = word.to_string() + " ";
+ let automaton = fst::automaton::Str::new(&q).starts_with();
+
+ let mut stream = self.rotated_ngrams.search(automaton).into_stream();
+
+ let mut contexts = Vec::new();
+
+ while let Some((ngram, freq)) = stream.next() {
+ if let Ok(ngram) = std::str::from_utf8(ngram) {
+ let mut ngram = tokenize(ngram);
+ ngram.rotate_right(1);
+ contexts.push((ngram, freq));
+ }
+ }
+
+ contexts
}
}
diff --git a/core/src/web_spell/term_freqs.rs b/core/src/web_spell/term_freqs.rs
index 0e420bb4..ed6b6562 100644
--- a/core/src/web_spell/term_freqs.rs
+++ b/core/src/web_spell/term_freqs.rs
@@ -97,7 +97,7 @@ impl StoredDict {
.iter()
.map(|d| MergePointer {
term: String::new(),
- freq: 0,
+ value: 0,
stream: d.map.stream(),
is_finished: false,
})
@@ -134,7 +134,7 @@ impl StoredDict {
}
if pointer.term == term {
- freq += pointer.freq;
+ freq += pointer.value;
pointer.advance();
}
}
diff --git a/core/src/web_spell/trainer.rs b/core/src/web_spell/trainer.rs
index 46a27118..26b18f43 100644
--- a/core/src/web_spell/trainer.rs
+++ b/core/src/web_spell/trainer.rs
@@ -14,9 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use indicatif::ParallelProgressIterator;
use rayon::prelude::*;
-use crate::web_spell::{error_model::ErrorModel, stupid_backoff::IntoMiddle};
+use crate::web_spell::error_model::ErrorModel;
use super::{stupid_backoff::StupidBackoffTrainer, tokenize, Result};
use std::{
@@ -112,10 +113,12 @@ impl SecondTrainer {
}
pub fn train(self) -> Result<()> {
+ tracing::info!("training error model");
let terms = self.term_dict.terms();
let errors: Vec<_> = terms
.into_par_iter()
+ .progress()
.map(|term| {
// one edit for words of
// up to four characters, two edits for up to twelve
@@ -128,37 +131,46 @@ impl SecondTrainer {
3
};
- let possible_errors = self
+ let possible_corrections = self
.term_dict
.search(&term, max_edit_distance)
.into_iter()
- .filter(|incorrect| {
- incorrect != &term
- && 10 * self.term_dict.freq(incorrect).unwrap_or_default()
- > self.term_dict.freq(&term).unwrap_or_default()
+ .filter(|correction| {
+ correction != &term
+ && 10 * self.term_dict.freq(&term).unwrap_or_default()
+ > self.term_dict.freq(correction).unwrap_or_default()
})
.collect::<Vec<_>>();
- (term, possible_errors)
+ (term, possible_corrections)
})
.filter(|(_, errors)| !errors.is_empty())
- .map(|(term, possible_errors)| {
+ .map(|(term, possible_corrections)| {
let contexts = self.lm_model.contexts(&term);
let best_terms = contexts
.into_iter()
+ .filter_map(
+ |(context, count)| {
+ if count < 10 {
+ None
+ } else {
+ Some(context)
+ }
+ },
+ )
.filter(|c| c.len() != 3)
.map(|context| {
- let most_probable = possible_errors
+ let most_probable = possible_corrections
.iter()
.chain(vec![&term])
.map(|t| {
let mut words = context.clone();
words[1] = t.clone();
- (self.lm_model.log_prob(&words, IntoMiddle::default()), t)
+ (self.lm_model.freq(&words), t)
})
- .max_by(|(a, _), (b, _)| a.total_cmp(b))
+ .max_by(|(a, _), (b, _)| a.cmp(b))
.unwrap()
.1
.clone();
@@ -180,17 +192,16 @@ impl SecondTrainer {
let mut error_model = ErrorModel::new();
- for (term, correction) in errors
+ for (term, possible_correction) in errors
.into_iter()
- .flat_map(|(term, correction)| {
- correction
- .keys()
- .map(|other| (term.clone(), other.clone()))
- .collect::<Vec<_>>()
+ .flat_map(|(term, possible_corrections)| {
+ possible_corrections
+ .into_keys()
+ .map(move |other| (term.clone(), other.clone()))
})
.filter(|(a, b)| a != b)
{
- error_model.add(&term, &correction);
+ error_model.add(&term, &possible_correction);
}
error_model.save(self.path.join("error_model.json"))?;
diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.py
index fef78a75..510c0c63 100644
--- a/ltr/auto_annotate.py
+++ b/ltr/auto_annotate.py
@@ -1,24 +1,29 @@
-from openai import OpenAI
-from os import environ
+import numpy as np
import re
+from llama_cpp import Llama
import sqlite3
-import requests
from tqdm import tqdm
import json
import time
+import requests
-NUM_RESULTS_PER_QUERY = 10
-PROMPT = """You are an expert data annotator employed at Google. Your task is to evaluate search results on an integer scale of 0-4, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source or niche blog and answer the users query.
-First briefly provide the reasoning for the evaluation and then give the relevancy on the form "Relevancy: {{}}" on a new line.
+NUM_RESULTS_PER_QUERY = 20
+NUM_ENSEMBLE_PREDS = 1
+PROMPT = """<|im_start|>system
+Perform the task to the best of your ability.<|im_end|>
+<|im_start|>user
+Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-4, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.
Query: "{}"
-URL: "{}"
+Url: "{}"
Title: "{}"
-Snippet: "{}"
-Explanation:"""
-
-client = OpenAI(
- api_key=environ.get("OPENAI_API_KEY"),
+Snippet: "{}"<|im_end|>
+<|im_start|>assistant
+"""
+llm = Llama(
+ n_gpu_layers=-1,
+ n_ctx=8000,
+ model_path="data/neuralhermes-2.5-mistral-7b.Q5_K_M.gguf",
)
@@ -60,6 +65,16 @@ db = setup_db()
cur = db.cursor()
for query in all_queries:
+ if len(query) < 3:
+ continue
+
+ # check if query has large percentage of non-alphanumeric characters
+ if sum([c.isalnum() for c in query]) / len(query) < 0.5:
+ continue
+
+ if len(query) > 100:
+ continue
+
cur.execute(
"""
INSERT OR IGNORE INTO queries (query)
@@ -169,23 +184,30 @@ for qid, query in tqdm(unannotated_queries.items()):
for url, orig_rank, webpage_json in tqdm(unnanotated_results):
webpage = json.loads(webpage_json)
prompt = get_prompt(query, url, webpage["title"], webpage["snippet"])
- res = (
- client.chat.completions.create(
- messages=[
- {
- "role": "system",
- "content": prompt,
- }
- ],
- model="gpt-4-1106-preview",
+ res = 0
+ n = 0
+ for _ in range(NUM_ENSEMBLE_PREDS):
+ output = llm.create_completion(
+ prompt,
+ max_tokens=-1,
+ echo=False,
+ temperature=0.4,
+ stop=["<|im_start|>", "<|im_end|>"],
)
- .choices[0]
- .message.content
- )
+ output = output["choices"][0]["text"]
+
+ relevancy = get_relevancy(output)
+ if relevancy is None:
+ continue
+
+ n += 1
+ res += relevancy
- relevancy = get_relevancy(res)
- if relevancy is None:
+ if n == 0:
+ print("No relevancy annotation for", query, url)
continue
+ relevancy = np.round(res / n).astype(int)
+ relevancy = max(0, min(4, int(relevancy)))
cur.execute(
"""
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index 3c856b28..6b2bdcc6 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -37,7 +37,7 @@ for qid in queries:
if w["label"] is not None
]
- urls = sorted(urls, key=lambda x: x[1], reverse=True)
+ urls = sorted(urls, key=lambda x: int(x[1]), reverse=True)
queries[qid]["urls"] = urls
@@ -105,9 +105,9 @@ model = lgb.LGBMRanker(
objective="lambdarank",
metric="ndcg",
importance_type="gain",
- num_leaves=20,
+ num_leaves=15,
n_estimators=n_estimators,
- max_depth=10,
+ max_depth=3,
learning_rate=0.1,
label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],
)
@@ -144,8 +144,8 @@ for i in range(len(X_test)):
assert model.predict(X_test[i : i + 1]) == saved_model.predict(X_test[i : i + 1])
# print an example
-print("Example:")
-t = X_test[0]
-print("Features:")
-pprint({id2feature[i]: v for i, v in enumerate(t)})
-print("Score:", model.predict(t.reshape(1, -1))[0])
+# print("Example:")
+# t = X_test[0]
+# print("Features:")
+# pprint({id2feature[i]: v for i, v in enumerate(t)})
+# print("Score:", model.predict(t.reshape(1, -1))[0])