web search done right
[web-spell] fix training oom store term_dict and ngrams from language model after each warc file has been parsed
Mikkel Denker612522c85cf3
diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rsindex a51668cc..33b187fc 100644--- a/core/src/entrypoint/web_spell.rs+++ b/core/src/entrypoint/web_spell.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::Path;+use std::path::{Path, PathBuf};use anyhow::Result;use fnv::FnvHashMap;@@ -32,21 +32,18 @@ use crate::{};pub struct SpellWorker {- trainer: FnvHashMap<Lang, FirstTrainer>,+ prev_trained: Vec<FnvHashMap<Lang, FirstTrainerResult>>,+ languages: Vec<Lang>,+ path: PathBuf,}impl SpellWorker {pub fn new<P: AsRef<Path>>(languages: &[Lang], path: P) -> Result<Self> {- let mut trainer = FnvHashMap::default();-- for lang in languages {- trainer.insert(- *lang,- FirstTrainer::new(path.as_ref().join(lang.to_string()))?,- );- }-- Ok(Self { trainer })+ Ok(Self {+ path: path.as_ref().to_path_buf(),+ languages: languages.to_vec(),+ prev_trained: Vec::new(),+ })}pub fn process(&mut self, job: Job) -> Result<()> {@@ -54,6 +51,16 @@ impl SpellWorker {info!("processing {}", name);+ let mut trainer_has_inserts = false;+ let mut trainer: FnvHashMap<Lang, FirstTrainer> = Default::default();++ for lang in &self.languages {+ trainer.insert(+ *lang,+ FirstTrainer::new(self.path.as_path().join(name).join(lang.to_string()))?,+ );+ }+let source = job.source_config.clone();let paths = vec![job.warc_path.clone()];@@ -75,31 +82,39 @@ impl SpellWorker {None => continue,};- match (webpage.clean_text(), self.trainer.get_mut(&lang)) {+ match (webpage.clean_text(), trainer.get_mut(&lang)) {(Some(text), Some(trainer)) => {trainer.add(text);+ trainer_has_inserts = true;}_ => continue,}}}+ if trainer_has_inserts {+ self.finish_training_step(trainer)?;+ }info!("finished processing {}", name);Ok(())}- fn next_training_step(self) -> Result<FnvHashMap<Lang, FirstTrainerResult>> {+ fn finish_training_step(&mut self, trainer: FnvHashMap<Lang, FirstTrainer>) -> Result<()> {debug!("next training step");- let mut result = FnvHashMap::default();+ let mut trained = FnvHashMap::default();- for (lang, trainer) in self.trainer {- result.insert(lang, trainer.next_training_step()?);+ for (lang, trainer) in trainer {+ trained.insert(lang, trainer.next_training_step()?);}+ self.prev_trained.push(trained);debug!("next training step created");+ Ok(())+ }- Ok(result)+ fn next_training_steps(self) -> Vec<FnvHashMap<Lang, FirstTrainerResult>> {+ self.prev_trained}}@@ -137,7 +152,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {worker.process(job).unwrap();}- worker.next_training_step().unwrap()+ worker.next_training_steps()}));}@@ -145,8 +160,10 @@ pub fn run(config: WebSpellConfig) -> Result<()> {for handler in handlers {let trained = handler.join().unwrap();- for (lang, result) in trained.into_iter() {- combined.entry(lang).or_default().push(result);+ for res in trained {+ for (lang, result) in res {+ combined.entry(lang).or_default().push(result);+ }}}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 8e6b5e5f..af260369 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -32,7 +32,6 @@ use crate::{fastfield_reader::FastFieldReader,search_ctx::Ctx,searcher::NUM_RESULTS_PER_PAGE,- webpage::region::Region,};pub use self::signal::*;@@ -65,11 +64,6 @@ impl Ranker {}}- pub fn with_region(mut self, region: Region) -> Self {- self.aggregator.set_selected_region(region);- self- }-pub fn with_offset(mut self, offset: usize) -> Self {self.offset = Some(offset);selfdiff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 6142548d..f6c556df 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -213,13 +213,17 @@ fn score_link_density(link_density: f64) -> f64 {fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {match aggregator.region_count.as_ref() {Some(region_count) => {- let boost = aggregator.selected_region.map_or(0.0, |region| {- if region == webpage_region {- 50.0- } else {- 0.0- }- });+ let boost = aggregator+ .query_data+ .as_ref()+ .and_then(|q| q.selected_region)+ .map_or(0.0, |region| {+ if region != Region::All && region == webpage_region {+ 50.0+ } else {+ 0.0+ }+ });boost + region_count.score(&webpage_region)}@@ -692,10 +696,11 @@ struct SegmentReader {struct QueryData {simple_terms: Vec<String>,optic_rules: Vec<optics::Rule>,+ selected_region: Option<Region>,}pub struct SignalAggregator {- query: Option<QueryData>,+ query_data: Option<QueryData>,query_signal_coefficients: Option<SignalCoefficient>,segment_reader: Option<SegmentReader>,inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,@@ -703,7 +708,6 @@ pub struct SignalAggregator {update_time_cache: Vec<f64>,query_centrality: Option<RefCell<query_centrality::Scorer>>,region_count: Option<Arc<RegionCount>>,- selected_region: Option<Region>,current_timestamp: Option<usize>,linear_regression: Option<Arc<LinearRegression>>,}@@ -721,7 +725,7 @@ impl Clone for SignalAggregator {.map(|scorer| RefCell::new(scorer.borrow().clone()));Self {- query: self.query.clone(),+ query_data: self.query_data.clone(),query_signal_coefficients: self.query_signal_coefficients.clone(),segment_reader: None,inbound_similarity,@@ -729,7 +733,6 @@ impl Clone for SignalAggregator {update_time_cache: self.update_time_cache.clone(),query_centrality,region_count: self.region_count.clone(),- selected_region: self.selected_region,current_timestamp: self.current_timestamp,linear_regression: self.linear_regression.clone(),}@@ -742,7 +745,7 @@ impl std::fmt::Debug for SignalAggregator {.field("query",&self- .query+ .query_data.as_ref().map(|q| q.simple_terms.clone()).unwrap_or_default(),@@ -775,6 +778,7 @@ impl SignalAggregator {}).cloned().collect(),+ selected_region: q.region().cloned(),});let mut s = Self {@@ -785,10 +789,9 @@ impl SignalAggregator {update_time_cache,query_centrality: None,region_count: None,- selected_region: None,current_timestamp: None,linear_regression: None,- query,+ query_data: query,};s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);@@ -804,7 +807,7 @@ impl SignalAggregator {let mut text_fields = EnumMap::new();let schema = tv_searcher.schema();- if let Some(query) = &self.query {+ if let Some(query) = &self.query_data {if !query.simple_terms.is_empty() {for signal in ALL_SIGNALS {if let Some(text_field) = signal.as_textfield() {@@ -866,7 +869,7 @@ impl SignalAggregator {) -> Vec<RuleBoost> {let mut optic_rule_boosts = Vec::new();- if let Some(query) = &self.query {+ if let Some(query) = &self.query_data {optic_rule_boosts = query.optic_rules.iter()@@ -922,10 +925,6 @@ impl SignalAggregator {self.region_count = Some(Arc::new(region_count));}- pub fn set_selected_region(&mut self, region: Region) {- self.selected_region = Some(region);- }-pub fn set_current_timestamp(&mut self, current_timestamp: usize) {self.current_timestamp = Some(current_timestamp);}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex cc449870..d47185a5 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -38,7 +38,6 @@ use crate::schema::TextField;use crate::search_ctx::Ctx;use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};use crate::webgraph::Node;-use crate::webpage::region::Region;use crate::{inverted_index, live_index, Error, Result};use super::WebsitesResult;@@ -218,12 +217,6 @@ whereself.collector_config.clone(),);- if let Some(region) = query.region() {- if *region != Region::All {- ranker = ranker.with_region(*region);- }- }-ranker.de_rank_similar(de_rank_similar);if query_centrality_coeff > 0.0 {diff --git a/core/src/warc.rs b/core/src/warc.rsindex ee95fcae..e9f5d945 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -27,6 +27,7 @@ use std::time::Duration;use flate2::read::MultiGzDecoder;use flate2::write::GzEncoder;use flate2::Compression;+use fnv::FnvHashSet;#[cfg(test)]use proptest_derive::Arbitrary;@@ -522,7 +523,7 @@ impl<R: Read> Iterator for RecordIterator<R> {pub struct DeduplicatedWarcWriter {writer: WarcWriter,- seen_url_hashes: std::collections::HashSet<md5::Digest>,+ seen_url_hashes: FnvHashSet<md5::Digest>,}impl Default for DeduplicatedWarcWriter {@@ -535,7 +536,7 @@ impl DeduplicatedWarcWriter {pub fn new() -> Self {Self {writer: WarcWriter::new(),- seen_url_hashes: std::collections::HashSet::new(),+ seen_url_hashes: FnvHashSet::default(),}}diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex 3e605a8e..0f06e0ad 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -149,7 +149,7 @@ def add_results(query):for i, result in enumerate(results):cur.execute("""- INSERT INTO search_results (qid, url, orig_rank, webpage_json)+ INSERT OR IGNORE INTO search_results (qid, url, orig_rank, webpage_json)VALUES (?, ?, ?, ?)""",(qid, result["url"], i, json.dumps(result)),diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 8680f440..4de29c78 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -4,6 +4,8 @@ import jsonfrom pprint import pprintimport sqlite3+CATEGORICAL_FEATURES = ["is_homepage"]+con = sqlite3.connect("data/auto-ranking-annotation.sqlite")cur = con.cursor()@@ -77,7 +79,8 @@ q_train = []for query, urls in items[:train_size]:q_train.append(query)for url, data in urls.items():- X_train.append([data["features"].get(k, 0) for k in id2feature])+ x = [data["features"].get(k, 0) for k in id2feature]+ X_train.append(x)y_train.append(data["score"])X_test = []@@ -110,8 +113,9 @@ model = lgb.LGBMRanker(importance_type="gain",num_leaves=50,n_estimators=n_estimators,+ categorical_features=[feature2id[k] for k in CATEGORICAL_FEATURES],max_depth=10,- learning_rate=0.01,+ learning_rate=0.1,label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],)model.fit(diff --git a/ltr/linear_model.py b/ltr/linear_model.pyindex bf913b1f..64c3252d 100644--- a/ltr/linear_model.py+++ b/ltr/linear_model.py@@ -26,23 +26,32 @@ queries = {qid: {"query": query} for qid, query in res.fetchall()}for qid in queries:res = cur.execute("""- SELECT qid, url, annotation, webpage_json+ SELECT qid, url, annotation, orig_rank, webpage_jsonFROM search_resultsWHERE qid = ?""",(qid,),)urls = {- url: {"label": label, "signals": json.loads(page)["rankingSignals"]}- for _, url, label, page in res.fetchall()+ url: {+ "label": label,+ "orig_rank": orig_rank,+ "signals": json.loads(page)["rankingSignals"],+ }+ for _, url, label, orig_rank, page in res.fetchall()}urls = [- (url, w["label"], w["signals"])+ (url, w["label"], w["orig_rank"], w["signals"])for url, w in urls.items()if w["label"] is not None]- urls = sorted(urls, key=lambda x: int(x[1]), reverse=True)+ sorting_key = lambda x: (+ -x[1],+ x[2],+ )++ urls = sorted(urls, key=sorting_key)queries[qid]["urls"] = urls@@ -50,7 +59,7 @@ feature2id = {}id2feature = {}for qid, data in queries.items():- for url, score, signals in data["urls"]:+ for url, score, _, signals in data["urls"]:for feature, value in signals.items():if feature not in feature2id:id = len(feature2id)@@ -65,16 +74,12 @@ random.shuffle(scores)sorted_features = sorted(feature2id.items(), key=lambda x: x[1])for qid, data in queries.items():- for url, score, signals in data["urls"]:+ for i, (url, score, _, signals) in enumerate(data["urls"]):signals = {feature2id[k]: v for k, v in signals.items()}signals = [signals[k] for _, k in sorted_features]- scores.append({"ranking_signals": signals, "score": score})---for score in scores:- score["score"] = np.power(2, score["score"])+ scores.append({"ranking_signals": signals, "score": 20 / (i + 1)})train = scores[: int(len(scores) * TRAIN_PERCENT)]test = scores[int(len(scores) * TRAIN_PERCENT) :]