Skip to content

web search done right

[web-spell] fix training oom store term_dict and ngrams from language model after each warc file has been parsed

Mikkel Denker612522c85cf3
diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rs
index a51668cc..33b187fc 100644
--- a/core/src/entrypoint/web_spell.rs
+++ b/core/src/entrypoint/web_spell.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::path::Path;
+use std::path::{Path, PathBuf};
use anyhow::Result;
use fnv::FnvHashMap;
@@ -32,21 +32,18 @@ use crate::{
};
pub struct SpellWorker {
- trainer: FnvHashMap<Lang, FirstTrainer>,
+ prev_trained: Vec<FnvHashMap<Lang, FirstTrainerResult>>,
+ languages: Vec<Lang>,
+ path: PathBuf,
}
impl SpellWorker {
pub fn new<P: AsRef<Path>>(languages: &[Lang], path: P) -> Result<Self> {
- let mut trainer = FnvHashMap::default();
-
- for lang in languages {
- trainer.insert(
- *lang,
- FirstTrainer::new(path.as_ref().join(lang.to_string()))?,
- );
- }
-
- Ok(Self { trainer })
+ Ok(Self {
+ path: path.as_ref().to_path_buf(),
+ languages: languages.to_vec(),
+ prev_trained: Vec::new(),
+ })
}
pub fn process(&mut self, job: Job) -> Result<()> {
@@ -54,6 +51,16 @@ impl SpellWorker {
info!("processing {}", name);
+ let mut trainer_has_inserts = false;
+ let mut trainer: FnvHashMap<Lang, FirstTrainer> = Default::default();
+
+ for lang in &self.languages {
+ trainer.insert(
+ *lang,
+ FirstTrainer::new(self.path.as_path().join(name).join(lang.to_string()))?,
+ );
+ }
+
let source = job.source_config.clone();
let paths = vec![job.warc_path.clone()];
@@ -75,31 +82,39 @@ impl SpellWorker {
None => continue,
};
- match (webpage.clean_text(), self.trainer.get_mut(&lang)) {
+ match (webpage.clean_text(), trainer.get_mut(&lang)) {
(Some(text), Some(trainer)) => {
trainer.add(text);
+ trainer_has_inserts = true;
}
_ => continue,
}
}
}
+ if trainer_has_inserts {
+ self.finish_training_step(trainer)?;
+ }
info!("finished processing {}", name);
Ok(())
}
- fn next_training_step(self) -> Result<FnvHashMap<Lang, FirstTrainerResult>> {
+ fn finish_training_step(&mut self, trainer: FnvHashMap<Lang, FirstTrainer>) -> Result<()> {
debug!("next training step");
- let mut result = FnvHashMap::default();
+ let mut trained = FnvHashMap::default();
- for (lang, trainer) in self.trainer {
- result.insert(lang, trainer.next_training_step()?);
+ for (lang, trainer) in trainer {
+ trained.insert(lang, trainer.next_training_step()?);
}
+ self.prev_trained.push(trained);
debug!("next training step created");
+ Ok(())
+ }
- Ok(result)
+ fn next_training_steps(self) -> Vec<FnvHashMap<Lang, FirstTrainerResult>> {
+ self.prev_trained
}
}
@@ -137,7 +152,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {
worker.process(job).unwrap();
}
- worker.next_training_step().unwrap()
+ worker.next_training_steps()
}));
}
@@ -145,8 +160,10 @@ pub fn run(config: WebSpellConfig) -> Result<()> {
for handler in handlers {
let trained = handler.join().unwrap();
- for (lang, result) in trained.into_iter() {
- combined.entry(lang).or_default().push(result);
+ for res in trained {
+ for (lang, result) in res {
+ combined.entry(lang).or_default().push(result);
+ }
}
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 8e6b5e5f..af260369 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -32,7 +32,6 @@ use crate::{
fastfield_reader::FastFieldReader,
search_ctx::Ctx,
searcher::NUM_RESULTS_PER_PAGE,
- webpage::region::Region,
};
pub use self::signal::*;
@@ -65,11 +64,6 @@ impl Ranker {
}
}
- pub fn with_region(mut self, region: Region) -> Self {
- self.aggregator.set_selected_region(region);
- self
- }
-
pub fn with_offset(mut self, offset: usize) -> Self {
self.offset = Some(offset);
self
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 6142548d..f6c556df 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -213,13 +213,17 @@ fn score_link_density(link_density: f64) -> f64 {
fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
match aggregator.region_count.as_ref() {
Some(region_count) => {
- let boost = aggregator.selected_region.map_or(0.0, |region| {
- if region == webpage_region {
- 50.0
- } else {
- 0.0
- }
- });
+ let boost = aggregator
+ .query_data
+ .as_ref()
+ .and_then(|q| q.selected_region)
+ .map_or(0.0, |region| {
+ if region != Region::All && region == webpage_region {
+ 50.0
+ } else {
+ 0.0
+ }
+ });
boost + region_count.score(&webpage_region)
}
@@ -692,10 +696,11 @@ struct SegmentReader {
struct QueryData {
simple_terms: Vec<String>,
optic_rules: Vec<optics::Rule>,
+ selected_region: Option<Region>,
}
pub struct SignalAggregator {
- query: Option<QueryData>,
+ query_data: Option<QueryData>,
query_signal_coefficients: Option<SignalCoefficient>,
segment_reader: Option<SegmentReader>,
inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,
@@ -703,7 +708,6 @@ pub struct SignalAggregator {
update_time_cache: Vec<f64>,
query_centrality: Option<RefCell<query_centrality::Scorer>>,
region_count: Option<Arc<RegionCount>>,
- selected_region: Option<Region>,
current_timestamp: Option<usize>,
linear_regression: Option<Arc<LinearRegression>>,
}
@@ -721,7 +725,7 @@ impl Clone for SignalAggregator {
.map(|scorer| RefCell::new(scorer.borrow().clone()));
Self {
- query: self.query.clone(),
+ query_data: self.query_data.clone(),
query_signal_coefficients: self.query_signal_coefficients.clone(),
segment_reader: None,
inbound_similarity,
@@ -729,7 +733,6 @@ impl Clone for SignalAggregator {
update_time_cache: self.update_time_cache.clone(),
query_centrality,
region_count: self.region_count.clone(),
- selected_region: self.selected_region,
current_timestamp: self.current_timestamp,
linear_regression: self.linear_regression.clone(),
}
@@ -742,7 +745,7 @@ impl std::fmt::Debug for SignalAggregator {
.field(
"query",
&self
- .query
+ .query_data
.as_ref()
.map(|q| q.simple_terms.clone())
.unwrap_or_default(),
@@ -775,6 +778,7 @@ impl SignalAggregator {
})
.cloned()
.collect(),
+ selected_region: q.region().cloned(),
});
let mut s = Self {
@@ -785,10 +789,9 @@ impl SignalAggregator {
update_time_cache,
query_centrality: None,
region_count: None,
- selected_region: None,
current_timestamp: None,
linear_regression: None,
- query,
+ query_data: query,
};
s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);
@@ -804,7 +807,7 @@ impl SignalAggregator {
let mut text_fields = EnumMap::new();
let schema = tv_searcher.schema();
- if let Some(query) = &self.query {
+ if let Some(query) = &self.query_data {
if !query.simple_terms.is_empty() {
for signal in ALL_SIGNALS {
if let Some(text_field) = signal.as_textfield() {
@@ -866,7 +869,7 @@ impl SignalAggregator {
) -> Vec<RuleBoost> {
let mut optic_rule_boosts = Vec::new();
- if let Some(query) = &self.query {
+ if let Some(query) = &self.query_data {
optic_rule_boosts = query
.optic_rules
.iter()
@@ -922,10 +925,6 @@ impl SignalAggregator {
self.region_count = Some(Arc::new(region_count));
}
- pub fn set_selected_region(&mut self, region: Region) {
- self.selected_region = Some(region);
- }
-
pub fn set_current_timestamp(&mut self, current_timestamp: usize) {
self.current_timestamp = Some(current_timestamp);
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index cc449870..d47185a5 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -38,7 +38,6 @@ use crate::schema::TextField;
use crate::search_ctx::Ctx;
use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
use crate::webgraph::Node;
-use crate::webpage::region::Region;
use crate::{inverted_index, live_index, Error, Result};
use super::WebsitesResult;
@@ -218,12 +217,6 @@ where
self.collector_config.clone(),
);
- if let Some(region) = query.region() {
- if *region != Region::All {
- ranker = ranker.with_region(*region);
- }
- }
-
ranker.de_rank_similar(de_rank_similar);
if query_centrality_coeff > 0.0 {
diff --git a/core/src/warc.rs b/core/src/warc.rs
index ee95fcae..e9f5d945 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -27,6 +27,7 @@ use std::time::Duration;
use flate2::read::MultiGzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
+use fnv::FnvHashSet;
#[cfg(test)]
use proptest_derive::Arbitrary;
@@ -522,7 +523,7 @@ impl<R: Read> Iterator for RecordIterator<R> {
pub struct DeduplicatedWarcWriter {
writer: WarcWriter,
- seen_url_hashes: std::collections::HashSet<md5::Digest>,
+ seen_url_hashes: FnvHashSet<md5::Digest>,
}
impl Default for DeduplicatedWarcWriter {
@@ -535,7 +536,7 @@ impl DeduplicatedWarcWriter {
pub fn new() -> Self {
Self {
writer: WarcWriter::new(),
- seen_url_hashes: std::collections::HashSet::new(),
+ seen_url_hashes: FnvHashSet::default(),
}
}
diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.py
index 3e605a8e..0f06e0ad 100644
--- a/ltr/auto_annotate.py
+++ b/ltr/auto_annotate.py
@@ -149,7 +149,7 @@ def add_results(query):
for i, result in enumerate(results):
cur.execute(
"""
- INSERT INTO search_results (qid, url, orig_rank, webpage_json)
+ INSERT OR IGNORE INTO search_results (qid, url, orig_rank, webpage_json)
VALUES (?, ?, ?, ?)
""",
(qid, result["url"], i, json.dumps(result)),
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index 8680f440..4de29c78 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -4,6 +4,8 @@ import json
from pprint import pprint
import sqlite3
+CATEGORICAL_FEATURES = ["is_homepage"]
+
con = sqlite3.connect("data/auto-ranking-annotation.sqlite")
cur = con.cursor()
@@ -77,7 +79,8 @@ q_train = []
for query, urls in items[:train_size]:
q_train.append(query)
for url, data in urls.items():
- X_train.append([data["features"].get(k, 0) for k in id2feature])
+ x = [data["features"].get(k, 0) for k in id2feature]
+ X_train.append(x)
y_train.append(data["score"])
X_test = []
@@ -110,8 +113,9 @@ model = lgb.LGBMRanker(
importance_type="gain",
num_leaves=50,
n_estimators=n_estimators,
+ categorical_features=[feature2id[k] for k in CATEGORICAL_FEATURES],
max_depth=10,
- learning_rate=0.01,
+ learning_rate=0.1,
label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],
)
model.fit(
diff --git a/ltr/linear_model.py b/ltr/linear_model.py
index bf913b1f..64c3252d 100644
--- a/ltr/linear_model.py
+++ b/ltr/linear_model.py
@@ -26,23 +26,32 @@ queries = {qid: {"query": query} for qid, query in res.fetchall()}
for qid in queries:
res = cur.execute(
"""
- SELECT qid, url, annotation, webpage_json
+ SELECT qid, url, annotation, orig_rank, webpage_json
FROM search_results
WHERE qid = ?
""",
(qid,),
)
urls = {
- url: {"label": label, "signals": json.loads(page)["rankingSignals"]}
- for _, url, label, page in res.fetchall()
+ url: {
+ "label": label,
+ "orig_rank": orig_rank,
+ "signals": json.loads(page)["rankingSignals"],
+ }
+ for _, url, label, orig_rank, page in res.fetchall()
}
urls = [
- (url, w["label"], w["signals"])
+ (url, w["label"], w["orig_rank"], w["signals"])
for url, w in urls.items()
if w["label"] is not None
]
- urls = sorted(urls, key=lambda x: int(x[1]), reverse=True)
+ sorting_key = lambda x: (
+ -x[1],
+ x[2],
+ )
+
+ urls = sorted(urls, key=sorting_key)
queries[qid]["urls"] = urls
@@ -50,7 +59,7 @@ feature2id = {}
id2feature = {}
for qid, data in queries.items():
- for url, score, signals in data["urls"]:
+ for url, score, _, signals in data["urls"]:
for feature, value in signals.items():
if feature not in feature2id:
id = len(feature2id)
@@ -65,16 +74,12 @@ random.shuffle(scores)
sorted_features = sorted(feature2id.items(), key=lambda x: x[1])
for qid, data in queries.items():
- for url, score, signals in data["urls"]:
+ for i, (url, score, _, signals) in enumerate(data["urls"]):
signals = {feature2id[k]: v for k, v in signals.items()}
signals = [signals[k] for _, k in sorted_features]
- scores.append({"ranking_signals": signals, "score": score})
-
-
-for score in scores:
- score["score"] = np.power(2, score["score"])
+ scores.append({"ranking_signals": signals, "score": 20 / (i + 1)})
train = scores[: int(len(scores) * TRAIN_PERCENT)]
test = scores[int(len(scores) * TRAIN_PERCENT) :]