web search done right
link density ranking signal
Mikkel Denker75e4d5ddce28
diff --git a/configs/api.toml b/configs/api.tomlindex 74c83910..2b9664d9 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -6,7 +6,7 @@ host = "0.0.0.0:3000"lambda_model_path = "data/lambdamart.txt"prometheus_host = "0.0.0.0:3001"queries_csv_path = "data/queries_us.csv"-spell_corrector_path = "data/web_spell/corrector"+spell_checker_path = "data/web_spell/checker"# qa_model_path = "data/qa_model"bangs_path = "data/bangs.json"summarizer_path = "data/summarizer"diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 4e4e2200..8829b676 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -197,7 +197,7 @@ pub struct ApiConfig {pub crossencoder_model_path: Option<String>,pub lambda_model_path: Option<String>,pub qa_model_path: Option<String>,- pub spell_corrector_path: Option<String>,+ pub spell_checker_path: Option<String>,pub bangs_path: String,pub summarizer_path: String,pub query_store_db_host: Option<String>,diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex ec6713b4..5a1b6cc5 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -51,7 +51,7 @@ pub enum WarcWriterMessage {Finish,}-async fn commit(writer: warc::WarcWriter, s3: config::S3Config) {+async fn commit(writer: warc::DeduplicatedWarcWriter, s3: config::S3Config) {let filename = format!("{}_{}.warc.gz",chrono::Utc::now().to_rfc3339(),@@ -94,7 +94,7 @@ async fn commit(writer: warc::WarcWriter, s3: config::S3Config) {}async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3: S3Config) {- let mut writer = warc::WarcWriter::new();+ let mut writer = warc::DeduplicatedWarcWriter::new();while let Some(message) = rx.recv().await {match message {@@ -126,7 +126,7 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:if writer.num_bytes() > 1_000_000_000 {commit(writer, s3.clone()).await;- writer = warc::WarcWriter::new();+ writer = warc::DeduplicatedWarcWriter::new();}}WarcWriterMessage::Finish => {diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex da502782..6ed56289 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -36,7 +36,7 @@ use crate::{distributed::{retry_strategy::ExponentialBackoff, sonic},entrypoint::crawler::router::{NewJob, RouterService},warc,- webpage::Html,+ webpage::{url_ext::UrlExt, Html},};use super::{@@ -202,6 +202,10 @@ impl<S: DatumStream> JobExecutor<S> {async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {while let Some(retryable_url) = urls.pop_front() {+ if Domain::from(&retryable_url.url) != self.job.domain {+ continue;+ }+if self.crawled_urls.contains(&retryable_url.url) {continue;}@@ -248,7 +252,7 @@ impl<S: DatumStream> JobExecutor<S> {continue;}- if new_url.host_str() != retryable_url.url.host_str() {+ if new_url.root_domain() != retryable_url.url.root_domain() {continue;}diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rsindex 77546f07..a51668cc 100644--- a/core/src/entrypoint/web_spell.rs+++ b/core/src/entrypoint/web_spell.rs@@ -155,7 +155,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {let second_trainer = SecondTrainer::new(results,Path::new(&config.output_path)- .join("corrector")+ .join("checker").join(lang.code()),)?;debug!("second trainer created");diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 24a4b2c6..6142548d 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -124,6 +124,8 @@ pub enum Signal {UrlDigits,#[serde(rename = "url_slashes")]UrlSlashes,+ #[serde(rename = "link_density")]+ LinkDensity,}impl From<Signal> for usize {@@ -132,7 +134,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 33] = [+pub const ALL_SIGNALS: [Signal; 34] = [Signal::Bm25Title,Signal::Bm25TitleBigrams,Signal::Bm25TitleTrigrams,@@ -166,6 +168,7 @@ pub const ALL_SIGNALS: [Signal; 33] = [Signal::LambdaMART,Signal::UrlDigits,Signal::UrlSlashes,+ Signal::LinkDensity,];fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f64 {@@ -183,18 +186,30 @@ fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f6}}+#[inline]fn score_trackers(num_trackers: f64) -> f64 {1.0 / (num_trackers + 1.0)}+#[inline]fn score_digits(num_digits: f64) -> f64 {1.0 / (num_digits + 1.0)}+#[inline]fn score_slashes(num_slashes: f64) -> f64 {1.0 / (num_slashes + 1.0)}+#[inline]+fn score_link_density(link_density: f64) -> f64 {+ if link_density > 0.5 {+ 0.0+ } else {+ 1.0 - link_density+ }+}+fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {match aggregator.region_count.as_ref() {Some(region_count) => {@@ -272,6 +287,7 @@ impl Signal {Signal::LambdaMART => 10.0,Signal::UrlSlashes => 0.01,Signal::UrlDigits => 0.01,+ Signal::LinkDensity => 0.00,}}@@ -337,6 +353,15 @@ impl Signal {field_value.map(|val| val as f64)}+ Signal::LinkDensity => {+ let field_value: Option<u64> = self+ .fastfield_value(signal_aggregator, doc)+ .and_then(|val| val.into());++ field_value+ .map(|val| val as f64 / FLOAT_SCALING as f64)+ .map(score_link_density)+ }Signal::FetchTimeMs => {let field_value: Option<u64> = self.fastfield_value(signal_aggregator, doc)@@ -500,6 +525,10 @@ impl Signal {.count() as f64;Some(score_slashes(num_slashes))}+ Signal::LinkDensity => {+ let link_density = webpage.html.link_density();+ Some(score_link_density(link_density))+ }Signal::Bm25Title| Signal::Bm25TitleBigrams| Signal::Bm25TitleTrigrams@@ -549,6 +578,7 @@ impl Signal {Signal::Region => Some(FastField::Region),Signal::UrlSlashes => Some(FastField::NumPathAndQuerySlashes),Signal::UrlDigits => Some(FastField::NumPathAndQueryDigits),+ Signal::LinkDensity => Some(FastField::LinkDensity),_ => None,}}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 5e451c1b..5db77aa3 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -230,6 +230,7 @@ pub enum FastField {NumPathAndQueryDigits,LikelyHasAds,LikelyHasPaywall,+ LinkDensity,}impl FastField {@@ -270,6 +271,7 @@ impl FastField {FastField::NumPathAndQueryDigits => "num_path_and_query_digits",FastField::LikelyHasAds => "likely_has_ads",FastField::LikelyHasPaywall => "likely_has_paywall",+ FastField::LinkDensity => "link_density",}}}@@ -595,6 +597,9 @@ impl Field {.set_indexed().set_stored(),),+ Field::Fast(FastField::LinkDensity) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_stored())+ }}}@@ -713,6 +718,7 @@ impl FastField {FastField::NumPathAndQueryDigits => DataType::U64,FastField::LikelyHasAds => DataType::U64,FastField::LikelyHasPaywall => DataType::U64,+ FastField::LinkDensity => DataType::U64,}}}diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex 499d5e6f..85135ba2 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -120,7 +120,7 @@ impl ApiSearcher {thresholds: config.thresholds,widgets: Widgets::new(config.widgets).unwrap(),spell_checker: config- .spell_corrector_path+ .spell_checker_path.map(|c| SpellChecker::open(c, config.correction_config).unwrap()),}}@@ -141,7 +141,7 @@ impl ApiSearcher {thresholds: config.thresholds,widgets: Widgets::new(config.widgets).unwrap(),spell_checker: config- .spell_corrector_path+ .spell_checker_path.map(|c| SpellChecker::open(c, config.correction_config).unwrap()),}}diff --git a/core/src/warc.rs b/core/src/warc.rsindex 4c1b6af0..ee95fcae 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -520,6 +520,49 @@ impl<R: Read> Iterator for RecordIterator<R> {}}+pub struct DeduplicatedWarcWriter {+ writer: WarcWriter,+ seen_url_hashes: std::collections::HashSet<md5::Digest>,+}++impl Default for DeduplicatedWarcWriter {+ fn default() -> Self {+ Self::new()+ }+}++impl DeduplicatedWarcWriter {+ pub fn new() -> Self {+ Self {+ writer: WarcWriter::new(),+ seen_url_hashes: std::collections::HashSet::new(),+ }+ }++ pub fn write(&mut self, record: &WarcRecord) -> Result<()> {+ let url_hash = md5::compute(&record.request.url);+ if self.seen_url_hashes.contains(&url_hash) {+ return Ok(());+ }++ self.seen_url_hashes.insert(url_hash);++ self.writer.write(record)+ }++ pub fn finish(self) -> Result<Vec<u8>> {+ self.writer.finish()+ }++ pub fn num_bytes(&self) -> usize {+ self.writer.num_bytes()+ }++ pub fn num_writes(&self) -> usize {+ self.writer.num_writes()+ }+}+pub struct WarcWriter {num_writes: usize,writer: GzEncoder<Vec<u8>>,diff --git a/core/src/web_spell/mod.rs b/core/src/web_spell/mod.rsindex 0df274e1..c7e54acf 100644--- a/core/src/web_spell/mod.rs+++ b/core/src/web_spell/mod.rs@@ -52,8 +52,8 @@ pub enum Error {#[error("Bincode error: {0}")]Bincode(#[from] bincode::Error),- #[error("Corrector not found")]- CorrectorNotFound,+ #[error("Checker not found")]+ CheckerNotFound,}pub type Result<T, E = Error> = std::result::Result<T, E>;diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rsindex 7eb3c615..2e601f53 100644--- a/core/src/web_spell/spell_checker.rs+++ b/core/src/web_spell/spell_checker.rs@@ -63,6 +63,62 @@ impl LangSpellChecker {self.term_dict.search(term, max_edit_distance)}+ fn lm_logprob(&self, term_idx: usize, context: &[String]) -> f64 {+ if term_idx == 0 {+ let strat = RightToLeft;+ self.language_model.log_prob(context, strat)+ } else if term_idx == context.len() - 1 {+ let strat = LeftToRight;+ self.language_model.log_prob(context, strat)+ } else {+ let strat = IntoMiddle::default();+ self.language_model.log_prob(context, strat)+ }+ }++ fn score_candidates(+ &self,+ term: &str,+ candidates: &[String],+ context: Vec<String>,+ term_idx: usize,+ ) -> Option<(String, f64)> {+ let mut best_term: Option<(String, f64)> = None;+ let mut context = context;++ for candidate in candidates {+ if candidate.as_str() == term {+ continue;+ }++ context[term_idx] = candidate.clone();++ let log_prob = self.lm_logprob(term_idx, &context);++ let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;++ let error_log_prob = if candidate.as_str() == term {+ match error_model::possible_errors(term, candidate) {+ Some(error_seq) => {+ (1.0 - self.config.misspelled_prob).log2()+ + self.error_model.log_prob(&error_seq)+ }+ None => 0.0,+ }+ } else {+ self.config.misspelled_prob+ };++ let score = scaled_lm_log_prob + error_log_prob;++ if best_term.is_none() || score > best_term.as_ref().unwrap().1 {+ best_term = Some((candidate.clone(), score));+ }+ }++ best_term+ }+fn correct_once(&self, text: &str) -> Option<Correction> {let orig_terms = super::tokenize(text);let mut terms = orig_terms.clone();@@ -96,62 +152,14 @@ impl LangSpellChecker {}let this_term_context_idx = this_term_context_idx.unwrap();- let term_log_prob = if this_term_context_idx == 0 {- let strat = RightToLeft;- self.language_model.log_prob(&context, strat)- } else if this_term_context_idx == context.len() - 1 {- let strat = LeftToRight;- self.language_model.log_prob(&context, strat)- } else {- let strat = IntoMiddle::default();- self.language_model.log_prob(&context, strat)- };+ let term_log_prob = self.lm_logprob(this_term_context_idx, &context);let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob;tracing::debug!(?term, ?scaled_term_log_prob);- let mut best_term: Option<(String, f64)> = None;-- for candidate in candidates {- if candidate == *term {- continue;- }-- context[this_term_context_idx] = candidate.clone();-- let log_prob = if this_term_context_idx == 0 {- let strat = RightToLeft;- self.language_model.log_prob(&context, strat)- } else if this_term_context_idx == context.len() - 1 {- let strat = LeftToRight;- self.language_model.log_prob(&context, strat)- } else {- let strat = IntoMiddle::default();- self.language_model.log_prob(&context, strat)- };-- let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;-- let error_log_prob = if &candidate == term {- match error_model::possible_errors(term, &candidate) {- Some(error_seq) => {- (1.0 - self.config.misspelled_prob).log2()- + self.error_model.log_prob(&error_seq)- }- None => 0.0,- }- } else {- self.config.misspelled_prob- };-- let score = scaled_lm_log_prob + error_log_prob;-- if best_term.is_none() || score > best_term.as_ref().unwrap().1 {- best_term = Some((candidate, score));- }- }-- if let Some((best_term, score)) = best_term {+ if let Some((best_term, score)) =+ self.score_candidates(term, &candidates, context, this_term_context_idx)+ {let diff = score.abs() - scaled_term_log_prob.abs();tracing::debug!(?best_term, ?score, ?diff);if diff > self.config.correction_threshold {@@ -179,6 +187,10 @@ impl LangSpellChecker {}fn correct(&self, text: &str) -> Option<Correction> {+ // TODO:+ // sometimes the text should be corrected more than once.+ // we should make sure to only correct each term once so we don't+ // get corrections to the corrections.self.correct_once(text.to_lowercase().as_str())}}@@ -190,11 +202,11 @@ pub struct SpellChecker {impl SpellChecker {pub fn open<P: AsRef<Path>>(path: P, config: CorrectionConfig) -> Result<Self> {if !path.as_ref().exists() {- return Err(Error::CorrectorNotFound);+ return Err(Error::CheckerNotFound);}if !path.as_ref().is_dir() {- return Err(Error::CorrectorNotFound);+ return Err(Error::CheckerNotFound);}let mut lang_spell_checkers = FnvHashMap::default();@@ -234,3 +246,41 @@ impl SpellChecker {.and_then(|s| s.correct(text))}}++#[cfg(test)]+mod tests {+ use super::*;++ fn correction(orig: &str, corrected: &str) -> Correction {+ let mut res = Correction::empty(orig.to_string());++ for (orig, corrected) in super::super::tokenize(orig)+ .into_iter()+ .zip(super::super::tokenize(corrected))+ {+ if orig == corrected {+ res.push(CorrectionTerm::NotCorrected(orig));+ } else {+ res.push(CorrectionTerm::Corrected(corrected));+ }+ }++ res+ }+ #[test]+ fn simple() {+ let path = Path::new("../data/web_spell/checker");++ if !path.exists() {+ return;+ }++ let spell_checker = SpellChecker::open(path, CorrectionConfig::default()).unwrap();++ assert_eq!(spell_checker.correct("hello", &Lang::Eng), None);+ assert_eq!(+ spell_checker.correct("dudw", &Lang::Eng),+ Some(correction("dudw", "dude"))+ );+ }+}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex be1879ae..c895adb9 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -479,6 +479,16 @@ impl Html {self.lang.as_ref()}+ pub fn link_density(&self) -> f64 {+ (1.0 + self.anchor_links().len() as f64)+ / (1.0+ + self+ .clean_text+ .as_ref()+ .map(|s| s.len())+ .unwrap_or_default() as f64)+ }+pub fn canonical_url(&self) -> Option<Url> {let mut canonical_url = None;@@ -1365,6 +1375,12 @@ impl Html {Field::Fast(FastField::LikelyHasPaywall) => {doc.add_u64(tantivy_field, self.likely_has_paywall() as u64);}+ Field::Fast(FastField::LinkDensity) => {+ doc.add_u64(+ tantivy_field,+ (self.link_density() * FLOAT_SCALING as f64) as u64,+ );+ }Field::Text(TextField::BacklinkText)| Field::Text(TextField::SafetyClassification)| Field::Text(TextField::InsertionTimestamp)