Skip to content

web search done right

link density ranking signal

Mikkel Denker75e4d5ddce28
diff --git a/configs/api.toml b/configs/api.toml
index 74c83910..2b9664d9 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -6,7 +6,7 @@ host = "0.0.0.0:3000"
lambda_model_path = "data/lambdamart.txt"
prometheus_host = "0.0.0.0:3001"
queries_csv_path = "data/queries_us.csv"
-spell_corrector_path = "data/web_spell/corrector"
+spell_checker_path = "data/web_spell/checker"
# qa_model_path = "data/qa_model"
bangs_path = "data/bangs.json"
summarizer_path = "data/summarizer"
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 4e4e2200..8829b676 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -197,7 +197,7 @@ pub struct ApiConfig {
pub crossencoder_model_path: Option<String>,
pub lambda_model_path: Option<String>,
pub qa_model_path: Option<String>,
- pub spell_corrector_path: Option<String>,
+ pub spell_checker_path: Option<String>,
pub bangs_path: String,
pub summarizer_path: String,
pub query_store_db_host: Option<String>,
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index ec6713b4..5a1b6cc5 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -51,7 +51,7 @@ pub enum WarcWriterMessage {
Finish,
}
-async fn commit(writer: warc::WarcWriter, s3: config::S3Config) {
+async fn commit(writer: warc::DeduplicatedWarcWriter, s3: config::S3Config) {
let filename = format!(
"{}_{}.warc.gz",
chrono::Utc::now().to_rfc3339(),
@@ -94,7 +94,7 @@ async fn commit(writer: warc::WarcWriter, s3: config::S3Config) {
}
async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3: S3Config) {
- let mut writer = warc::WarcWriter::new();
+ let mut writer = warc::DeduplicatedWarcWriter::new();
while let Some(message) = rx.recv().await {
match message {
@@ -126,7 +126,7 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:
if writer.num_bytes() > 1_000_000_000 {
commit(writer, s3.clone()).await;
- writer = warc::WarcWriter::new();
+ writer = warc::DeduplicatedWarcWriter::new();
}
}
WarcWriterMessage::Finish => {
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index da502782..6ed56289 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -36,7 +36,7 @@ use crate::{
distributed::{retry_strategy::ExponentialBackoff, sonic},
entrypoint::crawler::router::{NewJob, RouterService},
warc,
- webpage::Html,
+ webpage::{url_ext::UrlExt, Html},
};
use super::{
@@ -202,6 +202,10 @@ impl<S: DatumStream> JobExecutor<S> {
async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {
while let Some(retryable_url) = urls.pop_front() {
+ if Domain::from(&retryable_url.url) != self.job.domain {
+ continue;
+ }
+
if self.crawled_urls.contains(&retryable_url.url) {
continue;
}
@@ -248,7 +252,7 @@ impl<S: DatumStream> JobExecutor<S> {
continue;
}
- if new_url.host_str() != retryable_url.url.host_str() {
+ if new_url.root_domain() != retryable_url.url.root_domain() {
continue;
}
diff --git a/core/src/entrypoint/web_spell.rs b/core/src/entrypoint/web_spell.rs
index 77546f07..a51668cc 100644
--- a/core/src/entrypoint/web_spell.rs
+++ b/core/src/entrypoint/web_spell.rs
@@ -155,7 +155,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {
let second_trainer = SecondTrainer::new(
results,
Path::new(&config.output_path)
- .join("corrector")
+ .join("checker")
.join(lang.code()),
)?;
debug!("second trainer created");
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 24a4b2c6..6142548d 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -124,6 +124,8 @@ pub enum Signal {
UrlDigits,
#[serde(rename = "url_slashes")]
UrlSlashes,
+ #[serde(rename = "link_density")]
+ LinkDensity,
}
impl From<Signal> for usize {
@@ -132,7 +134,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 33] = [
+pub const ALL_SIGNALS: [Signal; 34] = [
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
Signal::Bm25TitleTrigrams,
@@ -166,6 +168,7 @@ pub const ALL_SIGNALS: [Signal; 33] = [
Signal::LambdaMART,
Signal::UrlDigits,
Signal::UrlSlashes,
+ Signal::LinkDensity,
];
fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f64 {
@@ -183,18 +186,30 @@ fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f6
}
}
+#[inline]
fn score_trackers(num_trackers: f64) -> f64 {
1.0 / (num_trackers + 1.0)
}
+#[inline]
fn score_digits(num_digits: f64) -> f64 {
1.0 / (num_digits + 1.0)
}
+#[inline]
fn score_slashes(num_slashes: f64) -> f64 {
1.0 / (num_slashes + 1.0)
}
+#[inline]
+fn score_link_density(link_density: f64) -> f64 {
+ if link_density > 0.5 {
+ 0.0
+ } else {
+ 1.0 - link_density
+ }
+}
+
fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
match aggregator.region_count.as_ref() {
Some(region_count) => {
@@ -272,6 +287,7 @@ impl Signal {
Signal::LambdaMART => 10.0,
Signal::UrlSlashes => 0.01,
Signal::UrlDigits => 0.01,
+ Signal::LinkDensity => 0.00,
}
}
@@ -337,6 +353,15 @@ impl Signal {
field_value.map(|val| val as f64)
}
+ Signal::LinkDensity => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value
+ .map(|val| val as f64 / FLOAT_SCALING as f64)
+ .map(score_link_density)
+ }
Signal::FetchTimeMs => {
let field_value: Option<u64> = self
.fastfield_value(signal_aggregator, doc)
@@ -500,6 +525,10 @@ impl Signal {
.count() as f64;
Some(score_slashes(num_slashes))
}
+ Signal::LinkDensity => {
+ let link_density = webpage.html.link_density();
+ Some(score_link_density(link_density))
+ }
Signal::Bm25Title
| Signal::Bm25TitleBigrams
| Signal::Bm25TitleTrigrams
@@ -549,6 +578,7 @@ impl Signal {
Signal::Region => Some(FastField::Region),
Signal::UrlSlashes => Some(FastField::NumPathAndQuerySlashes),
Signal::UrlDigits => Some(FastField::NumPathAndQueryDigits),
+ Signal::LinkDensity => Some(FastField::LinkDensity),
_ => None,
}
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 5e451c1b..5db77aa3 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -230,6 +230,7 @@ pub enum FastField {
NumPathAndQueryDigits,
LikelyHasAds,
LikelyHasPaywall,
+ LinkDensity,
}
impl FastField {
@@ -270,6 +271,7 @@ impl FastField {
FastField::NumPathAndQueryDigits => "num_path_and_query_digits",
FastField::LikelyHasAds => "likely_has_ads",
FastField::LikelyHasPaywall => "likely_has_paywall",
+ FastField::LinkDensity => "link_density",
}
}
}
@@ -595,6 +597,9 @@ impl Field {
.set_indexed()
.set_stored(),
),
+ Field::Fast(FastField::LinkDensity) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_stored())
+ }
}
}
@@ -713,6 +718,7 @@ impl FastField {
FastField::NumPathAndQueryDigits => DataType::U64,
FastField::LikelyHasAds => DataType::U64,
FastField::LikelyHasPaywall => DataType::U64,
+ FastField::LinkDensity => DataType::U64,
}
}
}
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
index 499d5e6f..85135ba2 100644
--- a/core/src/searcher/api.rs
+++ b/core/src/searcher/api.rs
@@ -120,7 +120,7 @@ impl ApiSearcher {
thresholds: config.thresholds,
widgets: Widgets::new(config.widgets).unwrap(),
spell_checker: config
- .spell_corrector_path
+ .spell_checker_path
.map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
}
}
@@ -141,7 +141,7 @@ impl ApiSearcher {
thresholds: config.thresholds,
widgets: Widgets::new(config.widgets).unwrap(),
spell_checker: config
- .spell_corrector_path
+ .spell_checker_path
.map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
}
}
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 4c1b6af0..ee95fcae 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -520,6 +520,49 @@ impl<R: Read> Iterator for RecordIterator<R> {
}
}
+pub struct DeduplicatedWarcWriter {
+ writer: WarcWriter,
+ seen_url_hashes: std::collections::HashSet<md5::Digest>,
+}
+
+impl Default for DeduplicatedWarcWriter {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl DeduplicatedWarcWriter {
+ pub fn new() -> Self {
+ Self {
+ writer: WarcWriter::new(),
+ seen_url_hashes: std::collections::HashSet::new(),
+ }
+ }
+
+ pub fn write(&mut self, record: &WarcRecord) -> Result<()> {
+ let url_hash = md5::compute(&record.request.url);
+ if self.seen_url_hashes.contains(&url_hash) {
+ return Ok(());
+ }
+
+ self.seen_url_hashes.insert(url_hash);
+
+ self.writer.write(record)
+ }
+
+ pub fn finish(self) -> Result<Vec<u8>> {
+ self.writer.finish()
+ }
+
+ pub fn num_bytes(&self) -> usize {
+ self.writer.num_bytes()
+ }
+
+ pub fn num_writes(&self) -> usize {
+ self.writer.num_writes()
+ }
+}
+
pub struct WarcWriter {
num_writes: usize,
writer: GzEncoder<Vec<u8>>,
diff --git a/core/src/web_spell/mod.rs b/core/src/web_spell/mod.rs
index 0df274e1..c7e54acf 100644
--- a/core/src/web_spell/mod.rs
+++ b/core/src/web_spell/mod.rs
@@ -52,8 +52,8 @@ pub enum Error {
#[error("Bincode error: {0}")]
Bincode(#[from] bincode::Error),
- #[error("Corrector not found")]
- CorrectorNotFound,
+ #[error("Checker not found")]
+ CheckerNotFound,
}
pub type Result<T, E = Error> = std::result::Result<T, E>;
diff --git a/core/src/web_spell/spell_checker.rs b/core/src/web_spell/spell_checker.rs
index 7eb3c615..2e601f53 100644
--- a/core/src/web_spell/spell_checker.rs
+++ b/core/src/web_spell/spell_checker.rs
@@ -63,6 +63,62 @@ impl LangSpellChecker {
self.term_dict.search(term, max_edit_distance)
}
+ fn lm_logprob(&self, term_idx: usize, context: &[String]) -> f64 {
+ if term_idx == 0 {
+ let strat = RightToLeft;
+ self.language_model.log_prob(context, strat)
+ } else if term_idx == context.len() - 1 {
+ let strat = LeftToRight;
+ self.language_model.log_prob(context, strat)
+ } else {
+ let strat = IntoMiddle::default();
+ self.language_model.log_prob(context, strat)
+ }
+ }
+
+ fn score_candidates(
+ &self,
+ term: &str,
+ candidates: &[String],
+ context: Vec<String>,
+ term_idx: usize,
+ ) -> Option<(String, f64)> {
+ let mut best_term: Option<(String, f64)> = None;
+ let mut context = context;
+
+ for candidate in candidates {
+ if candidate.as_str() == term {
+ continue;
+ }
+
+ context[term_idx] = candidate.clone();
+
+ let log_prob = self.lm_logprob(term_idx, &context);
+
+ let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;
+
+ let error_log_prob = if candidate.as_str() == term {
+ match error_model::possible_errors(term, candidate) {
+ Some(error_seq) => {
+ (1.0 - self.config.misspelled_prob).log2()
+ + self.error_model.log_prob(&error_seq)
+ }
+ None => 0.0,
+ }
+ } else {
+ self.config.misspelled_prob
+ };
+
+ let score = scaled_lm_log_prob + error_log_prob;
+
+ if best_term.is_none() || score > best_term.as_ref().unwrap().1 {
+ best_term = Some((candidate.clone(), score));
+ }
+ }
+
+ best_term
+ }
+
fn correct_once(&self, text: &str) -> Option<Correction> {
let orig_terms = super::tokenize(text);
let mut terms = orig_terms.clone();
@@ -96,62 +152,14 @@ impl LangSpellChecker {
}
let this_term_context_idx = this_term_context_idx.unwrap();
- let term_log_prob = if this_term_context_idx == 0 {
- let strat = RightToLeft;
- self.language_model.log_prob(&context, strat)
- } else if this_term_context_idx == context.len() - 1 {
- let strat = LeftToRight;
- self.language_model.log_prob(&context, strat)
- } else {
- let strat = IntoMiddle::default();
- self.language_model.log_prob(&context, strat)
- };
+ let term_log_prob = self.lm_logprob(this_term_context_idx, &context);
let scaled_term_log_prob = self.config.lm_prob_weight * term_log_prob;
tracing::debug!(?term, ?scaled_term_log_prob);
- let mut best_term: Option<(String, f64)> = None;
-
- for candidate in candidates {
- if candidate == *term {
- continue;
- }
-
- context[this_term_context_idx] = candidate.clone();
-
- let log_prob = if this_term_context_idx == 0 {
- let strat = RightToLeft;
- self.language_model.log_prob(&context, strat)
- } else if this_term_context_idx == context.len() - 1 {
- let strat = LeftToRight;
- self.language_model.log_prob(&context, strat)
- } else {
- let strat = IntoMiddle::default();
- self.language_model.log_prob(&context, strat)
- };
-
- let scaled_lm_log_prob = self.config.lm_prob_weight * log_prob;
-
- let error_log_prob = if &candidate == term {
- match error_model::possible_errors(term, &candidate) {
- Some(error_seq) => {
- (1.0 - self.config.misspelled_prob).log2()
- + self.error_model.log_prob(&error_seq)
- }
- None => 0.0,
- }
- } else {
- self.config.misspelled_prob
- };
-
- let score = scaled_lm_log_prob + error_log_prob;
-
- if best_term.is_none() || score > best_term.as_ref().unwrap().1 {
- best_term = Some((candidate, score));
- }
- }
-
- if let Some((best_term, score)) = best_term {
+ if let Some((best_term, score)) =
+ self.score_candidates(term, &candidates, context, this_term_context_idx)
+ {
let diff = score.abs() - scaled_term_log_prob.abs();
tracing::debug!(?best_term, ?score, ?diff);
if diff > self.config.correction_threshold {
@@ -179,6 +187,10 @@ impl LangSpellChecker {
}
fn correct(&self, text: &str) -> Option<Correction> {
+ // TODO:
+ // sometimes the text should be corrected more than once.
+ // we should make sure to only correct each term once so we don't
+ // get corrections to the corrections.
self.correct_once(text.to_lowercase().as_str())
}
}
@@ -190,11 +202,11 @@ pub struct SpellChecker {
impl SpellChecker {
pub fn open<P: AsRef<Path>>(path: P, config: CorrectionConfig) -> Result<Self> {
if !path.as_ref().exists() {
- return Err(Error::CorrectorNotFound);
+ return Err(Error::CheckerNotFound);
}
if !path.as_ref().is_dir() {
- return Err(Error::CorrectorNotFound);
+ return Err(Error::CheckerNotFound);
}
let mut lang_spell_checkers = FnvHashMap::default();
@@ -234,3 +246,41 @@ impl SpellChecker {
.and_then(|s| s.correct(text))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn correction(orig: &str, corrected: &str) -> Correction {
+ let mut res = Correction::empty(orig.to_string());
+
+ for (orig, corrected) in super::super::tokenize(orig)
+ .into_iter()
+ .zip(super::super::tokenize(corrected))
+ {
+ if orig == corrected {
+ res.push(CorrectionTerm::NotCorrected(orig));
+ } else {
+ res.push(CorrectionTerm::Corrected(corrected));
+ }
+ }
+
+ res
+ }
+ #[test]
+ fn simple() {
+ let path = Path::new("../data/web_spell/checker");
+
+ if !path.exists() {
+ return;
+ }
+
+ let spell_checker = SpellChecker::open(path, CorrectionConfig::default()).unwrap();
+
+ assert_eq!(spell_checker.correct("hello", &Lang::Eng), None);
+ assert_eq!(
+ spell_checker.correct("dudw", &Lang::Eng),
+ Some(correction("dudw", "dude"))
+ );
+ }
+}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index be1879ae..c895adb9 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -479,6 +479,16 @@ impl Html {
self.lang.as_ref()
}
+ pub fn link_density(&self) -> f64 {
+ (1.0 + self.anchor_links().len() as f64)
+ / (1.0
+ + self
+ .clean_text
+ .as_ref()
+ .map(|s| s.len())
+ .unwrap_or_default() as f64)
+ }
+
pub fn canonical_url(&self) -> Option<Url> {
let mut canonical_url = None;
@@ -1365,6 +1375,12 @@ impl Html {
Field::Fast(FastField::LikelyHasPaywall) => {
doc.add_u64(tantivy_field, self.likely_has_paywall() as u64);
}
+ Field::Fast(FastField::LinkDensity) => {
+ doc.add_u64(
+ tantivy_field,
+ (self.link_density() * FLOAT_SCALING as f64) as u64,
+ );
+ }
Field::Text(TextField::BacklinkText)
| Field::Text(TextField::SafetyClassification)
| Field::Text(TextField::InsertionTimestamp)