web search done right
use fetch time during ranking
Mikkel Denkerc3c1cb7f005d
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 7b451a08..420d6120 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -78,6 +78,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {let html = Html::parse(&record.response.body, &record.request.url);let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraphlet centrality = worker.centrality_store.get(html.host()).unwrap_or_default();+ let fetch_time_ms = record.metadata.fetch_time_ms as u64;trace!("inserting webpage: {:?}", html.url());@@ -88,6 +89,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {html,backlinks,centrality,+ fetch_time_ms,};if let Err(err) = index.insert(webpage) {diff --git a/src/index.rs b/src/index.rsindex 186ff713..ad2d85c3 100644--- a/src/index.rs+++ b/src/index.rs@@ -236,7 +236,8 @@ impl From<Document> for RetrievedWebpage {| Field::StemmedBody| Field::Domain| Field::DomainIfHomepage- | Field::IsHomepage => {}+ | Field::IsHomepage+ | Field::FetchTimeMs => {}}}@@ -318,6 +319,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -354,6 +356,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -388,6 +391,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -422,6 +426,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -455,6 +460,7 @@ mod tests {"https://www.a.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index@@ -478,6 +484,7 @@ mod tests {text: "B site is great".to_string(),}],1.0,+ 500,)).expect("failed to parse webpage");@@ -520,6 +527,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");}@@ -555,6 +563,7 @@ mod tests {"https://www.dr.dk",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -587,6 +596,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");@@ -630,6 +640,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");@@ -652,6 +663,7 @@ mod tests {"https://www.example.com",vec![],1.,+ 500,)).expect("failed to parse webpage");@@ -699,6 +711,7 @@ mod tests {"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -734,6 +747,7 @@ mod tests {"https://www.first.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index@@ -753,6 +767,7 @@ mod tests {"https://www.third.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index@@ -772,6 +787,7 @@ mod tests {"https://www.second.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -785,4 +801,61 @@ mod tests {assert_eq!(result.documents[1].url, "https://www.second.com");assert_eq!(result.documents[2].url, "https://www.third.com");}++ #[test]+ fn fetch_time_ranking() {+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("test", index.schema()).expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());++ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.first.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.second.com",+ vec![],+ 1.0,+ 5000,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 2);+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.first.com");+ assert_eq!(result.documents[1].url, "https://www.second.com");+ }}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 83359216..45d7c8eb 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -25,6 +25,7 @@ pub(crate) struct InitialScoreTweaker {}pub(crate) struct InitialSegmentScoreTweaker {centrality_reader: DynamicFastFieldReader<u64>,is_homepage_reader: DynamicFastFieldReader<u64>,+ fetch_time_ms_reader: DynamicFastFieldReader<u64>,}impl ScoreTweaker<f64> for InitialScoreTweaker {@@ -49,9 +50,19 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {.u64(is_homepage_field).expect("Failed to get is_homepage fast-field reader");+ let fetch_time_ms_field = segment_reader+ .schema()+ .get_field(Field::FetchTimeMs.as_str())+ .expect("Faild to load fetch_time_ms field");+ let fetch_time_ms_reader = segment_reader+ .fast_fields()+ .u64(fetch_time_ms_field)+ .expect("Failed to get fetch_time_ms fast-field reader");+Ok(InitialSegmentScoreTweaker {centrality_reader,is_homepage_reader,+ fetch_time_ms_reader,})}}@@ -61,7 +72,8 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {let score = score as f64;let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;let is_homepage = self.is_homepage_reader.get(doc) as f64;+ let fetch_time_ms = self.fetch_time_ms_reader.get(doc) as f64;- score + 400.0 * centrality + 5.0 * is_homepage+ score + (400.0 * centrality) + (5.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 12b12652..a482f358 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -65,6 +65,7 @@ mod tests {"https://www.a.com",vec![],0.0,+ 500,)).expect("failed to parse webpage");index@@ -88,6 +89,7 @@ mod tests {text: "B site is great".to_string(),}],5.0,+ 500,)).expect("failed to parse webpage");@@ -121,6 +123,7 @@ mod tests {"https://www.dr.dk",vec![],0.0,+ 500,)).expect("failed to parse webpage");index@@ -140,6 +143,7 @@ mod tests {"https://www.dr.dk/whatever",vec![],0.0,+ 500,)).expect("failed to parse webpage");index@@ -157,6 +161,7 @@ mod tests {"https://www.b.com",vec![],0.003,+ 500,)).expect("failed to parse webpage");diff --git a/src/schema.rs b/src/schema.rsindex 83314c46..21f1037e 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -35,8 +35,9 @@ pub enum Field {IsHomepage,BacklinkText,Centrality,+ FetchTimeMs,}-pub static ALL_FIELDS: [Field; 11] = [+pub static ALL_FIELDS: [Field; 12] = [Field::Title,Field::Body,Field::StemmedTitle,@@ -48,6 +49,7 @@ pub static ALL_FIELDS: [Field; 11] = [Field::IsHomepage,Field::BacklinkText,Field::Centrality,+ Field::FetchTimeMs,];impl Field {@@ -91,6 +93,11 @@ impl Field {Field::StemmedBody => IndexingOption::Text(self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),),+ Field::FetchTimeMs => IndexingOption::Numeric(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed(),+ ),}}@@ -107,6 +114,7 @@ impl Field {Field::Domain => "domain",Field::DomainIfHomepage => "domain_if_homepage",Field::IsHomepage => "is_homepage",+ Field::FetchTimeMs => "fetch_time_ms",}}@@ -121,7 +129,8 @@ impl Field {| Field::Centrality| Field::Url| Field::Domain- | Field::IsHomepage => None,+ | Field::IsHomepage+ | Field::FetchTimeMs => None,}}}diff --git a/src/snippet.rs b/src/snippet.rsindex 7b2ec269..d1ea49d3 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -301,6 +301,7 @@ Survey in 2016, 2017, and 2018."#;"https://www.example.com",vec![],1.0,+ 500,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 8a3d89dc..ce46eff1 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -144,16 +144,24 @@ pub struct Webpage<'a> {pub html: Html<'a>,pub backlinks: Vec<Link>,pub centrality: f64,+ pub fetch_time_ms: u64,}impl<'a> Webpage<'a> {- pub fn new(html: &'a str, url: &str, backlinks: Vec<Link>, centrality: f64) -> Self {+ pub fn new(+ html: &'a str,+ url: &str,+ backlinks: Vec<Link>,+ centrality: f64,+ fetch_time_ms: u64,+ ) -> Self {let html = Html::parse(html, url);Self {html,backlinks,centrality,+ fetch_time_ms,}}@@ -180,6 +188,13 @@ impl<'a> Webpage<'a> {(self.centrality * CENTRALITY_SCALING as f64) as u64,);+ doc.add_u64(+ schema+ .get_field(Field::FetchTimeMs.as_str())+ .expect("Failed to get fetch_time_ms field"),+ self.fetch_time_ms,+ );+Ok(doc)}}@@ -405,7 +420,7 @@ impl<'a> Html<'a> {Field::IsHomepage => {doc.add_u64(tantivy_field, self.is_homepage().then(|| 1).unwrap_or(0))}- Field::BacklinkText | Field::Centrality => {}+ Field::BacklinkText | Field::Centrality | Field::FetchTimeMs => {}}}