Skip to content

web search done right

use fetch time during ranking

Mikkel Denkerc3c1cb7f005d
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 7b451a08..420d6120 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -78,6 +78,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
let html = Html::parse(&record.response.body, &record.request.url);
let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph
let centrality = worker.centrality_store.get(html.host()).unwrap_or_default();
+ let fetch_time_ms = record.metadata.fetch_time_ms as u64;
trace!("inserting webpage: {:?}", html.url());
@@ -88,6 +89,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
html,
backlinks,
centrality,
+ fetch_time_ms,
};
if let Err(err) = index.insert(webpage) {
diff --git a/src/index.rs b/src/index.rs
index 186ff713..ad2d85c3 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -236,7 +236,8 @@ impl From<Document> for RetrievedWebpage {
| Field::StemmedBody
| Field::Domain
| Field::DomainIfHomepage
- | Field::IsHomepage => {}
+ | Field::IsHomepage
+ | Field::FetchTimeMs => {}
}
}
@@ -318,6 +319,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -354,6 +356,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -388,6 +391,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -422,6 +426,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -455,6 +460,7 @@ mod tests {
"https://www.a.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index
@@ -478,6 +484,7 @@ mod tests {
text: "B site is great".to_string(),
}],
1.0,
+ 500,
))
.expect("failed to parse webpage");
@@ -520,6 +527,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
}
@@ -555,6 +563,7 @@ mod tests {
"https://www.dr.dk",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -587,6 +596,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
@@ -630,6 +640,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
@@ -652,6 +663,7 @@ mod tests {
"https://www.example.com",
vec![],
1.,
+ 500,
))
.expect("failed to parse webpage");
@@ -699,6 +711,7 @@ mod tests {
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -734,6 +747,7 @@ mod tests {
"https://www.first.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index
@@ -753,6 +767,7 @@ mod tests {
"https://www.third.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index
@@ -772,6 +787,7 @@ mod tests {
"https://www.second.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -785,4 +801,61 @@ mod tests {
assert_eq!(result.documents[1].url, "https://www.second.com");
assert_eq!(result.documents[2].url, "https://www.third.com");
}
+
+ #[test]
+ fn fetch_time_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("test", index.schema()).expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.first.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.second.com",
+ vec![],
+ 1.0,
+ 5000,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 2);
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+ assert_eq!(result.documents[1].url, "https://www.second.com");
+ }
}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 83359216..45d7c8eb 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -25,6 +25,7 @@ pub(crate) struct InitialScoreTweaker {}
pub(crate) struct InitialSegmentScoreTweaker {
centrality_reader: DynamicFastFieldReader<u64>,
is_homepage_reader: DynamicFastFieldReader<u64>,
+ fetch_time_ms_reader: DynamicFastFieldReader<u64>,
}
impl ScoreTweaker<f64> for InitialScoreTweaker {
@@ -49,9 +50,19 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
.u64(is_homepage_field)
.expect("Failed to get is_homepage fast-field reader");
+ let fetch_time_ms_field = segment_reader
+ .schema()
+ .get_field(Field::FetchTimeMs.as_str())
+ .expect("Faild to load fetch_time_ms field");
+ let fetch_time_ms_reader = segment_reader
+ .fast_fields()
+ .u64(fetch_time_ms_field)
+ .expect("Failed to get fetch_time_ms fast-field reader");
+
Ok(InitialSegmentScoreTweaker {
centrality_reader,
is_homepage_reader,
+ fetch_time_ms_reader,
})
}
}
@@ -61,7 +72,8 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
let score = score as f64;
let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;
let is_homepage = self.is_homepage_reader.get(doc) as f64;
+ let fetch_time_ms = self.fetch_time_ms_reader.get(doc) as f64;
- score + 400.0 * centrality + 5.0 * is_homepage
+ score + (400.0 * centrality) + (5.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 12b12652..a482f358 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -65,6 +65,7 @@ mod tests {
"https://www.a.com",
vec![],
0.0,
+ 500,
))
.expect("failed to parse webpage");
index
@@ -88,6 +89,7 @@ mod tests {
text: "B site is great".to_string(),
}],
5.0,
+ 500,
))
.expect("failed to parse webpage");
@@ -121,6 +123,7 @@ mod tests {
"https://www.dr.dk",
vec![],
0.0,
+ 500,
))
.expect("failed to parse webpage");
index
@@ -140,6 +143,7 @@ mod tests {
"https://www.dr.dk/whatever",
vec![],
0.0,
+ 500,
))
.expect("failed to parse webpage");
index
@@ -157,6 +161,7 @@ mod tests {
"https://www.b.com",
vec![],
0.003,
+ 500,
))
.expect("failed to parse webpage");
diff --git a/src/schema.rs b/src/schema.rs
index 83314c46..21f1037e 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -35,8 +35,9 @@ pub enum Field {
IsHomepage,
BacklinkText,
Centrality,
+ FetchTimeMs,
}
-pub static ALL_FIELDS: [Field; 11] = [
+pub static ALL_FIELDS: [Field; 12] = [
Field::Title,
Field::Body,
Field::StemmedTitle,
@@ -48,6 +49,7 @@ pub static ALL_FIELDS: [Field; 11] = [
Field::IsHomepage,
Field::BacklinkText,
Field::Centrality,
+ Field::FetchTimeMs,
];
impl Field {
@@ -91,6 +93,11 @@ impl Field {
Field::StemmedBody => IndexingOption::Text(
self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),
),
+ Field::FetchTimeMs => IndexingOption::Numeric(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed(),
+ ),
}
}
@@ -107,6 +114,7 @@ impl Field {
Field::Domain => "domain",
Field::DomainIfHomepage => "domain_if_homepage",
Field::IsHomepage => "is_homepage",
+ Field::FetchTimeMs => "fetch_time_ms",
}
}
@@ -121,7 +129,8 @@ impl Field {
| Field::Centrality
| Field::Url
| Field::Domain
- | Field::IsHomepage => None,
+ | Field::IsHomepage
+ | Field::FetchTimeMs => None,
}
}
}
diff --git a/src/snippet.rs b/src/snippet.rs
index 7b2ec269..d1ea49d3 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -301,6 +301,7 @@ Survey in 2016, 2017, and 2018."#;
"https://www.example.com",
vec![],
1.0,
+ 500,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 8a3d89dc..ce46eff1 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -144,16 +144,24 @@ pub struct Webpage<'a> {
pub html: Html<'a>,
pub backlinks: Vec<Link>,
pub centrality: f64,
+ pub fetch_time_ms: u64,
}
impl<'a> Webpage<'a> {
- pub fn new(html: &'a str, url: &str, backlinks: Vec<Link>, centrality: f64) -> Self {
+ pub fn new(
+ html: &'a str,
+ url: &str,
+ backlinks: Vec<Link>,
+ centrality: f64,
+ fetch_time_ms: u64,
+ ) -> Self {
let html = Html::parse(html, url);
Self {
html,
backlinks,
centrality,
+ fetch_time_ms,
}
}
@@ -180,6 +188,13 @@ impl<'a> Webpage<'a> {
(self.centrality * CENTRALITY_SCALING as f64) as u64,
);
+ doc.add_u64(
+ schema
+ .get_field(Field::FetchTimeMs.as_str())
+ .expect("Failed to get fetch_time_ms field"),
+ self.fetch_time_ms,
+ );
+
Ok(doc)
}
}
@@ -405,7 +420,7 @@ impl<'a> Html<'a> {
Field::IsHomepage => {
doc.add_u64(tantivy_field, self.is_homepage().then(|| 1).unwrap_or(0))
}
- Field::BacklinkText | Field::Centrality => {}
+ Field::BacklinkText | Field::Centrality | Field::FetchTimeMs => {}
}
}