web search done right
take freshness into account during ranking
Mikkel Denker3861386da6ac
diff --git a/src/index.rs b/src/index.rsindex bedd4033..99f2d8f1 100644--- a/src/index.rs+++ b/src/index.rs@@ -40,12 +40,13 @@ const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;-#[derive(Clone, Hash, PartialEq, Eq)]+#[derive(Clone, Hash, PartialEq, Eq, Debug)]enum ImageDownloadJob {Favicon(Url),PrimaryImage { key: Uuid, url: Url },}+#[derive(Debug)]struct DownloadedImage {image: Image,key: String,@@ -149,9 +150,8 @@ impl Index {}if let Some(favicon) = webpage.html.favicon() {- self.image_download_jobs.insert(ImageDownloadJob::Favicon(- favicon.link.domain().to_string().into(),- ));+ self.image_download_jobs+ .insert(ImageDownloadJob::Favicon(favicon.link));}}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 493a77e6..17ddd856 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::schema::{Field, CENTRALITY_SCALING};+use chrono::Utc;use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};use tantivy::fastfield::{DynamicFastFieldReader, FastFieldReader};use tantivy::{DocId, Score, SegmentReader};@@ -26,6 +27,8 @@ pub(crate) struct InitialSegmentScoreTweaker {centrality_reader: DynamicFastFieldReader<u64>,is_homepage_reader: DynamicFastFieldReader<u64>,fetch_time_ms_reader: DynamicFastFieldReader<u64>,+ update_timestamp_reader: DynamicFastFieldReader<u64>,+ current_timestamp: f64,}impl ScoreTweaker<f64> for InitialScoreTweaker {@@ -59,21 +62,44 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {.u64(fetch_time_ms_field).expect("Failed to get fetch_time_ms fast-field reader");+ let update_timestamp_field = segment_reader+ .schema()+ .get_field(Field::LastUpdated.as_str())+ .expect("Faild to load last_updated field");+ let update_timestamp_reader = segment_reader+ .fast_fields()+ .u64(update_timestamp_field)+ .expect("Failed to get last_updated fast-field reader");++ let current_timestamp = Utc::now().timestamp() as f64;+Ok(InitialSegmentScoreTweaker {centrality_reader,is_homepage_reader,fetch_time_ms_reader,+ update_timestamp_reader,+ current_timestamp,})}}+fn time_to_score(time: f64) -> f64 {+ 1.0 / ((time + 1.0).log2())+}+impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {fn score(&mut self, doc: DocId, score: Score) -> f64 {let score = score as f64;let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;let is_homepage = self.is_homepage_reader.get(doc) as f64;let fetch_time_ms = self.fetch_time_ms_reader.get(doc) as f64;+ let update_timestamp = self.update_timestamp_reader.get(doc) as f64;+ let hours_since_update = (self.current_timestamp - update_timestamp).max(0.000001) / 3600.0;- (3.0 * score) + (1000.0 * centrality) + (1.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))+ (3.0 * score)+ + (4000.0 * centrality)+ + (1.0 * is_homepage)+ + (10.0 / (fetch_time_ms + 1.0))+ + (2000.0 * time_to_score(hours_since_update))}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex e6ac2b1e..e9745a00 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -170,4 +170,58 @@ mod tests {assert_eq!(result.documents.len(), 3);assert_eq!(result.documents[0].url, "https://www.dr.dk");}++ #[test]+ fn freshness_ranking() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Title</title>+ <meta property="og:updated_time" content="1999-06-22T19:37:34+00:00" />+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.old.com",+ vec![],+ 0.1,+ 500,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Title</title>+ <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.new.com",+ vec![],+ 0.09,+ 500,+ ))+ .expect("failed to parse webpage");++ index.commit().expect("failed to commit index");+ let searcher = Searcher::from(index);+ let result = searcher.search("title").expect("Search failed");++ assert_eq!(result.documents[0].url, "https://www.new.com");+ }}diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex 69fbe63a..ef3902a2 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -132,8 +132,12 @@ impl Url {};}- pub fn full(&self) -> &str {- &self.0+ pub fn full(&self) -> String {+ if self.find_protocol_end() == 0 {+ "https://".to_string() + &self.0+ } else {+ self.0.clone()+ }}pub async fn download_bytes(&self, timeout: Duration) -> Option<Vec<u8>> {