Skip to content

web search done right

take freshness into account during ranking

Mikkel Denker3861386da6ac
diff --git a/src/index.rs b/src/index.rs
index bedd4033..99f2d8f1 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -40,12 +40,13 @@ const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";
const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";
const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;
-#[derive(Clone, Hash, PartialEq, Eq)]
+#[derive(Clone, Hash, PartialEq, Eq, Debug)]
enum ImageDownloadJob {
Favicon(Url),
PrimaryImage { key: Uuid, url: Url },
}
+#[derive(Debug)]
struct DownloadedImage {
image: Image,
key: String,
@@ -149,9 +150,8 @@ impl Index {
}
if let Some(favicon) = webpage.html.favicon() {
- self.image_download_jobs.insert(ImageDownloadJob::Favicon(
- favicon.link.domain().to_string().into(),
- ));
+ self.image_download_jobs
+ .insert(ImageDownloadJob::Favicon(favicon.link));
}
}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 493a77e6..17ddd856 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::schema::{Field, CENTRALITY_SCALING};
+use chrono::Utc;
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
use tantivy::fastfield::{DynamicFastFieldReader, FastFieldReader};
use tantivy::{DocId, Score, SegmentReader};
@@ -26,6 +27,8 @@ pub(crate) struct InitialSegmentScoreTweaker {
centrality_reader: DynamicFastFieldReader<u64>,
is_homepage_reader: DynamicFastFieldReader<u64>,
fetch_time_ms_reader: DynamicFastFieldReader<u64>,
+ update_timestamp_reader: DynamicFastFieldReader<u64>,
+ current_timestamp: f64,
}
impl ScoreTweaker<f64> for InitialScoreTweaker {
@@ -59,21 +62,44 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
.u64(fetch_time_ms_field)
.expect("Failed to get fetch_time_ms fast-field reader");
+ let update_timestamp_field = segment_reader
+ .schema()
+ .get_field(Field::LastUpdated.as_str())
+ .expect("Faild to load last_updated field");
+ let update_timestamp_reader = segment_reader
+ .fast_fields()
+ .u64(update_timestamp_field)
+ .expect("Failed to get last_updated fast-field reader");
+
+ let current_timestamp = Utc::now().timestamp() as f64;
+
Ok(InitialSegmentScoreTweaker {
centrality_reader,
is_homepage_reader,
fetch_time_ms_reader,
+ update_timestamp_reader,
+ current_timestamp,
})
}
}
+fn time_to_score(time: f64) -> f64 {
+ 1.0 / ((time + 1.0).log2())
+}
+
impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
fn score(&mut self, doc: DocId, score: Score) -> f64 {
let score = score as f64;
let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;
let is_homepage = self.is_homepage_reader.get(doc) as f64;
let fetch_time_ms = self.fetch_time_ms_reader.get(doc) as f64;
+ let update_timestamp = self.update_timestamp_reader.get(doc) as f64;
+ let hours_since_update = (self.current_timestamp - update_timestamp).max(0.000001) / 3600.0;
- (3.0 * score) + (1000.0 * centrality) + (1.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))
+ (3.0 * score)
+ + (4000.0 * centrality)
+ + (1.0 * is_homepage)
+ + (10.0 / (fetch_time_ms + 1.0))
+ + (2000.0 * time_to_score(hours_since_update))
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index e6ac2b1e..e9745a00 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -170,4 +170,58 @@ mod tests {
assert_eq!(result.documents.len(), 3);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
+
+ #[test]
+ fn freshness_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Title</title>
+ <meta property="og:updated_time" content="1999-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.old.com",
+ vec![],
+ 0.1,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Title</title>
+ <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.new.com",
+ vec![],
+ 0.09,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::from(index);
+ let result = searcher.search("title").expect("Search failed");
+
+ assert_eq!(result.documents[0].url, "https://www.new.com");
+ }
}
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index 69fbe63a..ef3902a2 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -132,8 +132,12 @@ impl Url {
};
}
- pub fn full(&self) -> &str {
- &self.0
+ pub fn full(&self) -> String {
+ if self.find_protocol_end() == 0 {
+ "https://".to_string() + &self.0
+ } else {
+ self.0.clone()
+ }
}
pub async fn download_bytes(&self, timeout: Duration) -> Option<Vec<u8>> {