web search done right
show update time in search result
Mikkel Denker4f55c72dd2c2
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex f90857ce..a89333fb 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -74,7 +74,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {let file = WarcFile::download(source, &self.warc_path).unwrap();debug!("finished downloading");- for record in file.records().flatten().take(200) {+ for record in file.records().flatten().take(2000) {let html = Html::parse(&record.response.body, &record.request.url);let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraphlet centrality = worker.centrality_store.get(html.host()).unwrap_or_default();diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 4a98e9a6..b47a4e44 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use axum::Extension;+use chrono::{NaiveDateTime, Utc};use crate::{index::RetrievedWebpage, webpage::Url};use std::collections::HashMap;@@ -39,31 +40,65 @@ pub struct DisplayedWebpage {pub snippet: String,pub body: String,pub primary_image_uuid: Option<String>,+ pub last_updated: Option<String>,}const MAX_PRETTY_URL_LEN: usize = 50;const MAX_TITLE_LEN: usize = 50;-impl From<RetrievedWebpage> for DisplayedWebpage {- fn from(webpage: RetrievedWebpage) -> Self {- let url: Url = webpage.url.clone().into();- let domain = url.domain().to_string();- let mut pretty_url = url.strip_query().to_string();+fn prettify_url(url: Url) -> String {+ let mut pretty_url = url.strip_query().to_string();- if pretty_url.ends_with('/') {- pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();- }+ if pretty_url.ends_with('/') {+ pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();+ }++ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";+ pretty_url = Url::from(pretty_url.clone())+ .strip_protocol()+ .replace('/', " › ");+ pretty_url = protocol + &pretty_url;- let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";- pretty_url = Url::from(pretty_url.clone())- .strip_protocol()- .replace('/', " › ");- pretty_url = protocol + &pretty_url;+ if pretty_url.len() > MAX_PRETTY_URL_LEN {+ pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();+ pretty_url += "...";+ }- if pretty_url.len() > MAX_PRETTY_URL_LEN {- pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();- pretty_url += "...";+ pretty_url+}++fn prettify_date(date: NaiveDateTime) -> String {+ let current_time = Utc::now().naive_utc();+ let diff = current_time.signed_duration_since(date);++ let num_hours = diff.num_hours() + 1;+ if num_hours < 24 {+ if num_hours <= 1 {+ return "1 hour ago".to_string();+ } else {+ return format!("{num_hours} hours ago");+ }+ }++ let num_days = diff.num_days();+ if num_days < 30 {+ if num_days <= 1 {+ return "1 day ago".to_string();+ } else {+ return format!("{num_days} days ago");}+ }++ format!("{}", date.format("%d. %b. %Y"))+}++impl From<RetrievedWebpage> for DisplayedWebpage {+ fn from(webpage: RetrievedWebpage) -> Self {+ let last_updated = webpage.updated_time.map(|date| prettify_date(date));++ let url: Url = webpage.url.clone().into();+ let domain = url.domain().to_string();+ let pretty_url = prettify_url(url);let mut title = html_escape(&webpage.title);@@ -80,6 +115,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {snippet: webpage.snippet, // snippet has already been html-escaped.body: webpage.body,primary_image_uuid: webpage.primary_image_uuid,+ last_updated,}}}@@ -115,3 +151,36 @@ pub async fn route(};HtmlTemplate(template)}++#[cfg(test)]+mod tests {+ use chrono::{DateTime, Utc};++ use super::*;++ #[test]+ fn prettify_date_in_hours() {+ let date = Utc::now().naive_utc();+ assert_eq!(prettify_date(date), "1 hour ago".to_string());++ let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();+ assert_eq!(prettify_date(date), "2 hours ago".to_string());+ }++ #[test]+ fn prettify_date_days() {+ let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();+ assert_eq!(prettify_date(date), "1 day ago".to_string());++ let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();+ assert_eq!(prettify_date(date), "2 days ago".to_string());+ }++ #[test]+ fn prettify_date_rest() {+ let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")+ .unwrap()+ .naive_local();+ assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());+ }+}diff --git a/src/index.rs b/src/index.rsindex 0cd7ca26..493dfc01 100644--- a/src/index.rs+++ b/src/index.rs@@ -1,3 +1,4 @@+use chrono::NaiveDateTime;use serde::{Deserialize, Serialize};// Cuely is an open source web search engine.// Copyright (C) 2022 Cuely ApS@@ -101,10 +102,10 @@ impl Index {}pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {- self.maybe_insert_favicon(&webpage);- if let Some(uuid) = self.maybe_insert_primary_image(&webpage) {- webpage.set_primary_image_uuid(uuid);- }+ // self.maybe_insert_favicon(&webpage);+ // if let Some(uuid) = self.maybe_insert_primary_image(&webpage) {+ // webpage.set_primary_image_uuid(uuid);+ // }self.writer.add_document(webpage.into_tantivy(&self.schema)?)?;Ok(())@@ -257,6 +258,7 @@ pub struct RetrievedWebpage {pub body: String,pub favicon: Option<Image>,pub primary_image_uuid: Option<String>,+ pub updated_time: Option<NaiveDateTime>,}impl From<Document> for RetrievedWebpage {@@ -301,6 +303,16 @@ impl From<Document> for RetrievedWebpage {}}}+ Field::LastUpdated => {+ webpage.updated_time = {+ let timestamp = value.value.as_u64().unwrap() as i64;+ if timestamp == 0 {+ None+ } else {+ Some(NaiveDateTime::from_timestamp(timestamp, 0))+ }+ }+ }Field::BacklinkText| Field::Centrality| Field::Host@@ -309,7 +321,6 @@ impl From<Document> for RetrievedWebpage {| Field::Domain| Field::DomainIfHomepage| Field::IsHomepage- | Field::LastUpdated| Field::FetchTimeMs => {}}}diff --git a/src/schema.rs b/src/schema.rsindex 2e6fb2c6..a432acab 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -109,6 +109,7 @@ impl Field {&Field::LastUpdated => IndexingOption::Numeric(NumericOptions::default().set_fast(Cardinality::SingleValue)+ .set_stored().set_indexed(),),}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 72b4dbf3..c1e47c3b 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -166,7 +166,6 @@ impl<'a> Webpage<'a> {pub(crate) fn download_primary_image(&self) -> Option<Image> {self.html.primary_image().and_then(|url| {- dbg!(&url);url.download_bytes(Duration::from_secs(5)).and_then(|bytes| Image::from_bytes(bytes).ok())})@@ -460,9 +459,9 @@ impl<'a> Html<'a> {}Field::LastUpdated => doc.add_u64(tantivy_field,- self.updated_time()+ dbg!(dbg!(self.updated_time()).map(|time| time.timestamp().max(0) as u64)- .unwrap_or(0),+ .unwrap_or(0)),),Field::BacklinkText| Field::Centralitydiff --git a/templates/search.html b/templates/search.htmlindex 060b6f37..78b08dd1 100644--- a/templates/search.html+++ b/templates/search.html@@ -29,6 +29,13 @@<a class="title-link" href="{{ item.url }}">{{ item.title }}</a></div><div class="text-sm">+ {% match item.last_updated %}+ {% when Some with (val) %}+ <span class="text-gray-500">+ {{ val }}+ </span> —+ {% when None %}+ {% endmatch %}{{ item.snippet }}</div></div>