Skip to content

web search done right

show update time in search result

Mikkel Denker4f55c72dd2c2
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index f90857ce..a89333fb 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -74,7 +74,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
let file = WarcFile::download(source, &self.warc_path).unwrap();
debug!("finished downloading");
- for record in file.records().flatten().take(200) {
+ for record in file.records().flatten().take(2000) {
let html = Html::parse(&record.response.body, &record.request.url);
let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph
let centrality = worker.centrality_store.get(html.host()).unwrap_or_default();
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 4a98e9a6..b47a4e44 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use axum::Extension;
+use chrono::{NaiveDateTime, Utc};
use crate::{index::RetrievedWebpage, webpage::Url};
use std::collections::HashMap;
@@ -39,31 +40,65 @@ pub struct DisplayedWebpage {
pub snippet: String,
pub body: String,
pub primary_image_uuid: Option<String>,
+ pub last_updated: Option<String>,
}
const MAX_PRETTY_URL_LEN: usize = 50;
const MAX_TITLE_LEN: usize = 50;
-impl From<RetrievedWebpage> for DisplayedWebpage {
- fn from(webpage: RetrievedWebpage) -> Self {
- let url: Url = webpage.url.clone().into();
- let domain = url.domain().to_string();
- let mut pretty_url = url.strip_query().to_string();
+fn prettify_url(url: Url) -> String {
+ let mut pretty_url = url.strip_query().to_string();
- if pretty_url.ends_with('/') {
- pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
- }
+ if pretty_url.ends_with('/') {
+ pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
+ }
+
+ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
+ pretty_url = Url::from(pretty_url.clone())
+ .strip_protocol()
+ .replace('/', " › ");
+ pretty_url = protocol + &pretty_url;
- let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
- pretty_url = Url::from(pretty_url.clone())
- .strip_protocol()
- .replace('/', " › ");
- pretty_url = protocol + &pretty_url;
+ if pretty_url.len() > MAX_PRETTY_URL_LEN {
+ pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();
+ pretty_url += "...";
+ }
- if pretty_url.len() > MAX_PRETTY_URL_LEN {
- pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();
- pretty_url += "...";
+ pretty_url
+}
+
+fn prettify_date(date: NaiveDateTime) -> String {
+ let current_time = Utc::now().naive_utc();
+ let diff = current_time.signed_duration_since(date);
+
+ let num_hours = diff.num_hours() + 1;
+ if num_hours < 24 {
+ if num_hours <= 1 {
+ return "1 hour ago".to_string();
+ } else {
+ return format!("{num_hours} hours ago");
+ }
+ }
+
+ let num_days = diff.num_days();
+ if num_days < 30 {
+ if num_days <= 1 {
+ return "1 day ago".to_string();
+ } else {
+ return format!("{num_days} days ago");
}
+ }
+
+ format!("{}", date.format("%d. %b. %Y"))
+}
+
+impl From<RetrievedWebpage> for DisplayedWebpage {
+ fn from(webpage: RetrievedWebpage) -> Self {
+ let last_updated = webpage.updated_time.map(|date| prettify_date(date));
+
+ let url: Url = webpage.url.clone().into();
+ let domain = url.domain().to_string();
+ let pretty_url = prettify_url(url);
let mut title = html_escape(&webpage.title);
@@ -80,6 +115,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
snippet: webpage.snippet, // snippet has already been html-escaped.
body: webpage.body,
primary_image_uuid: webpage.primary_image_uuid,
+ last_updated,
}
}
}
@@ -115,3 +151,36 @@ pub async fn route(
};
HtmlTemplate(template)
}
+
+#[cfg(test)]
+mod tests {
+ use chrono::{DateTime, Utc};
+
+ use super::*;
+
+ #[test]
+ fn prettify_date_in_hours() {
+ let date = Utc::now().naive_utc();
+ assert_eq!(prettify_date(date), "1 hour ago".to_string());
+
+ let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();
+ assert_eq!(prettify_date(date), "2 hours ago".to_string());
+ }
+
+ #[test]
+ fn prettify_date_days() {
+ let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();
+ assert_eq!(prettify_date(date), "1 day ago".to_string());
+
+ let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();
+ assert_eq!(prettify_date(date), "2 days ago".to_string());
+ }
+
+ #[test]
+ fn prettify_date_rest() {
+ let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")
+ .unwrap()
+ .naive_local();
+ assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());
+ }
+}
diff --git a/src/index.rs b/src/index.rs
index 0cd7ca26..493dfc01 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -1,3 +1,4 @@
+use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
// Cuely is an open source web search engine.
// Copyright (C) 2022 Cuely ApS
@@ -101,10 +102,10 @@ impl Index {
}
pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {
- self.maybe_insert_favicon(&webpage);
- if let Some(uuid) = self.maybe_insert_primary_image(&webpage) {
- webpage.set_primary_image_uuid(uuid);
- }
+ // self.maybe_insert_favicon(&webpage);
+ // if let Some(uuid) = self.maybe_insert_primary_image(&webpage) {
+ // webpage.set_primary_image_uuid(uuid);
+ // }
self.writer
.add_document(webpage.into_tantivy(&self.schema)?)?;
Ok(())
@@ -257,6 +258,7 @@ pub struct RetrievedWebpage {
pub body: String,
pub favicon: Option<Image>,
pub primary_image_uuid: Option<String>,
+ pub updated_time: Option<NaiveDateTime>,
}
impl From<Document> for RetrievedWebpage {
@@ -301,6 +303,16 @@ impl From<Document> for RetrievedWebpage {
}
}
}
+ Field::LastUpdated => {
+ webpage.updated_time = {
+ let timestamp = value.value.as_u64().unwrap() as i64;
+ if timestamp == 0 {
+ None
+ } else {
+ Some(NaiveDateTime::from_timestamp(timestamp, 0))
+ }
+ }
+ }
Field::BacklinkText
| Field::Centrality
| Field::Host
@@ -309,7 +321,6 @@ impl From<Document> for RetrievedWebpage {
| Field::Domain
| Field::DomainIfHomepage
| Field::IsHomepage
- | Field::LastUpdated
| Field::FetchTimeMs => {}
}
}
diff --git a/src/schema.rs b/src/schema.rs
index 2e6fb2c6..a432acab 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -109,6 +109,7 @@ impl Field {
&Field::LastUpdated => IndexingOption::Numeric(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
+ .set_stored()
.set_indexed(),
),
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 72b4dbf3..c1e47c3b 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -166,7 +166,6 @@ impl<'a> Webpage<'a> {
pub(crate) fn download_primary_image(&self) -> Option<Image> {
self.html.primary_image().and_then(|url| {
- dbg!(&url);
url.download_bytes(Duration::from_secs(5))
.and_then(|bytes| Image::from_bytes(bytes).ok())
})
@@ -460,9 +459,9 @@ impl<'a> Html<'a> {
}
Field::LastUpdated => doc.add_u64(
tantivy_field,
- self.updated_time()
+ dbg!(dbg!(self.updated_time())
.map(|time| time.timestamp().max(0) as u64)
- .unwrap_or(0),
+ .unwrap_or(0)),
),
Field::BacklinkText
| Field::Centrality
diff --git a/templates/search.html b/templates/search.html
index 060b6f37..78b08dd1 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -29,6 +29,13 @@
<a class="title-link" href="{{ item.url }}">{{ item.title }}</a>
</div>
<div class="text-sm">
+ {% match item.last_updated %}
+ {% when Some with (val) %}
+ <span class="text-gray-500">
+ {{ val }}
+ </span> —
+ {% when None %}
+ {% endmatch %}
{{ item.snippet }}
</div>
</div>