web search done right
show primary image in search results
Mikkel Denker549d3fb07499
diff --git a/Cargo.toml b/Cargo.tomlindex 36650937..de0ea424 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -46,6 +46,7 @@ lazy_static = "1.4.0"rocksdb = "0.18.0"image = "0.24.3"chrono = "0.4.20"+uuid = "1.1.2"[dev-dependencies]maplit = "1.0.2"diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 420d6120..f90857ce 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -74,7 +74,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {let file = WarcFile::download(source, &self.warc_path).unwrap();debug!("finished downloading");- for record in file.records().flatten() {+ for record in file.records().flatten().take(200) {let html = Html::parse(&record.response.body, &record.request.url);let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraphlet centrality = worker.centrality_store.get(html.host()).unwrap_or_default();@@ -90,6 +90,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {backlinks,centrality,fetch_time_ms,+ primary_image_uuid: None,};if let Err(err) = index.insert(webpage) {diff --git a/src/frontend/mod.rs b/src/frontend/mod.rsindex d72e7a2e..c26f68f9 100644--- a/src/frontend/mod.rs+++ b/src/frontend/mod.rs@@ -31,6 +31,7 @@ use axum_extra::routing::SpaRouter;mod autosuggest;mod favicons;mod index;+mod primary_image;pub mod search;pub struct HtmlTemplate<T>(T);@@ -71,6 +72,7 @@ pub fn router(index_path: &str, queries_csv_path: &str) -> Result<Router> {.route("/search", get(search::route)).route("/autosuggest", get(autosuggest::route)).route("/favicons/:site", get(favicons::route))+ .route("/image/:uuid", get(primary_image::route)).merge(SpaRouter::new("/static", "static")).layer(Extension(state)))}diff --git a/src/frontend/primary_image.rs b/src/frontend/primary_image.rsnew file mode 100644index 00000000..90ecc190--- /dev/null+++ b/src/frontend/primary_image.rs@@ -0,0 +1,44 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use axum::body::Body;+use axum::extract::Path;+use axum::http::Response;+use axum::response::IntoResponse;+use axum::Extension;+use reqwest::StatusCode;++use super::State;++pub async fn route(+ Path(uuid): Path<String>,+ Extension(state): Extension<Arc<State>>,+) -> impl IntoResponse {+ let img = state.searcher.primary_image(uuid);++ match img {+ Some(img) => Response::builder()+ .status(StatusCode::OK)+ .body(Body::from(img.as_raw_bytes()))+ .unwrap(),+ None => Response::builder()+ .status(StatusCode::NOT_FOUND)+ .body(Body::empty())+ .unwrap(),+ }+}diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 5d40da37..4a98e9a6 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -38,6 +38,7 @@ pub struct DisplayedWebpage {pub pretty_url: String,pub snippet: String,pub body: String,+ pub primary_image_uuid: Option<String>,}const MAX_PRETTY_URL_LEN: usize = 50;@@ -78,6 +79,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {domain,snippet: webpage.snippet, // snippet has already been html-escaped.body: webpage.body,+ primary_image_uuid: webpage.primary_image_uuid,}}}diff --git a/src/image_store.rs b/src/image_store.rsindex e2a2b011..8a806857 100644--- a/src/image_store.rs+++ b/src/image_store.rs@@ -21,6 +21,7 @@ use image::{DynamicImage, ImageOutputFormat};use serde::{de, ser::SerializeStruct, Serialize};use std::io::{Cursor, Read, Seek, SeekFrom};use std::path::Path;+use uuid::Uuid;const FAVICON_SIZE: u32 = 32;@@ -161,11 +162,51 @@ impl FaviconStore {self.0.get(key)}- pub fn merge(&mut self, other: FaviconStore) {+ pub fn merge(&mut self, other: Self) {self.0.merge(other.0)}}+pub struct PrimaryImageStore(ImageStore);++impl PrimaryImageStore {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ let store = ImageStore::open_with_filters(+ path,+ vec![Box::new(ResizeFilter {+ width: 200,+ height: 100,+ })],+ );++ Self(store)+ }++ pub fn insert(&mut self, image: Image) -> Uuid {+ let uuid = self.generate_uuid();+ self.0.insert(&uuid.to_string(), image);+ uuid+ }++ pub fn get(&self, uuid: &Uuid) -> Option<Image> {+ self.0.get(uuid.to_string().as_str())+ }++ pub fn merge(&mut self, other: Self) {+ self.0.merge(other.0)+ }++ fn generate_uuid(&self) -> Uuid {+ let mut uuid = Uuid::new_v4();++ while self.0.contains(&uuid.to_string()) {+ uuid = Uuid::new_v4();+ }++ uuid+ }+}+impl Image {pub(crate) fn from_bytes(bytes: Vec<u8>) -> Result<Image> {Ok(Self(image::load_from_memory(&bytes)?))diff --git a/src/index.rs b/src/index.rsindex e875ac43..0cd7ca26 100644--- a/src/index.rs+++ b/src/index.rs@@ -18,9 +18,10 @@ use tantivy::collector::{Collector, Count};use tantivy::merge_policy::NoMergePolicy;use tantivy::schema::Schema;use tantivy::{DocAddress, Document, IndexReader, IndexWriter, LeasedItem};+use uuid::Uuid;use crate::directory::{self, DirEntry};-use crate::image_store::{FaviconStore, Image};+use crate::image_store::{FaviconStore, Image, PrimaryImageStore};use crate::query::Query;use crate::schema::{Field, ALL_FIELDS};use crate::searcher::SearchResult;@@ -34,6 +35,7 @@ use std::sync::Arc;const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";+const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";pub struct Index {pub path: String,@@ -42,6 +44,7 @@ pub struct Index {reader: IndexReader,schema: Arc<Schema>,favicon_store: FaviconStore,+ primary_image_store: PrimaryImageStore,}impl Index {@@ -68,6 +71,8 @@ impl Index {};let favicon_store = FaviconStore::open(path.as_ref().join(FAVICON_STORE_SUBFOLDER_NAME));+ let primary_image_store =+ PrimaryImageStore::open(path.as_ref().join(PRIMARY_IMAGE_STORE_SUBFOLDER_NAME));let tokenizer = Tokenizer::default();tantivy_index@@ -90,12 +95,16 @@ impl Index {schema: Arc::new(schema),path: path.as_ref().to_str().unwrap().to_string(),favicon_store,+ primary_image_store,tantivy_index,})}- pub fn insert(&mut self, webpage: Webpage) -> Result<()> {+ pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {self.maybe_insert_favicon(&webpage);+ if let Some(uuid) = self.maybe_insert_primary_image(&webpage) {+ webpage.set_primary_image_uuid(uuid);+ }self.writer.add_document(webpage.into_tantivy(&self.schema)?)?;Ok(())@@ -205,6 +214,9 @@ impl Index {self.favicon_store.merge(other.favicon_store);drop(self.favicon_store);+ self.primary_image_store.merge(other.primary_image_store);+ drop(self.primary_image_store);+Self::open(path).expect("failed to open index")}@@ -222,9 +234,19 @@ impl Index {}}+ fn maybe_insert_primary_image(&mut self, webpage: &Webpage) -> Option<Uuid> {+ webpage+ .download_primary_image()+ .map(|image| self.primary_image_store.insert(image))+ }+pub fn retrieve_favicon(&self, url: &Url) -> Option<Image> {self.favicon_store.get(url.domain())}++ pub fn retrieve_primary_image(&self, uuid: &Uuid) -> Option<Image> {+ self.primary_image_store.get(uuid)+ }}#[derive(Default, Debug)]@@ -234,6 +256,7 @@ pub struct RetrievedWebpage {pub snippet: String,pub body: String,pub favicon: Option<Image>,+ pub primary_image_uuid: Option<String>,}impl From<Document> for RetrievedWebpage {@@ -263,6 +286,21 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string()}+ Field::PrimaryImageUuid => {+ webpage.primary_image_uuid = {+ let s = value+ .value+ .as_text()+ .expect("Primary image uuid field should be text")+ .to_string();++ if s.is_empty() {+ None+ } else {+ Some(s)+ }+ }+ }Field::BacklinkText| Field::Centrality| Field::Host@@ -271,6 +309,7 @@ impl From<Document> for RetrievedWebpage {| Field::Domain| Field::DomainIfHomepage| Field::IsHomepage+ | Field::LastUpdated| Field::FetchTimeMs => {}}}diff --git a/src/schema.rs b/src/schema.rsindex 29b51511..2e6fb2c6 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -36,8 +36,10 @@ pub enum Field {BacklinkText,Centrality,FetchTimeMs,+ PrimaryImageUuid,+ LastUpdated,}-pub static ALL_FIELDS: [Field; 12] = [+pub static ALL_FIELDS: [Field; 14] = [Field::Title,Field::Body,Field::StemmedTitle,@@ -50,6 +52,8 @@ pub static ALL_FIELDS: [Field; 12] = [Field::BacklinkText,Field::Centrality,Field::FetchTimeMs,+ Field::PrimaryImageUuid,+ Field::LastUpdated,];impl Field {@@ -99,6 +103,14 @@ impl Field {.set_fast(Cardinality::SingleValue).set_indexed(),),+ Field::PrimaryImageUuid => {+ IndexingOption::Text(self.default_text_options().set_stored())+ }+ &Field::LastUpdated => IndexingOption::Numeric(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed(),+ ),}}@@ -116,6 +128,8 @@ impl Field {Field::DomainIfHomepage => "domain_if_homepage",Field::IsHomepage => "is_homepage",Field::FetchTimeMs => "fetch_time_ms",+ Field::PrimaryImageUuid => "primary_image_uuid",+ Field::LastUpdated => "last_updated",}}@@ -131,14 +145,14 @@ impl Field {| Field::Url| Field::Domain| Field::IsHomepage- | Field::FetchTimeMs => None,+ | Field::PrimaryImageUuid+ | Field::FetchTimeMs+ | Field::LastUpdated => None,}}pub fn is_searchable(&self) -> bool {- match self {- _ => true,- }+ !matches!(self, Field::PrimaryImageUuid)}}diff --git a/src/searcher.rs b/src/searcher.rsindex 33eac1cb..1895702d 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -14,6 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::str::FromStr;++use uuid::Uuid;+use crate::image_store::Image;use crate::index::{Index, RetrievedWebpage};use crate::query::Query;@@ -47,4 +51,11 @@ impl Searcher {pub fn favicon(&self, site: &Url) -> Option<Image> {self.index.retrieve_favicon(site)}++ pub fn primary_image(&self, uuid: String) -> Option<Image> {+ if let Ok(uuid) = Uuid::from_str(uuid.as_str()) {+ return self.index.retrieve_primary_image(&uuid);+ }+ None+ }}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 84e197d1..72b4dbf3 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -17,6 +17,7 @@ use crate::{image_store::Image, schema_org::SchemaOrg, Error, Result};use chrono::{DateTime, FixedOffset};use logos::{Lexer, Logos};use std::{collections::HashMap, time::Duration};+use uuid::Uuid;mod just_text;mod lexer;@@ -88,6 +89,7 @@ pub struct Webpage<'a> {pub backlinks: Vec<Link>,pub centrality: f64,pub fetch_time_ms: u64,+ pub primary_image_uuid: Option<Uuid>,}impl<'a> Webpage<'a> {@@ -105,6 +107,7 @@ impl<'a> Webpage<'a> {backlinks,centrality,fetch_time_ms,+ primary_image_uuid: None,}}@@ -138,29 +141,40 @@ impl<'a> Webpage<'a> {self.fetch_time_ms,);+ let uuid = self+ .primary_image_uuid+ .map(|uuid| uuid.to_string())+ .unwrap_or_default();+ doc.add_text(+ schema+ .get_field(Field::PrimaryImageUuid.as_str())+ .expect("Failed to get primary_image_uuid field"),+ uuid,+ );+Ok(doc)}pub(crate) fn download_favicon(&self) -> Option<Image> {self.html.favicon().and_then(|favicon_link| {- let client = reqwest::blocking::Client::builder()- .timeout(Duration::from_secs(1))- .build()- .unwrap();-- match client.get(favicon_link.link.full()).send() {- Ok(mut res) => {- let mut bytes = Vec::new();- if res.copy_to(&mut bytes).is_err() {- None- } else {- Image::from_bytes(bytes).ok()- }- }- Err(_) => None,- }+ favicon_link+ .link+ .download_bytes(Duration::from_secs(1))+ .and_then(|bytes| Image::from_bytes(bytes).ok())})}++ pub(crate) fn download_primary_image(&self) -> Option<Image> {+ self.html.primary_image().and_then(|url| {+ dbg!(&url);+ url.download_bytes(Duration::from_secs(5))+ .and_then(|bytes| Image::from_bytes(bytes).ok())+ })+ }++ pub(crate) fn set_primary_image_uuid(&mut self, uuid: Uuid) {+ self.primary_image_uuid = Some(uuid);+ }}#[derive(Debug)]@@ -258,7 +272,7 @@ impl<'a> Html<'a> {let (width, height) = match tag.attributes().get("sizes") {Some(size) => {if let Some((width, height)) = size.split_once('x') {- (Some(width.parse().unwrap()), Some(height.parse().unwrap()))+ (width.parse().ok(), height.parse().ok())} else {(None, None)}@@ -444,7 +458,16 @@ impl<'a> Html<'a> {Field::IsHomepage => {doc.add_u64(tantivy_field, self.is_homepage().then(|| 1).unwrap_or(0))}- Field::BacklinkText | Field::Centrality | Field::FetchTimeMs => {}+ Field::LastUpdated => doc.add_u64(+ tantivy_field,+ self.updated_time()+ .map(|time| time.timestamp().max(0) as u64)+ .unwrap_or(0),+ ),+ Field::BacklinkText+ | Field::Centrality+ | Field::FetchTimeMs+ | Field::PrimaryImageUuid => {}}}@@ -566,6 +589,12 @@ impl<'a> Html<'a> {pub fn primary_image(&self) -> Option<Url> {self.og_image().or_else(|| self.schema_org_images().first().cloned())+ .map(|mut url| {+ if !url.is_full_path() {+ url.prefix_with(&self.url);+ }+ url+ })}pub fn description(&self) -> Option<String> {@@ -1061,11 +1090,11 @@ mod tests {</body></html>"#;- let html = Html::parse(html, "example.com");+ let html = Html::parse(html, "https://example.com");assert_eq!(html.primary_image(),- Some("mexico-beach.jpg".to_string().into())+ Some("https://example.com/mexico-beach.jpg".to_string().into()));}diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex a5fc480f..5b6b3b67 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::fmt::Display;+use std::{fmt::Display, time::Duration};#[derive(Debug, PartialEq, Eq, Clone)]pub struct Url(String);@@ -133,4 +133,23 @@ impl Url {pub fn full(&self) -> &str {&self.0}++ pub fn download_bytes(&self, timeout: Duration) -> Option<Vec<u8>> {+ let client = reqwest::blocking::Client::builder()+ .timeout(timeout)+ .build()+ .unwrap();++ match client.get(self.full()).send() {+ Ok(mut res) => {+ let mut bytes = Vec::new();+ if res.copy_to(&mut bytes).is_err() {+ None+ } else {+ Some(bytes)+ }+ }+ Err(_) => None,+ }+ }}diff --git a/templates/search.html b/templates/search.htmlindex d0a5b1e3..060b6f37 100644--- a/templates/search.html+++ b/templates/search.html@@ -15,21 +15,30 @@<div class="search-result flex flex-col space-y-3 pt-8 pb-8 pl-40">{% for item in search_result %}- <div class="search-result-item bg-white max-w-2xl p-3">- <div class="text-sm mb-1 flex">- <div class="flex items-center mr-2">- <img src="/favicons/{{ item.domain }}" width="13px" />+ <div class="flex max-w-2xl p-3">+ <div class="flex flex-col">+ <div class="text-sm mb-1 flex">+ <div class="flex items-center mr-2">+ <img src="/favicons/{{ item.domain }}" width="13px" />+ </div>+ <div>+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>+ </div></div>- <div>- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>+ <div class="text-lg font-medium mb-1">+ <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>+ </div>+ <div class="text-sm">+ {{ item.snippet }}</div></div>- <div class="text-lg font-medium mb-1">- <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>- </div>- <div class="text-sm">- {{ item.snippet }}+ {% match item.primary_image_uuid %}+ {% when Some with (val) %}+ <div class="aspect-square h-full ml-3">+ <img class="w-full h-full object-cover rounded-full" src="/image/{{ val }}" /></div>+ {% when None %}+ {% endmatch %}</div>{% endfor %}</div>@@ -49,10 +58,6 @@font-smooth: grayscale;-moz-osx-font-smoothing: grayscale;}-- .search-result-item {- border-color: rgb(241 245 249);- }</style>{% endblock %}\ No newline at end of file