Skip to content

web search done right

show primary image in search results

Mikkel Denker549d3fb07499
diff --git a/Cargo.toml b/Cargo.toml
index 36650937..de0ea424 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -46,6 +46,7 @@ lazy_static = "1.4.0"
rocksdb = "0.18.0"
image = "0.24.3"
chrono = "0.4.20"
+uuid = "1.1.2"
[dev-dependencies]
maplit = "1.0.2"
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 420d6120..f90857ce 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -74,7 +74,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
let file = WarcFile::download(source, &self.warc_path).unwrap();
debug!("finished downloading");
- for record in file.records().flatten() {
+ for record in file.records().flatten().take(200) {
let html = Html::parse(&record.response.body, &record.request.url);
let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph
let centrality = worker.centrality_store.get(html.host()).unwrap_or_default();
@@ -90,6 +90,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
backlinks,
centrality,
fetch_time_ms,
+ primary_image_uuid: None,
};
if let Err(err) = index.insert(webpage) {
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index d72e7a2e..c26f68f9 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -31,6 +31,7 @@ use axum_extra::routing::SpaRouter;
mod autosuggest;
mod favicons;
mod index;
+mod primary_image;
pub mod search;
pub struct HtmlTemplate<T>(T);
@@ -71,6 +72,7 @@ pub fn router(index_path: &str, queries_csv_path: &str) -> Result<Router> {
.route("/search", get(search::route))
.route("/autosuggest", get(autosuggest::route))
.route("/favicons/:site", get(favicons::route))
+ .route("/image/:uuid", get(primary_image::route))
.merge(SpaRouter::new("/static", "static"))
.layer(Extension(state)))
}
diff --git a/src/frontend/primary_image.rs b/src/frontend/primary_image.rs
new file mode 100644
index 00000000..90ecc190
--- /dev/null
+++ b/src/frontend/primary_image.rs
@@ -0,0 +1,44 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use axum::body::Body;
+use axum::extract::Path;
+use axum::http::Response;
+use axum::response::IntoResponse;
+use axum::Extension;
+use reqwest::StatusCode;
+
+use super::State;
+
+pub async fn route(
+ Path(uuid): Path<String>,
+ Extension(state): Extension<Arc<State>>,
+) -> impl IntoResponse {
+ let img = state.searcher.primary_image(uuid);
+
+ match img {
+ Some(img) => Response::builder()
+ .status(StatusCode::OK)
+ .body(Body::from(img.as_raw_bytes()))
+ .unwrap(),
+ None => Response::builder()
+ .status(StatusCode::NOT_FOUND)
+ .body(Body::empty())
+ .unwrap(),
+ }
+}
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 5d40da37..4a98e9a6 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -38,6 +38,7 @@ pub struct DisplayedWebpage {
pub pretty_url: String,
pub snippet: String,
pub body: String,
+ pub primary_image_uuid: Option<String>,
}
const MAX_PRETTY_URL_LEN: usize = 50;
@@ -78,6 +79,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
domain,
snippet: webpage.snippet, // snippet has already been html-escaped.
body: webpage.body,
+ primary_image_uuid: webpage.primary_image_uuid,
}
}
}
diff --git a/src/image_store.rs b/src/image_store.rs
index e2a2b011..8a806857 100644
--- a/src/image_store.rs
+++ b/src/image_store.rs
@@ -21,6 +21,7 @@ use image::{DynamicImage, ImageOutputFormat};
use serde::{de, ser::SerializeStruct, Serialize};
use std::io::{Cursor, Read, Seek, SeekFrom};
use std::path::Path;
+use uuid::Uuid;
const FAVICON_SIZE: u32 = 32;
@@ -161,11 +162,51 @@ impl FaviconStore {
self.0.get(key)
}
- pub fn merge(&mut self, other: FaviconStore) {
+ pub fn merge(&mut self, other: Self) {
self.0.merge(other.0)
}
}
+pub struct PrimaryImageStore(ImageStore);
+
+impl PrimaryImageStore {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ let store = ImageStore::open_with_filters(
+ path,
+ vec![Box::new(ResizeFilter {
+ width: 200,
+ height: 100,
+ })],
+ );
+
+ Self(store)
+ }
+
+ pub fn insert(&mut self, image: Image) -> Uuid {
+ let uuid = self.generate_uuid();
+ self.0.insert(&uuid.to_string(), image);
+ uuid
+ }
+
+ pub fn get(&self, uuid: &Uuid) -> Option<Image> {
+ self.0.get(uuid.to_string().as_str())
+ }
+
+ pub fn merge(&mut self, other: Self) {
+ self.0.merge(other.0)
+ }
+
+ fn generate_uuid(&self) -> Uuid {
+ let mut uuid = Uuid::new_v4();
+
+ while self.0.contains(&uuid.to_string()) {
+ uuid = Uuid::new_v4();
+ }
+
+ uuid
+ }
+}
+
impl Image {
pub(crate) fn from_bytes(bytes: Vec<u8>) -> Result<Image> {
Ok(Self(image::load_from_memory(&bytes)?))
diff --git a/src/index.rs b/src/index.rs
index e875ac43..0cd7ca26 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -18,9 +18,10 @@ use tantivy::collector::{Collector, Count};
use tantivy::merge_policy::NoMergePolicy;
use tantivy::schema::Schema;
use tantivy::{DocAddress, Document, IndexReader, IndexWriter, LeasedItem};
+use uuid::Uuid;
use crate::directory::{self, DirEntry};
-use crate::image_store::{FaviconStore, Image};
+use crate::image_store::{FaviconStore, Image, PrimaryImageStore};
use crate::query::Query;
use crate::schema::{Field, ALL_FIELDS};
use crate::searcher::SearchResult;
@@ -34,6 +35,7 @@ use std::sync::Arc;
const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";
const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";
+const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";
pub struct Index {
pub path: String,
@@ -42,6 +44,7 @@ pub struct Index {
reader: IndexReader,
schema: Arc<Schema>,
favicon_store: FaviconStore,
+ primary_image_store: PrimaryImageStore,
}
impl Index {
@@ -68,6 +71,8 @@ impl Index {
};
let favicon_store = FaviconStore::open(path.as_ref().join(FAVICON_STORE_SUBFOLDER_NAME));
+ let primary_image_store =
+ PrimaryImageStore::open(path.as_ref().join(PRIMARY_IMAGE_STORE_SUBFOLDER_NAME));
let tokenizer = Tokenizer::default();
tantivy_index
@@ -90,12 +95,16 @@ impl Index {
schema: Arc::new(schema),
path: path.as_ref().to_str().unwrap().to_string(),
favicon_store,
+ primary_image_store,
tantivy_index,
})
}
- pub fn insert(&mut self, webpage: Webpage) -> Result<()> {
+ pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {
self.maybe_insert_favicon(&webpage);
+ if let Some(uuid) = self.maybe_insert_primary_image(&webpage) {
+ webpage.set_primary_image_uuid(uuid);
+ }
self.writer
.add_document(webpage.into_tantivy(&self.schema)?)?;
Ok(())
@@ -205,6 +214,9 @@ impl Index {
self.favicon_store.merge(other.favicon_store);
drop(self.favicon_store);
+ self.primary_image_store.merge(other.primary_image_store);
+ drop(self.primary_image_store);
+
Self::open(path).expect("failed to open index")
}
@@ -222,9 +234,19 @@ impl Index {
}
}
+ fn maybe_insert_primary_image(&mut self, webpage: &Webpage) -> Option<Uuid> {
+ webpage
+ .download_primary_image()
+ .map(|image| self.primary_image_store.insert(image))
+ }
+
pub fn retrieve_favicon(&self, url: &Url) -> Option<Image> {
self.favicon_store.get(url.domain())
}
+
+ pub fn retrieve_primary_image(&self, uuid: &Uuid) -> Option<Image> {
+ self.primary_image_store.get(uuid)
+ }
}
#[derive(Default, Debug)]
@@ -234,6 +256,7 @@ pub struct RetrievedWebpage {
pub snippet: String,
pub body: String,
pub favicon: Option<Image>,
+ pub primary_image_uuid: Option<String>,
}
impl From<Document> for RetrievedWebpage {
@@ -263,6 +286,21 @@ impl From<Document> for RetrievedWebpage {
.expect("Url field should be text")
.to_string()
}
+ Field::PrimaryImageUuid => {
+ webpage.primary_image_uuid = {
+ let s = value
+ .value
+ .as_text()
+ .expect("Primary image uuid field should be text")
+ .to_string();
+
+ if s.is_empty() {
+ None
+ } else {
+ Some(s)
+ }
+ }
+ }
Field::BacklinkText
| Field::Centrality
| Field::Host
@@ -271,6 +309,7 @@ impl From<Document> for RetrievedWebpage {
| Field::Domain
| Field::DomainIfHomepage
| Field::IsHomepage
+ | Field::LastUpdated
| Field::FetchTimeMs => {}
}
}
diff --git a/src/schema.rs b/src/schema.rs
index 29b51511..2e6fb2c6 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -36,8 +36,10 @@ pub enum Field {
BacklinkText,
Centrality,
FetchTimeMs,
+ PrimaryImageUuid,
+ LastUpdated,
}
-pub static ALL_FIELDS: [Field; 12] = [
+pub static ALL_FIELDS: [Field; 14] = [
Field::Title,
Field::Body,
Field::StemmedTitle,
@@ -50,6 +52,8 @@ pub static ALL_FIELDS: [Field; 12] = [
Field::BacklinkText,
Field::Centrality,
Field::FetchTimeMs,
+ Field::PrimaryImageUuid,
+ Field::LastUpdated,
];
impl Field {
@@ -99,6 +103,14 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
+ Field::PrimaryImageUuid => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
+ &Field::LastUpdated => IndexingOption::Numeric(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed(),
+ ),
}
}
@@ -116,6 +128,8 @@ impl Field {
Field::DomainIfHomepage => "domain_if_homepage",
Field::IsHomepage => "is_homepage",
Field::FetchTimeMs => "fetch_time_ms",
+ Field::PrimaryImageUuid => "primary_image_uuid",
+ Field::LastUpdated => "last_updated",
}
}
@@ -131,14 +145,14 @@ impl Field {
| Field::Url
| Field::Domain
| Field::IsHomepage
- | Field::FetchTimeMs => None,
+ | Field::PrimaryImageUuid
+ | Field::FetchTimeMs
+ | Field::LastUpdated => None,
}
}
pub fn is_searchable(&self) -> bool {
- match self {
- _ => true,
- }
+ !matches!(self, Field::PrimaryImageUuid)
}
}
diff --git a/src/searcher.rs b/src/searcher.rs
index 33eac1cb..1895702d 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -14,6 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::str::FromStr;
+
+use uuid::Uuid;
+
use crate::image_store::Image;
use crate::index::{Index, RetrievedWebpage};
use crate::query::Query;
@@ -47,4 +51,11 @@ impl Searcher {
pub fn favicon(&self, site: &Url) -> Option<Image> {
self.index.retrieve_favicon(site)
}
+
+ pub fn primary_image(&self, uuid: String) -> Option<Image> {
+ if let Ok(uuid) = Uuid::from_str(uuid.as_str()) {
+ return self.index.retrieve_primary_image(&uuid);
+ }
+ None
+ }
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 84e197d1..72b4dbf3 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -17,6 +17,7 @@ use crate::{image_store::Image, schema_org::SchemaOrg, Error, Result};
use chrono::{DateTime, FixedOffset};
use logos::{Lexer, Logos};
use std::{collections::HashMap, time::Duration};
+use uuid::Uuid;
mod just_text;
mod lexer;
@@ -88,6 +89,7 @@ pub struct Webpage<'a> {
pub backlinks: Vec<Link>,
pub centrality: f64,
pub fetch_time_ms: u64,
+ pub primary_image_uuid: Option<Uuid>,
}
impl<'a> Webpage<'a> {
@@ -105,6 +107,7 @@ impl<'a> Webpage<'a> {
backlinks,
centrality,
fetch_time_ms,
+ primary_image_uuid: None,
}
}
@@ -138,29 +141,40 @@ impl<'a> Webpage<'a> {
self.fetch_time_ms,
);
+ let uuid = self
+ .primary_image_uuid
+ .map(|uuid| uuid.to_string())
+ .unwrap_or_default();
+ doc.add_text(
+ schema
+ .get_field(Field::PrimaryImageUuid.as_str())
+ .expect("Failed to get primary_image_uuid field"),
+ uuid,
+ );
+
Ok(doc)
}
pub(crate) fn download_favicon(&self) -> Option<Image> {
self.html.favicon().and_then(|favicon_link| {
- let client = reqwest::blocking::Client::builder()
- .timeout(Duration::from_secs(1))
- .build()
- .unwrap();
-
- match client.get(favicon_link.link.full()).send() {
- Ok(mut res) => {
- let mut bytes = Vec::new();
- if res.copy_to(&mut bytes).is_err() {
- None
- } else {
- Image::from_bytes(bytes).ok()
- }
- }
- Err(_) => None,
- }
+ favicon_link
+ .link
+ .download_bytes(Duration::from_secs(1))
+ .and_then(|bytes| Image::from_bytes(bytes).ok())
})
}
+
+ pub(crate) fn download_primary_image(&self) -> Option<Image> {
+ self.html.primary_image().and_then(|url| {
+ dbg!(&url);
+ url.download_bytes(Duration::from_secs(5))
+ .and_then(|bytes| Image::from_bytes(bytes).ok())
+ })
+ }
+
+ pub(crate) fn set_primary_image_uuid(&mut self, uuid: Uuid) {
+ self.primary_image_uuid = Some(uuid);
+ }
}
#[derive(Debug)]
@@ -258,7 +272,7 @@ impl<'a> Html<'a> {
let (width, height) = match tag.attributes().get("sizes") {
Some(size) => {
if let Some((width, height)) = size.split_once('x') {
- (Some(width.parse().unwrap()), Some(height.parse().unwrap()))
+ (width.parse().ok(), height.parse().ok())
} else {
(None, None)
}
@@ -444,7 +458,16 @@ impl<'a> Html<'a> {
Field::IsHomepage => {
doc.add_u64(tantivy_field, self.is_homepage().then(|| 1).unwrap_or(0))
}
- Field::BacklinkText | Field::Centrality | Field::FetchTimeMs => {}
+ Field::LastUpdated => doc.add_u64(
+ tantivy_field,
+ self.updated_time()
+ .map(|time| time.timestamp().max(0) as u64)
+ .unwrap_or(0),
+ ),
+ Field::BacklinkText
+ | Field::Centrality
+ | Field::FetchTimeMs
+ | Field::PrimaryImageUuid => {}
}
}
@@ -566,6 +589,12 @@ impl<'a> Html<'a> {
pub fn primary_image(&self) -> Option<Url> {
self.og_image()
.or_else(|| self.schema_org_images().first().cloned())
+ .map(|mut url| {
+ if !url.is_full_path() {
+ url.prefix_with(&self.url);
+ }
+ url
+ })
}
pub fn description(&self) -> Option<String> {
@@ -1061,11 +1090,11 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "https://example.com");
assert_eq!(
html.primary_image(),
- Some("mexico-beach.jpg".to_string().into())
+ Some("https://example.com/mexico-beach.jpg".to_string().into())
);
}
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index a5fc480f..5b6b3b67 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::fmt::Display;
+use std::{fmt::Display, time::Duration};
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Url(String);
@@ -133,4 +133,23 @@ impl Url {
pub fn full(&self) -> &str {
&self.0
}
+
+ pub fn download_bytes(&self, timeout: Duration) -> Option<Vec<u8>> {
+ let client = reqwest::blocking::Client::builder()
+ .timeout(timeout)
+ .build()
+ .unwrap();
+
+ match client.get(self.full()).send() {
+ Ok(mut res) => {
+ let mut bytes = Vec::new();
+ if res.copy_to(&mut bytes).is_err() {
+ None
+ } else {
+ Some(bytes)
+ }
+ }
+ Err(_) => None,
+ }
+ }
}
diff --git a/templates/search.html b/templates/search.html
index d0a5b1e3..060b6f37 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -15,21 +15,30 @@
<div class="search-result flex flex-col space-y-3 pt-8 pb-8 pl-40">
{% for item in search_result %}
- <div class="search-result-item bg-white max-w-2xl p-3">
- <div class="text-sm mb-1 flex">
- <div class="flex items-center mr-2">
- <img src="/favicons/{{ item.domain }}" width="13px" />
+ <div class="flex max-w-2xl p-3">
+ <div class="flex flex-col">
+ <div class="text-sm mb-1 flex">
+ <div class="flex items-center mr-2">
+ <img src="/favicons/{{ item.domain }}" width="13px" />
+ </div>
+ <div>
+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>
+ </div>
</div>
- <div>
- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>
+ <div class="text-lg font-medium mb-1">
+ <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>
+ </div>
+ <div class="text-sm">
+ {{ item.snippet }}
</div>
</div>
- <div class="text-lg font-medium mb-1">
- <a class="title-link" href="{{ item.url }}">{{ item.title }}</a>
- </div>
- <div class="text-sm">
- {{ item.snippet }}
+ {% match item.primary_image_uuid %}
+ {% when Some with (val) %}
+ <div class="aspect-square h-full ml-3">
+ <img class="w-full h-full object-cover rounded-full" src="/image/{{ val }}" />
</div>
+ {% when None %}
+ {% endmatch %}
</div>
{% endfor %}
</div>
@@ -49,10 +58,6 @@
font-smooth: grayscale;
-moz-osx-font-smoothing: grayscale;
}
-
- .search-result-item {
- border-color: rgb(241 245 249);
- }
</style>
{% endblock %}
\ No newline at end of file