Skip to content

web search done right

show favicons in search result

Mikkel Denker70163f970c79
diff --git a/src/frontend/favicons.rs b/src/frontend/favicons.rs
new file mode 100644
index 00000000..9a4b9ce2
--- /dev/null
+++ b/src/frontend/favicons.rs
@@ -0,0 +1,45 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use axum::body::Body;
+use axum::extract::Path;
+use axum::http::Response;
+use axum::response::IntoResponse;
+use axum::Extension;
+use reqwest::StatusCode;
+
+use crate::webpage::Url;
+
+use super::State;
+
+pub async fn route(
+ Path(site): Path<String>,
+ Extension(state): Extension<Arc<State>>,
+) -> impl IntoResponse {
+ let img = state.searcher.favicon(&Url::from(site));
+
+ let bytes = match img {
+ Some(img) => img.as_raw_bytes(),
+ None => include_bytes!("../../static/images/globe.png").to_vec(),
+ };
+
+ Response::builder()
+ .status(StatusCode::OK)
+ .body(Body::from(bytes))
+ .unwrap()
+}
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index 8fbaecd8..d72e7a2e 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -29,6 +29,7 @@ use axum::{
use axum_extra::routing::SpaRouter;
mod autosuggest;
+mod favicons;
mod index;
pub mod search;
@@ -69,6 +70,7 @@ pub fn router(index_path: &str, queries_csv_path: &str) -> Result<Router> {
.route("/", get(index::route))
.route("/search", get(search::route))
.route("/autosuggest", get(autosuggest::route))
+ .route("/favicons/:site", get(favicons::route))
.merge(SpaRouter::new("/static", "static"))
.layer(Extension(state)))
}
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 0d133cc6..5d40da37 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -16,10 +16,7 @@
use axum::Extension;
-use crate::{
- index::RetrievedWebpage,
- webpage::{strip_protocol, strip_query, Url},
-};
+use crate::{index::RetrievedWebpage, webpage::Url};
use std::collections::HashMap;
use std::sync::Arc;
@@ -37,6 +34,7 @@ pub fn html_escape(s: &str) -> String {
pub struct DisplayedWebpage {
pub title: String,
pub url: String,
+ pub domain: String,
pub pretty_url: String,
pub snippet: String,
pub body: String,
@@ -47,14 +45,18 @@ const MAX_TITLE_LEN: usize = 50;
impl From<RetrievedWebpage> for DisplayedWebpage {
fn from(webpage: RetrievedWebpage) -> Self {
- let mut pretty_url = strip_query(&webpage.url).to_string();
+ let url: Url = webpage.url.clone().into();
+ let domain = url.domain().to_string();
+ let mut pretty_url = url.strip_query().to_string();
if pretty_url.ends_with('/') {
pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
}
let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
- pretty_url = strip_protocol(&pretty_url).replace('/', " › ");
+ pretty_url = Url::from(pretty_url.clone())
+ .strip_protocol()
+ .replace('/', " › ");
pretty_url = protocol + &pretty_url;
if pretty_url.len() > MAX_PRETTY_URL_LEN {
@@ -73,6 +75,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
title,
url: webpage.url,
pretty_url,
+ domain,
snippet: webpage.snippet, // snippet has already been html-escaped.
body: webpage.body,
}
diff --git a/src/image_store.rs b/src/image_store.rs
index c59d25c4..e2a2b011 100644
--- a/src/image_store.rs
+++ b/src/image_store.rs
@@ -15,28 +15,24 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::kv::{rocksdb_store::RocksDbStore, Kv};
+use crate::Result;
use image::imageops::FilterType;
use image::{DynamicImage, ImageOutputFormat};
use serde::{de, ser::SerializeStruct, Serialize};
use std::io::{Cursor, Read, Seek, SeekFrom};
use std::path::Path;
+const FAVICON_SIZE: u32 = 32;
+
#[derive(PartialEq, Debug, Clone)]
pub struct Image(DynamicImage);
impl Serialize for Image {
- fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
- let mut cursor = Cursor::new(Vec::new());
- self.0
- .write_to(&mut cursor, ImageOutputFormat::Png)
- .unwrap();
- cursor.seek(SeekFrom::Start(0)).unwrap();
-
- let mut bytes = Vec::new();
- cursor.read_to_end(&mut bytes).unwrap();
+ let bytes = self.as_raw_bytes();
let mut image = serializer.serialize_struct("image", 1)?;
image.serialize_field("0", &bytes)?;
@@ -45,7 +41,7 @@ impl Serialize for Image {
}
impl<'de> de::Deserialize<'de> for Image {
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
@@ -58,7 +54,7 @@ impl<'de> de::Deserialize<'de> for Image {
formatter.write_str("a serialized `Image` struct")
}
- fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+ fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'de>,
{
@@ -85,6 +81,7 @@ struct ImageStore {
}
impl ImageStore {
+ #[cfg(test)]
fn open<P: AsRef<Path>>(path: P) -> Self {
Self::open_with_filters(path, Vec::new())
}
@@ -110,9 +107,15 @@ impl ImageStore {
fn get(&self, key: &str) -> Option<Image> {
self.store.get(&key.to_string())
}
+
+ fn merge(&mut self, other: ImageStore) {
+ for (key, image) in other.store.iter() {
+ self.insert(&key, image);
+ }
+ }
}
-trait ImageFilter {
+trait ImageFilter: Send + Sync {
fn transform(&self, image: Image) -> Image;
}
@@ -138,8 +141,8 @@ impl FaviconStore {
let store = ImageStore::open_with_filters(
path,
vec![Box::new(ResizeFilter {
- width: 16,
- height: 16,
+ width: FAVICON_SIZE,
+ height: FAVICON_SIZE,
})],
);
@@ -157,6 +160,29 @@ impl FaviconStore {
pub fn get(&self, key: &str) -> Option<Image> {
self.0.get(key)
}
+
+ pub fn merge(&mut self, other: FaviconStore) {
+ self.0.merge(other.0)
+ }
+}
+
+impl Image {
+ pub(crate) fn from_bytes(bytes: Vec<u8>) -> Result<Image> {
+ Ok(Self(image::load_from_memory(&bytes)?))
+ }
+
+ pub(crate) fn as_raw_bytes(&self) -> Vec<u8> {
+ let mut cursor = Cursor::new(Vec::new());
+ self.0
+ .write_to(&mut cursor, ImageOutputFormat::Png)
+ .unwrap();
+ cursor.seek(SeekFrom::Start(0)).unwrap();
+
+ let mut bytes = Vec::new();
+ cursor.read_to_end(&mut bytes).unwrap();
+
+ bytes
+ }
}
#[cfg(test)]
@@ -214,11 +240,15 @@ mod tests {
#[test]
fn favicon_store() {
let image = Image(
- ImageBuffer::from_pixel(32, 32, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]))
- .into(),
+ ImageBuffer::from_pixel(
+ FAVICON_SIZE * 2,
+ FAVICON_SIZE * 2,
+ image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]),
+ )
+ .into(),
);
- assert_eq!(image.0.width(), 32);
- assert_eq!(image.0.height(), 32);
+ assert_eq!(image.0.width(), FAVICON_SIZE * 2);
+ assert_eq!(image.0.height(), FAVICON_SIZE * 2);
let mut store = FaviconStore::open(crate::gen_temp_path());
@@ -230,7 +260,7 @@ mod tests {
assert!(store.contains(key));
let retrieved_image = store.get(key).unwrap();
- assert_eq!(retrieved_image.0.width(), 16);
- assert_eq!(retrieved_image.0.height(), 16);
+ assert_eq!(retrieved_image.0.width(), FAVICON_SIZE);
+ assert_eq!(retrieved_image.0.height(), FAVICON_SIZE);
}
}
diff --git a/src/index.rs b/src/index.rs
index 3d47126a..e875ac43 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -20,23 +20,28 @@ use tantivy::schema::Schema;
use tantivy::{DocAddress, Document, IndexReader, IndexWriter, LeasedItem};
use crate::directory::{self, DirEntry};
+use crate::image_store::{FaviconStore, Image};
use crate::query::Query;
use crate::schema::{Field, ALL_FIELDS};
use crate::searcher::SearchResult;
use crate::snippet;
-use crate::webpage::Webpage;
+use crate::webpage::{Url, Webpage};
use crate::Result;
use crate::{schema::create_schema, tokenizer::Tokenizer};
use std::fs;
use std::path::Path;
use std::sync::Arc;
+const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";
+const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";
+
pub struct Index {
pub path: String,
tantivy_index: tantivy::Index,
writer: IndexWriter,
reader: IndexReader,
schema: Arc<Schema>,
+ favicon_store: FaviconStore,
}
impl Index {
@@ -49,13 +54,21 @@ impl Index {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let schema = create_schema();
- let tantivy_index = if path.as_ref().exists() {
- tantivy::Index::open_in_dir(path.as_ref())?
- } else {
+ if !path.as_ref().exists() {
fs::create_dir_all(path.as_ref())?;
- tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?
+ }
+
+ let tv_path = path.as_ref().join(INVERTED_INDEX_SUBFOLDER_NAME);
+
+ let tantivy_index = if tv_path.exists() {
+ tantivy::Index::open_in_dir(tv_path)?
+ } else {
+ fs::create_dir_all(&tv_path)?;
+ tantivy::Index::create_in_dir(tv_path, schema.clone())?
};
+ let favicon_store = FaviconStore::open(path.as_ref().join(FAVICON_STORE_SUBFOLDER_NAME));
+
let tokenizer = Tokenizer::default();
tantivy_index
.tokenizers()
@@ -76,11 +89,13 @@ impl Index {
reader: tantivy_index.reader()?,
schema: Arc::new(schema),
path: path.as_ref().to_str().unwrap().to_string(),
+ favicon_store,
tantivy_index,
})
}
pub fn insert(&mut self, webpage: Webpage) -> Result<()> {
+ self.maybe_insert_favicon(&webpage);
self.writer
.add_document(webpage.into_tantivy(&self.schema)?)?;
Ok(())
@@ -103,7 +118,7 @@ impl Index {
let mut webpages: Vec<RetrievedWebpage> = docs
.into_iter()
- .map(|(_score, doc_address)| Index::retrieve_doc(doc_address, &searcher))
+ .map(|(_score, doc_address)| self.retrieve_doc(doc_address, &searcher))
.filter(|page| page.is_ok())
.map(|page| page.unwrap())
.collect();
@@ -133,6 +148,7 @@ impl Index {
}
fn retrieve_doc(
+ &self,
doc_address: DocAddress,
searcher: &LeasedItem<tantivy::Searcher>,
) -> Result<RetrievedWebpage> {
@@ -155,11 +171,11 @@ impl Index {
.expect("failed to laod tantivy metadata for index");
let x = other.path.clone();
- let other_path = Path::new(x.as_str());
+ let other_path = Path::new(x.as_str()).join(INVERTED_INDEX_SUBFOLDER_NAME);
other.writer.wait_merging_threads().unwrap();
let path = self.path.clone();
- let self_path = Path::new(path.as_str());
+ let self_path = Path::new(path.as_str()).join(INVERTED_INDEX_SUBFOLDER_NAME);
self.writer.wait_merging_threads().unwrap();
for segment in other_meta.segments {
@@ -178,7 +194,7 @@ impl Index {
fs::remove_dir_all(other_path).unwrap();
- let self_path = Path::new(&path);
+ let self_path = Path::new(&path).join(INVERTED_INDEX_SUBFOLDER_NAME);
std::fs::write(
self_path.join("meta.json"),
@@ -186,12 +202,29 @@ impl Index {
)
.unwrap();
+ self.favicon_store.merge(other.favicon_store);
+ drop(self.favicon_store);
+
Self::open(path).expect("failed to open index")
}
pub fn schema(&self) -> Arc<Schema> {
Arc::clone(&self.schema)
}
+
+ fn maybe_insert_favicon(&mut self, webpage: &Webpage) {
+ if !webpage.html.is_homepage() || self.favicon_store.contains(webpage.html.domain()) {
+ return;
+ }
+
+ if let Some(favicon) = webpage.download_favicon() {
+ self.favicon_store.insert(webpage.html.domain(), favicon);
+ }
+ }
+
+ pub fn retrieve_favicon(&self, url: &Url) -> Option<Image> {
+ self.favicon_store.get(url.domain())
+ }
}
#[derive(Default, Debug)]
@@ -200,6 +233,7 @@ pub struct RetrievedWebpage {
pub url: String,
pub snippet: String,
pub body: String,
+ pub favicon: Option<Image>,
}
impl From<Document> for RetrievedWebpage {
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 1b3be914..71d4f542 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -23,6 +23,7 @@ pub trait Kv<K, V>
where
K: Serialize + DeserializeOwned,
V: Serialize + DeserializeOwned,
+ Self: Send + Sync,
{
fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>>;
fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>);
diff --git a/src/lib.rs b/src/lib.rs
index dc7ef3e2..7801c2c0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -168,6 +168,9 @@ pub enum Error {
#[error("Encountered an error in the FST crate")]
Fst(#[from] fst::Error),
+
+ #[error("Image error")]
+ Image(#[from] image::ImageError),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 8ca01145..493a77e6 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -74,6 +74,6 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
let is_homepage = self.is_homepage_reader.get(doc) as f64;
let fetch_time_ms = self.fetch_time_ms_reader.get(doc) as f64;
- (2.0 * score) + (400.0 * centrality) + (5.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))
+ (3.0 * score) + (1000.0 * centrality) + (1.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index a482f358..c2de6f8e 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -46,60 +46,58 @@ mod tests {
#[test]
fn harmonic_ranking() {
- for _ in 0..10 {
- let mut index = Index::temporary().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Website A</title>
- </head>
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ {CONTENT}
+ <a href="https://www.b.com">B site is great</a>
+ </html>
+ "#
+ ),
+ "https://www.a.com",
+ vec![],
+ 0.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ <body>
{CONTENT}
- <a href="https://www.b.com">B site is great</a>
- </html>
- "#
- ),
- "https://www.a.com",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Website B</title>
- </head>
- <body>
- {CONTENT}
- </body>
- </html>
- "#
- ),
- "https://www.b.com",
- vec![Link {
- source: "https://www.a.com".to_string().into(),
- destination: "https://www.b.com".to_string().into(),
- text: "B site is great".to_string(),
- }],
- 5.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.b.com",
+ vec![Link {
+ source: "https://www.a.com".to_string().into(),
+ destination: "https://www.b.com".to_string().into(),
+ text: "B site is great".to_string(),
+ }],
+ 5.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
- index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
- let result = searcher.search("great site").expect("Search failed");
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.b.com");
- assert_eq!(result.documents[1].url, "https://www.a.com");
- }
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::from(index);
+ let result = searcher.search("great site").expect("Search failed");
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.b.com");
+ assert_eq!(result.documents[1].url, "https://www.a.com");
}
#[test]
@@ -171,7 +169,7 @@ mod tests {
assert_eq!(result.documents.len(), 3);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
- assert_eq!(result.documents[1].url, "https://www.b.com");
- assert_eq!(result.documents[2].url, "https://www.dr.dk/whatever");
+ assert_eq!(result.documents[1].url, "https://www.dr.dk/whatever");
+ assert_eq!(result.documents[2].url, "https://www.b.com");
}
}
diff --git a/src/searcher.rs b/src/searcher.rs
index 2d53ebae..33eac1cb 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -14,9 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::image_store::Image;
use crate::index::{Index, RetrievedWebpage};
use crate::query::Query;
use crate::ranking::Ranker;
+use crate::webpage::Url;
use crate::Result;
#[derive(Debug)]
@@ -41,4 +43,8 @@ impl Searcher {
let ranker = Ranker::new(query.clone());
self.index.search(&query, ranker.collector())
}
+
+ pub fn favicon(&self, site: &Url) -> Option<Image> {
+ self.index.retrieve_favicon(site)
+ }
}
diff --git a/src/webpage/lexer.rs b/src/webpage/lexer.rs
index 99dea3d2..867a8c38 100644
--- a/src/webpage/lexer.rs
+++ b/src/webpage/lexer.rs
@@ -30,14 +30,14 @@ impl<'a> Tag<'a> {
for tok in self.raw.split_whitespace().skip(1) {
if tok.contains('=') {
if let Some((key, value)) = tok.split_once('=') {
- attributes.insert(key.trim(), trim_html_stuff(value));
+ attributes.insert(key.trim(), trim_html_stuff(value, false));
}
} else {
if tok == ">" || tok.starts_with(self.name.as_ref()) || tok == "/>" {
continue;
}
- attributes.insert(trim_html_stuff(tok), "");
+ attributes.insert(trim_html_stuff(tok, true), "");
}
}
@@ -49,7 +49,7 @@ impl<'a> Tag<'a> {
}
}
-fn trim_html_stuff(s: &str) -> &str {
+fn trim_html_stuff(s: &str, break_on_dash: bool) -> &str {
let mut start_val = 0;
let mut end_val = s.len();
@@ -62,7 +62,11 @@ fn trim_html_stuff(s: &str) -> &str {
for (idx, c) in s[start_val..].char_indices() {
end_val = start_val + idx;
- if c == ' ' || c == '"' || c == '>' || c == '/' {
+ if break_on_dash && c == '/' {
+ break;
+ }
+
+ if c == ' ' || c == '"' || c == '>' {
break;
}
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 350eccf5..8f5c2b73 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -13,9 +13,9 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{Error, Result};
+use crate::{image_store::Image, Error, Result};
use logos::{Lexer, Logos};
-use std::{collections::HashMap, fmt::Display};
+use std::{collections::HashMap, fmt::Display, time::Duration};
mod just_text;
mod lexer;
@@ -24,74 +24,6 @@ use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};
use self::{just_text::JustText, lexer::Token};
-pub fn find_protocol(url: &str) -> &'_ str {
- let mut start_host = 0;
- if url.starts_with("http://") || url.starts_with("https://") {
- start_host = url
- .find(':')
- .expect("It was checked that url starts with protocol");
- }
-
- &url[..start_host]
-}
-
-pub fn strip_protocol(url: &str) -> &'_ str {
- let mut start_host = 0;
- if url.starts_with("http://") || url.starts_with("https://") {
- start_host = url
- .find('/')
- .expect("It was checked that url starts with protocol");
- start_host += 2; // skip the two '/'
- }
-
- &url[start_host..]
-}
-
-pub fn strip_query(url: &str) -> &'_ str {
- let mut start_query = url.len();
- if url.contains('?') {
- start_query = url.find('?').expect("The url contains atleast 1 '?'");
- }
-
- &url[..start_query]
-}
-
-pub fn host(url: &str) -> &'_ str {
- let url = strip_protocol(url);
-
- let mut end_host = url.len();
- if url.contains('/') {
- end_host = url.find('/').expect("The url contains atleast 1 '/'");
- }
-
- &url[..end_host]
-}
-
-pub fn is_homepage(url: &str) -> bool {
- let url = strip_protocol(url);
- match url.find('/') {
- Some(idx) => idx == url.len() - 1,
- None => true,
- }
-}
-
-pub fn domain(url: &str) -> &'_ str {
- let host = host(url);
- let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();
- if num_punctuations > 1 {
- let domain_index = host.rfind('.').unwrap();
- let mut start_index = host[..domain_index].rfind('.').unwrap();
-
- if &host[start_index + 1..] == "co.uk" {
- start_index = host[start_index..].rfind('.').unwrap();
- }
-
- &host[start_index + 1..]
- } else {
- host
- }
-}
-
#[derive(PartialEq, Eq, Debug)]
pub struct FaviconLink {
link: Url,
@@ -205,6 +137,27 @@ impl<'a> Webpage<'a> {
Ok(doc)
}
+
+ pub(crate) fn download_favicon(&self) -> Option<Image> {
+ self.html.favicon().and_then(|favicon_link| {
+ let client = reqwest::blocking::Client::builder()
+ .timeout(Duration::from_secs(1))
+ .build()
+ .unwrap();
+
+ match client.get(&favicon_link.link.0).send() {
+ Ok(mut res) => {
+ let mut bytes = Vec::new();
+ if res.copy_to(&mut bytes).is_err() {
+ None
+ } else {
+ Image::from_bytes(bytes).ok()
+ }
+ }
+ Err(_) => None,
+ }
+ })
+ }
}
#[derive(Debug)]
@@ -311,9 +264,14 @@ impl<'a> Html<'a> {
};
let image_type = tag.attributes().get("type").map(|t| t.to_string());
+ let mut link: Url = link.to_string().into();
+
+ if !link.is_full_path() {
+ link.prefix_with(&self.url);
+ }
let favicon = FaviconLink {
- link: link.to_string().into(),
+ link,
image_type,
width,
height,
@@ -507,20 +465,102 @@ impl From<String> for Url {
}
impl Url {
+ pub fn strip_protocol(&self) -> &str {
+ let mut start_host = 0;
+ let url = &self.0;
+ if url.starts_with("http://") || url.starts_with("https://") {
+ start_host = url
+ .find('/')
+ .expect("It was checked that url starts with protocol");
+ start_host += 2; // skip the two '/'
+ }
+
+ &url[start_host..]
+ }
+
+ pub fn strip_query(&self) -> &str {
+ let url = &self.0;
+ let mut start_query = url.len();
+ if url.contains('?') {
+ start_query = url.find('?').expect("The url contains atleast 1 '?'");
+ }
+
+ &url[..start_query]
+ }
+
pub fn host(&self) -> &str {
- host(&self.0)
+ let url = self.strip_protocol();
+
+ let mut end_host = url.len();
+ if url.contains('/') {
+ end_host = url.find('/').expect("The url contains atleast 1 '/'");
+ }
+
+ &url[..end_host]
}
pub fn domain(&self) -> &str {
- domain(&self.0)
+ let host = self.host();
+ let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();
+ if num_punctuations > 1 {
+ let domain_index = host.rfind('.').unwrap();
+ let mut start_index = host[..domain_index].rfind('.').unwrap();
+
+ if &host[start_index + 1..] == "co.uk" {
+ start_index = host[start_index..].rfind('.').unwrap();
+ }
+
+ &host[start_index + 1..]
+ } else {
+ host
+ }
}
pub fn is_homepage(&self) -> bool {
- is_homepage(&self.0)
+ let url = self.strip_protocol();
+ match url.find('/') {
+ Some(idx) => idx == url.len() - 1,
+ None => true,
+ }
}
+ fn find_protocol_end(&self) -> usize {
+ let mut start_host = 0;
+ let url = &self.0;
+ if url.starts_with("http://") || url.starts_with("https://") {
+ start_host = url
+ .find(':')
+ .expect("It was checked that url starts with protocol");
+ }
+ start_host
+ }
pub fn protocol(&self) -> &str {
- find_protocol(&self.0)
+ &self.0[..self.find_protocol_end()]
+ }
+
+ pub fn site(&self) -> &str {
+ let start = self.find_protocol_end() + 3;
+ let url = &self.0[start..];
+
+ let mut end_host = url.len();
+ if url.contains('/') {
+ end_host = url.find('/').expect("The url contains atleast 1 '/'");
+ }
+
+ &self.0[..end_host + start]
+ }
+
+ fn is_full_path(&self) -> bool {
+ matches!(self.protocol(), "http" | "https" | "pdf")
+ }
+
+ fn prefix_with(&mut self, url: &Url) {
+ self.0 = match (url.0.ends_with('/'), self.0.starts_with('/')) {
+ (true, true) => url.site().to_string() + &self.0,
+ (true, false) => url.0.clone() + &self.0,
+ (false, true) => url.site().to_string() + &self.0,
+ (false, false) => url.0.clone() + "/" + &self.0,
+ };
}
}
@@ -723,16 +763,22 @@ mod tests {
#[test]
fn test_find_protocol() {
- assert_eq!(find_protocol("https://example.com"), "https");
- assert_eq!(find_protocol("http://example.com"), "http");
+ assert_eq!(
+ Url::from("https://example.com".to_string()).protocol(),
+ "https"
+ );
+ assert_eq!(
+ Url::from("http://example.com".to_string()).protocol(),
+ "http"
+ );
}
#[test]
- fn favicon() {
+ fn simple_favicon() {
let raw = r#"
<html>
<head>
- <link rel="icon" sizes="192x192" href="favicon_link.png" />
+ <link rel="icon" sizes="192x192" href="https://example.com/favicon.png" />
</head>
</html>
"#
@@ -742,11 +788,74 @@ mod tests {
assert_eq!(
webpage.favicon(),
Some(FaviconLink {
- link: "favicon_link.png".to_string().into(),
+ link: "https://example.com/favicon.png".to_string().into(),
width: Some(192),
height: Some(192),
image_type: None
})
)
}
+
+ fn full_link_favicon(href: &str, site_url: &str, expected: &str) {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <link rel="icon" sizes="192x192" href="{href}" />
+ </head>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, site_url);
+ assert_eq!(
+ webpage.favicon(),
+ Some(FaviconLink {
+ link: expected.to_string().into(),
+ width: Some(192),
+ height: Some(192),
+ image_type: None
+ })
+ );
+ }
+
+ #[test]
+ fn test_full_link_favicon_simple() {
+ full_link_favicon(
+ "/favicon.png",
+ "https://www.example.com/",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "/favicon.png",
+ "https://www.example.com",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "favicon.png",
+ "https://www.example.com",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "favicon.png",
+ "https://www.example.com/",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "favicon.png",
+ "https://www.example.com/test",
+ "https://www.example.com/test/favicon.png",
+ );
+ full_link_favicon(
+ "/favicon.png",
+ "https://www.example.com/test",
+ "https://www.example.com/favicon.png",
+ );
+ }
+
+ #[test]
+ fn domain_from_domain_url() {
+ let url: Url = "example.com".to_string().into();
+ assert_eq!(url.domain(), "example.com");
+ }
}
diff --git a/static/images/globe.png b/static/images/globe.png
new file mode 100644
index 00000000..3b81ae5c
Binary files /dev/null and b/static/images/globe.png differ
diff --git a/templates/search.html b/templates/search.html
index e9185e03..d0a5b1e3 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -16,8 +16,13 @@
<div class="search-result flex flex-col space-y-3 pt-8 pb-8 pl-40">
{% for item in search_result %}
<div class="search-result-item bg-white max-w-2xl p-3">
- <div class="text-sm mb-1">
- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>
+ <div class="text-sm mb-1 flex">
+ <div class="flex items-center mr-2">
+ <img src="/favicons/{{ item.domain }}" width="13px" />
+ </div>
+ <div>
+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>
+ </div>
</div>
<div class="text-lg font-medium mb-1">
<a class="title-link" href="{{ item.url }}">{{ item.title }}</a>