web search done right
show favicons in search result
Mikkel Denker70163f970c79
diff --git a/src/frontend/favicons.rs b/src/frontend/favicons.rsnew file mode 100644index 00000000..9a4b9ce2--- /dev/null+++ b/src/frontend/favicons.rs@@ -0,0 +1,45 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use axum::body::Body;+use axum::extract::Path;+use axum::http::Response;+use axum::response::IntoResponse;+use axum::Extension;+use reqwest::StatusCode;++use crate::webpage::Url;++use super::State;++pub async fn route(+ Path(site): Path<String>,+ Extension(state): Extension<Arc<State>>,+) -> impl IntoResponse {+ let img = state.searcher.favicon(&Url::from(site));++ let bytes = match img {+ Some(img) => img.as_raw_bytes(),+ None => include_bytes!("../../static/images/globe.png").to_vec(),+ };++ Response::builder()+ .status(StatusCode::OK)+ .body(Body::from(bytes))+ .unwrap()+}diff --git a/src/frontend/mod.rs b/src/frontend/mod.rsindex 8fbaecd8..d72e7a2e 100644--- a/src/frontend/mod.rs+++ b/src/frontend/mod.rs@@ -29,6 +29,7 @@ use axum::{use axum_extra::routing::SpaRouter;mod autosuggest;+mod favicons;mod index;pub mod search;@@ -69,6 +70,7 @@ pub fn router(index_path: &str, queries_csv_path: &str) -> Result<Router> {.route("/", get(index::route)).route("/search", get(search::route)).route("/autosuggest", get(autosuggest::route))+ .route("/favicons/:site", get(favicons::route)).merge(SpaRouter::new("/static", "static")).layer(Extension(state)))}diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 0d133cc6..5d40da37 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -16,10 +16,7 @@use axum::Extension;-use crate::{- index::RetrievedWebpage,- webpage::{strip_protocol, strip_query, Url},-};+use crate::{index::RetrievedWebpage, webpage::Url};use std::collections::HashMap;use std::sync::Arc;@@ -37,6 +34,7 @@ pub fn html_escape(s: &str) -> String {pub struct DisplayedWebpage {pub title: String,pub url: String,+ pub domain: String,pub pretty_url: String,pub snippet: String,pub body: String,@@ -47,14 +45,18 @@ const MAX_TITLE_LEN: usize = 50;impl From<RetrievedWebpage> for DisplayedWebpage {fn from(webpage: RetrievedWebpage) -> Self {- let mut pretty_url = strip_query(&webpage.url).to_string();+ let url: Url = webpage.url.clone().into();+ let domain = url.domain().to_string();+ let mut pretty_url = url.strip_query().to_string();if pretty_url.ends_with('/') {pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();}let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";- pretty_url = strip_protocol(&pretty_url).replace('/', " › ");+ pretty_url = Url::from(pretty_url.clone())+ .strip_protocol()+ .replace('/', " › ");pretty_url = protocol + &pretty_url;if pretty_url.len() > MAX_PRETTY_URL_LEN {@@ -73,6 +75,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {title,url: webpage.url,pretty_url,+ domain,snippet: webpage.snippet, // snippet has already been html-escaped.body: webpage.body,}diff --git a/src/image_store.rs b/src/image_store.rsindex c59d25c4..e2a2b011 100644--- a/src/image_store.rs+++ b/src/image_store.rs@@ -15,28 +15,24 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::kv::{rocksdb_store::RocksDbStore, Kv};+use crate::Result;use image::imageops::FilterType;use image::{DynamicImage, ImageOutputFormat};use serde::{de, ser::SerializeStruct, Serialize};use std::io::{Cursor, Read, Seek, SeekFrom};use std::path::Path;+const FAVICON_SIZE: u32 = 32;+#[derive(PartialEq, Debug, Clone)]pub struct Image(DynamicImage);impl Serialize for Image {- fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>whereS: serde::Serializer,{- let mut cursor = Cursor::new(Vec::new());- self.0- .write_to(&mut cursor, ImageOutputFormat::Png)- .unwrap();- cursor.seek(SeekFrom::Start(0)).unwrap();-- let mut bytes = Vec::new();- cursor.read_to_end(&mut bytes).unwrap();+ let bytes = self.as_raw_bytes();let mut image = serializer.serialize_struct("image", 1)?;image.serialize_field("0", &bytes)?;@@ -45,7 +41,7 @@ impl Serialize for Image {}impl<'de> de::Deserialize<'de> for Image {- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>whereD: serde::Deserializer<'de>,{@@ -58,7 +54,7 @@ impl<'de> de::Deserialize<'de> for Image {formatter.write_str("a serialized `Image` struct")}- fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>+ fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>whereA: serde::de::SeqAccess<'de>,{@@ -85,6 +81,7 @@ struct ImageStore {}impl ImageStore {+ #[cfg(test)]fn open<P: AsRef<Path>>(path: P) -> Self {Self::open_with_filters(path, Vec::new())}@@ -110,9 +107,15 @@ impl ImageStore {fn get(&self, key: &str) -> Option<Image> {self.store.get(&key.to_string())}++ fn merge(&mut self, other: ImageStore) {+ for (key, image) in other.store.iter() {+ self.insert(&key, image);+ }+ }}-trait ImageFilter {+trait ImageFilter: Send + Sync {fn transform(&self, image: Image) -> Image;}@@ -138,8 +141,8 @@ impl FaviconStore {let store = ImageStore::open_with_filters(path,vec![Box::new(ResizeFilter {- width: 16,- height: 16,+ width: FAVICON_SIZE,+ height: FAVICON_SIZE,})],);@@ -157,6 +160,29 @@ impl FaviconStore {pub fn get(&self, key: &str) -> Option<Image> {self.0.get(key)}++ pub fn merge(&mut self, other: FaviconStore) {+ self.0.merge(other.0)+ }+}++impl Image {+ pub(crate) fn from_bytes(bytes: Vec<u8>) -> Result<Image> {+ Ok(Self(image::load_from_memory(&bytes)?))+ }++ pub(crate) fn as_raw_bytes(&self) -> Vec<u8> {+ let mut cursor = Cursor::new(Vec::new());+ self.0+ .write_to(&mut cursor, ImageOutputFormat::Png)+ .unwrap();+ cursor.seek(SeekFrom::Start(0)).unwrap();++ let mut bytes = Vec::new();+ cursor.read_to_end(&mut bytes).unwrap();++ bytes+ }}#[cfg(test)]@@ -214,11 +240,15 @@ mod tests {#[test]fn favicon_store() {let image = Image(- ImageBuffer::from_pixel(32, 32, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]))- .into(),+ ImageBuffer::from_pixel(+ FAVICON_SIZE * 2,+ FAVICON_SIZE * 2,+ image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]),+ )+ .into(),);- assert_eq!(image.0.width(), 32);- assert_eq!(image.0.height(), 32);+ assert_eq!(image.0.width(), FAVICON_SIZE * 2);+ assert_eq!(image.0.height(), FAVICON_SIZE * 2);let mut store = FaviconStore::open(crate::gen_temp_path());@@ -230,7 +260,7 @@ mod tests {assert!(store.contains(key));let retrieved_image = store.get(key).unwrap();- assert_eq!(retrieved_image.0.width(), 16);- assert_eq!(retrieved_image.0.height(), 16);+ assert_eq!(retrieved_image.0.width(), FAVICON_SIZE);+ assert_eq!(retrieved_image.0.height(), FAVICON_SIZE);}}diff --git a/src/index.rs b/src/index.rsindex 3d47126a..e875ac43 100644--- a/src/index.rs+++ b/src/index.rs@@ -20,23 +20,28 @@ use tantivy::schema::Schema;use tantivy::{DocAddress, Document, IndexReader, IndexWriter, LeasedItem};use crate::directory::{self, DirEntry};+use crate::image_store::{FaviconStore, Image};use crate::query::Query;use crate::schema::{Field, ALL_FIELDS};use crate::searcher::SearchResult;use crate::snippet;-use crate::webpage::Webpage;+use crate::webpage::{Url, Webpage};use crate::Result;use crate::{schema::create_schema, tokenizer::Tokenizer};use std::fs;use std::path::Path;use std::sync::Arc;+const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";+const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";+pub struct Index {pub path: String,tantivy_index: tantivy::Index,writer: IndexWriter,reader: IndexReader,schema: Arc<Schema>,+ favicon_store: FaviconStore,}impl Index {@@ -49,13 +54,21 @@ impl Index {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let schema = create_schema();- let tantivy_index = if path.as_ref().exists() {- tantivy::Index::open_in_dir(path.as_ref())?- } else {+ if !path.as_ref().exists() {fs::create_dir_all(path.as_ref())?;- tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?+ }++ let tv_path = path.as_ref().join(INVERTED_INDEX_SUBFOLDER_NAME);++ let tantivy_index = if tv_path.exists() {+ tantivy::Index::open_in_dir(tv_path)?+ } else {+ fs::create_dir_all(&tv_path)?;+ tantivy::Index::create_in_dir(tv_path, schema.clone())?};+ let favicon_store = FaviconStore::open(path.as_ref().join(FAVICON_STORE_SUBFOLDER_NAME));+let tokenizer = Tokenizer::default();tantivy_index.tokenizers()@@ -76,11 +89,13 @@ impl Index {reader: tantivy_index.reader()?,schema: Arc::new(schema),path: path.as_ref().to_str().unwrap().to_string(),+ favicon_store,tantivy_index,})}pub fn insert(&mut self, webpage: Webpage) -> Result<()> {+ self.maybe_insert_favicon(&webpage);self.writer.add_document(webpage.into_tantivy(&self.schema)?)?;Ok(())@@ -103,7 +118,7 @@ impl Index {let mut webpages: Vec<RetrievedWebpage> = docs.into_iter()- .map(|(_score, doc_address)| Index::retrieve_doc(doc_address, &searcher))+ .map(|(_score, doc_address)| self.retrieve_doc(doc_address, &searcher)).filter(|page| page.is_ok()).map(|page| page.unwrap()).collect();@@ -133,6 +148,7 @@ impl Index {}fn retrieve_doc(+ &self,doc_address: DocAddress,searcher: &LeasedItem<tantivy::Searcher>,) -> Result<RetrievedWebpage> {@@ -155,11 +171,11 @@ impl Index {.expect("failed to laod tantivy metadata for index");let x = other.path.clone();- let other_path = Path::new(x.as_str());+ let other_path = Path::new(x.as_str()).join(INVERTED_INDEX_SUBFOLDER_NAME);other.writer.wait_merging_threads().unwrap();let path = self.path.clone();- let self_path = Path::new(path.as_str());+ let self_path = Path::new(path.as_str()).join(INVERTED_INDEX_SUBFOLDER_NAME);self.writer.wait_merging_threads().unwrap();for segment in other_meta.segments {@@ -178,7 +194,7 @@ impl Index {fs::remove_dir_all(other_path).unwrap();- let self_path = Path::new(&path);+ let self_path = Path::new(&path).join(INVERTED_INDEX_SUBFOLDER_NAME);std::fs::write(self_path.join("meta.json"),@@ -186,12 +202,29 @@ impl Index {).unwrap();+ self.favicon_store.merge(other.favicon_store);+ drop(self.favicon_store);+Self::open(path).expect("failed to open index")}pub fn schema(&self) -> Arc<Schema> {Arc::clone(&self.schema)}++ fn maybe_insert_favicon(&mut self, webpage: &Webpage) {+ if !webpage.html.is_homepage() || self.favicon_store.contains(webpage.html.domain()) {+ return;+ }++ if let Some(favicon) = webpage.download_favicon() {+ self.favicon_store.insert(webpage.html.domain(), favicon);+ }+ }++ pub fn retrieve_favicon(&self, url: &Url) -> Option<Image> {+ self.favicon_store.get(url.domain())+ }}#[derive(Default, Debug)]@@ -200,6 +233,7 @@ pub struct RetrievedWebpage {pub url: String,pub snippet: String,pub body: String,+ pub favicon: Option<Image>,}impl From<Document> for RetrievedWebpage {diff --git a/src/kv/mod.rs b/src/kv/mod.rsindex 1b3be914..71d4f542 100644--- a/src/kv/mod.rs+++ b/src/kv/mod.rs@@ -23,6 +23,7 @@ pub trait Kv<K, V>whereK: Serialize + DeserializeOwned,V: Serialize + DeserializeOwned,+ Self: Send + Sync,{fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>>;fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>);diff --git a/src/lib.rs b/src/lib.rsindex dc7ef3e2..7801c2c0 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -168,6 +168,9 @@ pub enum Error {#[error("Encountered an error in the FST crate")]Fst(#[from] fst::Error),++ #[error("Image error")]+ Image(#[from] image::ImageError),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 8ca01145..493a77e6 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -74,6 +74,6 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {let is_homepage = self.is_homepage_reader.get(doc) as f64;let fetch_time_ms = self.fetch_time_ms_reader.get(doc) as f64;- (2.0 * score) + (400.0 * centrality) + (5.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))+ (3.0 * score) + (1000.0 * centrality) + (1.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex a482f358..c2de6f8e 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -46,60 +46,58 @@ mod tests {#[test]fn harmonic_ranking() {- for _ in 0..10 {- let mut index = Index::temporary().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Website A</title>- </head>+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ {CONTENT}+ <a href="https://www.b.com">B site is great</a>+ </html>+ "#+ ),+ "https://www.a.com",+ vec![],+ 0.0,+ 500,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ <body>{CONTENT}- <a href="https://www.b.com">B site is great</a>- </html>- "#- ),- "https://www.a.com",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Website B</title>- </head>- <body>- {CONTENT}- </body>- </html>- "#- ),- "https://www.b.com",- vec![Link {- source: "https://www.a.com".to_string().into(),- destination: "https://www.b.com".to_string().into(),- text: "B site is great".to_string(),- }],- 5.0,- 500,- ))- .expect("failed to parse webpage");+ </body>+ </html>+ "#+ ),+ "https://www.b.com",+ vec![Link {+ source: "https://www.a.com".to_string().into(),+ destination: "https://www.b.com".to_string().into(),+ text: "B site is great".to_string(),+ }],+ 5.0,+ 500,+ ))+ .expect("failed to parse webpage");- index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);- let result = searcher.search("great site").expect("Search failed");- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.b.com");- assert_eq!(result.documents[1].url, "https://www.a.com");- }+ index.commit().expect("failed to commit index");+ let searcher = Searcher::from(index);+ let result = searcher.search("great site").expect("Search failed");+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.b.com");+ assert_eq!(result.documents[1].url, "https://www.a.com");}#[test]@@ -171,7 +169,7 @@ mod tests {assert_eq!(result.documents.len(), 3);assert_eq!(result.documents[0].url, "https://www.dr.dk");- assert_eq!(result.documents[1].url, "https://www.b.com");- assert_eq!(result.documents[2].url, "https://www.dr.dk/whatever");+ assert_eq!(result.documents[1].url, "https://www.dr.dk/whatever");+ assert_eq!(result.documents[2].url, "https://www.b.com");}}diff --git a/src/searcher.rs b/src/searcher.rsindex 2d53ebae..33eac1cb 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -14,9 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::image_store::Image;use crate::index::{Index, RetrievedWebpage};use crate::query::Query;use crate::ranking::Ranker;+use crate::webpage::Url;use crate::Result;#[derive(Debug)]@@ -41,4 +43,8 @@ impl Searcher {let ranker = Ranker::new(query.clone());self.index.search(&query, ranker.collector())}++ pub fn favicon(&self, site: &Url) -> Option<Image> {+ self.index.retrieve_favicon(site)+ }}diff --git a/src/webpage/lexer.rs b/src/webpage/lexer.rsindex 99dea3d2..867a8c38 100644--- a/src/webpage/lexer.rs+++ b/src/webpage/lexer.rs@@ -30,14 +30,14 @@ impl<'a> Tag<'a> {for tok in self.raw.split_whitespace().skip(1) {if tok.contains('=') {if let Some((key, value)) = tok.split_once('=') {- attributes.insert(key.trim(), trim_html_stuff(value));+ attributes.insert(key.trim(), trim_html_stuff(value, false));}} else {if tok == ">" || tok.starts_with(self.name.as_ref()) || tok == "/>" {continue;}- attributes.insert(trim_html_stuff(tok), "");+ attributes.insert(trim_html_stuff(tok, true), "");}}@@ -49,7 +49,7 @@ impl<'a> Tag<'a> {}}-fn trim_html_stuff(s: &str) -> &str {+fn trim_html_stuff(s: &str, break_on_dash: bool) -> &str {let mut start_val = 0;let mut end_val = s.len();@@ -62,7 +62,11 @@ fn trim_html_stuff(s: &str) -> &str {for (idx, c) in s[start_val..].char_indices() {end_val = start_val + idx;- if c == ' ' || c == '"' || c == '>' || c == '/' {+ if break_on_dash && c == '/' {+ break;+ }++ if c == ' ' || c == '"' || c == '>' {break;}}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 350eccf5..8f5c2b73 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -13,9 +13,9 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{Error, Result};+use crate::{image_store::Image, Error, Result};use logos::{Lexer, Logos};-use std::{collections::HashMap, fmt::Display};+use std::{collections::HashMap, fmt::Display, time::Duration};mod just_text;mod lexer;@@ -24,74 +24,6 @@ use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};use self::{just_text::JustText, lexer::Token};-pub fn find_protocol(url: &str) -> &'_ str {- let mut start_host = 0;- if url.starts_with("http://") || url.starts_with("https://") {- start_host = url- .find(':')- .expect("It was checked that url starts with protocol");- }-- &url[..start_host]-}--pub fn strip_protocol(url: &str) -> &'_ str {- let mut start_host = 0;- if url.starts_with("http://") || url.starts_with("https://") {- start_host = url- .find('/')- .expect("It was checked that url starts with protocol");- start_host += 2; // skip the two '/'- }-- &url[start_host..]-}--pub fn strip_query(url: &str) -> &'_ str {- let mut start_query = url.len();- if url.contains('?') {- start_query = url.find('?').expect("The url contains atleast 1 '?'");- }-- &url[..start_query]-}--pub fn host(url: &str) -> &'_ str {- let url = strip_protocol(url);-- let mut end_host = url.len();- if url.contains('/') {- end_host = url.find('/').expect("The url contains atleast 1 '/'");- }-- &url[..end_host]-}--pub fn is_homepage(url: &str) -> bool {- let url = strip_protocol(url);- match url.find('/') {- Some(idx) => idx == url.len() - 1,- None => true,- }-}--pub fn domain(url: &str) -> &'_ str {- let host = host(url);- let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();- if num_punctuations > 1 {- let domain_index = host.rfind('.').unwrap();- let mut start_index = host[..domain_index].rfind('.').unwrap();-- if &host[start_index + 1..] == "co.uk" {- start_index = host[start_index..].rfind('.').unwrap();- }-- &host[start_index + 1..]- } else {- host- }-}-#[derive(PartialEq, Eq, Debug)]pub struct FaviconLink {link: Url,@@ -205,6 +137,27 @@ impl<'a> Webpage<'a> {Ok(doc)}++ pub(crate) fn download_favicon(&self) -> Option<Image> {+ self.html.favicon().and_then(|favicon_link| {+ let client = reqwest::blocking::Client::builder()+ .timeout(Duration::from_secs(1))+ .build()+ .unwrap();++ match client.get(&favicon_link.link.0).send() {+ Ok(mut res) => {+ let mut bytes = Vec::new();+ if res.copy_to(&mut bytes).is_err() {+ None+ } else {+ Image::from_bytes(bytes).ok()+ }+ }+ Err(_) => None,+ }+ })+ }}#[derive(Debug)]@@ -311,9 +264,14 @@ impl<'a> Html<'a> {};let image_type = tag.attributes().get("type").map(|t| t.to_string());+ let mut link: Url = link.to_string().into();++ if !link.is_full_path() {+ link.prefix_with(&self.url);+ }let favicon = FaviconLink {- link: link.to_string().into(),+ link,image_type,width,height,@@ -507,20 +465,102 @@ impl From<String> for Url {}impl Url {+ pub fn strip_protocol(&self) -> &str {+ let mut start_host = 0;+ let url = &self.0;+ if url.starts_with("http://") || url.starts_with("https://") {+ start_host = url+ .find('/')+ .expect("It was checked that url starts with protocol");+ start_host += 2; // skip the two '/'+ }++ &url[start_host..]+ }++ pub fn strip_query(&self) -> &str {+ let url = &self.0;+ let mut start_query = url.len();+ if url.contains('?') {+ start_query = url.find('?').expect("The url contains atleast 1 '?'");+ }++ &url[..start_query]+ }+pub fn host(&self) -> &str {- host(&self.0)+ let url = self.strip_protocol();++ let mut end_host = url.len();+ if url.contains('/') {+ end_host = url.find('/').expect("The url contains atleast 1 '/'");+ }++ &url[..end_host]}pub fn domain(&self) -> &str {- domain(&self.0)+ let host = self.host();+ let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();+ if num_punctuations > 1 {+ let domain_index = host.rfind('.').unwrap();+ let mut start_index = host[..domain_index].rfind('.').unwrap();++ if &host[start_index + 1..] == "co.uk" {+ start_index = host[start_index..].rfind('.').unwrap();+ }++ &host[start_index + 1..]+ } else {+ host+ }}pub fn is_homepage(&self) -> bool {- is_homepage(&self.0)+ let url = self.strip_protocol();+ match url.find('/') {+ Some(idx) => idx == url.len() - 1,+ None => true,+ }}+ fn find_protocol_end(&self) -> usize {+ let mut start_host = 0;+ let url = &self.0;+ if url.starts_with("http://") || url.starts_with("https://") {+ start_host = url+ .find(':')+ .expect("It was checked that url starts with protocol");+ }+ start_host+ }pub fn protocol(&self) -> &str {- find_protocol(&self.0)+ &self.0[..self.find_protocol_end()]+ }++ pub fn site(&self) -> &str {+ let start = self.find_protocol_end() + 3;+ let url = &self.0[start..];++ let mut end_host = url.len();+ if url.contains('/') {+ end_host = url.find('/').expect("The url contains atleast 1 '/'");+ }++ &self.0[..end_host + start]+ }++ fn is_full_path(&self) -> bool {+ matches!(self.protocol(), "http" | "https" | "pdf")+ }++ fn prefix_with(&mut self, url: &Url) {+ self.0 = match (url.0.ends_with('/'), self.0.starts_with('/')) {+ (true, true) => url.site().to_string() + &self.0,+ (true, false) => url.0.clone() + &self.0,+ (false, true) => url.site().to_string() + &self.0,+ (false, false) => url.0.clone() + "/" + &self.0,+ };}}@@ -723,16 +763,22 @@ mod tests {#[test]fn test_find_protocol() {- assert_eq!(find_protocol("https://example.com"), "https");- assert_eq!(find_protocol("http://example.com"), "http");+ assert_eq!(+ Url::from("https://example.com".to_string()).protocol(),+ "https"+ );+ assert_eq!(+ Url::from("http://example.com".to_string()).protocol(),+ "http"+ );}#[test]- fn favicon() {+ fn simple_favicon() {let raw = r#"<html><head>- <link rel="icon" sizes="192x192" href="favicon_link.png" />+ <link rel="icon" sizes="192x192" href="https://example.com/favicon.png" /></head></html>"#@@ -742,11 +788,74 @@ mod tests {assert_eq!(webpage.favicon(),Some(FaviconLink {- link: "favicon_link.png".to_string().into(),+ link: "https://example.com/favicon.png".to_string().into(),width: Some(192),height: Some(192),image_type: None}))}++ fn full_link_favicon(href: &str, site_url: &str, expected: &str) {+ let raw = format!(+ r#"+ <html>+ <head>+ <link rel="icon" sizes="192x192" href="{href}" />+ </head>+ </html>+ "#+ );++ let webpage = Html::parse(&raw, site_url);+ assert_eq!(+ webpage.favicon(),+ Some(FaviconLink {+ link: expected.to_string().into(),+ width: Some(192),+ height: Some(192),+ image_type: None+ })+ );+ }++ #[test]+ fn test_full_link_favicon_simple() {+ full_link_favicon(+ "/favicon.png",+ "https://www.example.com/",+ "https://www.example.com/favicon.png",+ );+ full_link_favicon(+ "/favicon.png",+ "https://www.example.com",+ "https://www.example.com/favicon.png",+ );+ full_link_favicon(+ "favicon.png",+ "https://www.example.com",+ "https://www.example.com/favicon.png",+ );+ full_link_favicon(+ "favicon.png",+ "https://www.example.com/",+ "https://www.example.com/favicon.png",+ );+ full_link_favicon(+ "favicon.png",+ "https://www.example.com/test",+ "https://www.example.com/test/favicon.png",+ );+ full_link_favicon(+ "/favicon.png",+ "https://www.example.com/test",+ "https://www.example.com/favicon.png",+ );+ }++ #[test]+ fn domain_from_domain_url() {+ let url: Url = "example.com".to_string().into();+ assert_eq!(url.domain(), "example.com");+ }}diff --git a/static/images/globe.png b/static/images/globe.pngnew file mode 100644index 00000000..3b81ae5cBinary files /dev/null and b/static/images/globe.png differdiff --git a/templates/search.html b/templates/search.htmlindex e9185e03..d0a5b1e3 100644--- a/templates/search.html+++ b/templates/search.html@@ -16,8 +16,13 @@<div class="search-result flex flex-col space-y-3 pt-8 pb-8 pl-40">{% for item in search_result %}<div class="search-result-item bg-white max-w-2xl p-3">- <div class="text-sm mb-1">- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>+ <div class="text-sm mb-1 flex">+ <div class="flex items-center mr-2">+ <img src="/favicons/{{ item.domain }}" width="13px" />+ </div>+ <div>+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>+ </div></div><div class="text-lg font-medium mb-1"><a class="title-link" href="{{ item.url }}">{{ item.title }}</a>