web search done right
image store with filters
Mikkel Denker003b54b82074
diff --git a/Cargo.toml b/Cargo.tomlindex 224ce832..1dfc2e25 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -44,6 +44,7 @@ csv = "1.1.6"fst = "0.4.7"lazy_static = "1.4.0"rocksdb = "0.18.0"+image = "0.24.3"[dev-dependencies]maplit = "1.0.2"diff --git a/src/image_store.rs b/src/image_store.rsnew file mode 100644index 00000000..c59d25c4--- /dev/null+++ b/src/image_store.rs@@ -0,0 +1,236 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::kv::{rocksdb_store::RocksDbStore, Kv};+use image::imageops::FilterType;+use image::{DynamicImage, ImageOutputFormat};+use serde::{de, ser::SerializeStruct, Serialize};+use std::io::{Cursor, Read, Seek, SeekFrom};+use std::path::Path;++#[derive(PartialEq, Debug, Clone)]+pub struct Image(DynamicImage);++impl Serialize for Image {+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let mut cursor = Cursor::new(Vec::new());+ self.0+ .write_to(&mut cursor, ImageOutputFormat::Png)+ .unwrap();+ cursor.seek(SeekFrom::Start(0)).unwrap();++ let mut bytes = Vec::new();+ cursor.read_to_end(&mut bytes).unwrap();++ let mut image = serializer.serialize_struct("image", 1)?;+ image.serialize_field("0", &bytes)?;+ image.end()+ }+}++impl<'de> de::Deserialize<'de> for Image {+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct ImageVisitor;++ impl<'de> de::Visitor<'de> for ImageVisitor {+ type Value = Image;++ fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {+ formatter.write_str("a serialized `Image` struct")+ }++ fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>+ where+ A: serde::de::SeqAccess<'de>,+ {+ let raw = seq+ .next_element()?+ .ok_or_else(|| de::Error::invalid_length(0, &self))?;++ let image = Image(+ image::load_from_memory(raw)+ .expect("bytes does not seem to represent an image"),+ );++ Ok(image)+ }+ }++ deserializer.deserialize_struct("image", &["0"], ImageVisitor)+ }+}++struct ImageStore {+ store: Box<dyn Kv<String, Image>>,+ filters: Vec<Box<dyn ImageFilter>>,+}++impl ImageStore {+ fn open<P: AsRef<Path>>(path: P) -> Self {+ Self::open_with_filters(path, Vec::new())+ }++ fn open_with_filters<P: AsRef<Path>>(path: P, filters: Vec<Box<dyn ImageFilter>>) -> Self {+ let store = RocksDbStore::open(path);++ Self { store, filters }+ }++ fn insert(&mut self, key: &str, mut image: Image) {+ for filter in &self.filters {+ image = filter.transform(image);+ }++ self.store.insert(key.to_string(), image)+ }++ fn contains(&self, key: &str) -> bool {+ self.store.get(&key.to_string()).is_some()+ }++ fn get(&self, key: &str) -> Option<Image> {+ self.store.get(&key.to_string())+ }+}++trait ImageFilter {+ fn transform(&self, image: Image) -> Image;+}++struct ResizeFilter {+ width: u32,+ height: u32,+}++impl ImageFilter for ResizeFilter {+ fn transform(&self, image: Image) -> Image {+ Image(+ image+ .0+ .resize(self.width, self.height, FilterType::Gaussian),+ )+ }+}++pub struct FaviconStore(ImageStore);++impl FaviconStore {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ let store = ImageStore::open_with_filters(+ path,+ vec![Box::new(ResizeFilter {+ width: 16,+ height: 16,+ })],+ );++ Self(store)+ }++ pub fn insert(&mut self, key: &str, image: Image) {+ self.0.insert(key, image)+ }++ pub fn contains(&self, key: &str) -> bool {+ self.0.contains(key)+ }++ pub fn get(&self, key: &str) -> Option<Image> {+ self.0.get(key)+ }+}++#[cfg(test)]+mod tests {+ use image::ImageBuffer;++ use super::*;++ #[test]+ fn serialize_deserialize_image() {+ let image = Image(+ ImageBuffer::from_pixel(2, 2, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX])).into(),+ );++ let bytes = bincode::serialize(&image).unwrap();+ let decoded_image = bincode::deserialize(&bytes).unwrap();++ assert_eq!(image, decoded_image);+ }++ #[test]+ fn store_and_load_image() {+ let image = Image(+ ImageBuffer::from_pixel(2, 2, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX])).into(),+ );+ let key = "test";+ let mut store = ImageStore::open(crate::gen_temp_path());++ assert!(!store.contains(key));+ assert_eq!(store.get(key), None);+ store.insert(key, image.clone());+ assert!(store.contains(key));+ assert_eq!(store.get(key), Some(image));+ }++ #[test]+ fn resize_filter() {+ let image = Image(+ ImageBuffer::from_pixel(32, 32, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]))+ .into(),+ );+ assert_eq!(image.0.width(), 32);+ assert_eq!(image.0.height(), 32);++ let transformed_image = ResizeFilter {+ width: 16,+ height: 16,+ }+ .transform(image);++ assert_eq!(transformed_image.0.width(), 16);+ assert_eq!(transformed_image.0.height(), 16);+ }++ #[test]+ fn favicon_store() {+ let image = Image(+ ImageBuffer::from_pixel(32, 32, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX]))+ .into(),+ );+ assert_eq!(image.0.width(), 32);+ assert_eq!(image.0.height(), 32);++ let mut store = FaviconStore::open(crate::gen_temp_path());++ let key = "test";++ assert!(!store.contains(key));+ assert_eq!(store.get(key), None);+ store.insert(key, image);+ assert!(store.contains(key));++ let retrieved_image = store.get(key).unwrap();+ assert_eq!(retrieved_image.0.width(), 16);+ assert_eq!(retrieved_image.0.height(), 16);+ }+}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 45d7c8eb..8ca01145 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -74,6 +74,6 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {let is_homepage = self.is_homepage_reader.get(doc) as f64;let fetch_time_ms = self.fetch_time_ms_reader.get(doc) as f64;- score + (400.0 * centrality) + (5.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))+ (2.0 * score) + (400.0 * centrality) + (5.0 * is_homepage) + (10.0 / (fetch_time_ms + 1.0))}}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex ce46eff1..350eccf5 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -92,6 +92,14 @@ pub fn domain(url: &str) -> &'_ str {}}+#[derive(PartialEq, Eq, Debug)]+pub struct FaviconLink {+ link: Url,+ width: Option<u32>,+ height: Option<u32>,+ image_type: Option<String>,+}+pub struct Preprocessor<const N: usize> {removed_tags: [&'static str; N],num_open_tags: [i64; N],@@ -270,6 +278,61 @@ impl<'a> Html<'a> {links}+ pub fn favicon(&self) -> Option<FaviconLink> {+ let mut preprocessor = Preprocessor::new(["script", "style", "body"]);++ let tokens = self.tokens.clone();+ for tok in tokens {+ preprocessor.update(&tok);+ if preprocessor.is_inside_removed() {+ continue;+ }++ match tok {+ Token::StartTag(tag) | Token::SelfTerminatingTag(tag) if tag.name() == "link" => {+ let rel = tag.attributes().get("rel").cloned();+ if rel.is_none() {+ continue;+ }+ if rel.unwrap() != "icon" {+ continue;+ }++ if let Some(link) = tag.attributes().get("href") {+ let (width, height) = match tag.attributes().get("sizes") {+ Some(size) => {+ if let Some((width, height)) = size.split_once('x') {+ (Some(width.parse().unwrap()), Some(height.parse().unwrap()))+ } else {+ (None, None)+ }+ }+ _ => (None, None),+ };++ let image_type = tag.attributes().get("type").map(|t| t.to_string());++ let favicon = FaviconLink {+ link: link.to_string().into(),+ image_type,+ width,+ height,+ };+ return Some(favicon);+ }+ }+ Token::StartTag(_)+ | Token::EndTag(_)+ | Token::SelfTerminatingTag(_)+ | Token::BeginComment+ | Token::Error+ | Token::EndComment => {}+ }+ }++ None+ }+pub fn text(&self) -> Option<String> {let text = JustText::default().extract(self.raw);@@ -428,7 +491,7 @@ impl<'a> Html<'a> {}}-#[derive(Debug, PartialEq, Clone)]+#[derive(Debug, PartialEq, Eq, Clone)]pub struct Url(String);impl Display for Url {@@ -663,4 +726,27 @@ mod tests {assert_eq!(find_protocol("https://example.com"), "https");assert_eq!(find_protocol("http://example.com"), "http");}++ #[test]+ fn favicon() {+ let raw = r#"+ <html>+ <head>+ <link rel="icon" sizes="192x192" href="favicon_link.png" />+ </head>+ </html>+ "#+ .to_string();++ let webpage = Html::parse(&raw, "https://www.example.com");+ assert_eq!(+ webpage.favicon(),+ Some(FaviconLink {+ link: "favicon_link.png".to_string().into(),+ width: Some(192),+ height: Some(192),+ image_type: None+ })+ )+ }}