web search done right
only show images if the query matches the title or description
Mikkel Denkerd8bcd409c40e
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 33e4fe87..2b40e877 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -128,7 +128,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {backlinks,centrality,fetch_time_ms,- primary_image_uuid: None,+ primary_image: None,};if let Err(err) = index.insert(webpage) {debug!("{:?}", err);diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 47477a31..14598f0c 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -127,7 +127,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {domain,snippet: webpage.snippet, // snippet has already been html-escaped.body: webpage.body,- primary_image_uuid: webpage.primary_image_uuid,+ primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),last_updated,}}@@ -277,9 +277,13 @@ pub async fn route(.documents.into_iter().map(|mut webpage| {- webpage.primary_image_uuid = webpage.primary_image_uuid.and_then(|uuid| {- if state.searcher.primary_image(uuid.clone()).is_some() {- Some(uuid)+ webpage.primary_image = webpage.primary_image.and_then(|image| {+ if state+ .searcher+ .primary_image(image.uuid.clone().to_string())+ .is_some()+ {+ Some(image)} else {None}diff --git a/src/index.rs b/src/index.rsindex 3028c1a1..41e37c84 100644--- a/src/index.rs+++ b/src/index.rs@@ -164,10 +164,11 @@ impl Index {_ => return,}- if let Some(url) = webpage.html.primary_image() {+ if let Some(image) = webpage.html.primary_image() {+ let url = image.url.clone();if url.is_valid_uri() {let uuid = self.primary_image_store.generate_uuid();- webpage.set_primary_image_uuid(uuid);+ webpage.set_primary_image(uuid, image);self.primary_image_downloader.schedule(ImageDownloadJob {key: uuid,diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex e0b3376c..4963d613 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -25,7 +25,7 @@ use crate::image_store::Image;use crate::query::Query;use crate::schema::{Field, ALL_FIELDS};use crate::snippet;-use crate::webpage::Webpage;+use crate::webpage::{StoredPrimaryImage, Webpage};use crate::Result;use crate::{schema::create_schema, tokenizer::Tokenizer};use std::fs;@@ -110,6 +110,22 @@ impl InvertedIndex {.into_iter().map(|(_score, doc_address)| self.retrieve_doc(doc_address, &searcher)).filter_map(|page| page.ok())+ .map(|mut doc| {+ if let Some(image) = doc.primary_image.as_ref() {+ if !query.simple_terms().into_iter().all(|term| {+ image+ .title_terms+ .contains(term.to_ascii_lowercase().as_str())+ || image+ .description_terms+ .contains(term.to_ascii_lowercase().as_str())+ }) {+ doc.primary_image = None;+ }+ }++ doc+ }).collect();for page in &mut webpages {@@ -226,7 +242,7 @@ pub struct RetrievedWebpage {pub dirty_body: String,pub description: Option<String>,pub favicon: Option<Image>,- pub primary_image_uuid: Option<String>,+ pub primary_image: Option<StoredPrimaryImage>,pub updated_time: Option<NaiveDateTime>,}@@ -266,19 +282,14 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string()}- Field::PrimaryImageUuid => {- webpage.primary_image_uuid = {- let s = value+ Field::PrimaryImage => {+ webpage.primary_image = {+ let bytes = value.value- .as_text()- .expect("Primary image uuid field should be text")- .to_string();+ .as_bytes()+ .expect("Primary image field should be bytes");- if s.is_empty() {- None- } else {- Some(s)- }+ bincode::deserialize(bytes).unwrap()}}Field::LastUpdated => {@@ -319,6 +330,8 @@ impl From<Document> for RetrievedWebpage {#[cfg(test)]mod tests {+ use maplit::hashset;+use crate::{ranking::Ranker,webpage::{region::RegionCount, Link},@@ -786,4 +799,68 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.first.com");assert_eq!(result.documents[1].url, "https://www.second.com");}++ #[test]+ fn only_show_primary_images_when_relevant() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");+ let ranker = Ranker::new(RegionCount::default());++ let mut webpage = Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <meta property="og:image" content="https://example.com/link_to_image.html" />+ <meta property="og:description" content="This is an image for the test website" />+ <meta property="og:title" content="title" />+ <title>Test website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.example.com",+ vec![],+ 1.0,+ 500,+ );+ let uuid = uuid::uuid!("00000000-0000-0000-0000-ffff00000000");+ webpage.set_primary_image(uuid, webpage.html.primary_image().unwrap());++ index.insert(webpage).expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let query = Query::parse("website", index.schema(), index.tokenizers())+ .expect("Failed to parse query");++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");++ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.example.com");+ assert_eq!(+ result.documents[0].primary_image,+ Some(StoredPrimaryImage {+ uuid,+ title_terms: hashset! {"title".to_string()},+ description_terms: hashset! {"this".to_string(), "is".to_string(), "an".to_string(), "image".to_string(), "for".to_string(), "the".to_string(), "test".to_string(), "website".to_string()}+ })+ );++ let query = Query::parse("best website", index.schema(), index.tokenizers())+ .expect("Failed to parse query");++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");++ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.example.com");+ assert_eq!(result.documents[0].primary_image, None);+ }}diff --git a/src/schema.rs b/src/schema.rsindex 6ca34087..764a609e 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use tantivy::schema::{- Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,+ BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};use crate::tokenizer::{Normal, Stemmed};@@ -38,7 +38,7 @@ pub enum Field {BacklinkText,Centrality,FetchTimeMs,- PrimaryImageUuid,+ PrimaryImage,LastUpdated,Description,NumTrackers,@@ -59,7 +59,7 @@ pub static ALL_FIELDS: [Field; 19] = [Field::BacklinkText,Field::Centrality,Field::FetchTimeMs,- Field::PrimaryImageUuid,+ Field::PrimaryImage,Field::LastUpdated,Field::Description,Field::NumTrackers,@@ -123,9 +123,7 @@ impl Field {.set_fast(Cardinality::SingleValue).set_indexed(),),- Field::PrimaryImageUuid => {- IndexingOption::Text(self.default_text_options().set_stored())- }+ Field::PrimaryImage => IndexingOption::Bytes(BytesOptions::default().set_stored()),Field::LastUpdated => IndexingOption::Numeric(NumericOptions::default().set_fast(Cardinality::SingleValue)@@ -155,7 +153,7 @@ impl Field {Field::DomainIfHomepage => "domain_if_homepage",Field::IsHomepage => "is_homepage",Field::FetchTimeMs => "fetch_time_ms",- Field::PrimaryImageUuid => "primary_image_uuid",+ Field::PrimaryImage => "primary_image_uuid",Field::LastUpdated => "last_updated",Field::Description => "description",Field::AllBody => "all_body",@@ -179,7 +177,7 @@ impl Field {Field::BacklinkText => Some(4.0),Field::Centrality| Field::IsHomepage- | Field::PrimaryImageUuid+ | Field::PrimaryImage| Field::FetchTimeMs| Field::Description| Field::NumTrackers@@ -189,7 +187,7 @@ impl Field {}pub fn is_searchable(&self) -> bool {- !matches!(self, Field::PrimaryImageUuid | Field::BacklinkText)+ !matches!(self, Field::PrimaryImage | Field::BacklinkText)}}@@ -200,6 +198,7 @@ pub fn create_schema() -> tantivy::schema::Schema {match field.options() {IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),IndexingOption::Numeric(options) => builder.add_u64_field(field.as_str(), options),+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),};}@@ -209,4 +208,5 @@ pub fn create_schema() -> tantivy::schema::Schema {pub enum IndexingOption {Text(tantivy::schema::TextOptions),Numeric(tantivy::schema::NumericOptions),+ Bytes(tantivy::schema::BytesOptions),}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 68ae3ec2..57654b0d 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -13,12 +13,17 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{schema_org::SchemaOrg, Error, Result};+use crate::{schema_org::SchemaOrg, tokenizer, Error, Result};use chrono::{DateTime, FixedOffset};use itertools::Itertools;use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};use regex::Regex;-use std::{borrow::Borrow, collections::HashMap};+use serde::{Deserialize, Serialize};+use std::{+ borrow::Borrow,+ collections::{HashMap, HashSet},+};+use tantivy::tokenizer::Tokenizer;use uuid::Uuid;mod just_text;@@ -42,6 +47,13 @@ pub struct FaviconLink {image_type: Option<String>,}+#[derive(PartialEq, Eq, Debug)]+pub struct ImageLink {+ pub url: Url,+ pub title: Option<String>,+ pub description: Option<String>,+}+pub struct Preprocessor<const N: usize> {removed_tags: [&'static str; N],num_open_tags: [i64; N],@@ -91,12 +103,19 @@ impl<const N: usize> Preprocessor<N> {}}+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct StoredPrimaryImage {+ pub uuid: Uuid,+ pub title_terms: HashSet<String>,+ pub description_terms: HashSet<String>,+}+pub struct Webpage {pub html: Html,pub backlinks: Vec<Link>,pub centrality: f64,pub fetch_time_ms: u64,- pub primary_image_uuid: Option<Uuid>,+ pub primary_image: Option<StoredPrimaryImage>,}impl Webpage {@@ -115,7 +134,7 @@ impl Webpage {backlinks,centrality,fetch_time_ms,- primary_image_uuid: None,+ primary_image: None,}}@@ -167,22 +186,41 @@ impl Webpage {self.fetch_time_ms,);- let uuid = self- .primary_image_uuid- .map(|uuid| uuid.to_string())- .unwrap_or_default();- doc.add_text(+ let image = bincode::serialize(&self.primary_image).unwrap();+ doc.add_bytes(schema- .get_field(Field::PrimaryImageUuid.as_str())- .expect("Failed to get primary_image_uuid field"),- uuid,+ .get_field(Field::PrimaryImage.as_str())+ .expect("Failed to get primary_image field"),+ image,);Ok(doc)}- pub(crate) fn set_primary_image_uuid(&mut self, uuid: Uuid) {- self.primary_image_uuid = Some(uuid);+ pub(crate) fn set_primary_image(&mut self, uuid: Uuid, image: ImageLink) {+ let mut title_terms = HashSet::new();++ if let Some(title) = image.title {+ let mut tokenizer = tokenizer::Normal::default().token_stream(title.as_str());+ while let Some(token) = tokenizer.next() {+ title_terms.insert(token.text.clone());+ }+ }++ let mut description_terms = HashSet::new();++ if let Some(description) = image.description {+ let mut tokenizer = tokenizer::Normal::default().token_stream(description.as_str());+ while let Some(token) = tokenizer.next() {+ description_terms.insert(token.text.clone());+ }+ }++ self.primary_image = Some(StoredPrimaryImage {+ uuid,+ title_terms,+ description_terms,+ });}}@@ -453,7 +491,7 @@ impl Html {| Field::Centrality| Field::FetchTimeMs| Field::Region- | Field::PrimaryImageUuid => {}+ | Field::PrimaryImage => {}}}@@ -565,7 +603,7 @@ impl Html {})}- fn og_image(&self) -> Option<Url> {+ fn og_image(&self) -> Option<ImageLink> {self.metadata().into_iter().find(|metadata| {@@ -576,6 +614,11 @@ impl Html {}}).and_then(|metadata| metadata.get("content").map(|link| link.clone().into()))+ .map(|url| ImageLink {+ url,+ title: self.og_title(),+ description: self.description(),+ })}#[allow(unreachable_patterns)]@@ -597,14 +640,24 @@ impl Html {.or_else(|| self.article_modified_time())}- pub fn primary_image(&self) -> Option<Url> {+ pub fn primary_image(&self) -> Option<ImageLink> {self.og_image()- .or_else(|| self.schema_org_images().first().cloned())- .map(|mut url| {- if !url.is_full_path() {- url.prefix_with(&self.url);+ .or_else(|| {+ self.schema_org_images()+ .first()+ .cloned()+ .map(|url| ImageLink {+ url,+ title: self.og_title(),+ description: self.description(),+ })+ })+ .map(|mut image| {+ if !image.url.is_full_path() {+ image.url.prefix_with(&self.url);}- url++ image})}@@ -620,6 +673,19 @@ impl Html {}).and_then(|metadata| metadata.get("content").cloned())}++ pub fn og_title(&self) -> Option<String> {+ self.metadata()+ .into_iter()+ .find(|metadata| {+ if let Some(property) = metadata.get("property") {+ property == &String::from("og:title")+ } else {+ false+ }+ })+ .and_then(|metadata| metadata.get("content").cloned())+ }}#[derive(Debug, PartialEq, Eq)]@@ -1077,6 +1143,8 @@ mod tests {<html><head><meta property="og:image" content="https://example.com/link_to_image.html" />+ <meta property="og:description" content="desc" />+ <meta property="og:title" content="title" /></head><body></body>@@ -1086,7 +1154,11 @@ mod tests {assert_eq!(html.primary_image(),- Some("https://example.com/link_to_image.html".to_string().into())+ Some(ImageLink {+ url: "https://example.com/link_to_image.html".to_string().into(),+ title: Some("title".to_string()),+ description: Some("desc".to_string())+ }));let html = r#"@@ -1125,7 +1197,11 @@ mod tests {assert_eq!(html.primary_image(),- Some("https://example.com/mexico-beach.jpg".to_string().into())+ Some(ImageLink {+ url: "https://example.com/mexico-beach.jpg".to_string().into(),+ title: None,+ description: None+ }));}