Skip to content

web search done right

only show images if the query matches the title or description

Mikkel Denkerd8bcd409c40e
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 33e4fe87..2b40e877 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -128,7 +128,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
backlinks,
centrality,
fetch_time_ms,
- primary_image_uuid: None,
+ primary_image: None,
};
if let Err(err) = index.insert(webpage) {
debug!("{:?}", err);
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 47477a31..14598f0c 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -127,7 +127,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
domain,
snippet: webpage.snippet, // snippet has already been html-escaped.
body: webpage.body,
- primary_image_uuid: webpage.primary_image_uuid,
+ primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),
last_updated,
}
}
@@ -277,9 +277,13 @@ pub async fn route(
.documents
.into_iter()
.map(|mut webpage| {
- webpage.primary_image_uuid = webpage.primary_image_uuid.and_then(|uuid| {
- if state.searcher.primary_image(uuid.clone()).is_some() {
- Some(uuid)
+ webpage.primary_image = webpage.primary_image.and_then(|image| {
+ if state
+ .searcher
+ .primary_image(image.uuid.clone().to_string())
+ .is_some()
+ {
+ Some(image)
} else {
None
}
diff --git a/src/index.rs b/src/index.rs
index 3028c1a1..41e37c84 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -164,10 +164,11 @@ impl Index {
_ => return,
}
- if let Some(url) = webpage.html.primary_image() {
+ if let Some(image) = webpage.html.primary_image() {
+ let url = image.url.clone();
if url.is_valid_uri() {
let uuid = self.primary_image_store.generate_uuid();
- webpage.set_primary_image_uuid(uuid);
+ webpage.set_primary_image(uuid, image);
self.primary_image_downloader.schedule(ImageDownloadJob {
key: uuid,
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index e0b3376c..4963d613 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -25,7 +25,7 @@ use crate::image_store::Image;
use crate::query::Query;
use crate::schema::{Field, ALL_FIELDS};
use crate::snippet;
-use crate::webpage::Webpage;
+use crate::webpage::{StoredPrimaryImage, Webpage};
use crate::Result;
use crate::{schema::create_schema, tokenizer::Tokenizer};
use std::fs;
@@ -110,6 +110,22 @@ impl InvertedIndex {
.into_iter()
.map(|(_score, doc_address)| self.retrieve_doc(doc_address, &searcher))
.filter_map(|page| page.ok())
+ .map(|mut doc| {
+ if let Some(image) = doc.primary_image.as_ref() {
+ if !query.simple_terms().into_iter().all(|term| {
+ image
+ .title_terms
+ .contains(term.to_ascii_lowercase().as_str())
+ || image
+ .description_terms
+ .contains(term.to_ascii_lowercase().as_str())
+ }) {
+ doc.primary_image = None;
+ }
+ }
+
+ doc
+ })
.collect();
for page in &mut webpages {
@@ -226,7 +242,7 @@ pub struct RetrievedWebpage {
pub dirty_body: String,
pub description: Option<String>,
pub favicon: Option<Image>,
- pub primary_image_uuid: Option<String>,
+ pub primary_image: Option<StoredPrimaryImage>,
pub updated_time: Option<NaiveDateTime>,
}
@@ -266,19 +282,14 @@ impl From<Document> for RetrievedWebpage {
.expect("Url field should be text")
.to_string()
}
- Field::PrimaryImageUuid => {
- webpage.primary_image_uuid = {
- let s = value
+ Field::PrimaryImage => {
+ webpage.primary_image = {
+ let bytes = value
.value
- .as_text()
- .expect("Primary image uuid field should be text")
- .to_string();
+ .as_bytes()
+ .expect("Primary image field should be bytes");
- if s.is_empty() {
- None
- } else {
- Some(s)
- }
+ bincode::deserialize(bytes).unwrap()
}
}
Field::LastUpdated => {
@@ -319,6 +330,8 @@ impl From<Document> for RetrievedWebpage {
#[cfg(test)]
mod tests {
+ use maplit::hashset;
+
use crate::{
ranking::Ranker,
webpage::{region::RegionCount, Link},
@@ -786,4 +799,68 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.first.com");
assert_eq!(result.documents[1].url, "https://www.second.com");
}
+
+ #[test]
+ fn only_show_primary_images_when_relevant() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ranker = Ranker::new(RegionCount::default());
+
+ let mut webpage = Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <meta property="og:image" content="https://example.com/link_to_image.html" />
+ <meta property="og:description" content="This is an image for the test website" />
+ <meta property="og:title" content="title" />
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ 500,
+ );
+ let uuid = uuid::uuid!("00000000-0000-0000-0000-ffff00000000");
+ webpage.set_primary_image(uuid, webpage.html.primary_image().unwrap());
+
+ index.insert(webpage).expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let query = Query::parse("website", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(
+ result.documents[0].primary_image,
+ Some(StoredPrimaryImage {
+ uuid,
+ title_terms: hashset! {"title".to_string()},
+ description_terms: hashset! {"this".to_string(), "is".to_string(), "an".to_string(), "image".to_string(), "for".to_string(), "the".to_string(), "test".to_string(), "website".to_string()}
+ })
+ );
+
+ let query = Query::parse("best website", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[0].primary_image, None);
+ }
}
diff --git a/src/schema.rs b/src/schema.rs
index 6ca34087..764a609e 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use tantivy::schema::{
- Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
+ BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
};
use crate::tokenizer::{Normal, Stemmed};
@@ -38,7 +38,7 @@ pub enum Field {
BacklinkText,
Centrality,
FetchTimeMs,
- PrimaryImageUuid,
+ PrimaryImage,
LastUpdated,
Description,
NumTrackers,
@@ -59,7 +59,7 @@ pub static ALL_FIELDS: [Field; 19] = [
Field::BacklinkText,
Field::Centrality,
Field::FetchTimeMs,
- Field::PrimaryImageUuid,
+ Field::PrimaryImage,
Field::LastUpdated,
Field::Description,
Field::NumTrackers,
@@ -123,9 +123,7 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::PrimaryImageUuid => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
+ Field::PrimaryImage => IndexingOption::Bytes(BytesOptions::default().set_stored()),
Field::LastUpdated => IndexingOption::Numeric(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
@@ -155,7 +153,7 @@ impl Field {
Field::DomainIfHomepage => "domain_if_homepage",
Field::IsHomepage => "is_homepage",
Field::FetchTimeMs => "fetch_time_ms",
- Field::PrimaryImageUuid => "primary_image_uuid",
+ Field::PrimaryImage => "primary_image_uuid",
Field::LastUpdated => "last_updated",
Field::Description => "description",
Field::AllBody => "all_body",
@@ -179,7 +177,7 @@ impl Field {
Field::BacklinkText => Some(4.0),
Field::Centrality
| Field::IsHomepage
- | Field::PrimaryImageUuid
+ | Field::PrimaryImage
| Field::FetchTimeMs
| Field::Description
| Field::NumTrackers
@@ -189,7 +187,7 @@ impl Field {
}
pub fn is_searchable(&self) -> bool {
- !matches!(self, Field::PrimaryImageUuid | Field::BacklinkText)
+ !matches!(self, Field::PrimaryImage | Field::BacklinkText)
}
}
@@ -200,6 +198,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
match field.options() {
IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),
IndexingOption::Numeric(options) => builder.add_u64_field(field.as_str(), options),
+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),
};
}
@@ -209,4 +208,5 @@ pub fn create_schema() -> tantivy::schema::Schema {
pub enum IndexingOption {
Text(tantivy::schema::TextOptions),
Numeric(tantivy::schema::NumericOptions),
+ Bytes(tantivy::schema::BytesOptions),
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 68ae3ec2..57654b0d 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -13,12 +13,17 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{schema_org::SchemaOrg, Error, Result};
+use crate::{schema_org::SchemaOrg, tokenizer, Error, Result};
use chrono::{DateTime, FixedOffset};
use itertools::Itertools;
use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
use regex::Regex;
-use std::{borrow::Borrow, collections::HashMap};
+use serde::{Deserialize, Serialize};
+use std::{
+ borrow::Borrow,
+ collections::{HashMap, HashSet},
+};
+use tantivy::tokenizer::Tokenizer;
use uuid::Uuid;
mod just_text;
@@ -42,6 +47,13 @@ pub struct FaviconLink {
image_type: Option<String>,
}
+#[derive(PartialEq, Eq, Debug)]
+pub struct ImageLink {
+ pub url: Url,
+ pub title: Option<String>,
+ pub description: Option<String>,
+}
+
pub struct Preprocessor<const N: usize> {
removed_tags: [&'static str; N],
num_open_tags: [i64; N],
@@ -91,12 +103,19 @@ impl<const N: usize> Preprocessor<N> {
}
}
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct StoredPrimaryImage {
+ pub uuid: Uuid,
+ pub title_terms: HashSet<String>,
+ pub description_terms: HashSet<String>,
+}
+
pub struct Webpage {
pub html: Html,
pub backlinks: Vec<Link>,
pub centrality: f64,
pub fetch_time_ms: u64,
- pub primary_image_uuid: Option<Uuid>,
+ pub primary_image: Option<StoredPrimaryImage>,
}
impl Webpage {
@@ -115,7 +134,7 @@ impl Webpage {
backlinks,
centrality,
fetch_time_ms,
- primary_image_uuid: None,
+ primary_image: None,
}
}
@@ -167,22 +186,41 @@ impl Webpage {
self.fetch_time_ms,
);
- let uuid = self
- .primary_image_uuid
- .map(|uuid| uuid.to_string())
- .unwrap_or_default();
- doc.add_text(
+ let image = bincode::serialize(&self.primary_image).unwrap();
+ doc.add_bytes(
schema
- .get_field(Field::PrimaryImageUuid.as_str())
- .expect("Failed to get primary_image_uuid field"),
- uuid,
+ .get_field(Field::PrimaryImage.as_str())
+ .expect("Failed to get primary_image field"),
+ image,
);
Ok(doc)
}
- pub(crate) fn set_primary_image_uuid(&mut self, uuid: Uuid) {
- self.primary_image_uuid = Some(uuid);
+ pub(crate) fn set_primary_image(&mut self, uuid: Uuid, image: ImageLink) {
+ let mut title_terms = HashSet::new();
+
+ if let Some(title) = image.title {
+ let mut tokenizer = tokenizer::Normal::default().token_stream(title.as_str());
+ while let Some(token) = tokenizer.next() {
+ title_terms.insert(token.text.clone());
+ }
+ }
+
+ let mut description_terms = HashSet::new();
+
+ if let Some(description) = image.description {
+ let mut tokenizer = tokenizer::Normal::default().token_stream(description.as_str());
+ while let Some(token) = tokenizer.next() {
+ description_terms.insert(token.text.clone());
+ }
+ }
+
+ self.primary_image = Some(StoredPrimaryImage {
+ uuid,
+ title_terms,
+ description_terms,
+ });
}
}
@@ -453,7 +491,7 @@ impl Html {
| Field::Centrality
| Field::FetchTimeMs
| Field::Region
- | Field::PrimaryImageUuid => {}
+ | Field::PrimaryImage => {}
}
}
@@ -565,7 +603,7 @@ impl Html {
})
}
- fn og_image(&self) -> Option<Url> {
+ fn og_image(&self) -> Option<ImageLink> {
self.metadata()
.into_iter()
.find(|metadata| {
@@ -576,6 +614,11 @@ impl Html {
}
})
.and_then(|metadata| metadata.get("content").map(|link| link.clone().into()))
+ .map(|url| ImageLink {
+ url,
+ title: self.og_title(),
+ description: self.description(),
+ })
}
#[allow(unreachable_patterns)]
@@ -597,14 +640,24 @@ impl Html {
.or_else(|| self.article_modified_time())
}
- pub fn primary_image(&self) -> Option<Url> {
+ pub fn primary_image(&self) -> Option<ImageLink> {
self.og_image()
- .or_else(|| self.schema_org_images().first().cloned())
- .map(|mut url| {
- if !url.is_full_path() {
- url.prefix_with(&self.url);
+ .or_else(|| {
+ self.schema_org_images()
+ .first()
+ .cloned()
+ .map(|url| ImageLink {
+ url,
+ title: self.og_title(),
+ description: self.description(),
+ })
+ })
+ .map(|mut image| {
+ if !image.url.is_full_path() {
+ image.url.prefix_with(&self.url);
}
- url
+
+ image
})
}
@@ -620,6 +673,19 @@ impl Html {
})
.and_then(|metadata| metadata.get("content").cloned())
}
+
+ pub fn og_title(&self) -> Option<String> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("og:title")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| metadata.get("content").cloned())
+ }
}
#[derive(Debug, PartialEq, Eq)]
@@ -1077,6 +1143,8 @@ mod tests {
<html>
<head>
<meta property="og:image" content="https://example.com/link_to_image.html" />
+ <meta property="og:description" content="desc" />
+ <meta property="og:title" content="title" />
</head>
<body>
</body>
@@ -1086,7 +1154,11 @@ mod tests {
assert_eq!(
html.primary_image(),
- Some("https://example.com/link_to_image.html".to_string().into())
+ Some(ImageLink {
+ url: "https://example.com/link_to_image.html".to_string().into(),
+ title: Some("title".to_string()),
+ description: Some("desc".to_string())
+ })
);
let html = r#"
@@ -1125,7 +1197,11 @@ mod tests {
assert_eq!(
html.primary_image(),
- Some("https://example.com/mexico-beach.jpg".to_string().into())
+ Some(ImageLink {
+ url: "https://example.com/mexico-beach.jpg".to_string().into(),
+ title: None,
+ description: None
+ })
);
}