Skip to content

web search done right

specialized metadata functions

Mikkel Denker2fe0c1b5dc3d
diff --git a/src/schema_org.rs b/src/schema_org.rs
index 4f821863..a953b781 100644
--- a/src/schema_org.rs
+++ b/src/schema_org.rs
@@ -16,6 +16,7 @@
use serde::{Deserialize, Serialize};
+#[non_exhaustive]
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]
#[serde(tag = "@type")]
pub enum SchemaOrg {
diff --git a/src/webpage/lexer.rs b/src/webpage/lexer.rs
index d41e6579..d475f045 100644
--- a/src/webpage/lexer.rs
+++ b/src/webpage/lexer.rs
@@ -23,21 +23,62 @@ pub struct Tag<'a> {
raw: &'a str,
}
+fn addr_of(s: &str) -> usize {
+ s.as_ptr() as usize
+}
+
+fn split_whitespace_indices(s: &str) -> impl Iterator<Item = (usize, &str)> {
+ s.split_whitespace()
+ .map(move |sub| (addr_of(sub) - addr_of(s), sub))
+}
+
impl<'a> Tag<'a> {
pub fn attributes(&self) -> HashMap<&'a str, &'a str> {
let mut attributes = HashMap::new();
- for tok in self.raw.split_whitespace().skip(1) {
- if tok.contains('=') {
- if let Some((key, value)) = tok.split_once('=') {
- attributes.insert(key.trim(), trim_html_stuff(value, false));
+ let mut long_attribute: Option<usize> = None;
+ let mut is_first_token = true;
+ for (start_idx, tok) in split_whitespace_indices(self.raw) {
+ let num_quotes = tok.chars().filter(|c| *c == '"').count();
+ if (tok.contains('>') && num_quotes == 0 && is_first_token)
+ || tok.contains('<')
+ || tok == "/>"
+ || tok.starts_with(self.name.as_ref())
+ {
+ is_first_token = false;
+ continue;
+ }
+ is_first_token = false;
+
+ if num_quotes == 0 {
+ if long_attribute.is_none() {
+ // attribute without value (e.g. autofocus)
+ let to_insert = trim_html_stuff(tok, true);
+ if !to_insert.is_empty() {
+ attributes.insert(to_insert, "");
+ }
}
- } else {
- if tok == ">" || tok.starts_with(self.name.as_ref()) || tok == "/>" {
- continue;
+ } else if num_quotes == 1 {
+ if let Some(attribute_start_idx) = long_attribute {
+ // closing attribute
+ let tok = &self.raw[attribute_start_idx..start_idx + tok.len()];
+ if tok.contains('=') {
+ if let Some((key, value)) = tok.split_once('=') {
+ attributes.insert(key.trim(), trim_html_stuff(value, false));
+ }
+ }
+ long_attribute = None;
+ } else {
+ // opening attribute
+ long_attribute = Some(start_idx);
+ }
+ } else if num_quotes == 2 {
+ // self-containing attribute
+ if tok.contains('=') {
+ if let Some((key, value)) = tok.split_once('=') {
+ attributes.insert(key.trim(), trim_html_stuff(value, false));
+ }
}
-
- attributes.insert(trim_html_stuff(tok, true), "");
}
}
@@ -66,7 +107,7 @@ fn trim_html_stuff(s: &str, break_on_dash: bool) -> &str {
break;
}
- if c == ' ' || c == '"' || c == '>' {
+ if c == '"' || c == '>' {
break;
}
}
@@ -273,4 +314,28 @@ mod tests {
assert_eq!(tag.name(), "a");
assert_eq!(lex.next(), None);
}
+
+ #[test]
+ fn test_split_whitespace_indices() {
+ let mut iter = split_whitespace_indices(" Hello world");
+
+ assert_eq!(Some((1, "Hello")), iter.next());
+ assert_eq!(Some((7, "world")), iter.next());
+ }
+
+ #[test]
+ fn attribute_containing_spaces() {
+ let mut lex = Token::lexer("<meta description=\"this is a long attribute\">");
+
+ let tag = match lex.next().unwrap() {
+ Token::StartTag(tag) => tag,
+ _ => panic!(),
+ };
+
+ assert_eq!(
+ tag.attributes(),
+ hashmap!["description" => "this is a long attribute"]
+ );
+ assert_eq!(lex.next(), None);
+ }
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 52cf2014..84e197d1 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -497,7 +497,24 @@ impl<'a> Html<'a> {
schemas
}
- pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {
+ fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("article:modified_time")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| {
+ metadata
+ .get("content")
+ .and_then(|time| DateTime::parse_from_rfc3339(time).ok())
+ })
+ }
+
+ fn og_updated_time(&self) -> Option<DateTime<FixedOffset>> {
self.metadata()
.into_iter()
.find(|metadata| {
@@ -513,6 +530,56 @@ impl<'a> Html<'a> {
.and_then(|time| DateTime::parse_from_rfc3339(time).ok())
})
}
+
+ fn og_image(&self) -> Option<Url> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("og:image")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| metadata.get("content").map(|link| link.clone().into()))
+ }
+
+ #[allow(unreachable_patterns)]
+ fn schema_org_images(&self) -> Vec<Url> {
+ self.schema_org()
+ .into_iter()
+ .filter(|schema| matches!(schema, SchemaOrg::ImageObject(_)))
+ .flat_map(|schema| {
+ match schema {
+ SchemaOrg::ImageObject(image) => image.content_url.map(|url| url.into()),
+ _ => None, // has been filtered, so only image is possible
+ }
+ })
+ .collect()
+ }
+
+ pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {
+ self.og_updated_time()
+ .or_else(|| self.article_modified_time())
+ }
+
+ pub fn primary_image(&self) -> Option<Url> {
+ self.og_image()
+ .or_else(|| self.schema_org_images().first().cloned())
+ }
+
+ pub fn description(&self) -> Option<String> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("og:description")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| metadata.get("content").cloned())
+ }
}
#[derive(Debug, PartialEq, Eq)]
@@ -943,4 +1010,112 @@ mod tests {
assert_eq!(html.updated_time(), None);
}
+
+ #[test]
+ fn primary_image() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:image" content="https://example.com/link_to_image.html" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(
+ html.primary_image(),
+ Some("https://example.com/link_to_image.html".to_string().into())
+ );
+
+ let html = r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(html.primary_image(), None);
+
+ let html = r#"
+ <html>
+ <head>
+ <script type="application/ld+json">
+ {
+ "@context": "https://schema.org",
+ "@type": "ImageObject",
+ "author": "Jane Doe",
+ "contentLocation": "Puerto Vallarta, Mexico",
+ "contentUrl": "mexico-beach.jpg",
+ "datePublished": "2008-01-25",
+ "description": "I took this picture while on vacation last year.",
+ "name": "Beach in Mexico"
+ }
+ </script>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(
+ html.primary_image(),
+ Some("mexico-beach.jpg".to_string().into())
+ );
+ }
+
+ #[test]
+ fn description() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:description" content="This is a page description" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(
+ html.description(),
+ Some("This is a page description".to_string())
+ );
+
+ let html = r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(html.description(), None);
+ }
+
+ #[test]
+ fn article_modified_time() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="article:modified_time" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(
+ html.updated_time(),
+ Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())
+ );
+ }
}