web search done right
specialized metadata functions
Mikkel Denker2fe0c1b5dc3d
diff --git a/src/schema_org.rs b/src/schema_org.rsindex 4f821863..a953b781 100644--- a/src/schema_org.rs+++ b/src/schema_org.rs@@ -16,6 +16,7 @@use serde::{Deserialize, Serialize};+#[non_exhaustive]#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]#[serde(tag = "@type")]pub enum SchemaOrg {diff --git a/src/webpage/lexer.rs b/src/webpage/lexer.rsindex d41e6579..d475f045 100644--- a/src/webpage/lexer.rs+++ b/src/webpage/lexer.rs@@ -23,21 +23,62 @@ pub struct Tag<'a> {raw: &'a str,}+fn addr_of(s: &str) -> usize {+ s.as_ptr() as usize+}++fn split_whitespace_indices(s: &str) -> impl Iterator<Item = (usize, &str)> {+ s.split_whitespace()+ .map(move |sub| (addr_of(sub) - addr_of(s), sub))+}+impl<'a> Tag<'a> {pub fn attributes(&self) -> HashMap<&'a str, &'a str> {let mut attributes = HashMap::new();- for tok in self.raw.split_whitespace().skip(1) {- if tok.contains('=') {- if let Some((key, value)) = tok.split_once('=') {- attributes.insert(key.trim(), trim_html_stuff(value, false));+ let mut long_attribute: Option<usize> = None;+ let mut is_first_token = true;+ for (start_idx, tok) in split_whitespace_indices(self.raw) {+ let num_quotes = tok.chars().filter(|c| *c == '"').count();+ if (tok.contains('>') && num_quotes == 0 && is_first_token)+ || tok.contains('<')+ || tok == "/>"+ || tok.starts_with(self.name.as_ref())+ {+ is_first_token = false;+ continue;+ }+ is_first_token = false;++ if num_quotes == 0 {+ if long_attribute.is_none() {+ // attribute without value (e.g. autofocus)+ let to_insert = trim_html_stuff(tok, true);+ if !to_insert.is_empty() {+ attributes.insert(to_insert, "");+ }}- } else {- if tok == ">" || tok.starts_with(self.name.as_ref()) || tok == "/>" {- continue;+ } else if num_quotes == 1 {+ if let Some(attribute_start_idx) = long_attribute {+ // closing attribute+ let tok = &self.raw[attribute_start_idx..start_idx + tok.len()];+ if tok.contains('=') {+ if let Some((key, value)) = tok.split_once('=') {+ attributes.insert(key.trim(), trim_html_stuff(value, false));+ }+ }+ long_attribute = None;+ } else {+ // opening attribute+ long_attribute = Some(start_idx);+ }+ } else if num_quotes == 2 {+ // self-containing attribute+ if tok.contains('=') {+ if let Some((key, value)) = tok.split_once('=') {+ attributes.insert(key.trim(), trim_html_stuff(value, false));+ }}-- attributes.insert(trim_html_stuff(tok, true), "");}}@@ -66,7 +107,7 @@ fn trim_html_stuff(s: &str, break_on_dash: bool) -> &str {break;}- if c == ' ' || c == '"' || c == '>' {+ if c == '"' || c == '>' {break;}}@@ -273,4 +314,28 @@ mod tests {assert_eq!(tag.name(), "a");assert_eq!(lex.next(), None);}++ #[test]+ fn test_split_whitespace_indices() {+ let mut iter = split_whitespace_indices(" Hello world");++ assert_eq!(Some((1, "Hello")), iter.next());+ assert_eq!(Some((7, "world")), iter.next());+ }++ #[test]+ fn attribute_containing_spaces() {+ let mut lex = Token::lexer("<meta description=\"this is a long attribute\">");++ let tag = match lex.next().unwrap() {+ Token::StartTag(tag) => tag,+ _ => panic!(),+ };++ assert_eq!(+ tag.attributes(),+ hashmap!["description" => "this is a long attribute"]+ );+ assert_eq!(lex.next(), None);+ }}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 52cf2014..84e197d1 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -497,7 +497,24 @@ impl<'a> Html<'a> {schemas}- pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {+ fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {+ self.metadata()+ .into_iter()+ .find(|metadata| {+ if let Some(property) = metadata.get("property") {+ property == &String::from("article:modified_time")+ } else {+ false+ }+ })+ .and_then(|metadata| {+ metadata+ .get("content")+ .and_then(|time| DateTime::parse_from_rfc3339(time).ok())+ })+ }++ fn og_updated_time(&self) -> Option<DateTime<FixedOffset>> {self.metadata().into_iter().find(|metadata| {@@ -513,6 +530,56 @@ impl<'a> Html<'a> {.and_then(|time| DateTime::parse_from_rfc3339(time).ok())})}++ fn og_image(&self) -> Option<Url> {+ self.metadata()+ .into_iter()+ .find(|metadata| {+ if let Some(property) = metadata.get("property") {+ property == &String::from("og:image")+ } else {+ false+ }+ })+ .and_then(|metadata| metadata.get("content").map(|link| link.clone().into()))+ }++ #[allow(unreachable_patterns)]+ fn schema_org_images(&self) -> Vec<Url> {+ self.schema_org()+ .into_iter()+ .filter(|schema| matches!(schema, SchemaOrg::ImageObject(_)))+ .flat_map(|schema| {+ match schema {+ SchemaOrg::ImageObject(image) => image.content_url.map(|url| url.into()),+ _ => None, // has been filtered, so only image is possible+ }+ })+ .collect()+ }++ pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {+ self.og_updated_time()+ .or_else(|| self.article_modified_time())+ }++ pub fn primary_image(&self) -> Option<Url> {+ self.og_image()+ .or_else(|| self.schema_org_images().first().cloned())+ }++ pub fn description(&self) -> Option<String> {+ self.metadata()+ .into_iter()+ .find(|metadata| {+ if let Some(property) = metadata.get("property") {+ property == &String::from("og:description")+ } else {+ false+ }+ })+ .and_then(|metadata| metadata.get("content").cloned())+ }}#[derive(Debug, PartialEq, Eq)]@@ -943,4 +1010,112 @@ mod tests {assert_eq!(html.updated_time(), None);}++ #[test]+ fn primary_image() {+ let html = r#"+ <html>+ <head>+ <meta property="og:image" content="https://example.com/link_to_image.html" />+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(+ html.primary_image(),+ Some("https://example.com/link_to_image.html".to_string().into())+ );++ let html = r#"+ <html>+ <head>+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(html.primary_image(), None);++ let html = r#"+ <html>+ <head>+ <script type="application/ld+json">+ {+ "@context": "https://schema.org",+ "@type": "ImageObject",+ "author": "Jane Doe",+ "contentLocation": "Puerto Vallarta, Mexico",+ "contentUrl": "mexico-beach.jpg",+ "datePublished": "2008-01-25",+ "description": "I took this picture while on vacation last year.",+ "name": "Beach in Mexico"+ }+ </script>+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(+ html.primary_image(),+ Some("mexico-beach.jpg".to_string().into())+ );+ }++ #[test]+ fn description() {+ let html = r#"+ <html>+ <head>+ <meta property="og:description" content="This is a page description" />+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(+ html.description(),+ Some("This is a page description".to_string())+ );++ let html = r#"+ <html>+ <head>+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(html.description(), None);+ }++ #[test]+ fn article_modified_time() {+ let html = r#"+ <html>+ <head>+ <meta property="article:modified_time" content="2022-06-22T19:37:34+00:00" />+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(+ html.updated_time(),+ Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())+ );+ }}