web search done right
implement '<base>' tags for relative urls
Mikkel Denker36fab02c9daf
diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rsindex ffcced11..3df237ee 100644--- a/crates/core/src/webpage/html/links.rs+++ b/crates/core/src/webpage/html/links.rs@@ -185,7 +185,7 @@ impl Html {};let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());- let link = Url::parse_with_base_url(self.url(), link).ok()?;+ let link = Url::parse_with_base_url(self.base_url(), link).ok()?;let favicon = FaviconLink {link,@@ -214,7 +214,7 @@ impl Html {.and_then(|metadata| {metadata.get("content")- .and_then(|link| Url::parse_with_base_url(self.url(), link).ok())+ .and_then(|link| Url::parse_with_base_url(self.base_url(), link).ok())}).map(|url| ImageLink {url,@@ -279,7 +279,9 @@ impl Html {continue;}- if let Ok(dest) = Url::parse_with_base_url(self.url(), dest) {+ if let Ok(dest) =+ Url::parse_with_base_url(self.base_url(), dest)+ {let mut rel = RelFlags::from_html(&dest,&attributes.borrow(),@@ -327,7 +329,7 @@ impl Html {continue;}- if let Ok(dest) = Url::parse_with_base_url(self.url(), dest) {+ if let Ok(dest) = Url::parse_with_base_url(self.base_url(), dest) {let mut rel = RelFlags::from_html(&dest, &attributes.borrow(), &location);if icann_domain == dest.icann_domain() {@@ -356,7 +358,7 @@ impl Html {for node in self.root.select("link").unwrap() {if let Some(element) = node.as_node().as_element() {if let Some(href) = element.attributes.borrow().get("href") {- if let Ok(href) = Url::parse_with_base_url(self.url(), href) {+ if let Ok(href) = Url::parse_with_base_url(self.base_url(), href) {let mut rel =RelFlags::from_html(&href, &element.attributes.borrow(), &location);@@ -401,7 +403,7 @@ impl Html {) {if let Some(content) = metadata.get("content") {if let Ok(destination) =- Url::parse_with_base_url(self.url(), content.as_str())+ Url::parse_with_base_url(self.base_url(), content.as_str()){let mut rel = location.as_rel();@@ -432,7 +434,7 @@ impl Html {) {if let Some(content) = metadata.get("content") {if let Ok(destination) =- Url::parse_with_base_url(self.url(), content.as_str())+ Url::parse_with_base_url(self.base_url(), content.as_str()){let mut rel = location.as_rel();@@ -464,7 +466,8 @@ impl Html {links.extend(self.scripts().into_iter().filter_map(|script| {match script.attributes.get("src") {Some(url) => {- let script_url = Url::parse_with_base_url(self.url(), url.as_str()).ok()?;+ let script_url =+ Url::parse_with_base_url(self.base_url(), url.as_str()).ok()?;if script_url.root_domain() != root_domain {let mut rel = Location::SCRIPT.as_rel();diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex e6de556c..daf0ab01 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -51,6 +51,7 @@ pub struct Html {clean_text: Option<String>,lang: Option<Lang>,robots: Option<EnumSet<RobotsMeta>>,+ base_url: Url,}impl Html {@@ -70,17 +71,22 @@ impl Html {pub fn parse_without_text(html: &str, url: &str) -> Result<Self> {let root = kuchiki::parse_html().one(html);- let url = Url::parse(url)?;+ let url = Url::parse(url)?.normalize();let mut res = Self {root,all_text: None,clean_text: None,lang: None,+ base_url: url.clone(),url,robots: None,};+ if let Some(base_url) = res.find_base_url() {+ res.base_url = base_url;+ }+// TODO: need to figure out when to respect// the canonical url, it's not always the case that// the website use the tag correctly.@@ -94,12 +100,25 @@ impl Html {// }// }- res.url.normalize_in_place();res.robots = res.parse_robots_meta();Ok(res)}+ fn find_base_url(&self) -> Option<Url> {+ self.root.select("base").unwrap().next().and_then(|node| {+ if let Some(element) = node.as_node().as_element() {+ if let Some(href) = element.attributes.borrow().get("href") {+ if let Ok(url) = Url::parse_with_base_url(self.base_url(), href) {+ return Some(url);+ }+ }+ }++ None+ })+ }+pub fn content_type(&self) -> Option<String> {self.metadata().into_iter()@@ -129,7 +148,7 @@ impl Html {if let Some(rel) = element.attributes.borrow().get("rel") {if rel == "canonical" {if let Some(href) = element.attributes.borrow().get("href") {- if let Ok(url) = Url::parse_with_base_url(self.url(), href) {+ if let Ok(url) = Url::parse_with_base_url(self.base_url(), href) {canonical_url = Some(url);}}@@ -192,6 +211,10 @@ impl Html {&self.url}+ fn base_url(&self) -> &Url {+ &self.base_url+ }+pub fn metadata(&self) -> Vec<Meta> {let mut metas = Vec::new();@@ -245,13 +268,13 @@ impl Html {if let Some(link) = script.attributes.get("src")- .and_then(|link| Url::parse_with_base_url(self.url(), link).ok())+ .and_then(|link| Url::parse_with_base_url(self.base_url(), link).ok()){links.push(link);}for res in URL_REGEX.find_iter(&script.content) {- if let Ok(link) = Url::parse_with_base_url(self.url(), res.as_str()) {+ if let Ok(link) = Url::parse_with_base_url(self.base_url(), res.as_str()) {links.push(link);}}@@ -262,7 +285,7 @@ impl Html {.attributes.borrow().get("href")- .and_then(|link| Url::parse_with_base_url(self.url(), link).ok())+ .and_then(|link| Url::parse_with_base_url(self.base_url(), link).ok()){links.push(link);}@@ -396,7 +419,7 @@ impl Html {.many().into_iter().filter_map(|url| url.try_into_string())- .filter_map(|url| Url::parse_with_base_url(self.url(), &url).ok())+ .filter_map(|url| Url::parse_with_base_url(self.base_url(), &url).ok())})}).flatten()@@ -1184,4 +1207,75 @@ mod tests {assert!(html.likely_has_paywall());}++ #[test]+ fn test_base_url() {+ let html = Html::parse(+ r##"+ <html>+ <head>+ <base href="https://www.example.com/base/" />+ </head>+ <body>+ <a href="link1">Link 1</a>+ <a href="link2">Link 2</a>+ <a href="https://www.example.com/link3">Link 3</a>+ <a href="https://www.another.com/">Link 4</a>+ </body>+ </html>+ "##,+ "https://www.example.com/",+ )+ .unwrap();++ assert_eq!(+ html.base_url(),+ &Url::parse("https://www.example.com/base/").unwrap()+ );+ assert_eq!(+ html.all_links()+ .iter()+ .map(|l| l.destination.to_string())+ .collect::<Vec<String>>(),+ vec![+ "https://www.example.com/base/link1",+ "https://www.example.com/base/link2",+ "https://www.example.com/link3",+ "https://www.another.com/",+ ]+ );++ let html = Html::parse(+ r##"+ <html>+ <head>+ <div>+ <base href="https://www.example.com/base/" />+ </div>+ </head>+ <body>+ <a href="link1">Link 1</a>+ <a href="link2">Link 2</a>+ </body>+ </html>+ "##,+ "https://www.example.com/",+ )+ .unwrap();++ assert_eq!(+ html.base_url(),+ &Url::parse("https://www.example.com/base/").unwrap()+ );+ assert_eq!(+ html.all_links()+ .iter()+ .map(|l| l.destination.to_string())+ .collect::<Vec<String>>(),+ vec![+ "https://www.example.com/base/link1",+ "https://www.example.com/base/link2",+ ]+ );+ }}