Skip to content

web search done right

implement '<base>' tags for relative urls

Mikkel Denker36fab02c9daf
diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rs
index ffcced11..3df237ee 100644
--- a/crates/core/src/webpage/html/links.rs
+++ b/crates/core/src/webpage/html/links.rs
@@ -185,7 +185,7 @@ impl Html {
};
let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());
- let link = Url::parse_with_base_url(self.url(), link).ok()?;
+ let link = Url::parse_with_base_url(self.base_url(), link).ok()?;
let favicon = FaviconLink {
link,
@@ -214,7 +214,7 @@ impl Html {
.and_then(|metadata| {
metadata
.get("content")
- .and_then(|link| Url::parse_with_base_url(self.url(), link).ok())
+ .and_then(|link| Url::parse_with_base_url(self.base_url(), link).ok())
})
.map(|url| ImageLink {
url,
@@ -279,7 +279,9 @@ impl Html {
continue;
}
- if let Ok(dest) = Url::parse_with_base_url(self.url(), dest) {
+ if let Ok(dest) =
+ Url::parse_with_base_url(self.base_url(), dest)
+ {
let mut rel = RelFlags::from_html(
&dest,
&attributes.borrow(),
@@ -327,7 +329,7 @@ impl Html {
continue;
}
- if let Ok(dest) = Url::parse_with_base_url(self.url(), dest) {
+ if let Ok(dest) = Url::parse_with_base_url(self.base_url(), dest) {
let mut rel = RelFlags::from_html(&dest, &attributes.borrow(), &location);
if icann_domain == dest.icann_domain() {
@@ -356,7 +358,7 @@ impl Html {
for node in self.root.select("link").unwrap() {
if let Some(element) = node.as_node().as_element() {
if let Some(href) = element.attributes.borrow().get("href") {
- if let Ok(href) = Url::parse_with_base_url(self.url(), href) {
+ if let Ok(href) = Url::parse_with_base_url(self.base_url(), href) {
let mut rel =
RelFlags::from_html(&href, &element.attributes.borrow(), &location);
@@ -401,7 +403,7 @@ impl Html {
) {
if let Some(content) = metadata.get("content") {
if let Ok(destination) =
- Url::parse_with_base_url(self.url(), content.as_str())
+ Url::parse_with_base_url(self.base_url(), content.as_str())
{
let mut rel = location.as_rel();
@@ -432,7 +434,7 @@ impl Html {
) {
if let Some(content) = metadata.get("content") {
if let Ok(destination) =
- Url::parse_with_base_url(self.url(), content.as_str())
+ Url::parse_with_base_url(self.base_url(), content.as_str())
{
let mut rel = location.as_rel();
@@ -464,7 +466,8 @@ impl Html {
links.extend(self.scripts().into_iter().filter_map(|script| {
match script.attributes.get("src") {
Some(url) => {
- let script_url = Url::parse_with_base_url(self.url(), url.as_str()).ok()?;
+ let script_url =
+ Url::parse_with_base_url(self.base_url(), url.as_str()).ok()?;
if script_url.root_domain() != root_domain {
let mut rel = Location::SCRIPT.as_rel();
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index e6de556c..daf0ab01 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -51,6 +51,7 @@ pub struct Html {
clean_text: Option<String>,
lang: Option<Lang>,
robots: Option<EnumSet<RobotsMeta>>,
+ base_url: Url,
}
impl Html {
@@ -70,17 +71,22 @@ impl Html {
pub fn parse_without_text(html: &str, url: &str) -> Result<Self> {
let root = kuchiki::parse_html().one(html);
- let url = Url::parse(url)?;
+ let url = Url::parse(url)?.normalize();
let mut res = Self {
root,
all_text: None,
clean_text: None,
lang: None,
+ base_url: url.clone(),
url,
robots: None,
};
+ if let Some(base_url) = res.find_base_url() {
+ res.base_url = base_url;
+ }
+
// TODO: need to figure out when to respect
// the canonical url, it's not always the case that
// the website use the tag correctly.
@@ -94,12 +100,25 @@ impl Html {
// }
// }
- res.url.normalize_in_place();
res.robots = res.parse_robots_meta();
Ok(res)
}
+ fn find_base_url(&self) -> Option<Url> {
+ self.root.select("base").unwrap().next().and_then(|node| {
+ if let Some(element) = node.as_node().as_element() {
+ if let Some(href) = element.attributes.borrow().get("href") {
+ if let Ok(url) = Url::parse_with_base_url(self.base_url(), href) {
+ return Some(url);
+ }
+ }
+ }
+
+ None
+ })
+ }
+
pub fn content_type(&self) -> Option<String> {
self.metadata()
.into_iter()
@@ -129,7 +148,7 @@ impl Html {
if let Some(rel) = element.attributes.borrow().get("rel") {
if rel == "canonical" {
if let Some(href) = element.attributes.borrow().get("href") {
- if let Ok(url) = Url::parse_with_base_url(self.url(), href) {
+ if let Ok(url) = Url::parse_with_base_url(self.base_url(), href) {
canonical_url = Some(url);
}
}
@@ -192,6 +211,10 @@ impl Html {
&self.url
}
+ fn base_url(&self) -> &Url {
+ &self.base_url
+ }
+
pub fn metadata(&self) -> Vec<Meta> {
let mut metas = Vec::new();
@@ -245,13 +268,13 @@ impl Html {
if let Some(link) = script
.attributes
.get("src")
- .and_then(|link| Url::parse_with_base_url(self.url(), link).ok())
+ .and_then(|link| Url::parse_with_base_url(self.base_url(), link).ok())
{
links.push(link);
}
for res in URL_REGEX.find_iter(&script.content) {
- if let Ok(link) = Url::parse_with_base_url(self.url(), res.as_str()) {
+ if let Ok(link) = Url::parse_with_base_url(self.base_url(), res.as_str()) {
links.push(link);
}
}
@@ -262,7 +285,7 @@ impl Html {
.attributes
.borrow()
.get("href")
- .and_then(|link| Url::parse_with_base_url(self.url(), link).ok())
+ .and_then(|link| Url::parse_with_base_url(self.base_url(), link).ok())
{
links.push(link);
}
@@ -396,7 +419,7 @@ impl Html {
.many()
.into_iter()
.filter_map(|url| url.try_into_string())
- .filter_map(|url| Url::parse_with_base_url(self.url(), &url).ok())
+ .filter_map(|url| Url::parse_with_base_url(self.base_url(), &url).ok())
})
})
.flatten()
@@ -1184,4 +1207,75 @@ mod tests {
assert!(html.likely_has_paywall());
}
+
+ #[test]
+ fn test_base_url() {
+ let html = Html::parse(
+ r##"
+ <html>
+ <head>
+ <base href="https://www.example.com/base/" />
+ </head>
+ <body>
+ <a href="link1">Link 1</a>
+ <a href="link2">Link 2</a>
+ <a href="https://www.example.com/link3">Link 3</a>
+ <a href="https://www.another.com/">Link 4</a>
+ </body>
+ </html>
+ "##,
+ "https://www.example.com/",
+ )
+ .unwrap();
+
+ assert_eq!(
+ html.base_url(),
+ &Url::parse("https://www.example.com/base/").unwrap()
+ );
+ assert_eq!(
+ html.all_links()
+ .iter()
+ .map(|l| l.destination.to_string())
+ .collect::<Vec<String>>(),
+ vec![
+ "https://www.example.com/base/link1",
+ "https://www.example.com/base/link2",
+ "https://www.example.com/link3",
+ "https://www.another.com/",
+ ]
+ );
+
+ let html = Html::parse(
+ r##"
+ <html>
+ <head>
+ <div>
+ <base href="https://www.example.com/base/" />
+ </div>
+ </head>
+ <body>
+ <a href="link1">Link 1</a>
+ <a href="link2">Link 2</a>
+ </body>
+ </html>
+ "##,
+ "https://www.example.com/",
+ )
+ .unwrap();
+
+ assert_eq!(
+ html.base_url(),
+ &Url::parse("https://www.example.com/base/").unwrap()
+ );
+ assert_eq!(
+ html.all_links()
+ .iter()
+ .map(|l| l.destination.to_string())
+ .collect::<Vec<String>>(),
+ vec![
+ "https://www.example.com/base/link1",
+ "https://www.example.com/base/link2",
+ ]
+ );
+ }
}