Skip to content

web search done right

use canonical url when it is on the same domain

Mikkel Denker7e1a8beadcb9
diff --git a/core/src/webpage/html/mod.rs b/core/src/webpage/html/mod.rs
index 3f472e67..ff2aeb4d 100644
--- a/core/src/webpage/html/mod.rs
+++ b/core/src/webpage/html/mod.rs
@@ -40,12 +40,12 @@ pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(
#[derive(Debug)]
pub struct Html {
- pub(crate) url: Url,
- pub(crate) root: NodeRef, // this is reference counted (cheap to clone)
- pub(crate) all_text: Option<String>,
- pub(crate) clean_text: Option<String>,
- pub(crate) lang: Option<Lang>,
- pub(crate) robots: Option<EnumSet<RobotsMeta>>,
+ url: Url,
+ root: NodeRef, // this is reference counted (cheap to clone)
+ all_text: Option<String>,
+ clean_text: Option<String>,
+ lang: Option<Lang>,
+ robots: Option<EnumSet<RobotsMeta>>,
}
impl Html {
@@ -92,6 +92,12 @@ impl Html {
}
}
+ if let Some(canonical) = res.canonical_url() {
+ if canonical.root_domain() == res.url.root_domain() {
+ res.url = canonical;
+ }
+ }
+
if res.url.query().unwrap_or_default().is_empty() {
res.url.set_query(None);
}
@@ -1000,6 +1006,11 @@ mod tests {
Some(Url::parse("https://example.com/canonical.html").unwrap())
);
+ assert_eq!(
+ html.url(),
+ &Url::parse("https://example.com/canonical.html").unwrap()
+ );
+
let html = Html::parse(
r#"
<html>
@@ -1018,6 +1029,30 @@ mod tests {
html.url(),
&Url::parse("https://www.example.com/whatever").unwrap()
);
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <link rel="canonical" href="https://another-example.com/canonical.html" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert_eq!(
+ html.canonical_url(),
+ Some(Url::parse("https://another-example.com/canonical.html").unwrap())
+ );
+
+ assert_eq!(
+ html.url(),
+ &Url::parse("https://www.example.com/whatever").unwrap()
+ );
}
#[test]
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 588c1e99..d8584bf7 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -21,8 +21,6 @@ use crate::{
};
use chrono::{DateTime, Utc};
-#[cfg(test)]
-use kuchiki::traits::TendrilSink;
use std::collections::HashMap;
use tantivy::{time::OffsetDateTime, TantivyDocument};
use url::Url;
@@ -60,14 +58,7 @@ pub struct Webpage {
impl Default for Webpage {
fn default() -> Self {
Self {
- html: Html {
- url: Url::parse("https://example.com").expect("Failed to parse url"),
- root: kuchiki::parse_html().one("<html></html>"),
- all_text: None,
- clean_text: None,
- lang: None,
- robots: None,
- },
+ html: Html::parse_without_text("<html></html>", "https://example.com/").unwrap(),
backlink_labels: Default::default(),
host_centrality: Default::default(),
host_centrality_rank: u64::MAX as f64,
diff --git a/core/src/webpage/region.rs b/core/src/webpage/region.rs
index ef4c10fa..f7496226 100644
--- a/core/src/webpage/region.rs
+++ b/core/src/webpage/region.rs
@@ -94,7 +94,7 @@ impl Region {
}
pub fn guess_from(webpage: &Webpage) -> Result<Self> {
- match webpage.html.lang {
+ match webpage.html.lang() {
Some(lang) => match lang {
whatlang::Lang::Eng => Ok(Region::US),
whatlang::Lang::Spa => Ok(Region::Spain),