web search done right
use canonical url when it is on the same domain
Mikkel Denker7e1a8beadcb9
diff --git a/core/src/webpage/html/mod.rs b/core/src/webpage/html/mod.rsindex 3f472e67..ff2aeb4d 100644--- a/core/src/webpage/html/mod.rs+++ b/core/src/webpage/html/mod.rs@@ -40,12 +40,12 @@ pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(#[derive(Debug)]pub struct Html {- pub(crate) url: Url,- pub(crate) root: NodeRef, // this is reference counted (cheap to clone)- pub(crate) all_text: Option<String>,- pub(crate) clean_text: Option<String>,- pub(crate) lang: Option<Lang>,- pub(crate) robots: Option<EnumSet<RobotsMeta>>,+ url: Url,+ root: NodeRef, // this is reference counted (cheap to clone)+ all_text: Option<String>,+ clean_text: Option<String>,+ lang: Option<Lang>,+ robots: Option<EnumSet<RobotsMeta>>,}impl Html {@@ -92,6 +92,12 @@ impl Html {}}+ if let Some(canonical) = res.canonical_url() {+ if canonical.root_domain() == res.url.root_domain() {+ res.url = canonical;+ }+ }+if res.url.query().unwrap_or_default().is_empty() {res.url.set_query(None);}@@ -1000,6 +1006,11 @@ mod tests {Some(Url::parse("https://example.com/canonical.html").unwrap()));+ assert_eq!(+ html.url(),+ &Url::parse("https://example.com/canonical.html").unwrap()+ );+let html = Html::parse(r#"<html>@@ -1018,6 +1029,30 @@ mod tests {html.url(),&Url::parse("https://www.example.com/whatever").unwrap());++ let html = Html::parse(+ r#"+ <html>+ <head>+ <link rel="canonical" href="https://another-example.com/canonical.html" />+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ )+ .unwrap();++ assert_eq!(+ html.canonical_url(),+ Some(Url::parse("https://another-example.com/canonical.html").unwrap())+ );++ assert_eq!(+ html.url(),+ &Url::parse("https://www.example.com/whatever").unwrap()+ );}#[test]diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 588c1e99..d8584bf7 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -21,8 +21,6 @@ use crate::{};use chrono::{DateTime, Utc};-#[cfg(test)]-use kuchiki::traits::TendrilSink;use std::collections::HashMap;use tantivy::{time::OffsetDateTime, TantivyDocument};use url::Url;@@ -60,14 +58,7 @@ pub struct Webpage {impl Default for Webpage {fn default() -> Self {Self {- html: Html {- url: Url::parse("https://example.com").expect("Failed to parse url"),- root: kuchiki::parse_html().one("<html></html>"),- all_text: None,- clean_text: None,- lang: None,- robots: None,- },+ html: Html::parse_without_text("<html></html>", "https://example.com/").unwrap(),backlink_labels: Default::default(),host_centrality: Default::default(),host_centrality_rank: u64::MAX as f64,diff --git a/core/src/webpage/region.rs b/core/src/webpage/region.rsindex ef4c10fa..f7496226 100644--- a/core/src/webpage/region.rs+++ b/core/src/webpage/region.rs@@ -94,7 +94,7 @@ impl Region {}pub fn guess_from(webpage: &Webpage) -> Result<Self> {- match webpage.html.lang {+ match webpage.html.lang() {Some(lang) => match lang {whatlang::Lang::Eng => Ok(Region::US),whatlang::Lang::Spa => Ok(Region::Spain),