web search done right
respect robots meta tag and use canonical url
Mikkel Denker802bddedaed3
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex b7962351..ca065eec 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -129,6 +129,11 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}){let mut html = Html::parse_without_text(&record.response.body, &record.request.url);++ if html.is_no_index() {+ continue;+ }+let node = Node::from(html.url());let node_id = worker.host_centrality_storediff --git a/core/src/enum_map.rs b/core/src/enum_map.rsindex a42e166c..92fa9feb 100644--- a/core/src/enum_map.rs+++ b/core/src/enum_map.rs@@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize};#[derive(Clone, Debug, Serialize, Deserialize)]pub struct EnumMap<K: Into<usize>, V> {inner: Vec<Option<V>>,+ len: usize,_phantom: std::marker::PhantomData<K>,}@@ -35,6 +36,7 @@ wherepub fn new() -> Self {Self {inner: vec![],+ len: 0,_phantom: std::marker::PhantomData,}}@@ -46,9 +48,17 @@ whereself.inner.resize_with(key + 1, || None);}+ if self.inner[key].is_none() {+ self.len += 1;+ }+self.inner[key] = Some(value);}+ pub fn len(&self) -> usize {+ self.len+ }+pub fn get(&self, key: K) -> Option<&V> {let key = key.into();if key >= self.inner.len() {@@ -87,6 +97,37 @@ where}}+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct EnumSet<K: Into<usize>> {+ map: EnumMap<K, ()>,+}++impl<K: Into<usize>> Default for EnumSet<K> {+ fn default() -> Self {+ Self::new()+ }+}++impl<K: Into<usize>> EnumSet<K> {+ pub fn new() -> Self {+ Self {+ map: EnumMap::new(),+ }+ }++ pub fn insert(&mut self, key: K) {+ self.map.insert(key, ());+ }++ pub fn contains(&self, key: K) -> bool {+ self.map.get(key).is_some()+ }++ pub fn is_empty(&self) -> bool {+ self.map.len() == 0+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/core/src/lib.rs b/core/src/lib.rsindex d1429827..2bc9ec04 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -410,6 +410,9 @@ pub enum Error {#[error("S3")]S3(#[from] s3::error::S3Error),++ #[error("Unknown webpage robots meta tag")]+ UnknownRobotsMetaTag,}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex bea44cbe..89978985 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ enum_map::EnumSet,prehashed::{hash, split_u128},schema::{FastField, TextField},simhash, tokenizer,@@ -24,7 +25,7 @@ use chrono::{DateTime, FixedOffset, Utc};use itertools::Itertools;use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};use regex::Regex;-use std::{collections::HashMap, panic};+use std::{collections::HashMap, panic, str::FromStr};use tantivy::tokenizer::{PreTokenizedString, Tokenizer};use whatlang::Lang;@@ -240,6 +241,33 @@ struct Script {content: String,}+#[derive(Debug)]+enum RobotsMeta {+ NoIndex,+ NoFollow,+}++impl FromStr for RobotsMeta {+ type Err = Error;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ "noindex" => Ok(RobotsMeta::NoIndex),+ "nofollow" => Ok(RobotsMeta::NoFollow),+ _ => Err(Error::UnknownRobotsMetaTag),+ }+ }+}++impl From<RobotsMeta> for usize {+ fn from(val: RobotsMeta) -> Self {+ match val {+ RobotsMeta::NoIndex => 0,+ RobotsMeta::NoFollow => 1,+ }+ }+}+#[derive(Debug)]pub struct Html {url: Url,@@ -247,6 +275,7 @@ pub struct Html {all_text: Option<String>,clean_text: Option<String>,lang: Option<Lang>,+ robots: Option<EnumSet<RobotsMeta>>,}impl Html {@@ -268,13 +297,84 @@ impl Html {let mut url: Url = url.to_string().into();url = url.full_without_id_tags().into();- Self {+ let mut res = Self {root,all_text: None,clean_text: None,lang: None,url,+ robots: None,+ };++ if let Some(url) = res.canonical_url() {+ res.url = url;}++ res.robots = res.parse_robots_meta();++ res+ }++ fn canonical_url(&self) -> Option<Url> {+ let mut canonical_url = None;++ for node in self.root.select("link").unwrap() {+ if let Some(element) = node.as_node().as_element() {+ if let Some(rel) = element.attributes.borrow().get("rel") {+ if rel == "canonical" {+ if let Some(href) = element.attributes.borrow().get("href") {+ let url = Url::from(href).into_absolute(self.url());+ if url.is_valid_uri() {+ canonical_url = Some(url);+ }+ }+ }+ }+ }+ }++ canonical_url+ }++ fn parse_robots_meta(&self) -> Option<EnumSet<RobotsMeta>> {+ let mut robots = EnumSet::new();++ for node in self.root.select("meta").unwrap() {+ if let Some(element) = node.as_node().as_element() {+ if let Some(name) = element.attributes.borrow().get("name") {+ if name == "robots" {+ if let Some(content) = element.attributes.borrow().get("content") {+ for part in content.split(',') {+ let part = part.trim();+ if let Ok(meta) = part.parse::<RobotsMeta>() {+ robots.insert(meta);+ }+ }+ }+ }+ }+ }+ }++ if robots.is_empty() {+ None+ } else {+ Some(robots)+ }+ }++ pub fn is_no_index(&self) -> bool {+ self.robots+ .as_ref()+ .map(|robots| robots.contains(RobotsMeta::NoIndex))+ .unwrap_or(false)+ }++ pub fn is_no_follow(&self) -> bool {+ self.robots+ .as_ref()+ .map(|robots| robots.contains(RobotsMeta::NoFollow))+ .unwrap_or(false)}pub fn parse_text(&mut self) {@@ -298,6 +398,10 @@ impl Html {}pub fn anchor_links(&self) -> Vec<Link> {+ if self.is_no_follow() {+ return Vec::new();+ }+let mut links = Vec::new();let mut open_links = Vec::new();@@ -320,7 +424,7 @@ impl Html {}links.push(Link {- source: self.url.clone(),+ source: self.url().clone(),destination: Url::from(dest.to_string()).into_absolute(self.url()),text: text.trim().to_string(),@@ -354,7 +458,7 @@ impl Html {if let Some(dest) = attributes.borrow().get("href") {links.push(Link {- source: self.url.clone(),+ source: self.url().clone(),destination: Url::from(dest.to_string()).into_absolute(self.url()),text: text.trim().to_string(),});@@ -371,7 +475,7 @@ impl Html {if let Some(element) = node.as_node().as_element() {if let Some(href) = element.attributes.borrow().get("href") {links.push(Link {- source: self.url.clone(),+ source: self.url().clone(),destination: Url::from(href).into_absolute(self.url()),text: String::new(),})@@ -444,7 +548,7 @@ impl Html {let script_url = Url::from(url.as_str()).into_absolute(self.url());if self.url().domain() != script_url.domain() {Some(Link {- source: self.url.clone(),+ source: self.url().clone(),destination: script_url,text: String::new(),})@@ -1099,7 +1203,7 @@ impl Html {})}).map(|mut image| {- image.url = image.url.into_absolute(&self.url);+ image.url = image.url.into_absolute(self.url());image})@@ -1888,23 +1992,11 @@ mod tests {assert_eq!(webpage.anchor_links(),- vec![- Link {- source: "https://www.example.com/whatever".to_string().into(),- destination: "https://example.com".to_string().into(),- text: "Link to example".to_string()- },- // Link {- // source: "https://www.example.com/whatever".to_string().into(),- // destination: "test.com".to_string().into(),- // text: String::new()- // },- // Link {- // source: "https://www.example.com/whatever".to_string().into(),- // destination: "link.com".to_string().into(),- // text: String::new()- // },- ]+ vec![Link {+ source: "https://www.example.com/whatever".to_string().into(),+ destination: "https://example.com".to_string().into(),+ text: "Link to example".to_string()+ },]);}@@ -1915,4 +2007,130 @@ mod tests {assert!(html.clean_text().is_some());}++ #[test]+ fn canonical_url() {+ let html = Html::parse(+ r#"+ <html>+ <head>+ <link rel="canonical" href="https://example.com/canonical.html" />+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ );++ assert_eq!(+ html.canonical_url(),+ Some("https://example.com/canonical.html".to_string().into())+ );++ assert_eq!(+ html.url(),+ &"https://example.com/canonical.html".to_string().into()+ );++ let html = Html::parse(+ r#"+ <html>+ <head>+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ );++ assert_eq!(html.canonical_url(), None);+ assert_eq!(+ html.url(),+ &"https://www.example.com/whatever".to_string().into()+ );+ }++ #[test]+ fn robots_meta_tag() {+ let html = Html::parse(+ r#"+ <html>+ <head>+ <meta name="robots" content="noindex, nofollow" />+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ );++ assert!(html.is_no_index());+ assert!(html.is_no_follow());++ let html = Html::parse(+ r#"+ <html>+ <head>+ <meta name="robots" content="noindex,nofollow" />+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ );++ assert!(html.is_no_index());+ assert!(html.is_no_follow());++ let html = Html::parse(+ r#"+ <html>+ <head>+ <meta name="robots" content="noindex" />+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ );++ assert!(html.is_no_index());+ assert!(!html.is_no_follow());++ let html = Html::parse(+ r#"+ <html>+ <head>+ <meta name="robots" content="nofollow" />+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ );++ assert!(!html.is_no_index());+ assert!(html.is_no_follow());++ let html = Html::parse(+ r#"+ <html>+ <head>+ </head>+ <body>+ </body>+ </html>+ "#,+ "https://www.example.com/whatever",+ );++ assert!(!html.is_no_index());+ assert!(!html.is_no_follow());+ }}diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rsindex 3790b995..4bfef9f2 100644--- a/core/src/webpage/url.rs+++ b/core/src/webpage/url.rs@@ -20,6 +20,8 @@ use publicsuffix::Psl;use serde::{Deserialize, Serialize};use tracing::debug;+use crate::{ceil_char_boundary, floor_char_boundary};+use super::URL_REGEX;pub static LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {@@ -82,6 +84,7 @@ impl Url {start_host += 2; // skip the two '/'}+ let start_host = ceil_char_boundary(url, start_host);&url[start_host..]}@@ -92,6 +95,7 @@ impl Url {start_query = url.find('?').expect("The url contains atleast 1 '?'");}+ let start_query = ceil_char_boundary(url, start_query);&url[..start_query]}@@ -103,9 +107,11 @@ impl Url {end_site = url.find('/').expect("The url contains atleast 1 '/'");}+ let end_site = ceil_char_boundary(url, end_site);let res = &url[..end_site];if let Some(query_begin) = res.find('?') {+ let query_begin = ceil_char_boundary(res, query_begin);&res[..query_begin]} else {res@@ -145,7 +151,9 @@ impl Url {return "";}- &domain[..domain.len() - tld.len() - 1]+ let idx = domain.len() - tld.len() - 1;+ let idx = ceil_char_boundary(domain, idx);+ &domain[..idx]}}@@ -154,7 +162,8 @@ impl Url {if subdomain.is_empty() || subdomain == "." {None} else {- Some(&subdomain[..subdomain.len() - 1])+ let idx = floor_char_boundary(subdomain, subdomain.len() - 1);+ Some(&subdomain[..idx])}} else {None@@ -271,7 +280,8 @@ impl Url {pub fn without_query(&self) -> &str {if let Some(query_begin) = self.0.find('?') {- &self.0[..query_begin]+ let idx = floor_char_boundary(&self.0, query_begin);+ &self.0[..idx]} else {&self.0}@@ -280,7 +290,8 @@ impl Url {pub fn full_without_id_tags(&self) -> String {let full = self.full();if let Some(id_begin) = full.find('#') {- full[..id_begin].to_string()+ let idx = floor_char_boundary(&full, id_begin);+ full[..idx].to_string()} else {full}