Skip to content

web search done right

respect robots meta tag and use canonical url

Mikkel Denker802bddedaed3
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index b7962351..ca065eec 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -129,6 +129,11 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
})
{
let mut html = Html::parse_without_text(&record.response.body, &record.request.url);
+
+ if html.is_no_index() {
+ continue;
+ }
+
let node = Node::from(html.url());
let node_id = worker
.host_centrality_store
diff --git a/core/src/enum_map.rs b/core/src/enum_map.rs
index a42e166c..92fa9feb 100644
--- a/core/src/enum_map.rs
+++ b/core/src/enum_map.rs
@@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct EnumMap<K: Into<usize>, V> {
inner: Vec<Option<V>>,
+ len: usize,
_phantom: std::marker::PhantomData<K>,
}
@@ -35,6 +36,7 @@ where
pub fn new() -> Self {
Self {
inner: vec![],
+ len: 0,
_phantom: std::marker::PhantomData,
}
}
@@ -46,9 +48,17 @@ where
self.inner.resize_with(key + 1, || None);
}
+ if self.inner[key].is_none() {
+ self.len += 1;
+ }
+
self.inner[key] = Some(value);
}
+ pub fn len(&self) -> usize {
+ self.len
+ }
+
pub fn get(&self, key: K) -> Option<&V> {
let key = key.into();
if key >= self.inner.len() {
@@ -87,6 +97,37 @@ where
}
}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct EnumSet<K: Into<usize>> {
+ map: EnumMap<K, ()>,
+}
+
+impl<K: Into<usize>> Default for EnumSet<K> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<K: Into<usize>> EnumSet<K> {
+ pub fn new() -> Self {
+ Self {
+ map: EnumMap::new(),
+ }
+ }
+
+ pub fn insert(&mut self, key: K) {
+ self.map.insert(key, ());
+ }
+
+ pub fn contains(&self, key: K) -> bool {
+ self.map.get(key).is_some()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.map.len() == 0
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/core/src/lib.rs b/core/src/lib.rs
index d1429827..2bc9ec04 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -410,6 +410,9 @@ pub enum Error {
#[error("S3")]
S3(#[from] s3::error::S3Error),
+
+ #[error("Unknown webpage robots meta tag")]
+ UnknownRobotsMetaTag,
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index bea44cbe..89978985 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ enum_map::EnumSet,
prehashed::{hash, split_u128},
schema::{FastField, TextField},
simhash, tokenizer,
@@ -24,7 +25,7 @@ use chrono::{DateTime, FixedOffset, Utc};
use itertools::Itertools;
use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
use regex::Regex;
-use std::{collections::HashMap, panic};
+use std::{collections::HashMap, panic, str::FromStr};
use tantivy::tokenizer::{PreTokenizedString, Tokenizer};
use whatlang::Lang;
@@ -240,6 +241,33 @@ struct Script {
content: String,
}
+#[derive(Debug)]
+enum RobotsMeta {
+ NoIndex,
+ NoFollow,
+}
+
+impl FromStr for RobotsMeta {
+ type Err = Error;
+
+ fn from_str(s: &str) -> Result<Self> {
+ match s {
+ "noindex" => Ok(RobotsMeta::NoIndex),
+ "nofollow" => Ok(RobotsMeta::NoFollow),
+ _ => Err(Error::UnknownRobotsMetaTag),
+ }
+ }
+}
+
+impl From<RobotsMeta> for usize {
+ fn from(val: RobotsMeta) -> Self {
+ match val {
+ RobotsMeta::NoIndex => 0,
+ RobotsMeta::NoFollow => 1,
+ }
+ }
+}
+
#[derive(Debug)]
pub struct Html {
url: Url,
@@ -247,6 +275,7 @@ pub struct Html {
all_text: Option<String>,
clean_text: Option<String>,
lang: Option<Lang>,
+ robots: Option<EnumSet<RobotsMeta>>,
}
impl Html {
@@ -268,13 +297,84 @@ impl Html {
let mut url: Url = url.to_string().into();
url = url.full_without_id_tags().into();
- Self {
+ let mut res = Self {
root,
all_text: None,
clean_text: None,
lang: None,
url,
+ robots: None,
+ };
+
+ if let Some(url) = res.canonical_url() {
+ res.url = url;
}
+
+ res.robots = res.parse_robots_meta();
+
+ res
+ }
+
+ fn canonical_url(&self) -> Option<Url> {
+ let mut canonical_url = None;
+
+ for node in self.root.select("link").unwrap() {
+ if let Some(element) = node.as_node().as_element() {
+ if let Some(rel) = element.attributes.borrow().get("rel") {
+ if rel == "canonical" {
+ if let Some(href) = element.attributes.borrow().get("href") {
+ let url = Url::from(href).into_absolute(self.url());
+ if url.is_valid_uri() {
+ canonical_url = Some(url);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ canonical_url
+ }
+
+ fn parse_robots_meta(&self) -> Option<EnumSet<RobotsMeta>> {
+ let mut robots = EnumSet::new();
+
+ for node in self.root.select("meta").unwrap() {
+ if let Some(element) = node.as_node().as_element() {
+ if let Some(name) = element.attributes.borrow().get("name") {
+ if name == "robots" {
+ if let Some(content) = element.attributes.borrow().get("content") {
+ for part in content.split(',') {
+ let part = part.trim();
+ if let Ok(meta) = part.parse::<RobotsMeta>() {
+ robots.insert(meta);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if robots.is_empty() {
+ None
+ } else {
+ Some(robots)
+ }
+ }
+
+ pub fn is_no_index(&self) -> bool {
+ self.robots
+ .as_ref()
+ .map(|robots| robots.contains(RobotsMeta::NoIndex))
+ .unwrap_or(false)
+ }
+
+ pub fn is_no_follow(&self) -> bool {
+ self.robots
+ .as_ref()
+ .map(|robots| robots.contains(RobotsMeta::NoFollow))
+ .unwrap_or(false)
}
pub fn parse_text(&mut self) {
@@ -298,6 +398,10 @@ impl Html {
}
pub fn anchor_links(&self) -> Vec<Link> {
+ if self.is_no_follow() {
+ return Vec::new();
+ }
+
let mut links = Vec::new();
let mut open_links = Vec::new();
@@ -320,7 +424,7 @@ impl Html {
}
links.push(Link {
- source: self.url.clone(),
+ source: self.url().clone(),
destination: Url::from(dest.to_string())
.into_absolute(self.url()),
text: text.trim().to_string(),
@@ -354,7 +458,7 @@ impl Html {
if let Some(dest) = attributes.borrow().get("href") {
links.push(Link {
- source: self.url.clone(),
+ source: self.url().clone(),
destination: Url::from(dest.to_string()).into_absolute(self.url()),
text: text.trim().to_string(),
});
@@ -371,7 +475,7 @@ impl Html {
if let Some(element) = node.as_node().as_element() {
if let Some(href) = element.attributes.borrow().get("href") {
links.push(Link {
- source: self.url.clone(),
+ source: self.url().clone(),
destination: Url::from(href).into_absolute(self.url()),
text: String::new(),
})
@@ -444,7 +548,7 @@ impl Html {
let script_url = Url::from(url.as_str()).into_absolute(self.url());
if self.url().domain() != script_url.domain() {
Some(Link {
- source: self.url.clone(),
+ source: self.url().clone(),
destination: script_url,
text: String::new(),
})
@@ -1099,7 +1203,7 @@ impl Html {
})
})
.map(|mut image| {
- image.url = image.url.into_absolute(&self.url);
+ image.url = image.url.into_absolute(self.url());
image
})
@@ -1888,23 +1992,11 @@ mod tests {
assert_eq!(
webpage.anchor_links(),
- vec![
- Link {
- source: "https://www.example.com/whatever".to_string().into(),
- destination: "https://example.com".to_string().into(),
- text: "Link to example".to_string()
- },
- // Link {
- // source: "https://www.example.com/whatever".to_string().into(),
- // destination: "test.com".to_string().into(),
- // text: String::new()
- // },
- // Link {
- // source: "https://www.example.com/whatever".to_string().into(),
- // destination: "link.com".to_string().into(),
- // text: String::new()
- // },
- ]
+ vec![Link {
+ source: "https://www.example.com/whatever".to_string().into(),
+ destination: "https://example.com".to_string().into(),
+ text: "Link to example".to_string()
+ },]
);
}
@@ -1915,4 +2007,130 @@ mod tests {
assert!(html.clean_text().is_some());
}
+
+ #[test]
+ fn canonical_url() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <link rel="canonical" href="https://example.com/canonical.html" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ );
+
+ assert_eq!(
+ html.canonical_url(),
+ Some("https://example.com/canonical.html".to_string().into())
+ );
+
+ assert_eq!(
+ html.url(),
+ &"https://example.com/canonical.html".to_string().into()
+ );
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ );
+
+ assert_eq!(html.canonical_url(), None);
+ assert_eq!(
+ html.url(),
+ &"https://www.example.com/whatever".to_string().into()
+ );
+ }
+
+ #[test]
+ fn robots_meta_tag() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noindex, nofollow" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ );
+
+ assert!(html.is_no_index());
+ assert!(html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noindex,nofollow" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ );
+
+ assert!(html.is_no_index());
+ assert!(html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noindex" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ );
+
+ assert!(html.is_no_index());
+ assert!(!html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="nofollow" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ );
+
+ assert!(!html.is_no_index());
+ assert!(html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ );
+
+ assert!(!html.is_no_index());
+ assert!(!html.is_no_follow());
+ }
}
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index 3790b995..4bfef9f2 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -20,6 +20,8 @@ use publicsuffix::Psl;
use serde::{Deserialize, Serialize};
use tracing::debug;
+use crate::{ceil_char_boundary, floor_char_boundary};
+
use super::URL_REGEX;
pub static LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {
@@ -82,6 +84,7 @@ impl Url {
start_host += 2; // skip the two '/'
}
+ let start_host = ceil_char_boundary(url, start_host);
&url[start_host..]
}
@@ -92,6 +95,7 @@ impl Url {
start_query = url.find('?').expect("The url contains atleast 1 '?'");
}
+ let start_query = ceil_char_boundary(url, start_query);
&url[..start_query]
}
@@ -103,9 +107,11 @@ impl Url {
end_site = url.find('/').expect("The url contains atleast 1 '/'");
}
+ let end_site = ceil_char_boundary(url, end_site);
let res = &url[..end_site];
if let Some(query_begin) = res.find('?') {
+ let query_begin = ceil_char_boundary(res, query_begin);
&res[..query_begin]
} else {
res
@@ -145,7 +151,9 @@ impl Url {
return "";
}
- &domain[..domain.len() - tld.len() - 1]
+ let idx = domain.len() - tld.len() - 1;
+ let idx = ceil_char_boundary(domain, idx);
+ &domain[..idx]
}
}
@@ -154,7 +162,8 @@ impl Url {
if subdomain.is_empty() || subdomain == "." {
None
} else {
- Some(&subdomain[..subdomain.len() - 1])
+ let idx = floor_char_boundary(subdomain, subdomain.len() - 1);
+ Some(&subdomain[..idx])
}
} else {
None
@@ -271,7 +280,8 @@ impl Url {
pub fn without_query(&self) -> &str {
if let Some(query_begin) = self.0.find('?') {
- &self.0[..query_begin]
+ let idx = floor_char_boundary(&self.0, query_begin);
+ &self.0[..idx]
} else {
&self.0
}
@@ -280,7 +290,8 @@ impl Url {
pub fn full_without_id_tags(&self) -> String {
let full = self.full();
if let Some(id_begin) = full.find('#') {
- full[..id_begin].to_string()
+ let idx = floor_char_boundary(&full, id_begin);
+ full[..idx].to_string()
} else {
full
}