web search done right
skip links to/from same domain when calculating harmonic centralities
Mikkel Denkere4ae26470ec5
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex fddba1d7..1008705c 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -20,16 +20,26 @@ use std::{cmp, collections::BTreeMap};use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};use crate::{- ampc::dht::upsert,- entrypoint::ampc::approximated_harmonic_centrality::DhtTable,- kahan_sum::KahanSum,- webgraph::{self, centrality::harmonic::SKIPPED_REL},+ ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,+ kahan_sum::KahanSum, webgraph, webpage::html::links::RelFlags,};use rayon::prelude::*;const BATCH_SIZE: usize = 1024;const MAX_OUTGOING_EDGES: usize = 128;+pub static SKIPPED_REL: once_cell::sync::Lazy<RelFlags> = once_cell::sync::Lazy::new(|| {+ RelFlags::TAG+ | RelFlags::NOFOLLOW+ | RelFlags::IS_IN_FOOTER+ | RelFlags::IS_IN_NAVIGATION+ | RelFlags::PRIVACY_POLICY+ | RelFlags::TERMS_OF_SERVICE+ | RelFlags::SEARCH+ | RelFlags::LINK_TAG+ | RelFlags::SCRIPT_TAG+});+#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]pub enum ApproxCentralityMapper {InitCentrality,diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex b6678d6d..14a21e09 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -43,6 +43,7 @@ pub static SKIPPED_REL: once_cell::sync::Lazy<RelFlags> = once_cell::sync::Lazy:| RelFlags::SEARCH| RelFlags::LINK_TAG| RelFlags::SCRIPT_TAG+ | RelFlags::SAME_ICANN_DOMAIN});type Counter = BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>>;@@ -562,4 +563,24 @@ mod tests {assert!(centrality.0.values().all(|&v| v == 0.0));}++ #[test]+ fn test_same_icann_domain_ignored() {+ let mut graph = WebgraphWriter::new(+ crate::gen_temp_path(),+ crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),+ None,+ );++ for (from, to, label) in test_edges() {+ graph.insert(from, to, label, RelFlags::SAME_ICANN_DOMAIN);+ }++ let graph = graph.finalize();++ let centrality = HarmonicCentrality::calculate(&graph);++ assert!(centrality.0.values().all(|&v| v == 0.0));+ }}diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rsindex d4b31d7c..f3de8484 100644--- a/crates/core/src/webpage/html/links.rs+++ b/crates/core/src/webpage/html/links.rs@@ -121,6 +121,7 @@ bitflags! {const LINK_TAG = 1 << 18;const SCRIPT_TAG = 1 << 19;const META_TAG = 1 << 20;+ const SAME_ICANN_DOMAIN = 1 << 21;}}@@ -254,6 +255,8 @@ impl Html {let mut open_links = Vec::new();let mut location = Location::empty();+ let icann_domain = self.url().icann_domain();+for edge in self.root.traverse() {match edge {NodeEdge::Start(node) => {@@ -279,14 +282,20 @@ impl Html {if let Ok(dest) =Url::parse(dest).or_else(|_| self.url().join(dest)){+ let mut rel = RelFlags::from_html(+ &dest,+ &attributes.borrow(),+ &location,+ );++ if icann_domain == dest.icann_domain() {+ rel |= RelFlags::SAME_ICANN_DOMAIN;+ }+links.push(Link {source: self.url().clone(),text: text.trim().to_string(),- rel: RelFlags::from_html(- &dest,- &attributes.borrow(),- &location,- ),+ rel,destination: dest,});}@@ -327,10 +336,16 @@ impl Html {}if let Ok(dest) = Url::parse(dest).or_else(|_| self.url().join(dest)) {+ let mut rel = RelFlags::from_html(&dest, &attributes.borrow(), &location);++ if icann_domain == dest.icann_domain() {+ rel |= RelFlags::SAME_ICANN_DOMAIN;+ }+links.push(Link {source: self.url().clone(),- rel: RelFlags::from_html(&dest, &attributes.borrow(), &location),destination: dest,+ rel,text: text.trim().to_string(),});}@@ -344,18 +359,22 @@ impl Html {let mut links = Vec::new();let location = Location::LINK;+ let icann_domain = self.url().icann_domain();for node in self.root.select("link").unwrap() {if let Some(element) = node.as_node().as_element() {if let Some(href) = element.attributes.borrow().get("href") {if let Ok(href) = Url::parse(href).or_else(|_| self.url().join(href)) {+ let mut rel =+ RelFlags::from_html(&href, &element.attributes.borrow(), &location);++ if icann_domain == href.icann_domain() {+ rel |= RelFlags::SAME_ICANN_DOMAIN;+ }+links.push(Link {source: self.url().clone(),- rel: RelFlags::from_html(- &href,- &element.attributes.borrow(),- &location,- ),+ rel,destination: href,text: String::new(),});@@ -369,6 +388,7 @@ impl Html {fn metadata_links(&self) -> Vec<Link> {let location = Location::META;+ let icann_domain = self.url().icann_domain();self.metadata().into_iter()@@ -391,11 +411,17 @@ impl Html {if let Ok(destination) = Url::parse(content.as_str()).or_else(|_| self.url().join(content.as_str())){+ let mut rel = location.as_rel();++ if destination.icann_domain() == icann_domain {+ rel |= RelFlags::SAME_ICANN_DOMAIN;+ }+return Some(Link {source: self.url().clone(),destination,text: String::new(),- rel: location.as_rel(),+ rel,});}}@@ -416,11 +442,17 @@ impl Html {if let Ok(destination) = Url::parse(content.as_str()).or_else(|_| self.url().join(content.as_str())){+ let mut rel = location.as_rel();++ if destination.icann_domain() == icann_domain {+ rel |= RelFlags::SAME_ICANN_DOMAIN;+ }+return Some(Link {source: self.url().clone(),destination,text: String::new(),- rel: location.as_rel(),+ rel,});}}@@ -434,6 +466,8 @@ impl Html {pub fn all_links(&self) -> Vec<Link> {let mut links = self.anchor_links();+ let icann_domain = self.url().icann_domain();+ let root_domain = self.url().root_domain();links.extend(self.scripts().into_iter().filter_map(|script| {match script.attributes.get("src") {@@ -442,12 +476,18 @@ impl Html {.or_else(|_| self.url().join(url.as_str())).ok()?;- if script_url.root_domain() != self.url().root_domain() {+ if script_url.root_domain() != root_domain {+ let mut rel = Location::SCRIPT.as_rel();++ if script_url.icann_domain() == icann_domain {+ rel |= RelFlags::SAME_ICANN_DOMAIN;+ }+Some(Link {source: self.url().clone(),destination: script_url,text: String::new(),- rel: Location::SCRIPT.as_rel(),+ rel,})} else {None@@ -649,7 +689,7 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com/tags/example").unwrap(),text: "Example".to_string(),- rel: RelFlags::TAG+ rel: RelFlags::TAG | RelFlags::SAME_ICANN_DOMAIN});@@ -659,7 +699,7 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com/tags/example").unwrap(),text: "Example".to_string(),- rel: RelFlags::TAG | RelFlags::NOFOLLOW+ rel: RelFlags::TAG | RelFlags::NOFOLLOW | RelFlags::SAME_ICANN_DOMAIN});@@ -669,7 +709,7 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com/tags/example").unwrap(),text: "Example".to_string(),- rel: RelFlags::TAG | RelFlags::SPONSORED+ rel: RelFlags::TAG | RelFlags::SPONSORED | RelFlags::SAME_ICANN_DOMAIN});@@ -679,7 +719,7 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com/authors/example").unwrap(),text: "Example".to_string(),- rel: RelFlags::AUTHOR+ rel: RelFlags::AUTHOR | RelFlags::SAME_ICANN_DOMAIN});@@ -689,7 +729,9 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com/terms-of-service").unwrap(),text: "Terms of service".to_string(),- rel: RelFlags::TERMS_OF_SERVICE | RelFlags::IS_IN_FOOTER+ rel: RelFlags::TERMS_OF_SERVICE+ | RelFlags::IS_IN_FOOTER+ | RelFlags::SAME_ICANN_DOMAIN});@@ -699,7 +741,9 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com/privacy-policy").unwrap(),text: "Privacy policy".to_string(),- rel: RelFlags::PRIVACY_POLICY | RelFlags::IS_IN_FOOTER+ rel: RelFlags::PRIVACY_POLICY+ | RelFlags::IS_IN_FOOTER+ | RelFlags::SAME_ICANN_DOMAIN});}diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex 70ff4fc4..eddea515 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -530,7 +530,7 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com").unwrap(),text: "Link to example".to_string(),- rel: RelFlags::empty(),+ rel: RelFlags::SAME_ICANN_DOMAIN,}]);assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));@@ -974,7 +974,7 @@ mod tests {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com").unwrap(),text: "Link to example".to_string(),- rel: RelFlags::empty(),+ rel: RelFlags::SAME_ICANN_DOMAIN,},]);}