Skip to content

web search done right

skip links to/from same domain when calculating harmonic centralities

Mikkel Denkere4ae26470ec5
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index fddba1d7..1008705c 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -20,16 +20,26 @@ use std::{cmp, collections::BTreeMap};
use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};
use crate::{
- ampc::dht::upsert,
- entrypoint::ampc::approximated_harmonic_centrality::DhtTable,
- kahan_sum::KahanSum,
- webgraph::{self, centrality::harmonic::SKIPPED_REL},
+ ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,
+ kahan_sum::KahanSum, webgraph, webpage::html::links::RelFlags,
};
use rayon::prelude::*;
const BATCH_SIZE: usize = 1024;
const MAX_OUTGOING_EDGES: usize = 128;
+pub static SKIPPED_REL: once_cell::sync::Lazy<RelFlags> = once_cell::sync::Lazy::new(|| {
+ RelFlags::TAG
+ | RelFlags::NOFOLLOW
+ | RelFlags::IS_IN_FOOTER
+ | RelFlags::IS_IN_NAVIGATION
+ | RelFlags::PRIVACY_POLICY
+ | RelFlags::TERMS_OF_SERVICE
+ | RelFlags::SEARCH
+ | RelFlags::LINK_TAG
+ | RelFlags::SCRIPT_TAG
+});
+
#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]
pub enum ApproxCentralityMapper {
InitCentrality,
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index b6678d6d..14a21e09 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -43,6 +43,7 @@ pub static SKIPPED_REL: once_cell::sync::Lazy<RelFlags> = once_cell::sync::Lazy:
| RelFlags::SEARCH
| RelFlags::LINK_TAG
| RelFlags::SCRIPT_TAG
+ | RelFlags::SAME_ICANN_DOMAIN
});
type Counter = BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>>;
@@ -562,4 +563,24 @@ mod tests {
assert!(centrality.0.values().all(|&v| v == 0.0));
}
+
+ #[test]
+ fn test_same_icann_domain_ignored() {
+ let mut graph = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
+ None,
+ );
+
+ for (from, to, label) in test_edges() {
+ graph.insert(from, to, label, RelFlags::SAME_ICANN_DOMAIN);
+ }
+
+ let graph = graph.finalize();
+
+ let centrality = HarmonicCentrality::calculate(&graph);
+
+ assert!(centrality.0.values().all(|&v| v == 0.0));
+ }
}
diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rs
index d4b31d7c..f3de8484 100644
--- a/crates/core/src/webpage/html/links.rs
+++ b/crates/core/src/webpage/html/links.rs
@@ -121,6 +121,7 @@ bitflags! {
const LINK_TAG = 1 << 18;
const SCRIPT_TAG = 1 << 19;
const META_TAG = 1 << 20;
+ const SAME_ICANN_DOMAIN = 1 << 21;
}
}
@@ -254,6 +255,8 @@ impl Html {
let mut open_links = Vec::new();
let mut location = Location::empty();
+ let icann_domain = self.url().icann_domain();
+
for edge in self.root.traverse() {
match edge {
NodeEdge::Start(node) => {
@@ -279,14 +282,20 @@ impl Html {
if let Ok(dest) =
Url::parse(dest).or_else(|_| self.url().join(dest))
{
+ let mut rel = RelFlags::from_html(
+ &dest,
+ &attributes.borrow(),
+ &location,
+ );
+
+ if icann_domain == dest.icann_domain() {
+ rel |= RelFlags::SAME_ICANN_DOMAIN;
+ }
+
links.push(Link {
source: self.url().clone(),
text: text.trim().to_string(),
- rel: RelFlags::from_html(
- &dest,
- &attributes.borrow(),
- &location,
- ),
+ rel,
destination: dest,
});
}
@@ -327,10 +336,16 @@ impl Html {
}
if let Ok(dest) = Url::parse(dest).or_else(|_| self.url().join(dest)) {
+ let mut rel = RelFlags::from_html(&dest, &attributes.borrow(), &location);
+
+ if icann_domain == dest.icann_domain() {
+ rel |= RelFlags::SAME_ICANN_DOMAIN;
+ }
+
links.push(Link {
source: self.url().clone(),
- rel: RelFlags::from_html(&dest, &attributes.borrow(), &location),
destination: dest,
+ rel,
text: text.trim().to_string(),
});
}
@@ -344,18 +359,22 @@ impl Html {
let mut links = Vec::new();
let location = Location::LINK;
+ let icann_domain = self.url().icann_domain();
for node in self.root.select("link").unwrap() {
if let Some(element) = node.as_node().as_element() {
if let Some(href) = element.attributes.borrow().get("href") {
if let Ok(href) = Url::parse(href).or_else(|_| self.url().join(href)) {
+ let mut rel =
+ RelFlags::from_html(&href, &element.attributes.borrow(), &location);
+
+ if icann_domain == href.icann_domain() {
+ rel |= RelFlags::SAME_ICANN_DOMAIN;
+ }
+
links.push(Link {
source: self.url().clone(),
- rel: RelFlags::from_html(
- &href,
- &element.attributes.borrow(),
- &location,
- ),
+ rel,
destination: href,
text: String::new(),
});
@@ -369,6 +388,7 @@ impl Html {
fn metadata_links(&self) -> Vec<Link> {
let location = Location::META;
+ let icann_domain = self.url().icann_domain();
self.metadata()
.into_iter()
@@ -391,11 +411,17 @@ impl Html {
if let Ok(destination) = Url::parse(content.as_str())
.or_else(|_| self.url().join(content.as_str()))
{
+ let mut rel = location.as_rel();
+
+ if destination.icann_domain() == icann_domain {
+ rel |= RelFlags::SAME_ICANN_DOMAIN;
+ }
+
return Some(Link {
source: self.url().clone(),
destination,
text: String::new(),
- rel: location.as_rel(),
+ rel,
});
}
}
@@ -416,11 +442,17 @@ impl Html {
if let Ok(destination) = Url::parse(content.as_str())
.or_else(|_| self.url().join(content.as_str()))
{
+ let mut rel = location.as_rel();
+
+ if destination.icann_domain() == icann_domain {
+ rel |= RelFlags::SAME_ICANN_DOMAIN;
+ }
+
return Some(Link {
source: self.url().clone(),
destination,
text: String::new(),
- rel: location.as_rel(),
+ rel,
});
}
}
@@ -434,6 +466,8 @@ impl Html {
pub fn all_links(&self) -> Vec<Link> {
let mut links = self.anchor_links();
+ let icann_domain = self.url().icann_domain();
+ let root_domain = self.url().root_domain();
links.extend(self.scripts().into_iter().filter_map(|script| {
match script.attributes.get("src") {
@@ -442,12 +476,18 @@ impl Html {
.or_else(|_| self.url().join(url.as_str()))
.ok()?;
- if script_url.root_domain() != self.url().root_domain() {
+ if script_url.root_domain() != root_domain {
+ let mut rel = Location::SCRIPT.as_rel();
+
+ if script_url.icann_domain() == icann_domain {
+ rel |= RelFlags::SAME_ICANN_DOMAIN;
+ }
+
Some(Link {
source: self.url().clone(),
destination: script_url,
text: String::new(),
- rel: Location::SCRIPT.as_rel(),
+ rel,
})
} else {
None
@@ -649,7 +689,7 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com/tags/example").unwrap(),
text: "Example".to_string(),
- rel: RelFlags::TAG
+ rel: RelFlags::TAG | RelFlags::SAME_ICANN_DOMAIN
}
);
@@ -659,7 +699,7 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com/tags/example").unwrap(),
text: "Example".to_string(),
- rel: RelFlags::TAG | RelFlags::NOFOLLOW
+ rel: RelFlags::TAG | RelFlags::NOFOLLOW | RelFlags::SAME_ICANN_DOMAIN
}
);
@@ -669,7 +709,7 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com/tags/example").unwrap(),
text: "Example".to_string(),
- rel: RelFlags::TAG | RelFlags::SPONSORED
+ rel: RelFlags::TAG | RelFlags::SPONSORED | RelFlags::SAME_ICANN_DOMAIN
}
);
@@ -679,7 +719,7 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com/authors/example").unwrap(),
text: "Example".to_string(),
- rel: RelFlags::AUTHOR
+ rel: RelFlags::AUTHOR | RelFlags::SAME_ICANN_DOMAIN
}
);
@@ -689,7 +729,9 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com/terms-of-service").unwrap(),
text: "Terms of service".to_string(),
- rel: RelFlags::TERMS_OF_SERVICE | RelFlags::IS_IN_FOOTER
+ rel: RelFlags::TERMS_OF_SERVICE
+ | RelFlags::IS_IN_FOOTER
+ | RelFlags::SAME_ICANN_DOMAIN
}
);
@@ -699,7 +741,9 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com/privacy-policy").unwrap(),
text: "Privacy policy".to_string(),
- rel: RelFlags::PRIVACY_POLICY | RelFlags::IS_IN_FOOTER
+ rel: RelFlags::PRIVACY_POLICY
+ | RelFlags::IS_IN_FOOTER
+ | RelFlags::SAME_ICANN_DOMAIN
}
);
}
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 70ff4fc4..eddea515 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -530,7 +530,7 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com").unwrap(),
text: "Link to example".to_string(),
- rel: RelFlags::empty(),
+ rel: RelFlags::SAME_ICANN_DOMAIN,
}]
);
assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
@@ -974,7 +974,7 @@ mod tests {
source: Url::parse("https://www.example.com/whatever").unwrap(),
destination: Url::parse("https://example.com").unwrap(),
text: "Link to example".to_string(),
- rel: RelFlags::empty(),
+ rel: RelFlags::SAME_ICANN_DOMAIN,
},]
);
}