web search done right
[webgraph] dont insert links between nodes of the same root domain into host webgraph. this should remove seo where a.site.com -> b.site.com -> c.site.com etc.
Mikkel Denker53f09a649c69
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 69a6f7d5..01e20295 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -19,7 +19,7 @@ use crate::{entrypoint::download_all_warc_files,mapreduce::Worker,webgraph::{self, Node, WebgraphWriter},- webpage::Html,+ webpage::{url_ext::UrlExt, Html},Result,};use itertools::Itertools;@@ -128,7 +128,14 @@ impl WebgraphWorker {source = source.into_host();destination = destination.into_host();- self.host_graph.insert(source, destination, link.text);+ let dest_domain = link.destination.root_domain();+ let source_domain = link.source.root_domain();+ if dest_domain.is_some()+ && source_domain.is_some()+ && dest_domain != source_domain+ {+ self.host_graph.insert(source, destination, link.text);+ }}}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 9c823f08..84e296af 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -28,6 +28,7 @@ use utoipa::ToSchema;use crate::executor::Executor;use crate::intmap;+use crate::webpage::url_ext::UrlExt;pub mod centrality;mod store;@@ -183,7 +184,7 @@ impl Node {match url {Ok(url) => {- let host = url.host_str().unwrap_or_default().to_string();+ let host = url.normalized_host().unwrap_or_default().to_string();Node { name: host }}Err(_) => Node {diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rsindex 60f7eb67..42f0eec0 100644--- a/core/src/webpage/url_ext.rs+++ b/core/src/webpage/url_ext.rs@@ -16,11 +16,12 @@use publicsuffix::Psl;-static FULL_LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {- include_str!("../../public_suffix_list.dat")- .parse()- .expect("Failed to parse public suffix list")-});+static PUBLIC_SUFFIX_LIST: once_cell::sync::Lazy<publicsuffix::List> =+ once_cell::sync::Lazy::new(|| {+ include_str!("../../public_suffix_list.dat")+ .parse()+ .expect("Failed to parse public suffix list")+ });static ICANN_LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {include_str!("../../public_icann_suffix.dat").parse()@@ -44,7 +45,8 @@ impl UrlExt for url::Url {fn root_domain(&self) -> Option<&str> {let host = self.host_str()?;- let suffix = std::str::from_utf8(FULL_LIST.domain(host.as_bytes())?.as_bytes()).ok()?;+ let suffix =+ std::str::from_utf8(PUBLIC_SUFFIX_LIST.domain(host.as_bytes())?.as_bytes()).ok()?;Some(suffix)}