Skip to content

web search done right

[webgraph] dont insert links between nodes of the same root domain into host webgraph. this should remove seo where a.site.com -> b.site.com -> c.site.com etc.

Mikkel Denker53f09a649c69
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 69a6f7d5..01e20295 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -19,7 +19,7 @@ use crate::{
entrypoint::download_all_warc_files,
mapreduce::Worker,
webgraph::{self, Node, WebgraphWriter},
- webpage::Html,
+ webpage::{url_ext::UrlExt, Html},
Result,
};
use itertools::Itertools;
@@ -128,7 +128,14 @@ impl WebgraphWorker {
source = source.into_host();
destination = destination.into_host();
- self.host_graph.insert(source, destination, link.text);
+ let dest_domain = link.destination.root_domain();
+ let source_domain = link.source.root_domain();
+ if dest_domain.is_some()
+ && source_domain.is_some()
+ && dest_domain != source_domain
+ {
+ self.host_graph.insert(source, destination, link.text);
+ }
}
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 9c823f08..84e296af 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -28,6 +28,7 @@ use utoipa::ToSchema;
use crate::executor::Executor;
use crate::intmap;
+use crate::webpage::url_ext::UrlExt;
pub mod centrality;
mod store;
@@ -183,7 +184,7 @@ impl Node {
match url {
Ok(url) => {
- let host = url.host_str().unwrap_or_default().to_string();
+ let host = url.normalized_host().unwrap_or_default().to_string();
Node { name: host }
}
Err(_) => Node {
diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rs
index 60f7eb67..42f0eec0 100644
--- a/core/src/webpage/url_ext.rs
+++ b/core/src/webpage/url_ext.rs
@@ -16,11 +16,12 @@
use publicsuffix::Psl;
-static FULL_LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {
- include_str!("../../public_suffix_list.dat")
- .parse()
- .expect("Failed to parse public suffix list")
-});
+static PUBLIC_SUFFIX_LIST: once_cell::sync::Lazy<publicsuffix::List> =
+ once_cell::sync::Lazy::new(|| {
+ include_str!("../../public_suffix_list.dat")
+ .parse()
+ .expect("Failed to parse public suffix list")
+ });
static ICANN_LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {
include_str!("../../public_icann_suffix.dat")
.parse()
@@ -44,7 +45,8 @@ impl UrlExt for url::Url {
fn root_domain(&self) -> Option<&str> {
let host = self.host_str()?;
- let suffix = std::str::from_utf8(FULL_LIST.domain(host.as_bytes())?.as_bytes()).ok()?;
+ let suffix =
+ std::str::from_utf8(PUBLIC_SUFFIX_LIST.domain(host.as_bytes())?.as_bytes()).ok()?;
Some(suffix)
}