Skip to content

web search done right

Some linkbuilders buy a lot of domains. Relying on root domain is insufficient

Mikkel Denkera4dbe7289ba1
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 72a9fbc2..bbadc3cf 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -113,19 +113,10 @@ impl WebgraphWorker {
.anchor_links()
.into_iter()
.filter(|link| matches!(link.destination.scheme(), "http" | "https"))
- .filter(|link| link.source.root_domain() != link.destination.root_domain())
- .filter(|link| {
- link.source.root_domain().is_some()
- && link.destination.root_domain().is_some()
- })
{
let source = link.source.clone();
let destination = link.destination.clone();
- if source.root_domain() == destination.root_domain() {
- continue;
- }
-
trace!("inserting link {:?}", link);
let mut source = Node::from(source);