web search done right
Some linkbuilders buy a lot of domains. Relying on root domain is insufficient
Mikkel Denkera4dbe7289ba1
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 72a9fbc2..bbadc3cf 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -113,19 +113,10 @@ impl WebgraphWorker {.anchor_links().into_iter().filter(|link| matches!(link.destination.scheme(), "http" | "https"))- .filter(|link| link.source.root_domain() != link.destination.root_domain())- .filter(|link| {- link.source.root_domain().is_some()- && link.destination.root_domain().is_some()- }){let source = link.source.clone();let destination = link.destination.clone();- if source.root_domain() == destination.root_domain() {- continue;- }-trace!("inserting link {:?}", link);let mut source = Node::from(source);