web search done right
remove some telemetry hosts from list of ad servers so pages aren't mistakenly tagged as containing ads (#209)
Mikkel Denkera1c9721b6cfe
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 108057a1..b818783f 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -234,6 +234,7 @@ mod tests {}#[tokio::test]+ #[allow(clippy::too_many_lines)]async fn it_ranks_search_results() {let mut wrt = WebgraphWriter::new(crate::gen_temp_path(),diff --git a/crates/core/src/webpage/adservers.txt b/crates/core/src/webpage/adservers.txtindex fd576a52..ca497bb3 100644--- a/crates/core/src/webpage/adservers.txt+++ b/crates/core/src/webpage/adservers.txt@@ -941,7 +941,6 @@ alclick.com,alenty.com,alertafterthought.com,alexa-sitestats.s3.amazonaws.com,-alexametrics.com,algorix.co,aliasanvil.com,alikeaddition.com,@@ -959,7 +958,6 @@ ambiguousquilt.com,ambitiousagreement.com,americash.com,ampleclick.com,-amung.us,ancientact.com,androiddownload.net,aniview.com,@@ -1021,7 +1019,6 @@ auctionads.com,auctionads.net,audience.media,audience2media.com,-audienceinsights.com,audit.median.hu,audit.webinform.hu,augur.io,@@ -1390,7 +1387,6 @@ clksite.com,cloisteredcord.com,cloisteredhydrant.com,closedcows.com,-cloudflareinsights.com,cloudiq.com,clrstm.com,cluster.adultworld.com,@@ -1744,7 +1740,6 @@ fast-redirecting.com,fastclick.com,fastclick.com.edgesuite.net,fastclick.net,-fastly-insights.com,faultycanvas.com,faultyfowl.com,fave.co,@@ -2002,8 +1997,6 @@ impulsehands.com,imrworldwide.com,inclk.com,incognitosearches.com,-incoming-telemetry.thunderbird.net,-incoming.telemetry.mozilla.org,indexexchange.com,indexstats.com,indexww.com,@@ -2306,7 +2299,6 @@ methodcash.com,metrics-logger.spot.im,metrics.api.drift.com,metrics.articulate.com,-metrics.cnn.com,metrics.consumerreports.org,metrics.ctv.ca,metrics.foxnews.com,@@ -3236,9 +3228,6 @@ teenyvolcano.com,teethfan.com,telaria.com,telemetrics.klaviyo.com,-telemetry.dropbox.com,-telemetry.malwarebytes.com,-telemetry.v.dropbox.com,temelio.com,tend.io,tendertest.com,@@ -3276,7 +3265,6 @@ tinybar.com,tinytendency.com,tiresomethunder.com,tkbo.com,-tls.telemetry.swe.quicinc.com,tlvmedia.com,tm.br.de,tnkexchange.com,@@ -3659,7 +3647,6 @@ zenkreka.com,zenzuu.com,zeus.developershed.com,zeusclicks.com,-zion-telemetry.api.cnn.io,zippingcare.com,zlp6s.pw,zm232.com,