web search done right
make sure no-follow links aren't used in /explore and to score host similarity
Mikkel Denkerd53313388e21
diff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rsindex ac393c71..c4462a75 100644--- a/crates/core/src/ranking/bitvec_similarity.rs+++ b/crates/core/src/ranking/bitvec_similarity.rs@@ -186,6 +186,12 @@ impl BitVec {#[cfg(test)]mod tests {+ use crate::{+ executor::Executor,+ webgraph::{Compression, Node, WebgraphWriter},+ webpage::html::links::RelFlags,+ };+use super::*;use std::iter::repeat;@@ -284,4 +290,31 @@ mod tests {assert!((expected - sim).abs() < 0.1);}++ #[tokio::test]+ async fn test_ignores_no_follow() {+ let temp_dir = crate::gen_temp_dir().unwrap();++ let mut writer = WebgraphWriter::new(+ &temp_dir,+ Executor::single_thread(),+ Compression::default(),+ None,+ );++ let a = Node::from("A");+ let b = Node::from("B");+ let c = Node::from("C");++ writer.insert(a.clone(), b.clone(), String::new(), RelFlags::NOFOLLOW);++ writer.insert(a.clone(), c.clone(), String::new(), RelFlags::default());++ let graph = writer.finalize();++ let bitvecs = BitVec::batch_new_for(&[b.id(), c.id()], &graph).await;++ assert_eq!(bitvecs.len(), 2);+ assert_eq!(bitvecs[0].sim(&bitvecs[1]), 0.0);+ }}diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 2ee6474d..042d4ddb 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -43,6 +43,7 @@ use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpuse crate::web_spell::SpellChecker;use crate::webgraph::remote::{RemoteWebgraph, WebgraphGranularity};use crate::webgraph::EdgeLimit;+use crate::webpage::html::links::RelFlags;use crate::widgets::{Widget, Widgets};use crate::{bangs::Bangs,@@ -223,7 +224,13 @@ whereself.batch_raw_ingoing(nodes, EdgeLimit::Limit(1024)).await.into_iter()- .map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect())+ .map(|edges| {+ edges+ .into_iter()+ .filter(|edge| !edge.rel.contains(RelFlags::NOFOLLOW))+ .map(|edge| edge.from.node())+ .collect()+ }).collect()}}diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex f60322bc..fbe00cba 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -26,7 +26,7 @@ use crate::{remote::{RemoteWebgraph, WebgraphGranularity},EdgeLimit, Node, NodeID,},- webpage::url_ext::UrlExt,+ webpage::{html::links::RelFlags, url_ext::UrlExt},SortableFloat,};@@ -86,6 +86,7 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {let backlink_nodes = in_edges.iter().flatten()+ .filter(|e| !e.rel.contains(RelFlags::NOFOLLOW)).map(|e| e.from.node()).unique().collect::<Vec<_>>();@@ -99,6 +100,7 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {let potential_nodes: Vec<_> = outgoing_edges.iter().flatten()+ .filter(|e| !e.rel.contains(RelFlags::NOFOLLOW)).map(|e| e.to.node()).unique().filter(|n| !nodes.contains(n))