Skip to content

web search done right

make sure no-follow links aren't used in /explore and to score host similarity

Mikkel Denkerd53313388e21
diff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rs
index ac393c71..c4462a75 100644
--- a/crates/core/src/ranking/bitvec_similarity.rs
+++ b/crates/core/src/ranking/bitvec_similarity.rs
@@ -186,6 +186,12 @@ impl BitVec {
#[cfg(test)]
mod tests {
+ use crate::{
+ executor::Executor,
+ webgraph::{Compression, Node, WebgraphWriter},
+ webpage::html::links::RelFlags,
+ };
+
use super::*;
use std::iter::repeat;
@@ -284,4 +290,31 @@ mod tests {
assert!((expected - sim).abs() < 0.1);
}
+
+ #[tokio::test]
+ async fn test_ignores_no_follow() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+
+ let mut writer = WebgraphWriter::new(
+ &temp_dir,
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+
+ let a = Node::from("A");
+ let b = Node::from("B");
+ let c = Node::from("C");
+
+ writer.insert(a.clone(), b.clone(), String::new(), RelFlags::NOFOLLOW);
+
+ writer.insert(a.clone(), c.clone(), String::new(), RelFlags::default());
+
+ let graph = writer.finalize();
+
+ let bitvecs = BitVec::batch_new_for(&[b.id(), c.id()], &graph).await;
+
+ assert_eq!(bitvecs.len(), 2);
+ assert_eq!(bitvecs[0].sim(&bitvecs[1]), 0.0);
+ }
}
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 2ee6474d..042d4ddb 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -43,6 +43,7 @@ use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSp
use crate::web_spell::SpellChecker;
use crate::webgraph::remote::{RemoteWebgraph, WebgraphGranularity};
use crate::webgraph::EdgeLimit;
+use crate::webpage::html::links::RelFlags;
use crate::widgets::{Widget, Widgets};
use crate::{
bangs::Bangs,
@@ -223,7 +224,13 @@ where
self.batch_raw_ingoing(nodes, EdgeLimit::Limit(1024))
.await
.into_iter()
- .map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect())
+ .map(|edges| {
+ edges
+ .into_iter()
+ .filter(|edge| !edge.rel.contains(RelFlags::NOFOLLOW))
+ .map(|edge| edge.from.node())
+ .collect()
+ })
.collect()
}
}
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index f60322bc..fbe00cba 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -26,7 +26,7 @@ use crate::{
remote::{RemoteWebgraph, WebgraphGranularity},
EdgeLimit, Node, NodeID,
},
- webpage::url_ext::UrlExt,
+ webpage::{html::links::RelFlags, url_ext::UrlExt},
SortableFloat,
};
@@ -86,6 +86,7 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
let backlink_nodes = in_edges
.iter()
.flatten()
+ .filter(|e| !e.rel.contains(RelFlags::NOFOLLOW))
.map(|e| e.from.node())
.unique()
.collect::<Vec<_>>();
@@ -99,6 +100,7 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
let potential_nodes: Vec<_> = outgoing_edges
.iter()
.flatten()
+ .filter(|e| !e.rel.contains(RelFlags::NOFOLLOW))
.map(|e| e.to.node())
.unique()
.filter(|n| !nodes.contains(n))