web search done right
support filters in similar hosts
Mikkel Denker6048d9f133da
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex 0e11b7d7..dc13e085 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -46,6 +46,7 @@ pub mod host {pub struct SimilarHostsQuery {pub hosts: Vec<String>,pub top_n: usize,+ pub filters: Option<Vec<String>>,}#[derive(serde::Serialize, serde::Deserialize, IntoParams)]@@ -78,7 +79,7 @@ pub mod host {Ok(Json(state.similar_hosts- .find_similar_hosts(&hosts, params.top_n)+ .find_similar_hosts(hosts, params.top_n, params.filters.unwrap_or_default()).await.into_iter().map(|node| ScoredHost {diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex e761a420..9affdcd3 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -24,7 +24,10 @@ use url::Url;use crate::{ranking::{bitvec_similarity, inbound_similarity},webgraph::{- query::{HostBacklinksQuery, HostForwardlinksQuery, Id2NodeQuery},+ query::{+ HostBacklinksQuery, HostForwardlinksQuery, Id2NodeQuery, NotFilter, OrFilter,+ RelFlagsFilter, TextFilter,+ },remote::RemoteWebgraph,EdgeLimit, Node, NodeID, SmallEdge,},@@ -61,13 +64,17 @@ impl SimilarHostsFinder {}async fn batch_ingoing_edges(&self, nodes: &[NodeID], limit: EdgeLimit) -> Vec<Vec<SmallEdge>> {+ let queries: Vec<_> = nodes+ .iter()+ .map(|n| {+ HostBacklinksQuery::new(*n)+ .with_limit(limit)+ .filter(NotFilter::new(RelFlagsFilter::from(RelFlags::NOFOLLOW)))+ })+ .collect();+self.webgraph- .batch_search(- nodes- .iter()- .map(|n| HostBacklinksQuery::new(*n).with_limit(limit))- .collect(),- )+ .batch_search(queries).await.unwrap_or_default()}@@ -76,19 +83,39 @@ impl SimilarHostsFinder {&self,nodes: &[NodeID],limit: EdgeLimit,+ filters: &[String],) -> Vec<Vec<SmallEdge>> {+ let queries: Vec<_> = nodes+ .iter()+ .map(|n| {+ let mut query = HostForwardlinksQuery::new(*n)+ .with_limit(limit)+ .filter(NotFilter::new(RelFlagsFilter::from(RelFlags::NOFOLLOW)));++ if !filters.is_empty() {+ let mut or_filter = OrFilter::new();++ for filter in filters {+ or_filter = or_filter.or(TextFilter::new(+ filter.clone(),+ crate::webgraph::schema::ToUrl,+ ));+ }++ query = query.filter(or_filter);+ }++ query+ })+ .collect();+self.webgraph- .batch_search(- nodes- .iter()- .map(|n| HostForwardlinksQuery::new(*n).with_limit(limit))- .collect(),- )+ .batch_search(queries).await.unwrap_or_default()}- async fn potential_nodes(&self, nodes: &[NodeID]) -> Vec<NodeID> {+ async fn potential_nodes(&self, nodes: &[NodeID], filters: &[String]) -> Vec<NodeID> {let in_edges = self.batch_ingoing_edges(nodes, EdgeLimit::Limit(128)).await;let backlink_nodes = in_edges@@ -102,7 +129,7 @@ impl SimilarHostsFinder {let num_backlink_nodes = backlink_nodes.len();let outgoing_edges = self- .batch_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(512))+ .batch_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(512), filters).await;let mut counts = FnvHashMap::default();@@ -137,9 +164,14 @@ impl SimilarHostsFinder {.collect()}- async fn scored_nodes(&self, nodes: &[NodeID], limit: usize) -> Vec<(NodeID, SortableFloat)> {+ async fn scored_nodes(+ &self,+ nodes: &[NodeID],+ limit: usize,+ filters: &[String],+ ) -> Vec<(NodeID, SortableFloat)> {let mut scorer = self.scorer(nodes).await;- let potential_nodes = self.potential_nodes(nodes).await;+ let potential_nodes = self.potential_nodes(nodes, filters).await;let inbounds = bitvec_similarity::BitVec::batch_new_for(&potential_nodes, &self.webgraph).await@@ -159,7 +191,12 @@ impl SimilarHostsFinder {.collect()}- pub async fn find_similar_hosts(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {+ pub async fn find_similar_hosts(+ &self,+ nodes: Vec<String>,+ limit: usize,+ filters: Vec<String>,+ ) -> Vec<ScoredNode> {const DEAD_LINKS_BUFFER: usize = 30;let orig_limit = limit.min(self.max_similar_hosts);let limit = orig_limit + nodes.len() + DEAD_LINKS_BUFFER;@@ -181,7 +218,7 @@ impl SimilarHostsFinder {let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();- let scored_nodes = self.scored_nodes(&nodes, limit).await;+ let scored_nodes = self.scored_nodes(&nodes, limit, &filters).await;let potential_nodes = scored_nodes.iter()