web search done right
filter selected domains from similar sites in explore
Mikkel Denkerd01ba415862c
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.tomlindex 226afc0d..6b62a4b3 100644--- a/configs/crawler/crawler.toml+++ b/configs/crawler/crawler.toml@@ -6,8 +6,8 @@ router_hosts = ["0.0.0.0:8181"]timeout_seconds = 30[user_agent]-full = "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)"-token = "StractBot"+full = "<user_agent>"+token = "<user_agent_token>" # e.g. 'StractBot'[s3]access_key = "<access_key>"diff --git a/core/src/api/explore.rs b/core/src/api/explore.rsindex 7f999e70..e59bf927 100644--- a/core/src/api/explore.rs+++ b/core/src/api/explore.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use axum::{extract, Json};+use axum::extract;use http::StatusCode;use optics::{Optic, SiteRankings};use utoipa::ToSchema;@@ -39,9 +39,10 @@ pub async fn explore_export_optic(chosen_sites,similar_sites,}): extract::Json<ExploreExportOpticParams>,-) -> Result<Json<String>, StatusCode> {+) -> Result<String, StatusCode> {let rules = similar_sites.into_iter()+ .chain(chosen_sites.clone().into_iter()).map(|site| optics::Rule {matches: vec![optics::Matching {pattern: vec![@@ -65,5 +66,5 @@ pub async fn explore_export_optic(..Default::default()};- Ok(Json(optic.to_string()))+ Ok(optic.to_string())}diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 4b13ba84..a9f32ecb 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -16,10 +16,14 @@use std::{cmp::Reverse, collections::BinaryHeap, sync::Arc};+use hashbrown::HashSet;+use url::Url;+use crate::{intmap::{IntMap, IntSet},ranking::inbound_similarity::InboundSimilarity,webgraph::{Node, NodeID, Webgraph},+ webpage::url_ext::UrlExt,};#[derive(serde::Serialize, serde::Deserialize, Debug)]@@ -73,14 +77,26 @@ impl SimilarSitesFinder {}pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {- let limit = limit.min(self.max_similar_sites);+ const SIMILAR_DOMAINS_BUFFER: usize = 30;+ let orig_limit = limit.min(self.max_similar_sites);+ let limit = orig_limit + nodes.len() + SIMILAR_DOMAINS_BUFFER;let nodes: Vec<_> = nodes.iter().map(|url| Node::from(url.to_string()).into_host())- .map(|node| node.id()).collect();+ let domains = nodes+ .iter()+ .filter_map(|node| {+ Url::parse(&format!("http://{}", &node.name))+ .ok()+ .and_then(|url| url.root_domain().map(|d| d.to_string()))+ })+ .collect::<HashSet<_>>();++ let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();+let scorer = self.inbound_similarity.scorer(&nodes, &[], true);let mut backlinks: IntMap<NodeID, f64> = IntMap::new();@@ -95,14 +111,11 @@ impl SimilarSitesFinder {let mut top_backlink_nodes: Vec<_> = backlinks.into_iter()- .map(|(node, score)| (node, score))+ .filter(|(_, score)| score.is_finite()).collect();- top_backlink_nodes.sort_unstable_by(|(_, score_a), (_, score_b)| {- score_a- .partial_cmp(score_b)- .unwrap_or(std::cmp::Ordering::Equal)- });+ top_backlink_nodes+ .sort_unstable_by(|(_, score_a), (_, score_b)| score_a.total_cmp(score_b));top_backlink_nodes.reverse();let mut scored_nodes = BinaryHeap::with_capacity(limit);@@ -142,10 +155,23 @@ impl SimilarSitesFinder {scored_nodes.into_iter()- .map(|ScoredNodeID { node_id, score }| {+ .filter_map(|ScoredNodeID { node_id, score }| {let node = self.webgraph.id2node(&node_id).unwrap();- ScoredNode { node, score }+ match Url::parse(&format!("http://{}", &node.name))+ .ok()+ .and_then(|url| url.root_domain().map(|s| s.to_string()))+ {+ Some(dom) => {+ if !domains.contains(&dom) {+ Some(ScoredNode { node, score })+ } else {+ None+ }+ }+ None => None,+ }})+ .take(orig_limit).collect()}