Skip to content

web search done right

filter selected domains from similar sites in explore

Mikkel Denkerd01ba415862c
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.toml
index 226afc0d..6b62a4b3 100644
--- a/configs/crawler/crawler.toml
+++ b/configs/crawler/crawler.toml
@@ -6,8 +6,8 @@ router_hosts = ["0.0.0.0:8181"]
timeout_seconds = 30
[user_agent]
-full = "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)"
-token = "StractBot"
+full = "<user_agent>"
+token = "<user_agent_token>" # e.g. 'StractBot'
[s3]
access_key = "<access_key>"
diff --git a/core/src/api/explore.rs b/core/src/api/explore.rs
index 7f999e70..e59bf927 100644
--- a/core/src/api/explore.rs
+++ b/core/src/api/explore.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use axum::{extract, Json};
+use axum::extract;
use http::StatusCode;
use optics::{Optic, SiteRankings};
use utoipa::ToSchema;
@@ -39,9 +39,10 @@ pub async fn explore_export_optic(
chosen_sites,
similar_sites,
}): extract::Json<ExploreExportOpticParams>,
-) -> Result<Json<String>, StatusCode> {
+) -> Result<String, StatusCode> {
let rules = similar_sites
.into_iter()
+ .chain(chosen_sites.clone().into_iter())
.map(|site| optics::Rule {
matches: vec![optics::Matching {
pattern: vec![
@@ -65,5 +66,5 @@ pub async fn explore_export_optic(
..Default::default()
};
- Ok(Json(optic.to_string()))
+ Ok(optic.to_string())
}
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index 4b13ba84..a9f32ecb 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -16,10 +16,14 @@
use std::{cmp::Reverse, collections::BinaryHeap, sync::Arc};
+use hashbrown::HashSet;
+use url::Url;
+
use crate::{
intmap::{IntMap, IntSet},
ranking::inbound_similarity::InboundSimilarity,
webgraph::{Node, NodeID, Webgraph},
+ webpage::url_ext::UrlExt,
};
#[derive(serde::Serialize, serde::Deserialize, Debug)]
@@ -73,14 +77,26 @@ impl SimilarSitesFinder {
}
pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {
- let limit = limit.min(self.max_similar_sites);
+ const SIMILAR_DOMAINS_BUFFER: usize = 30;
+ let orig_limit = limit.min(self.max_similar_sites);
+ let limit = orig_limit + nodes.len() + SIMILAR_DOMAINS_BUFFER;
let nodes: Vec<_> = nodes
.iter()
.map(|url| Node::from(url.to_string()).into_host())
- .map(|node| node.id())
.collect();
+ let domains = nodes
+ .iter()
+ .filter_map(|node| {
+ Url::parse(&format!("http://{}", &node.name))
+ .ok()
+ .and_then(|url| url.root_domain().map(|d| d.to_string()))
+ })
+ .collect::<HashSet<_>>();
+
+ let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();
+
let scorer = self.inbound_similarity.scorer(&nodes, &[], true);
let mut backlinks: IntMap<NodeID, f64> = IntMap::new();
@@ -95,14 +111,11 @@ impl SimilarSitesFinder {
let mut top_backlink_nodes: Vec<_> = backlinks
.into_iter()
- .map(|(node, score)| (node, score))
+ .filter(|(_, score)| score.is_finite())
.collect();
- top_backlink_nodes.sort_unstable_by(|(_, score_a), (_, score_b)| {
- score_a
- .partial_cmp(score_b)
- .unwrap_or(std::cmp::Ordering::Equal)
- });
+ top_backlink_nodes
+ .sort_unstable_by(|(_, score_a), (_, score_b)| score_a.total_cmp(score_b));
top_backlink_nodes.reverse();
let mut scored_nodes = BinaryHeap::with_capacity(limit);
@@ -142,10 +155,23 @@ impl SimilarSitesFinder {
scored_nodes
.into_iter()
- .map(|ScoredNodeID { node_id, score }| {
+ .filter_map(|ScoredNodeID { node_id, score }| {
let node = self.webgraph.id2node(&node_id).unwrap();
- ScoredNode { node, score }
+ match Url::parse(&format!("http://{}", &node.name))
+ .ok()
+ .and_then(|url| url.root_domain().map(|s| s.to_string()))
+ {
+ Some(dom) => {
+ if !domains.contains(&dom) {
+ Some(ScoredNode { node, score })
+ } else {
+ None
+ }
+ }
+ None => None,
+ }
})
+ .take(orig_limit)
.collect()
}