web search done right
boost and exclude domains in crawl plan
Mikkel Denker7f5a4e846d57
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 6b3d2324..a6b6c42c 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -445,6 +445,12 @@ pub struct TopHostsBudgetBoostConfig {pub reserved_budget_fraction: f64,}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]+pub struct CrawlPlannerDomainBoost {+ pub domain: String,+ pub boost: f64,+}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlPlannerConfig {pub host_harmonic_path: String,@@ -457,6 +463,9 @@ pub struct CrawlPlannerConfig {pub wander_fraction: f64,pub top_host_fraction: f64,+ pub excluded_domains: Option<Vec<String>>,+ pub domain_boosts: Option<Vec<CrawlPlannerDomainBoost>>,+pub top_hosts_budget_boost: Option<TopHostsBudgetBoostConfig>,pub gossip: GossipConfig,diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 1be5e640..f1ab3029 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -20,7 +20,7 @@ use indicatif::ProgressIterator;use itertools::Itertools;use rayon::{prelude::*, ThreadPoolBuilder};use std::cmp::Reverse;-use std::collections::BTreeMap;+use std::collections::{BTreeMap, HashMap, HashSet};use std::hash::{Hash, Hasher};use std::sync::Arc;use std::{@@ -151,6 +151,10 @@ pub struct CrawlPlanner {page_graph: RemoteWebgraph,host_graph: RemoteWebgraph,config: CrawlPlannerConfig,++ excluded_domains: HashSet<Domain>,++ domain_boosts: HashMap<Domain, f64>,}impl CrawlPlanner {@@ -165,11 +169,27 @@ impl CrawlPlanner {let page_graph = RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await;let host_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Host).await;+ let domain_boosts = config+ .domain_boosts+ .clone()+ .unwrap_or_default()+ .into_iter()+ .map(|domain_boost| (Domain(domain_boost.domain), domain_boost.boost))+ .collect();+Ok(Self {host_centrality,page_centrality,page_graph,host_graph,+ domain_boosts,+ excluded_domains: config+ .excluded_domains+ .clone()+ .unwrap_or_default()+ .into_iter()+ .map(Domain)+ .collect(),config,})}@@ -203,9 +223,11 @@ impl CrawlPlanner {.map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default()).sum();- let wander_budget = ((wander_budget * host_centrality) / total_host_centralities)- .max(1.0)- .round() as u64;+ let wander_budget = ((wander_budget * host_centrality) / total_host_centralities).max(1.0);++ let boost = *self.domain_boosts.get(&domain).unwrap_or(&1.0);+ let wander_budget = boost * wander_budget;+ let wander_budget = wander_budget.round() as u64;let mut urls: Vec<_> = pages.into_iter()@@ -464,7 +486,7 @@ impl CrawlPlanner {let next_queue = AtomicUsize::new(0);grouped_pages.into_par_iter().for_each(|(domain, pages)| {- if domain.as_str().is_empty() {+ if domain.as_str().is_empty() || self.excluded_domains.contains(&domain) {return;}