Skip to content

web search done right

boost and exclude domains in crawl plan

Mikkel Denker7f5a4e846d57
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 6b3d2324..a6b6c42c 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -445,6 +445,12 @@ pub struct TopHostsBudgetBoostConfig {
pub reserved_budget_fraction: f64,
}
+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
+pub struct CrawlPlannerDomainBoost {
+ pub domain: String,
+ pub boost: f64,
+}
+
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlPlannerConfig {
pub host_harmonic_path: String,
@@ -457,6 +463,9 @@ pub struct CrawlPlannerConfig {
pub wander_fraction: f64,
pub top_host_fraction: f64,
+ pub excluded_domains: Option<Vec<String>>,
+ pub domain_boosts: Option<Vec<CrawlPlannerDomainBoost>>,
+
pub top_hosts_budget_boost: Option<TopHostsBudgetBoostConfig>,
pub gossip: GossipConfig,
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index 1be5e640..f1ab3029 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -20,7 +20,7 @@ use indicatif::ProgressIterator;
use itertools::Itertools;
use rayon::{prelude::*, ThreadPoolBuilder};
use std::cmp::Reverse;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::{
@@ -151,6 +151,10 @@ pub struct CrawlPlanner {
page_graph: RemoteWebgraph,
host_graph: RemoteWebgraph,
config: CrawlPlannerConfig,
+
+ excluded_domains: HashSet<Domain>,
+
+ domain_boosts: HashMap<Domain, f64>,
}
impl CrawlPlanner {
@@ -165,11 +169,27 @@ impl CrawlPlanner {
let page_graph = RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await;
let host_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Host).await;
+ let domain_boosts = config
+ .domain_boosts
+ .clone()
+ .unwrap_or_default()
+ .into_iter()
+ .map(|domain_boost| (Domain(domain_boost.domain), domain_boost.boost))
+ .collect();
+
Ok(Self {
host_centrality,
page_centrality,
page_graph,
host_graph,
+ domain_boosts,
+ excluded_domains: config
+ .excluded_domains
+ .clone()
+ .unwrap_or_default()
+ .into_iter()
+ .map(Domain)
+ .collect(),
config,
})
}
@@ -203,9 +223,11 @@ impl CrawlPlanner {
.map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())
.sum();
- let wander_budget = ((wander_budget * host_centrality) / total_host_centralities)
- .max(1.0)
- .round() as u64;
+ let wander_budget = ((wander_budget * host_centrality) / total_host_centralities).max(1.0);
+
+ let boost = *self.domain_boosts.get(&domain).unwrap_or(&1.0);
+ let wander_budget = boost * wander_budget;
+ let wander_budget = wander_budget.round() as u64;
let mut urls: Vec<_> = pages
.into_iter()
@@ -464,7 +486,7 @@ impl CrawlPlanner {
let next_queue = AtomicUsize::new(0);
grouped_pages.into_par_iter().for_each(|(domain, pages)| {
- if domain.as_str().is_empty() {
+ if domain.as_str().is_empty() || self.excluded_domains.contains(&domain) {
return;
}