web search done right
group urls by domain centrality in planner and process groups by highest centrality first
Mikkel Denkerc5a465b7d663
diff --git a/configs/crawler/planner.toml b/configs/crawler/planner.tomlindex 81af0656..a065831d 100644--- a/configs/crawler/planner.toml+++ b/configs/crawler/planner.toml@@ -1,5 +1,6 @@host_graph_path = "data/webgraph_host"host_harmonic_path = "data/centrality/harmonic"+host_centrality_rank_store_path = "data/centrality/harmonic_rank"output_path = "data/crawlplan"page_graph_path = "data/webgraph_page"page_harmonic_path = "data/centrality_page/approx_harmonic"diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 05d6b2ef..87a18e57 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -546,6 +546,7 @@ pub struct CrawlPlannerDomainBoost {pub struct CrawlPlannerConfig {pub host_harmonic_path: String,pub page_harmonic_path: String,+ pub host_centrality_rank_store_path: String,pub output_path: String,pub num_job_queues: usize,diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex de407448..4b6a2e93 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -21,7 +21,7 @@ use itertools::Itertools;use rayon::{prelude::*, ThreadPoolBuilder};use std::cmp::Reverse;use std::collections::{BTreeMap, HashMap, HashSet};-use std::hash::{Hash, Hasher};+use std::hash::Hash;use std::sync::Arc;use std::{path::Path,@@ -32,6 +32,7 @@ use url::Url;use crate::crawler::WeightedUrl;use crate::distributed::cluster::Cluster;use crate::external_sort::ExternalSorter;+use crate::log_group::HarmonicRankGroup;use crate::webgraph::remote::{Host, Page, RemoteWebgraph};use crate::webgraph::Node;use crate::webpage::url_ext::UrlExt;@@ -72,10 +73,17 @@ impl From<StoredUrl> for Url {struct UrlGrouper {groups: Vec<speedy_kv::Db<StoredUrl, ()>>,folder: std::path::PathBuf,+ harmonic_group: Arc<HarmonicRankGroup>,+ harmonic_rank: Arc<speedy_kv::Db<NodeID, u64>>,}impl UrlGrouper {- pub fn new<P>(output_path: P, num_groups: usize) -> Self+ pub fn new<P>(+ output_path: P,+ num_groups: usize,+ harmonic_group: Arc<HarmonicRankGroup>,+ harmonic_rank: Arc<speedy_kv::Db<NodeID, u64>>,+ ) -> SelfwhereP: AsRef<std::path::Path>,{@@ -88,16 +96,32 @@ impl UrlGrouper {}).collect(),folder,+ harmonic_group,+ harmonic_rank,}}fn group(&mut self, url: &StoredUrl) -> usize {let domain = url.icann_domain().unwrap_or_default();- let mut hasher = std::collections::hash_map::DefaultHasher::new();- domain.hash(&mut hasher);+ let Ok(domain_url) = Url::parse(domain) else {+ return self.groups.len() - 1;+ };++ let id = Node::from(domain_url).id();- hasher.finish() as usize % self.groups.len()+ match self.harmonic_rank.get(&id) {+ Ok(Some(rank)) => {+ let group = self.harmonic_group.group(rank) as usize;+ if group >= self.groups.len() {+ self.groups.len() - 1+ } else {+ group+ }+ }+ Ok(None) => self.groups.len() - 1,+ Err(_) => self.groups.len() - 1,+ }}pub fn insert(&mut self, url: Url) {@@ -145,12 +169,15 @@ struct Budget {}pub struct CrawlPlanner {- host_centrality: speedy_kv::Db<NodeID, f64>,- page_centrality: speedy_kv::Db<NodeID, f64>,+ host_centrality: Arc<speedy_kv::Db<NodeID, f64>>,+ host_centrality_rank: Arc<speedy_kv::Db<NodeID, u64>>,+ page_centrality: Arc<speedy_kv::Db<NodeID, f64>>,page_graph: RemoteWebgraph<Page>,host_graph: RemoteWebgraph<Host>,config: CrawlPlannerConfig,+ harmonic_group: Arc<HarmonicRankGroup>,+excluded_domains: HashSet<Domain>,domain_boosts: HashMap<Domain, f64>,@@ -159,6 +186,7 @@ pub struct CrawlPlanner {impl CrawlPlanner {pub async fn new(host_centrality: speedy_kv::Db<NodeID, f64>,+ host_centrality_rank: speedy_kv::Db<NodeID, u64>,page_centrality: speedy_kv::Db<NodeID, f64>,cluster: Arc<Cluster>,config: CrawlPlannerConfig,@@ -179,8 +207,8 @@ impl CrawlPlanner {.collect();Ok(Self {- host_centrality,- page_centrality,+ host_centrality: Arc::new(host_centrality),+ page_centrality: Arc::new(page_centrality),page_graph,host_graph,domain_boosts,@@ -191,6 +219,11 @@ impl CrawlPlanner {.into_iter().map(Domain).collect(),+ harmonic_group: Arc::new(HarmonicRankGroup::new(+ host_centrality_rank.len() as u64,+ NUM_GROUPS as u64,+ )),+ host_centrality_rank: Arc::new(host_centrality_rank),config,})}@@ -346,7 +379,12 @@ impl CrawlPlanner {async fn pages_to_crawl(&self) -> (Vec<speedy_kv::Db<StoredUrl, ()>>, u64) {let mut budgets = self.assign_budgets();- let mut grouper = UrlGrouper::new(&self.config.output_path, NUM_GROUPS);+ let mut grouper = UrlGrouper::new(+ &self.config.output_path,+ NUM_GROUPS,+ self.harmonic_group.clone(),+ self.host_centrality_rank.clone(),+ );let mut futures = FuturesOrdered::new();let num_pages = self.page_centrality.len();diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rsindex 62951fea..e2d93a49 100644--- a/crates/core/src/entrypoint/crawler.rs+++ b/crates/core/src/entrypoint/crawler.rs@@ -62,8 +62,10 @@ pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {}pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {- let page_centrality = speedy_kv::Db::open_or_create(&config.page_harmonic_path).unwrap();- let host_centrality = speedy_kv::Db::open_or_create(&config.host_harmonic_path).unwrap();+ let page_centrality = speedy_kv::Db::open_or_create(&config.page_harmonic_path)?;+ let host_centrality = speedy_kv::Db::open_or_create(&config.host_harmonic_path)?;+ let host_centrality_rank =+ speedy_kv::Db::open_or_create(&config.host_centrality_rank_store_path)?;let gossip = config.gossip.clone();let cluster = Arc::new(@@ -74,7 +76,14 @@ pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {.await?,);- let planner = CrawlPlanner::new(host_centrality, page_centrality, cluster, config).await?;+ let planner = CrawlPlanner::new(+ host_centrality,+ host_centrality_rank,+ page_centrality,+ cluster,+ config,+ )+ .await?;planner.build().await?;