Skip to content

web search done right

group urls by domain centrality in planner and process groups by highest centrality first

Mikkel Denkerc5a465b7d663
diff --git a/configs/crawler/planner.toml b/configs/crawler/planner.toml
index 81af0656..a065831d 100644
--- a/configs/crawler/planner.toml
+++ b/configs/crawler/planner.toml
@@ -1,5 +1,6 @@
host_graph_path = "data/webgraph_host"
host_harmonic_path = "data/centrality/harmonic"
+host_centrality_rank_store_path = "data/centrality/harmonic_rank"
output_path = "data/crawlplan"
page_graph_path = "data/webgraph_page"
page_harmonic_path = "data/centrality_page/approx_harmonic"
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 05d6b2ef..87a18e57 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -546,6 +546,7 @@ pub struct CrawlPlannerDomainBoost {
pub struct CrawlPlannerConfig {
pub host_harmonic_path: String,
pub page_harmonic_path: String,
+ pub host_centrality_rank_store_path: String,
pub output_path: String,
pub num_job_queues: usize,
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index de407448..4b6a2e93 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -21,7 +21,7 @@ use itertools::Itertools;
use rayon::{prelude::*, ThreadPoolBuilder};
use std::cmp::Reverse;
use std::collections::{BTreeMap, HashMap, HashSet};
-use std::hash::{Hash, Hasher};
+use std::hash::Hash;
use std::sync::Arc;
use std::{
path::Path,
@@ -32,6 +32,7 @@ use url::Url;
use crate::crawler::WeightedUrl;
use crate::distributed::cluster::Cluster;
use crate::external_sort::ExternalSorter;
+use crate::log_group::HarmonicRankGroup;
use crate::webgraph::remote::{Host, Page, RemoteWebgraph};
use crate::webgraph::Node;
use crate::webpage::url_ext::UrlExt;
@@ -72,10 +73,17 @@ impl From<StoredUrl> for Url {
struct UrlGrouper {
groups: Vec<speedy_kv::Db<StoredUrl, ()>>,
folder: std::path::PathBuf,
+ harmonic_group: Arc<HarmonicRankGroup>,
+ harmonic_rank: Arc<speedy_kv::Db<NodeID, u64>>,
}
impl UrlGrouper {
- pub fn new<P>(output_path: P, num_groups: usize) -> Self
+ pub fn new<P>(
+ output_path: P,
+ num_groups: usize,
+ harmonic_group: Arc<HarmonicRankGroup>,
+ harmonic_rank: Arc<speedy_kv::Db<NodeID, u64>>,
+ ) -> Self
where
P: AsRef<std::path::Path>,
{
@@ -88,16 +96,32 @@ impl UrlGrouper {
})
.collect(),
folder,
+ harmonic_group,
+ harmonic_rank,
}
}
fn group(&mut self, url: &StoredUrl) -> usize {
let domain = url.icann_domain().unwrap_or_default();
- let mut hasher = std::collections::hash_map::DefaultHasher::new();
- domain.hash(&mut hasher);
+ let Ok(domain_url) = Url::parse(domain) else {
+ return self.groups.len() - 1;
+ };
+
+ let id = Node::from(domain_url).id();
- hasher.finish() as usize % self.groups.len()
+ match self.harmonic_rank.get(&id) {
+ Ok(Some(rank)) => {
+ let group = self.harmonic_group.group(rank) as usize;
+ if group >= self.groups.len() {
+ self.groups.len() - 1
+ } else {
+ group
+ }
+ }
+ Ok(None) => self.groups.len() - 1,
+ Err(_) => self.groups.len() - 1,
+ }
}
pub fn insert(&mut self, url: Url) {
@@ -145,12 +169,15 @@ struct Budget {
}
pub struct CrawlPlanner {
- host_centrality: speedy_kv::Db<NodeID, f64>,
- page_centrality: speedy_kv::Db<NodeID, f64>,
+ host_centrality: Arc<speedy_kv::Db<NodeID, f64>>,
+ host_centrality_rank: Arc<speedy_kv::Db<NodeID, u64>>,
+ page_centrality: Arc<speedy_kv::Db<NodeID, f64>>,
page_graph: RemoteWebgraph<Page>,
host_graph: RemoteWebgraph<Host>,
config: CrawlPlannerConfig,
+ harmonic_group: Arc<HarmonicRankGroup>,
+
excluded_domains: HashSet<Domain>,
domain_boosts: HashMap<Domain, f64>,
@@ -159,6 +186,7 @@ pub struct CrawlPlanner {
impl CrawlPlanner {
pub async fn new(
host_centrality: speedy_kv::Db<NodeID, f64>,
+ host_centrality_rank: speedy_kv::Db<NodeID, u64>,
page_centrality: speedy_kv::Db<NodeID, f64>,
cluster: Arc<Cluster>,
config: CrawlPlannerConfig,
@@ -179,8 +207,8 @@ impl CrawlPlanner {
.collect();
Ok(Self {
- host_centrality,
- page_centrality,
+ host_centrality: Arc::new(host_centrality),
+ page_centrality: Arc::new(page_centrality),
page_graph,
host_graph,
domain_boosts,
@@ -191,6 +219,11 @@ impl CrawlPlanner {
.into_iter()
.map(Domain)
.collect(),
+ harmonic_group: Arc::new(HarmonicRankGroup::new(
+ host_centrality_rank.len() as u64,
+ NUM_GROUPS as u64,
+ )),
+ host_centrality_rank: Arc::new(host_centrality_rank),
config,
})
}
@@ -346,7 +379,12 @@ impl CrawlPlanner {
async fn pages_to_crawl(&self) -> (Vec<speedy_kv::Db<StoredUrl, ()>>, u64) {
let mut budgets = self.assign_budgets();
- let mut grouper = UrlGrouper::new(&self.config.output_path, NUM_GROUPS);
+ let mut grouper = UrlGrouper::new(
+ &self.config.output_path,
+ NUM_GROUPS,
+ self.harmonic_group.clone(),
+ self.host_centrality_rank.clone(),
+ );
let mut futures = FuturesOrdered::new();
let num_pages = self.page_centrality.len();
diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rs
index 62951fea..e2d93a49 100644
--- a/crates/core/src/entrypoint/crawler.rs
+++ b/crates/core/src/entrypoint/crawler.rs
@@ -62,8 +62,10 @@ pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {
}
pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {
- let page_centrality = speedy_kv::Db::open_or_create(&config.page_harmonic_path).unwrap();
- let host_centrality = speedy_kv::Db::open_or_create(&config.host_harmonic_path).unwrap();
+ let page_centrality = speedy_kv::Db::open_or_create(&config.page_harmonic_path)?;
+ let host_centrality = speedy_kv::Db::open_or_create(&config.host_harmonic_path)?;
+ let host_centrality_rank =
+ speedy_kv::Db::open_or_create(&config.host_centrality_rank_store_path)?;
let gossip = config.gossip.clone();
let cluster = Arc::new(
@@ -74,7 +76,14 @@ pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {
.await?,
);
- let planner = CrawlPlanner::new(host_centrality, page_centrality, cluster, config).await?;
+ let planner = CrawlPlanner::new(
+ host_centrality,
+ host_centrality_rank,
+ page_centrality,
+ cluster,
+ config,
+ )
+ .await?;
planner.build().await?;