web search done right
use page centrality directly in crawl planner to prioritise pages.
Mikkel Denker5c94ded56756
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 540b461e..8c9925bd 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -439,20 +439,16 @@ impl Default for WidgetsConfig {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlPlannerConfig {- pub page_harmonic_path: String,pub host_harmonic_path: String,- pub page_graph_path: String,- pub host_graph_path: String,+ pub page_harmonic_path: String,pub output_path: String,pub num_job_queues: usize,pub crawl_budget: usize,- pub top_host_fraction: f64,pub wander_fraction: f64,- pub top_n_hosts_surplus: usize,- pub num_threads: Option<usize>,+ pub gossip: GossipConfig,}#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex 186c584e..412a8d1b 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -84,6 +84,8 @@ struct Site(String);Debug,PartialEq,Eq,+ PartialOrd,+ Ord,Hash,serde::Serialize,serde::Deserialize,diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 7b0a9530..72d66223 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -14,109 +14,58 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::{anyhow, Result};-use hashbrown::HashSet;use indicatif::ParallelProgressIterator;-use indicatif::ProgressIterator;use itertools::Itertools;-use rayon::prelude::*;-use rayon::ThreadPoolBuilder;+use rayon::{prelude::*, ThreadPoolBuilder};use std::collections::BTreeMap;-use std::collections::HashMap;-use std::iter;+use std::sync::Arc;use std::{path::Path,sync::{atomic::AtomicUsize, Mutex},};use url::Url;+use crate::config::WebgraphGranularity;use crate::crawler::WeightedUrl;+use crate::distributed::cluster::Cluster;use crate::webgraph::centrality::{top_nodes, TopNodes};-use crate::SortableFloat;+use crate::webgraph::remote::RemoteWebgraph;+use crate::webgraph::Node;+use crate::webpage::url_ext::UrlExt;use crate::{config::CrawlPlannerConfig,crawler::{file_queue::FileQueueWriter, Job},- webgraph::{NodeID, Webgraph},+ webgraph::NodeID,};use super::Domain;-const MAX_SURPLUS_BUDGET_ITERATIONS: usize = 100;-pub struct CrawlPlanner {host_centrality: speedy_kv::Db<NodeID, f64>,page_centrality: speedy_kv::Db<NodeID, f64>,- host_graph: Webgraph,- page_graph: Webgraph,+ page_graph: RemoteWebgraph,config: CrawlPlannerConfig,}impl CrawlPlanner {- pub fn new(+ pub async fn new(host_centrality: speedy_kv::Db<NodeID, f64>,page_centrality: speedy_kv::Db<NodeID, f64>,- host_graph: Webgraph,- page_graph: Webgraph,+ cluster: Arc<Cluster>,config: CrawlPlannerConfig,) -> Result<Self> {Self::check_config(&config)?;+ let page_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Page).await;+Ok(Self {host_centrality,page_centrality,- host_graph,page_graph,config,})}- fn all_pages(&self, host: NodeID) -> Vec<(NodeID, f64)> {- self.page_graph- .pages_by_host(&host)- .into_iter()- .map(|id| {- (- id,- self.page_centrality- .get(&id)- .ok()- .flatten()- .unwrap_or_default(),- )- })- .collect::<Vec<_>>()- }-- fn group_domain(&self, hosts: &[NodeID]) -> Vec<(Domain, Vec<NodeID>)> {- let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();-- for host in hosts {- let node = self.host_graph.id2node(host).unwrap();- if let Ok(url) = Url::parse(&format!("http://{}", node.as_str())) {- let domain = Domain::from(url);- domains.entry(domain).or_default().push(*host);- }- }-- domains- .into_iter()- .sorted_by_cached_key(|(_, hosts)| {- let s = hosts- .iter()- .map(|host| {- self.host_centrality- .get(host)- .ok()- .flatten()- .unwrap_or_default()- })- .sum::<f64>();-- SortableFloat::from(s)- })- .rev()- .collect()- }-fn check_config(config: &CrawlPlannerConfig) -> Result<()> {if !(0.0..=1.0).contains(&config.wander_fraction) {return Err(anyhow::anyhow!(@@ -124,210 +73,103 @@ impl CrawlPlanner {));}- if !(0.0..=1.0).contains(&config.top_host_fraction) {- return Err(anyhow::anyhow!(- "top_host_fraction must be in range [0.0, 1.0]"- ));- }-Ok(())}fn prepare_job(&self,domain: Domain,- hosts: &[NodeID],- host_budgets: &BTreeMap<NodeID, u64>,+ pages: Vec<Url>,+ wander_budget: f64,+ total_host_centralities: f64,) -> (Job, DomainStats) {- let mut total_wander_budget = 0;- let mut total_schedule_budget = 0;- let mut total_scheduled_urls = 0;- let mut total_known_urls = 0;- let mut urls = HashSet::new();-- for host in hosts {- let mut pages = self.all_pages(*host);- total_known_urls += pages.len();-- if pages.is_empty() {- continue;- }-- pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));- tracing::debug!("num pages: {}", pages.len());- let host_budget = host_budgets.get(host).copied().unwrap_or_default();-- tracing::debug!("host_budget: {host_budget}");- let schedule_budget = (host_budget as f64 * (1.0 - self.config.wander_fraction))- .round()- .max(0.0) as u64;+ let hosts = pages+ .iter()+ .map(|url| Node::from(url).into_host())+ .unique()+ .collect::<Vec<_>>();- tracing::debug!("schedule_budget: {schedule_budget}");- let wander_budget = (host_budget as f64 * self.config.wander_fraction)- .max(0.0)- .round() as u64;- tracing::debug!("wander_budget: {wander_budget}");+ let host_centrality: f64 = hosts+ .iter()+ .map(|host| host.id())+ .map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())+ .sum();- total_wander_budget += wander_budget;- total_schedule_budget += schedule_budget;- let host_name = self.host_graph.id2node(host).unwrap().as_str().to_string();+ let wander_budget = (wander_budget * (host_centrality / total_host_centralities))+ .max(1.0)+ .round() as u64;- let before = urls.len();- urls.extend(- pages+ let mut urls: Vec<_> = pages+ .into_iter()+ .chain(+ hosts.into_iter()- .filter_map(|(id, score)| self.page_graph.id2node(&id).map(|n| (n, score)))- .map(|(n, score)| (n.as_str().to_string(), score))- .filter_map(|(n, score)| {- Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))- })- .map(|(url, score)| WeightedUrl { url, weight: score })- .take(schedule_budget as usize)- .chain(- iter::once_with(|| {- Some(WeightedUrl {- url: format!("http://{host_name}").parse().ok()?,- weight: 1.0,- })- })- .flatten(),- ),- );-- total_scheduled_urls += urls.len() as u64 - before as u64;- }+ .map(|host| Url::parse(&format!("https://{}", host.as_str())).unwrap()),+ )+ .unique()+ .map(|url| WeightedUrl {+ url: url.clone(),+ weight: self+ .page_centrality+ .get(&Node::from(url).id())+ .unwrap()+ .unwrap_or_default(),+ })+ .collect();- tracing::trace!(- "domain: {:#?} hosts: {:#?} urls: {:#?}",- domain,- hosts,- urls- );+ urls.sort_by(|a, b| b.weight.total_cmp(&a.weight));+ urls.reverse();let job = Job {domain: domain.clone(),- urls: urls- .into_iter()- .sorted_by(|a, b| b.weight.total_cmp(&a.weight))- .collect(),- wandering_urls: total_wander_budget,+ urls: urls.into_iter().collect(),+ wandering_urls: wander_budget,};let domain_stats = DomainStats {domain,- num_hosts: hosts.len(),- schedule_budget: total_schedule_budget,- wander_budget: total_wander_budget,- scheduled_urls: total_scheduled_urls,- known_urls: total_known_urls,+ known_urls: job.urls.len(),+ num_hosts: job+ .urls+ .iter()+ .map(|url| url.url.host_str())+ .unique()+ .count(),+ scheduled_urls: job.urls.len() as u64,+ wander_budget,};(job, domain_stats)}- fn assign_host_budgets(&self, hosts: &[NodeID]) -> BTreeMap<NodeID, u64> {- let mut total_host_centrality = hosts- .iter()- .filter_map(|v| self.host_centrality.get(v).ok().flatten())- .sum::<f64>();-- let host_pages: BTreeMap<_, _> = hosts- .par_iter()- .progress_count(hosts.len() as u64)- .map(|host| {- let num_pages = self.page_graph.pages_by_host(host).len() as u64;- (*host, num_pages)- })- .collect();-- let mut host_budgets: BTreeMap<_, _> = hosts- .par_iter()- .progress_count(hosts.len() as u64)- .map(|host| {- let host_centrality = self- .host_centrality- .get(host)- .ok()- .flatten()- .unwrap_or_default();-- let host_budget = ((self.config.crawl_budget as f64 * host_centrality)- / total_host_centrality)- .round()- .max(0.0) as u64;-- let num_pages = host_pages.get(host).copied().unwrap_or_default();-- (*host, host_budget.min(num_pages))- })- .collect();-- let mut surplus_budget = self.config.crawl_budget as u64- - host_budgets- .values()- .sum::<u64>()- .min(self.config.crawl_budget as u64);-- // assign surplus budget- let mut has_updated = true;- let mut i = 0;- while surplus_budget > 0 && has_updated && i < MAX_SURPLUS_BUDGET_ITERATIONS {- tracing::info!("trying to schedule surplus budget: {}", surplus_budget);- i += 1;-- has_updated = false;- let mut new_surplus_budget = surplus_budget;- let mut new_total_host_centrality = 0.0;-- for host in hosts- .iter()- .take(self.config.top_n_hosts_surplus)- .progress()- {- if new_surplus_budget == 0 {- break;- }-- let num_pages = host_pages.get(host).copied().unwrap_or_default();- let host_budget = host_budgets.get_mut(host).unwrap();-- if num_pages > *host_budget {- let centrality = self- .host_centrality- .get(host)- .ok()- .flatten()- .unwrap_or_default();- new_total_host_centrality += centrality;-- let part = (centrality * surplus_budget as f64 / total_host_centrality)- .round()- .max(0.0) as u64;-- let diff = num_pages - *host_budget;- let extra = part.min(diff);-- if extra > 0 {- has_updated = true;- }-- let extra = ((extra as f64 / (1.0 - self.config.wander_fraction)) as u64)- .min(new_surplus_budget);+ async fn pages_to_crawl(&self) -> Vec<Url> {+ let page_ids = top_nodes(+ &self.page_centrality,+ TopNodes::Fraction(+ self.config.crawl_budget as f64 * (1.0 - self.config.wander_fraction),+ ),+ )+ .into_iter()+ .map(|(page, _)| page)+ .collect::<Vec<_>>();- *host_budget += extra;- new_surplus_budget -= extra;- }- }+ let mut pages = Vec::with_capacity(page_ids.len());- total_host_centrality = new_total_host_centrality;- surplus_budget = new_surplus_budget;+ for chunk in page_ids.into_iter().chunks(1024).into_iter() {+ let nodes = chunk.collect::<Vec<_>>();+ let nodes = self.page_graph.batch_get_node(&nodes).await.unwrap();+ pages.extend(+ nodes+ .into_iter()+ .flatten()+ .map(|n| Url::parse(n.as_str()).unwrap()),+ );}- tracing::info!("surplus done (remaining={})", surplus_budget);- host_budgets+ pages}- pub fn build<P: AsRef<Path>>(&self, output: P) -> Result<()> {+ pub async fn build<P: AsRef<Path>>(&self, output: P) -> Result<()> {if output.as_ref().exists() {return Err(anyhow!("output path already exists"));}@@ -335,17 +177,20 @@ impl CrawlPlanner {let queue_path = output.as_ref().join("job_queue");std::fs::create_dir_all(&queue_path)?;- let hosts = top_nodes(- &self.host_centrality,- TopNodes::Fraction(self.config.top_host_fraction),- )- .into_iter()- .map(|(host, _)| host)- .collect::<Vec<_>>();- tracing::info!("found {} hosts", hosts.len());+ tracing::info!("getting pages to crawl");++ let pages = self.pages_to_crawl().await;- let grouped_hosts = self.group_domain(&hosts);- let num_groups = grouped_hosts.len();+ let mut grouped_pages = BTreeMap::new();++ for page in pages {+ if let Some(domain) = page.icann_domain() {+ grouped_pages+ .entry(Domain::from(domain.to_string()))+ .or_insert_with(Vec::new)+ .push(page);+ }+ }let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..self.config.num_job_queues).map(|i| {@@ -359,31 +204,41 @@ impl CrawlPlanner {let stats = Mutex::new(Vec::new());- let next_queue = AtomicUsize::new(0);-- let num_threads = self- .config- .num_threads- .unwrap_or_else(|| usize::from(std::thread::available_parallelism().unwrap()));-let pool = ThreadPoolBuilder::new()- .num_threads(num_threads)+ .num_threads(usize::from(std::thread::available_parallelism().unwrap())).stack_size(80_000_000).thread_name(move |num| format!("crawl-planner-{num}")).build()?;pool.install(|| {- let host_budgets = self.assign_host_budgets(&hosts);+ let total_host_centralities: f64 = grouped_pages+ .par_iter()+ .map(|(_, pages)| {+ pages+ .iter()+ .map(|url| Node::from(url).into_host().id())+ .unique()+ .map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())+ .sum::<f64>()+ })+ .sum();++ let wander_budget = self.config.crawl_budget as f64 * self.config.wander_fraction;++ let next_queue = AtomicUsize::new(0);++ let num_groups = grouped_pages.len();- grouped_hosts+ grouped_pages.into_par_iter().progress_count(num_groups as u64)- .for_each(|(domain, hosts)| {+ .for_each(|(domain, pages)| {if domain.as_str().is_empty() {return;}- let (job, domain_stats) = self.prepare_job(domain, &hosts, &host_budgets);+ let (job, domain_stats) =+ self.prepare_job(domain, pages, wander_budget, total_host_centralities);stats.lock()@@ -408,7 +263,7 @@ impl CrawlPlanner {}let mut stats = stats.into_inner().unwrap_or_else(|e| e.into_inner());- stats.sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));+ stats.sort_by_key(|b| std::cmp::Reverse(b.schedule_budget()));let total_scheduled_urls: u64 = stats.iter().map(|d| d.scheduled_urls).sum();let total_wander_budget: u64 = stats.iter().map(|d| d.wander_budget).sum();@@ -437,200 +292,19 @@ struct DomainStats {domain: Domain,known_urls: usize,num_hosts: usize,- schedule_budget: u64,scheduled_urls: u64,wander_budget: u64,}+impl DomainStats {+ pub fn schedule_budget(&self) -> u64 {+ self.scheduled_urls + self.wander_budget+ }+}+#[derive(serde::Serialize)]struct Metadata {total_scheduled_urls: u64,total_wander_budget: u64,stats: Vec<DomainStats>,}--#[cfg(test)]-mod tests {- use super::*;- use crate::{- crawler::file_queue::FileQueue,- executor::Executor,- gen_temp_path,- webgraph::{centrality::harmonic::HarmonicCentrality, Compression, Node, WebgraphWriter},- };-- fn host_test_edges() -> Vec<(Node, Node, String)> {- // ┌────┐- // │ │- // ┌───A◄─┐ │- // │ │ │- // ▼ │ │- // B─────►C◄┘- // ▲- // │- // │- // D-- vec![- (- Node::from("http://a.com"),- Node::from("http://b.com"),- String::new(),- ),- (- Node::from("http://b.com"),- Node::from("http://c.com"),- String::new(),- ),- (- Node::from("http://a.com"),- Node::from("http://c.com"),- String::new(),- ),- (- Node::from("http://c.com"),- Node::from("http://a.com"),- String::new(),- ),- (- Node::from("http://d.com"),- Node::from("http://c.com"),- String::new(),- ),- ]- }-- fn page_test_edges() -> Vec<(Node, Node, String)> {- vec![- (- Node::from("http://a.com/"),- Node::from("http://b.com/123"),- String::new(),- ),- (- Node::from("http://b.com/123"),- Node::from("http://c.com/page"),- String::new(),- ),- (- Node::from("http://a.com/whut"),- Node::from("http://c.com/page"),- String::new(),- ),- (- Node::from("http://c.com/"),- Node::from("http://a.com/"),- String::new(),- ),- (- Node::from("http://d.com/321"),- Node::from("http://c.com/page"),- String::new(),- ),- ]- }-- fn test_graph(edges: Vec<(Node, Node, String)>) -> Webgraph {- let mut graph = WebgraphWriter::new(- crate::gen_temp_path(),- Executor::single_thread(),- Compression::default(),- None,- );-- for (from, to, label) in edges {- graph.insert(from, to, label);- }-- graph.commit();-- graph.finalize()- }-- fn test_plan() -> Vec<Job> {- let host_graph = test_graph(host_test_edges());- let page_graph = test_graph(page_test_edges());-- let centrality = HarmonicCentrality::calculate(&host_graph);- let mut host_centrality = speedy_kv::Db::open_or_create(crate::gen_temp_path()).unwrap();-- for (node, score) in centrality.iter() {- host_centrality.insert(*node, score).unwrap();- }- host_centrality.commit().unwrap();-- let centrality = HarmonicCentrality::calculate(&page_graph);- let mut page_centrality = speedy_kv::Db::open_or_create(crate::gen_temp_path()).unwrap();-- for (node, score) in centrality.iter() {- page_centrality.insert(*node, score).unwrap();- }- page_centrality.commit().unwrap();-- let planner = CrawlPlanner::new(- host_centrality,- page_centrality,- host_graph,- page_graph,- CrawlPlannerConfig {- crawl_budget: 100,- top_host_fraction: 1.0,- wander_fraction: 0.1,- top_n_hosts_surplus: 2,- num_job_queues: 1,- page_harmonic_path: String::new(),- host_harmonic_path: String::new(),- page_graph_path: String::new(),- host_graph_path: String::new(),- output_path: String::new(),- num_threads: Some(1),- },- )- .unwrap();-- let planner_path = gen_temp_path();- planner.build(planner_path.clone()).unwrap();-- let mut queue: FileQueue<Job> =- FileQueue::open(planner_path.join("job_queue").join("0.queue")).unwrap();-- let mut jobs = Vec::new();-- while let Some(job) = queue.pop().unwrap() {- jobs.push(job);- }-- jobs- }-- #[test]- fn test_ordered_by_centralities() {- let jobs = test_plan();-- assert_eq!(jobs.len(), 3);-- assert_eq!(jobs[0].domain.as_str(), "c.com");- assert_eq!(jobs[1].domain.as_str(), "a.com");- assert_eq!(jobs[2].domain.as_str(), "b.com");-- for job in jobs {- let urls: Vec<_> = job.urls.iter().collect();-- assert!(urls.windows(2).all(|w| w[0].weight >= w[1].weight));- }- }-- #[test]- fn test_contains_frontpage() {- let jobs = test_plan();-- for job in jobs {- let domain = job.domain.as_str();- assert!(job.urls.iter().any(|url| {- url.url.path() == "/"- && url.url.host_str() == Some(domain)- && url.url.query().is_none()- }))- }- }-}diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rsindex ea02e5ef..1807dfea 100644--- a/crates/core/src/entrypoint/crawler.rs+++ b/crates/core/src/entrypoint/crawler.rs@@ -20,7 +20,6 @@ use crate::{config,crawler::{self, planner::CrawlPlanner, CrawlCoordinator, Crawler},distributed::sonic::service::{sonic_service, Message},- webgraph::WebgraphBuilder,Result,};@@ -62,24 +61,25 @@ pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {}}-pub fn planner(config: config::CrawlPlannerConfig) -> Result<()> {+pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {let page_centrality = speedy_kv::Db::open_or_create(&config.page_harmonic_path).unwrap();let host_centrality = speedy_kv::Db::open_or_create(&config.host_harmonic_path).unwrap();- let page_graph = WebgraphBuilder::new(&config.page_graph_path)- .single_threaded()- .open();- let host_graph = WebgraphBuilder::new(&config.host_graph_path).open();++ let gossip = config.gossip.clone();+ let cluster = Arc::new(+ crate::distributed::cluster::Cluster::join_as_spectator(+ gossip.cluster_id,+ gossip.addr,+ gossip.seed_nodes.unwrap_or_default(),+ )+ .await?,+ );+let output_path = config.output_path.clone();- let planner = CrawlPlanner::new(- host_centrality,- page_centrality,- host_graph,- page_graph,- config,- )?;+ let planner = CrawlPlanner::new(host_centrality, page_centrality, cluster, config).await?;- planner.build(output_path)?;+ planner.build(output_path).await?;Ok(())}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 1564ff40..60afd253 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -415,7 +415,10 @@ fn main() -> Result<()> {Crawler::Plan { config_path } => {let config: config::CrawlPlannerConfig = load_toml_config(config_path);- entrypoint::crawler::planner(config)?;+ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::crawler::planner(config))?;}},Commands::SafetyClassifier { options } => match options {