web search done right
fix stack overflow in crawl planner
Mikkel Denker90066d4f811e
diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rsindex 5feff786..0a55a067 100644--- a/core/src/crawler/planner.rs+++ b/core/src/crawler/planner.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::{anyhow, Result};use rayon::prelude::*;+use rayon::ThreadPoolBuilder;use std::collections::HashMap;use std::{collections::VecDeque,@@ -97,7 +98,9 @@ pub fn make_crawl_plan<P: AsRef<Path>>(&host_centrality,TopHosts::Fraction(config.top_host_fraction),);- tracing::debug!("generating for {} hosts", hosts.len());+ tracing::info!("found {} hosts", hosts.len());++ let total_host_centrality = host_centrality.iter().map(|(_, v)| v).sum::<f64>();let grouped = group_domain(hosts, &host_graph);let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)@@ -114,67 +117,77 @@ pub fn make_crawl_plan<P: AsRef<Path>>(let next_queue = AtomicUsize::new(0);- grouped.into_par_iter().for_each(|(domain, hosts)| {- let mut total_wander_budget = 0;- let mut total_schedule_budget = 0;- let mut total_scheduled_urls = 0;- let mut urls = VecDeque::new();-- for host in hosts {- let mut pages = all_pages(&page_centrality, &page_graph, host);- pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));- tracing::debug!("num pages: {}", pages.len());- let host_centrality = host_centrality.get(&host).unwrap_or_default();-- let host_budget = (config.crawl_budget as f64 * host_centrality).max(1.0) as u64;- tracing::debug!("host_budget: {host_budget}");- let schedule_budget =- (host_budget as f64 * (1.0 - config.wander_fraction)).max(1.0) as u64;- tracing::debug!("schedule_budget: {schedule_budget}");- let wander_budget = (host_budget as f64 * config.wander_fraction).max(0.0) as u64;- tracing::debug!("wander_budget: {wander_budget}");-- total_wander_budget += wander_budget;- total_schedule_budget += schedule_budget;-- let before = urls.len();- urls.extend(- pages- .into_iter()- .map(|(id, _)| id)- .filter_map(|id| page_graph.id2node(&id))- .map(|n| n.name)- .filter_map(|n| Url::parse(&format!("http://{n}")).ok())- .take(schedule_budget as usize),- );-- total_scheduled_urls += urls.len() as u64 - before as u64;- }-- let job = Job {- domain: domain.clone(),- urls,- wandering_urls: total_wander_budget,- };-- let domain_stats = DomainStats {- domain,- schedule_budget: total_schedule_budget,- wander_budget: total_wander_budget,- scheduled_urls: total_scheduled_urls,- };- stats- .lock()- .unwrap_or_else(|e| e.into_inner())- .push(domain_stats);-- // `fetch_add` wraps around on overflow- let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);- job_queues[queue_index % job_queues.len()]- .lock()- .unwrap_or_else(|e| e.into_inner())- .push(job)- .unwrap();+ let pool = ThreadPoolBuilder::new()+ .num_threads(num_cpus::get())+ .stack_size(80_000_000)+ .thread_name(move |num| format!("crawl-planner-{num}"))+ .build()?;++ pool.install(|| {+ grouped.into_par_iter().for_each(|(domain, hosts)| {+ let mut total_wander_budget = 0;+ let mut total_schedule_budget = 0;+ let mut total_scheduled_urls = 0;+ let mut urls = VecDeque::new();++ for host in hosts {+ let mut pages = all_pages(&page_centrality, &page_graph, host);+ pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));+ tracing::debug!("num pages: {}", pages.len());+ let host_centrality = host_centrality.get(&host).unwrap_or_default();++ let host_budget = ((config.crawl_budget as f64 * host_centrality)+ / total_host_centrality)+ .max(1.0) as u64;+ tracing::debug!("host_budget: {host_budget}");+ let schedule_budget =+ (host_budget as f64 * (1.0 - config.wander_fraction)).max(1.0) as u64;+ tracing::debug!("schedule_budget: {schedule_budget}");+ let wander_budget = (host_budget as f64 * config.wander_fraction).max(0.0) as u64;+ tracing::debug!("wander_budget: {wander_budget}");++ total_wander_budget += wander_budget;+ total_schedule_budget += schedule_budget;++ let before = urls.len();+ urls.extend(+ pages+ .into_iter()+ .map(|(id, _)| id)+ .filter_map(|id| page_graph.id2node(&id))+ .map(|n| n.name)+ .filter_map(|n| Url::parse(&format!("http://{n}")).ok())+ .take(schedule_budget as usize),+ );++ total_scheduled_urls += urls.len() as u64 - before as u64;+ }++ let job = Job {+ domain: domain.clone(),+ urls,+ wandering_urls: total_wander_budget,+ };++ let domain_stats = DomainStats {+ domain,+ schedule_budget: total_schedule_budget,+ wander_budget: total_wander_budget,+ scheduled_urls: total_scheduled_urls,+ };+ stats+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .push(domain_stats);++ // `fetch_add` wraps around on overflow+ let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ job_queues[queue_index % job_queues.len()]+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .push(job)+ .unwrap();+ })});for queue in job_queues {@@ -184,10 +197,14 @@ pub fn make_crawl_plan<P: AsRef<Path>>(.finalize()?;}- let metadata = Metadata {+ let mut metadata = Metadata {stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),};+ metadata+ .stats+ .sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));+let metadata_path = output.as_ref().join("metadata.json");let metadata_file = std::fs::File::create(metadata_path)?;diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex fe3f96e8..d719fa17 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -69,7 +69,9 @@ pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {pub fn planner(config: config::CrawlPlannerConfig) -> Result<()> {let page_centrality = RocksDbStore::open(&config.page_harmonic_path);let host_centrality = RocksDbStore::open(&config.host_harmonic_path);- let page_graph = WebgraphBuilder::new(&config.page_graph_path).open();+ let page_graph = WebgraphBuilder::new(&config.page_graph_path)+ .single_threaded()+ .open();let host_graph = WebgraphBuilder::new(&config.host_graph_path).open();let output_path = config.output_path.clone();