Skip to content

web search done right

fix stack overflow in crawl planner

Mikkel Denker90066d4f811e
diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rs
index 5feff786..0a55a067 100644
--- a/core/src/crawler/planner.rs
+++ b/core/src/crawler/planner.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::{anyhow, Result};
use rayon::prelude::*;
+use rayon::ThreadPoolBuilder;
use std::collections::HashMap;
use std::{
collections::VecDeque,
@@ -97,7 +98,9 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
&host_centrality,
TopHosts::Fraction(config.top_host_fraction),
);
- tracing::debug!("generating for {} hosts", hosts.len());
+ tracing::info!("found {} hosts", hosts.len());
+
+ let total_host_centrality = host_centrality.iter().map(|(_, v)| v).sum::<f64>();
let grouped = group_domain(hosts, &host_graph);
let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)
@@ -114,67 +117,77 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
let next_queue = AtomicUsize::new(0);
- grouped.into_par_iter().for_each(|(domain, hosts)| {
- let mut total_wander_budget = 0;
- let mut total_schedule_budget = 0;
- let mut total_scheduled_urls = 0;
- let mut urls = VecDeque::new();
-
- for host in hosts {
- let mut pages = all_pages(&page_centrality, &page_graph, host);
- pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));
- tracing::debug!("num pages: {}", pages.len());
- let host_centrality = host_centrality.get(&host).unwrap_or_default();
-
- let host_budget = (config.crawl_budget as f64 * host_centrality).max(1.0) as u64;
- tracing::debug!("host_budget: {host_budget}");
- let schedule_budget =
- (host_budget as f64 * (1.0 - config.wander_fraction)).max(1.0) as u64;
- tracing::debug!("schedule_budget: {schedule_budget}");
- let wander_budget = (host_budget as f64 * config.wander_fraction).max(0.0) as u64;
- tracing::debug!("wander_budget: {wander_budget}");
-
- total_wander_budget += wander_budget;
- total_schedule_budget += schedule_budget;
-
- let before = urls.len();
- urls.extend(
- pages
- .into_iter()
- .map(|(id, _)| id)
- .filter_map(|id| page_graph.id2node(&id))
- .map(|n| n.name)
- .filter_map(|n| Url::parse(&format!("http://{n}")).ok())
- .take(schedule_budget as usize),
- );
-
- total_scheduled_urls += urls.len() as u64 - before as u64;
- }
-
- let job = Job {
- domain: domain.clone(),
- urls,
- wandering_urls: total_wander_budget,
- };
-
- let domain_stats = DomainStats {
- domain,
- schedule_budget: total_schedule_budget,
- wander_budget: total_wander_budget,
- scheduled_urls: total_scheduled_urls,
- };
- stats
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .push(domain_stats);
-
- // `fetch_add` wraps around on overflow
- let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
- job_queues[queue_index % job_queues.len()]
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .push(job)
- .unwrap();
+ let pool = ThreadPoolBuilder::new()
+ .num_threads(num_cpus::get())
+ .stack_size(80_000_000)
+ .thread_name(move |num| format!("crawl-planner-{num}"))
+ .build()?;
+
+ pool.install(|| {
+ grouped.into_par_iter().for_each(|(domain, hosts)| {
+ let mut total_wander_budget = 0;
+ let mut total_schedule_budget = 0;
+ let mut total_scheduled_urls = 0;
+ let mut urls = VecDeque::new();
+
+ for host in hosts {
+ let mut pages = all_pages(&page_centrality, &page_graph, host);
+ pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));
+ tracing::debug!("num pages: {}", pages.len());
+ let host_centrality = host_centrality.get(&host).unwrap_or_default();
+
+ let host_budget = ((config.crawl_budget as f64 * host_centrality)
+ / total_host_centrality)
+ .max(1.0) as u64;
+ tracing::debug!("host_budget: {host_budget}");
+ let schedule_budget =
+ (host_budget as f64 * (1.0 - config.wander_fraction)).max(1.0) as u64;
+ tracing::debug!("schedule_budget: {schedule_budget}");
+ let wander_budget = (host_budget as f64 * config.wander_fraction).max(0.0) as u64;
+ tracing::debug!("wander_budget: {wander_budget}");
+
+ total_wander_budget += wander_budget;
+ total_schedule_budget += schedule_budget;
+
+ let before = urls.len();
+ urls.extend(
+ pages
+ .into_iter()
+ .map(|(id, _)| id)
+ .filter_map(|id| page_graph.id2node(&id))
+ .map(|n| n.name)
+ .filter_map(|n| Url::parse(&format!("http://{n}")).ok())
+ .take(schedule_budget as usize),
+ );
+
+ total_scheduled_urls += urls.len() as u64 - before as u64;
+ }
+
+ let job = Job {
+ domain: domain.clone(),
+ urls,
+ wandering_urls: total_wander_budget,
+ };
+
+ let domain_stats = DomainStats {
+ domain,
+ schedule_budget: total_schedule_budget,
+ wander_budget: total_wander_budget,
+ scheduled_urls: total_scheduled_urls,
+ };
+ stats
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .push(domain_stats);
+
+ // `fetch_add` wraps around on overflow
+ let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ job_queues[queue_index % job_queues.len()]
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .push(job)
+ .unwrap();
+ })
});
for queue in job_queues {
@@ -184,10 +197,14 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
.finalize()?;
}
- let metadata = Metadata {
+ let mut metadata = Metadata {
stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),
};
+ metadata
+ .stats
+ .sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));
+
let metadata_path = output.as_ref().join("metadata.json");
let metadata_file = std::fs::File::create(metadata_path)?;
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index fe3f96e8..d719fa17 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -69,7 +69,9 @@ pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {
pub fn planner(config: config::CrawlPlannerConfig) -> Result<()> {
let page_centrality = RocksDbStore::open(&config.page_harmonic_path);
let host_centrality = RocksDbStore::open(&config.host_harmonic_path);
- let page_graph = WebgraphBuilder::new(&config.page_graph_path).open();
+ let page_graph = WebgraphBuilder::new(&config.page_graph_path)
+ .single_threaded()
+ .open();
let host_graph = WebgraphBuilder::new(&config.host_graph_path).open();
let output_path = config.output_path.clone();