web search done right
refactor crawl planner * make sure crawl plan has frontpage of all crawled sites * order crawl plan to first crawl sites with high harmonic centrality
Mikkel Denkerc7e596ac5b77
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex bcf5caf9..ffd4a7a1 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -394,6 +394,8 @@ pub struct CrawlPlannerConfig {pub top_host_fraction: f64,pub wander_fraction: f64,pub top_n_hosts_surplus: usize,++ pub num_threads: Option<usize>,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/crates/core/src/crawler/coordinator.rs b/crates/core/src/crawler/coordinator.rsindex cc4f20e3..b2b21049 100644--- a/crates/core/src/crawler/coordinator.rs+++ b/crates/core/src/crawler/coordinator.rs@@ -24,7 +24,7 @@ pub struct CrawlCoordinator {impl CrawlCoordinator {pub fn new<P: AsRef<Path>>(jobs_queue: P) -> Result<Self> {Ok(Self {- jobs: Mutex::new(FileQueue::new(jobs_queue)?),+ jobs: Mutex::new(FileQueue::open(jobs_queue)?),})}diff --git a/crates/core/src/crawler/file_queue.rs b/crates/core/src/crawler/file_queue.rsindex fb4397f9..5b0b07a9 100644--- a/crates/core/src/crawler/file_queue.rs+++ b/crates/core/src/crawler/file_queue.rs@@ -85,7 +85,7 @@ wherelet file = self.writer.into_inner()?;Ok(FileQueue {- pointer: FilePointer::new(self.path)?,+ pointer: FilePointer::open(self.path)?,file: unsafe { Mmap::map(&file)? },_marker: std::marker::PhantomData,})@@ -97,7 +97,7 @@ struct FilePointer {}impl FilePointer {- fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {+ fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {if !path.as_ref().exists() {std::fs::create_dir_all(path.as_ref())?;}@@ -142,7 +142,7 @@ impl<T> FileQueue<T>whereT: serde::Serialize + serde::de::DeserializeOwned,{- pub fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {+ pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {if !path.as_ref().exists() {std::fs::create_dir_all(path.as_ref())?;}@@ -151,7 +151,7 @@ wherelet file = unsafe { Mmap::map(&file)? };Ok(Self {- pointer: FilePointer::new(path)?,+ pointer: FilePointer::open(path)?,file,_marker: std::marker::PhantomData,})diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex a189a34b..9edefef8 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -102,6 +102,20 @@ pub struct WeightedUrl {pub weight: f64,}+impl PartialEq for WeightedUrl {+ fn eq(&self, other: &Self) -> bool {+ self.url == other.url+ }+}++impl Eq for WeightedUrl {}++impl std::hash::Hash for WeightedUrl {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.url.hash(state);+ }+}+/// All urls in a job must be from the same domain and only one job per domain./// at a time. This ensures that we stay polite when crawling.#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex ebddb907..08d434d2 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -14,14 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::{anyhow, Result};+use hashbrown::HashSet;use indicatif::ParallelProgressIterator;use indicatif::ProgressIterator;+use itertools::Itertools;use rayon::prelude::*;use rayon::ThreadPoolBuilder;use std::collections::BTreeMap;use std::collections::HashMap;+use std::iter;use std::{- collections::VecDeque,path::Path,sync::{atomic::AtomicUsize, Mutex},};@@ -29,6 +31,7 @@ use url::Url;use crate::crawler::WeightedUrl;use crate::webgraph::centrality::{top_hosts, TopHosts};+use crate::SortableFloat;use crate::{config::CrawlPlannerConfig,crawler::{file_queue::FileQueueWriter, Job},@@ -40,132 +43,213 @@ use super::Domain;const MAX_SURPLUS_BUDGET_ITERATIONS: usize = 100;-fn all_pages(- page_centrality: &RocksDbStore<NodeID, f64>,- page_graph: &Webgraph,- host: NodeID,-) -> Vec<(NodeID, f64)> {- page_graph- .pages_by_host(&host)- .into_iter()- .map(|id| (id, page_centrality.get(&id).unwrap_or_default()))- .collect::<Vec<_>>()+pub struct CrawlPlanner {+ host_centrality: RocksDbStore<NodeID, f64>,+ page_centrality: RocksDbStore<NodeID, f64>,+ host_graph: Webgraph,+ page_graph: Webgraph,+ config: CrawlPlannerConfig,}-fn group_domain(hosts: &[NodeID], host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {- let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();+impl CrawlPlanner {+ pub fn new(+ host_centrality: RocksDbStore<NodeID, f64>,+ page_centrality: RocksDbStore<NodeID, f64>,+ host_graph: Webgraph,+ page_graph: Webgraph,+ config: CrawlPlannerConfig,+ ) -> Result<Self> {+ Self::check_config(&config)?;++ Ok(Self {+ host_centrality,+ page_centrality,+ host_graph,+ page_graph,+ config,+ })+ }- for host in hosts {- let node = host_graph.id2node(host).unwrap();- if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {- let domain = Domain::from(url);- domains.entry(domain).or_default().push(*host);- }+ fn all_pages(&self, host: NodeID) -> Vec<(NodeID, f64)> {+ self.page_graph+ .pages_by_host(&host)+ .into_iter()+ .map(|id| (id, self.page_centrality.get(&id).unwrap_or_default()))+ .collect::<Vec<_>>()}- domains-}+ fn group_domain(&self, hosts: &[NodeID]) -> Vec<(Domain, Vec<NodeID>)> {+ let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();-fn check_config(config: &CrawlPlannerConfig) -> Result<()> {- if !(0.0..=1.0).contains(&config.wander_fraction) {- return Err(anyhow::anyhow!(- "top_host_fraction must be in range [0.0, 1.0]"- ));- }+ for host in hosts {+ let node = self.host_graph.id2node(host).unwrap();+ if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {+ let domain = Domain::from(url);+ domains.entry(domain).or_default().push(*host);+ }+ }++ domains+ .into_iter()+ .sorted_by_cached_key(|(_, hosts)| {+ let s = hosts+ .iter()+ .map(|host| self.host_centrality.get(host).unwrap_or_default())+ .sum::<f64>();- if !(0.0..=1.0).contains(&config.top_host_fraction) {- return Err(anyhow::anyhow!(- "top_host_fraction must be in range [0.0, 1.0]"- ));+ SortableFloat::from(s)+ })+ .rev()+ .collect()}- Ok(())-}+ fn check_config(config: &CrawlPlannerConfig) -> Result<()> {+ if !(0.0..=1.0).contains(&config.wander_fraction) {+ return Err(anyhow::anyhow!(+ "top_host_fraction must be in range [0.0, 1.0]"+ ));+ }-pub fn make_crawl_plan<P: AsRef<Path>>(- host_centrality: RocksDbStore<NodeID, f64>,- page_centrality: RocksDbStore<NodeID, f64>,- host_graph: Webgraph,- page_graph: Webgraph,- config: CrawlPlannerConfig,- output: P,-) -> Result<()> {- check_config(&config)?;+ if !(0.0..=1.0).contains(&config.top_host_fraction) {+ return Err(anyhow::anyhow!(+ "top_host_fraction must be in range [0.0, 1.0]"+ ));+ }- if output.as_ref().exists() {- return Err(anyhow!("output path already exists"));+ Ok(())}- let queue_path = output.as_ref().join("job_queue");- std::fs::create_dir_all(&queue_path)?;-- let hosts = top_hosts(- &host_centrality,- TopHosts::Fraction(config.top_host_fraction),- );- let num_hosts = hosts.len();- tracing::info!("found {} hosts", num_hosts);-- let mut total_host_centrality = hosts- .iter()- .filter_map(|v| host_centrality.get(v))- .sum::<f64>();- let grouped = group_domain(&hosts, &host_graph);- let num_groups = grouped.len();-- let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)- .map(|i| {- let path = queue_path.join(format!("{}.queue", i));- FileQueueWriter::new(path)- })- .collect::<Result<Vec<_>>>()?- .into_iter()- .map(Mutex::new)- .collect();+ fn prepare_job(+ &self,+ domain: Domain,+ hosts: &[NodeID],+ host_budgets: &BTreeMap<NodeID, u64>,+ ) -> (Job, DomainStats) {+ let mut total_wander_budget = 0;+ let mut total_schedule_budget = 0;+ let mut total_scheduled_urls = 0;+ let mut total_known_urls = 0;+ let mut urls = HashSet::new();++ for host in hosts {+ let mut pages = self.all_pages(*host);+ total_known_urls += pages.len();++ if pages.is_empty() {+ continue;+ }++ pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));+ tracing::debug!("num pages: {}", pages.len());+ let host_budget = host_budgets.get(host).copied().unwrap_or_default();++ tracing::debug!("host_budget: {host_budget}");+ let schedule_budget = (host_budget as f64 * (1.0 - self.config.wander_fraction))+ .round()+ .max(0.0) as u64;++ tracing::debug!("schedule_budget: {schedule_budget}");+ let wander_budget = (host_budget as f64 * self.config.wander_fraction)+ .max(0.0)+ .round() as u64;+ tracing::debug!("wander_budget: {wander_budget}");++ total_wander_budget += wander_budget;+ total_schedule_budget += schedule_budget;+ let host_name = self.host_graph.id2node(host).unwrap().name.clone();++ let before = urls.len();+ urls.extend(+ pages+ .into_iter()+ .filter_map(|(id, score)| self.page_graph.id2node(&id).map(|n| (n, score)))+ .map(|(n, score)| (n.name, score))+ .filter_map(|(n, score)| {+ Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))+ })+ .map(|(url, score)| WeightedUrl { url, weight: score })+ .take(schedule_budget as usize)+ .chain(+ iter::once_with(|| {+ Some(WeightedUrl {+ url: format!("http://{host_name}").parse().ok()?,+ weight: 1.0,+ })+ })+ .flatten(),+ ),+ );- let stats = Mutex::new(Vec::new());+ total_scheduled_urls += urls.len() as u64 - before as u64;+ }- let next_queue = AtomicUsize::new(0);+ tracing::trace!(+ "domain: {:#?} hosts: {:#?} urls: {:#?}",+ domain,+ hosts,+ urls+ );++ let job = Job {+ domain: domain.clone(),+ urls: urls+ .into_iter()+ .sorted_by(|a, b| b.weight.total_cmp(&a.weight))+ .collect(),+ wandering_urls: total_wander_budget,+ };++ let domain_stats = DomainStats {+ domain,+ num_hosts: hosts.len(),+ schedule_budget: total_schedule_budget,+ wander_budget: total_wander_budget,+ scheduled_urls: total_scheduled_urls,+ known_urls: total_known_urls,+ };++ (job, domain_stats)+ }- let pool = ThreadPoolBuilder::new()- .num_threads(num_cpus::get())- .stack_size(80_000_000)- .thread_name(move |num| format!("crawl-planner-{num}"))- .build()?;+ fn assign_host_budgets(&self, hosts: &[NodeID]) -> BTreeMap<NodeID, u64> {+ let mut total_host_centrality = hosts+ .iter()+ .filter_map(|v| self.host_centrality.get(v))+ .sum::<f64>();- pool.install(|| {let host_pages: BTreeMap<_, _> = hosts.par_iter()- .progress_count(num_hosts as u64)+ .progress_count(hosts.len() as u64).map(|host| {- let num_pages = page_graph.pages_by_host(host).len() as u64;+ let num_pages = self.page_graph.pages_by_host(host).len() as u64;(*host, num_pages)}).collect();let mut host_budgets: BTreeMap<_, _> = hosts.par_iter()- .progress_count(num_hosts as u64)+ .progress_count(hosts.len() as u64).map(|host| {- let host_centrality = host_centrality.get(host).unwrap_or_default();+ let host_centrality = self.host_centrality.get(host).unwrap_or_default();- let host_budget = ((config.crawl_budget as f64 * host_centrality)+ let host_budget = ((self.config.crawl_budget as f64 * host_centrality)/ total_host_centrality).round().max(0.0) as u64;let num_pages = host_pages.get(host).copied().unwrap_or_default();- (*host, host_budget.min(num_pages as u64))+ (*host, host_budget.min(num_pages))}).collect();- let mut surplus_budget = config.crawl_budget as u64+ let mut surplus_budget = self.config.crawl_budget as u64- host_budgets.values().sum::<u64>()- .min(config.crawl_budget as u64);+ .min(self.config.crawl_budget as u64);+ // assign surplus budgetlet mut has_updated = true;let mut i = 0;while surplus_budget > 0 && has_updated && i < MAX_SURPLUS_BUDGET_ITERATIONS {@@ -176,7 +260,11 @@ pub fn make_crawl_plan<P: AsRef<Path>>(let mut new_surplus_budget = surplus_budget;let mut new_total_host_centrality = 0.0;- for host in hosts.iter().take(config.top_n_hosts_surplus).progress() {+ for host in hosts+ .iter()+ .take(self.config.top_n_hosts_surplus)+ .progress()+ {if new_surplus_budget == 0 {break;}@@ -185,7 +273,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(let host_budget = host_budgets.get_mut(host).unwrap();if num_pages > *host_budget {- let centrality = host_centrality.get(host).unwrap_or_default();+ let centrality = self.host_centrality.get(host).unwrap_or_default();new_total_host_centrality += centrality;let part = (centrality * surplus_budget as f64 / total_host_centrality)@@ -199,7 +287,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(has_updated = true;}- let extra = ((extra as f64 / (1.0 - config.wander_fraction)) as u64)+ let extra = ((extra as f64 / (1.0 - self.config.wander_fraction)) as u64).min(new_surplus_budget);*host_budget += extra;@@ -212,130 +300,108 @@ pub fn make_crawl_plan<P: AsRef<Path>>(}tracing::info!("surplus done (remaining={})", surplus_budget);+ host_budgets+ }- grouped- .into_par_iter()- .progress_count(num_groups as u64)- .for_each(|(domain, hosts)| {- if domain.as_str().is_empty() {- return;- }+ pub fn build<P: AsRef<Path>>(&self, output: P) -> Result<()> {+ if output.as_ref().exists() {+ return Err(anyhow!("output path already exists"));+ }- let mut total_wander_budget = 0;- let mut total_schedule_budget = 0;- let mut total_scheduled_urls = 0;- let mut total_known_urls = 0;- let mut urls = VecDeque::new();+ let queue_path = output.as_ref().join("job_queue");+ std::fs::create_dir_all(&queue_path)?;- for host in &hosts {- let mut pages = all_pages(&page_centrality, &page_graph, *host);- total_known_urls += pages.len();+ let hosts = top_hosts(+ &self.host_centrality,+ TopHosts::Fraction(self.config.top_host_fraction),+ );+ tracing::info!("found {} hosts", hosts.len());- if pages.is_empty() {- continue;- }+ let grouped_hosts = self.group_domain(&hosts);+ let num_groups = grouped_hosts.len();- pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));- tracing::debug!("num pages: {}", pages.len());- let host_budget = host_budgets.get(host).copied().unwrap_or_default();+ let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..self.config.num_job_queues)+ .map(|i| {+ let path = queue_path.join(format!("{}.queue", i));+ FileQueueWriter::new(path)+ })+ .collect::<Result<Vec<_>>>()?+ .into_iter()+ .map(Mutex::new)+ .collect();- tracing::debug!("host_budget: {host_budget}");- let schedule_budget = (host_budget as f64 * (1.0 - config.wander_fraction))- .round()- .max(0.0) as u64;+ let stats = Mutex::new(Vec::new());- tracing::debug!("schedule_budget: {schedule_budget}");- let wander_budget = (host_budget as f64 * config.wander_fraction)- .max(0.0)- .round() as u64;- tracing::debug!("wander_budget: {wander_budget}");-- total_wander_budget += wander_budget;- total_schedule_budget += schedule_budget;-- let before = urls.len();- urls.extend(- pages- .into_iter()- .filter_map(|(id, score)| page_graph.id2node(&id).map(|n| (n, score)))- .map(|(n, score)| (n.name, score))- .filter_map(|(n, score)| {- Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))- })- .map(|(url, score)| WeightedUrl { url, weight: score })- .take(schedule_budget as usize),- );+ let next_queue = AtomicUsize::new(0);- total_scheduled_urls += urls.len() as u64 - before as u64;- }+ let num_threads = self.config.num_threads.unwrap_or_else(num_cpus::get);- tracing::trace!(- "domain: {:#?} hosts: {:#?} urls: {:#?}",- domain,- hosts,- urls- );-- let job = Job {- domain: domain.clone(),- urls,- wandering_urls: total_wander_budget,- };-- let domain_stats = DomainStats {- domain,- num_hosts: hosts.len(),- schedule_budget: total_schedule_budget,- wander_budget: total_wander_budget,- scheduled_urls: total_scheduled_urls,- known_urls: total_known_urls,- };- stats- .lock()- .unwrap_or_else(|e| e.into_inner())- .push(domain_stats);-- // `fetch_add` wraps around on overflow- let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);- job_queues[queue_index % job_queues.len()]- .lock()- .unwrap_or_else(|e| e.into_inner())- .push(job)- .unwrap();- })- });+ let pool = ThreadPoolBuilder::new()+ .num_threads(num_threads)+ .stack_size(80_000_000)+ .thread_name(move |num| format!("crawl-planner-{num}"))+ .build()?;- for queue in job_queues {- queue- .into_inner()- .unwrap_or_else(|e| e.into_inner())- .finalize()?;- }+ pool.install(|| {+ let host_budgets = self.assign_host_budgets(&hosts);- let mut metadata = Metadata {- stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),- };+ grouped_hosts+ .into_par_iter()+ .progress_count(num_groups as u64)+ .for_each(|(domain, hosts)| {+ if domain.as_str().is_empty() {+ return;+ }++ let (job, domain_stats) = self.prepare_job(domain, &hosts, &host_budgets);++ stats+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .push(domain_stats);++ // `fetch_add` wraps around on overflow+ let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ job_queues[queue_index % job_queues.len()]+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .push(job)+ .unwrap();+ })+ });++ for queue in job_queues {+ queue+ .into_inner()+ .unwrap_or_else(|e| e.into_inner())+ .finalize()?;+ }++ let mut metadata = Metadata {+ stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),+ };- tracing::info!(- "total scheduled urls: {}",- metadata.stats.iter().map(|d| d.scheduled_urls).sum::<u64>()- );+ tracing::info!(+ "total scheduled urls: {}",+ metadata.stats.iter().map(|d| d.scheduled_urls).sum::<u64>()+ );- tracing::info!(- "total wander budget: {}",- metadata.stats.iter().map(|d| d.wander_budget).sum::<u64>()- );+ tracing::info!(+ "total wander budget: {}",+ metadata.stats.iter().map(|d| d.wander_budget).sum::<u64>()+ );- metadata- .stats- .sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));+ metadata+ .stats+ .sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));- let metadata_path = output.as_ref().join("metadata.json");+ let metadata_path = output.as_ref().join("metadata.json");- let metadata_file = std::fs::File::create(metadata_path)?;- serde_json::to_writer_pretty(metadata_file, &metadata)?;+ let metadata_file = std::fs::File::create(metadata_path)?;+ serde_json::to_writer_pretty(metadata_file, &metadata)?;- Ok(())+ Ok(())+ }}#[derive(serde::Serialize)]@@ -352,3 +418,188 @@ struct DomainStats {struct Metadata {stats: Vec<DomainStats>,}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::{+ crawler::file_queue::FileQueue,+ executor::Executor,+ gen_temp_path,+ webgraph::{centrality::harmonic::HarmonicCentrality, Compression, Node, WebgraphWriter},+ };++ fn host_test_edges() -> Vec<(Node, Node, String)> {+ // ┌────┐+ // │ │+ // ┌───A◄─┐ │+ // │ │ │+ // ▼ │ │+ // B─────►C◄┘+ // ▲+ // │+ // │+ // D++ vec![+ (+ Node::from("http://a.com"),+ Node::from("http://b.com"),+ String::new(),+ ),+ (+ Node::from("http://b.com"),+ Node::from("http://c.com"),+ String::new(),+ ),+ (+ Node::from("http://a.com"),+ Node::from("http://c.com"),+ String::new(),+ ),+ (+ Node::from("http://c.com"),+ Node::from("http://a.com"),+ String::new(),+ ),+ (+ Node::from("http://d.com"),+ Node::from("http://c.com"),+ String::new(),+ ),+ ]+ }++ fn page_test_edges() -> Vec<(Node, Node, String)> {+ vec![+ (+ Node::from("http://a.com/"),+ Node::from("http://b.com/123"),+ String::new(),+ ),+ (+ Node::from("http://b.com/123"),+ Node::from("http://c.com/page"),+ String::new(),+ ),+ (+ Node::from("http://a.com/whut"),+ Node::from("http://c.com/page"),+ String::new(),+ ),+ (+ Node::from("http://c.com/"),+ Node::from("http://a.com/"),+ String::new(),+ ),+ (+ Node::from("http://d.com/321"),+ Node::from("http://c.com/page"),+ String::new(),+ ),+ ]+ }++ fn test_graph(edges: Vec<(Node, Node, String)>) -> Webgraph {+ let mut graph = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ );++ for (from, to, label) in edges {+ graph.insert(from, to, label);+ }++ graph.commit();++ graph.finalize()+ }++ fn test_plan() -> Vec<Job> {+ let host_graph = test_graph(host_test_edges());+ let page_graph = test_graph(page_test_edges());++ let centrality = HarmonicCentrality::calculate(&host_graph);+ let host_centrality = RocksDbStore::open(crate::gen_temp_path());++ for (node, score) in centrality.iter() {+ host_centrality.insert(*node, score);+ }+ host_centrality.flush();++ let centrality = HarmonicCentrality::calculate(&page_graph);+ let page_centrality = RocksDbStore::open(crate::gen_temp_path());++ for (node, score) in centrality.iter() {+ page_centrality.insert(*node, score);+ }+ page_centrality.flush();++ let planner = CrawlPlanner::new(+ host_centrality,+ page_centrality,+ host_graph,+ page_graph,+ CrawlPlannerConfig {+ crawl_budget: 100,+ top_host_fraction: 1.0,+ wander_fraction: 0.1,+ top_n_hosts_surplus: 2,+ num_job_queues: 1,+ page_harmonic_path: String::new(),+ host_harmonic_path: String::new(),+ page_graph_path: String::new(),+ host_graph_path: String::new(),+ output_path: String::new(),+ num_threads: Some(1),+ },+ )+ .unwrap();++ let planner_path = gen_temp_path();+ planner.build(planner_path.clone()).unwrap();++ let mut queue: FileQueue<Job> =+ FileQueue::open(planner_path.join("job_queue").join("0.queue")).unwrap();++ let mut jobs = Vec::new();++ while let Some(job) = queue.pop().unwrap() {+ jobs.push(job);+ }++ jobs+ }++ #[test]+ fn test_ordered_by_centralities() {+ let jobs = test_plan();++ assert_eq!(jobs.len(), 3);++ assert_eq!(jobs[0].domain.as_str(), "c.com");+ assert_eq!(jobs[1].domain.as_str(), "a.com");+ assert_eq!(jobs[2].domain.as_str(), "b.com");++ for job in jobs {+ let urls: Vec<_> = job.urls.iter().collect();++ assert!(urls.windows(2).all(|w| w[0].weight >= w[1].weight));+ }+ }++ #[test]+ fn test_contains_frontpage() {+ let jobs = test_plan();++ for job in jobs {+ let domain = job.domain.as_str();+ assert!(job.urls.iter().any(|url| {+ url.url.path() == "/"+ && url.url.host_str() == Some(domain)+ && url.url.query().is_none()+ }))+ }+ }+}diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rsindex bd040fea..acb1d44f 100644--- a/crates/core/src/entrypoint/centrality.rs+++ b/crates/core/src/entrypoint/centrality.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::Result;-use serde::{Deserialize, Serialize};use std::{cmp::Reverse, fs::File, path::Path};use crate::{@@ -26,6 +25,7 @@ use crate::{centrality::{approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality},Node, WebgraphBuilder,},+ SortableFloat,};fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {@@ -46,29 +46,6 @@ fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {wtr.flush().unwrap();}-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]-struct SortableFloat(f64);--impl PartialEq for SortableFloat {- fn eq(&self, other: &Self) -> bool {- self.0 == other.0- }-}--impl Eq for SortableFloat {}--impl PartialOrd for SortableFloat {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- Some(self.cmp(other))- }-}--impl Ord for SortableFloat {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.0.total_cmp(&other.0)- }-}-pub struct Centrality;impl Centrality {diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rsindex ebba3103..d56266b1 100644--- a/crates/core/src/entrypoint/crawler.rs+++ b/crates/core/src/entrypoint/crawler.rs@@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize};use crate::{config,- crawler::{self, planner::make_crawl_plan, CrawlCoordinator, Crawler},+ crawler::{self, planner::CrawlPlanner, CrawlCoordinator, Crawler},distributed::sonic::{self, service::Message},kv::rocksdb_store::RocksDbStore,sonic_service,@@ -75,15 +75,16 @@ pub fn planner(config: config::CrawlPlannerConfig) -> Result<()> {let host_graph = WebgraphBuilder::new(&config.host_graph_path).open();let output_path = config.output_path.clone();- make_crawl_plan(+ let planner = CrawlPlanner::new(host_centrality,page_centrality,host_graph,page_graph,config,- output_path,)?;+ planner.build(output_path)?;+Ok(())}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex e3fe7a44..1d32c8c0 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -23,6 +23,7 @@#![allow(clippy::cast_precision_loss)]#![allow(clippy::missing_errors_doc)]+use serde::{Deserialize, Serialize};use std::path::PathBuf;use thiserror::Error;@@ -191,6 +192,41 @@ pub fn combine_u64s(nums: [u64; 2]) -> u128 {((nums[0] as u128) << 64) | (nums[1] as u128)}+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+pub struct SortableFloat(f64);++impl From<f64> for SortableFloat {+ fn from(f: f64) -> Self {+ SortableFloat(f)+ }+}++impl From<SortableFloat> for f64 {+ fn from(f: SortableFloat) -> Self {+ f.0+ }+}++impl PartialEq for SortableFloat {+ fn eq(&self, other: &Self) -> bool {+ self.0 == other.0+ }+}++impl Eq for SortableFloat {}++impl PartialOrd for SortableFloat {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++impl Ord for SortableFloat {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.0.total_cmp(&other.0)+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/core/src/rake.rs b/crates/core/src/rake.rsindex 616fbb94..bd35b18f 100644--- a/crates/core/src/rake.rs+++ b/crates/core/src/rake.rs@@ -1,3 +1,19 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+//! Implementation of the RAKE keyword extraction algorithm.//!//! https://doi.org/10.1002/9780470689646.ch1diff --git a/crates/core/src/stopwords.rs b/crates/core/src/stopwords.rsindex 8b13eb69..b7d44d15 100644--- a/crates/core/src/stopwords.rs+++ b/crates/core/src/stopwords.rs@@ -1,3 +1,19 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use hashbrown::{HashMap, HashSet};use whatlang::Lang;diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex b23f6b78..125aefc5 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -16,6 +16,8 @@use std::path::{Path, PathBuf};+use itertools::Itertools;+use super::{store::{EdgeStore, EdgeStoreWriter},Compression, Edge, InnerEdge, NodeID,@@ -137,7 +139,10 @@ impl Segment {}pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {- self.reversed_adjacency.nodes_by_prefix(host_node)+ let rev = self.reversed_adjacency.nodes_by_prefix(host_node);+ let fwd = self.adjacency.nodes_by_prefix(host_node);++ rev.into_iter().chain(fwd).sorted().dedup().collect()}pub fn id(&self) -> String {