Skip to content

web search done right

refactor crawl planner * make sure crawl plan has frontpage of all crawled sites * order crawl plan to first crawl sites with high harmonic centrality

Mikkel Denkerc7e596ac5b77
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index bcf5caf9..ffd4a7a1 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -394,6 +394,8 @@ pub struct CrawlPlannerConfig {
pub top_host_fraction: f64,
pub wander_fraction: f64,
pub top_n_hosts_surplus: usize,
+
+ pub num_threads: Option<usize>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/crates/core/src/crawler/coordinator.rs b/crates/core/src/crawler/coordinator.rs
index cc4f20e3..b2b21049 100644
--- a/crates/core/src/crawler/coordinator.rs
+++ b/crates/core/src/crawler/coordinator.rs
@@ -24,7 +24,7 @@ pub struct CrawlCoordinator {
impl CrawlCoordinator {
pub fn new<P: AsRef<Path>>(jobs_queue: P) -> Result<Self> {
Ok(Self {
- jobs: Mutex::new(FileQueue::new(jobs_queue)?),
+ jobs: Mutex::new(FileQueue::open(jobs_queue)?),
})
}
diff --git a/crates/core/src/crawler/file_queue.rs b/crates/core/src/crawler/file_queue.rs
index fb4397f9..5b0b07a9 100644
--- a/crates/core/src/crawler/file_queue.rs
+++ b/crates/core/src/crawler/file_queue.rs
@@ -85,7 +85,7 @@ where
let file = self.writer.into_inner()?;
Ok(FileQueue {
- pointer: FilePointer::new(self.path)?,
+ pointer: FilePointer::open(self.path)?,
file: unsafe { Mmap::map(&file)? },
_marker: std::marker::PhantomData,
})
@@ -97,7 +97,7 @@ struct FilePointer {
}
impl FilePointer {
- fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
+ fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
if !path.as_ref().exists() {
std::fs::create_dir_all(path.as_ref())?;
}
@@ -142,7 +142,7 @@ impl<T> FileQueue<T>
where
T: serde::Serialize + serde::de::DeserializeOwned,
{
- pub fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
+ pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
if !path.as_ref().exists() {
std::fs::create_dir_all(path.as_ref())?;
}
@@ -151,7 +151,7 @@ where
let file = unsafe { Mmap::map(&file)? };
Ok(Self {
- pointer: FilePointer::new(path)?,
+ pointer: FilePointer::open(path)?,
file,
_marker: std::marker::PhantomData,
})
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rs
index a189a34b..9edefef8 100644
--- a/crates/core/src/crawler/mod.rs
+++ b/crates/core/src/crawler/mod.rs
@@ -102,6 +102,20 @@ pub struct WeightedUrl {
pub weight: f64,
}
+impl PartialEq for WeightedUrl {
+ fn eq(&self, other: &Self) -> bool {
+ self.url == other.url
+ }
+}
+
+impl Eq for WeightedUrl {}
+
+impl std::hash::Hash for WeightedUrl {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.url.hash(state);
+ }
+}
+
/// All urls in a job must be from the same domain and only one job per domain.
/// at a time. This ensures that we stay polite when crawling.
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index ebddb907..08d434d2 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -14,14 +14,16 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::{anyhow, Result};
+use hashbrown::HashSet;
use indicatif::ParallelProgressIterator;
use indicatif::ProgressIterator;
+use itertools::Itertools;
use rayon::prelude::*;
use rayon::ThreadPoolBuilder;
use std::collections::BTreeMap;
use std::collections::HashMap;
+use std::iter;
use std::{
- collections::VecDeque,
path::Path,
sync::{atomic::AtomicUsize, Mutex},
};
@@ -29,6 +31,7 @@ use url::Url;
use crate::crawler::WeightedUrl;
use crate::webgraph::centrality::{top_hosts, TopHosts};
+use crate::SortableFloat;
use crate::{
config::CrawlPlannerConfig,
crawler::{file_queue::FileQueueWriter, Job},
@@ -40,132 +43,213 @@ use super::Domain;
const MAX_SURPLUS_BUDGET_ITERATIONS: usize = 100;
-fn all_pages(
- page_centrality: &RocksDbStore<NodeID, f64>,
- page_graph: &Webgraph,
- host: NodeID,
-) -> Vec<(NodeID, f64)> {
- page_graph
- .pages_by_host(&host)
- .into_iter()
- .map(|id| (id, page_centrality.get(&id).unwrap_or_default()))
- .collect::<Vec<_>>()
+pub struct CrawlPlanner {
+ host_centrality: RocksDbStore<NodeID, f64>,
+ page_centrality: RocksDbStore<NodeID, f64>,
+ host_graph: Webgraph,
+ page_graph: Webgraph,
+ config: CrawlPlannerConfig,
}
-fn group_domain(hosts: &[NodeID], host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {
- let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();
+impl CrawlPlanner {
+ pub fn new(
+ host_centrality: RocksDbStore<NodeID, f64>,
+ page_centrality: RocksDbStore<NodeID, f64>,
+ host_graph: Webgraph,
+ page_graph: Webgraph,
+ config: CrawlPlannerConfig,
+ ) -> Result<Self> {
+ Self::check_config(&config)?;
+
+ Ok(Self {
+ host_centrality,
+ page_centrality,
+ host_graph,
+ page_graph,
+ config,
+ })
+ }
- for host in hosts {
- let node = host_graph.id2node(host).unwrap();
- if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {
- let domain = Domain::from(url);
- domains.entry(domain).or_default().push(*host);
- }
+ fn all_pages(&self, host: NodeID) -> Vec<(NodeID, f64)> {
+ self.page_graph
+ .pages_by_host(&host)
+ .into_iter()
+ .map(|id| (id, self.page_centrality.get(&id).unwrap_or_default()))
+ .collect::<Vec<_>>()
}
- domains
-}
+ fn group_domain(&self, hosts: &[NodeID]) -> Vec<(Domain, Vec<NodeID>)> {
+ let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();
-fn check_config(config: &CrawlPlannerConfig) -> Result<()> {
- if !(0.0..=1.0).contains(&config.wander_fraction) {
- return Err(anyhow::anyhow!(
- "top_host_fraction must be in range [0.0, 1.0]"
- ));
- }
+ for host in hosts {
+ let node = self.host_graph.id2node(host).unwrap();
+ if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {
+ let domain = Domain::from(url);
+ domains.entry(domain).or_default().push(*host);
+ }
+ }
+
+ domains
+ .into_iter()
+ .sorted_by_cached_key(|(_, hosts)| {
+ let s = hosts
+ .iter()
+ .map(|host| self.host_centrality.get(host).unwrap_or_default())
+ .sum::<f64>();
- if !(0.0..=1.0).contains(&config.top_host_fraction) {
- return Err(anyhow::anyhow!(
- "top_host_fraction must be in range [0.0, 1.0]"
- ));
+ SortableFloat::from(s)
+ })
+ .rev()
+ .collect()
}
- Ok(())
-}
+ fn check_config(config: &CrawlPlannerConfig) -> Result<()> {
+ if !(0.0..=1.0).contains(&config.wander_fraction) {
+ return Err(anyhow::anyhow!(
+ "top_host_fraction must be in range [0.0, 1.0]"
+ ));
+ }
-pub fn make_crawl_plan<P: AsRef<Path>>(
- host_centrality: RocksDbStore<NodeID, f64>,
- page_centrality: RocksDbStore<NodeID, f64>,
- host_graph: Webgraph,
- page_graph: Webgraph,
- config: CrawlPlannerConfig,
- output: P,
-) -> Result<()> {
- check_config(&config)?;
+ if !(0.0..=1.0).contains(&config.top_host_fraction) {
+ return Err(anyhow::anyhow!(
+ "top_host_fraction must be in range [0.0, 1.0]"
+ ));
+ }
- if output.as_ref().exists() {
- return Err(anyhow!("output path already exists"));
+ Ok(())
}
- let queue_path = output.as_ref().join("job_queue");
- std::fs::create_dir_all(&queue_path)?;
-
- let hosts = top_hosts(
- &host_centrality,
- TopHosts::Fraction(config.top_host_fraction),
- );
- let num_hosts = hosts.len();
- tracing::info!("found {} hosts", num_hosts);
-
- let mut total_host_centrality = hosts
- .iter()
- .filter_map(|v| host_centrality.get(v))
- .sum::<f64>();
- let grouped = group_domain(&hosts, &host_graph);
- let num_groups = grouped.len();
-
- let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)
- .map(|i| {
- let path = queue_path.join(format!("{}.queue", i));
- FileQueueWriter::new(path)
- })
- .collect::<Result<Vec<_>>>()?
- .into_iter()
- .map(Mutex::new)
- .collect();
+ fn prepare_job(
+ &self,
+ domain: Domain,
+ hosts: &[NodeID],
+ host_budgets: &BTreeMap<NodeID, u64>,
+ ) -> (Job, DomainStats) {
+ let mut total_wander_budget = 0;
+ let mut total_schedule_budget = 0;
+ let mut total_scheduled_urls = 0;
+ let mut total_known_urls = 0;
+ let mut urls = HashSet::new();
+
+ for host in hosts {
+ let mut pages = self.all_pages(*host);
+ total_known_urls += pages.len();
+
+ if pages.is_empty() {
+ continue;
+ }
+
+ pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));
+ tracing::debug!("num pages: {}", pages.len());
+ let host_budget = host_budgets.get(host).copied().unwrap_or_default();
+
+ tracing::debug!("host_budget: {host_budget}");
+ let schedule_budget = (host_budget as f64 * (1.0 - self.config.wander_fraction))
+ .round()
+ .max(0.0) as u64;
+
+ tracing::debug!("schedule_budget: {schedule_budget}");
+ let wander_budget = (host_budget as f64 * self.config.wander_fraction)
+ .max(0.0)
+ .round() as u64;
+ tracing::debug!("wander_budget: {wander_budget}");
+
+ total_wander_budget += wander_budget;
+ total_schedule_budget += schedule_budget;
+ let host_name = self.host_graph.id2node(host).unwrap().name.clone();
+
+ let before = urls.len();
+ urls.extend(
+ pages
+ .into_iter()
+ .filter_map(|(id, score)| self.page_graph.id2node(&id).map(|n| (n, score)))
+ .map(|(n, score)| (n.name, score))
+ .filter_map(|(n, score)| {
+ Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))
+ })
+ .map(|(url, score)| WeightedUrl { url, weight: score })
+ .take(schedule_budget as usize)
+ .chain(
+ iter::once_with(|| {
+ Some(WeightedUrl {
+ url: format!("http://{host_name}").parse().ok()?,
+ weight: 1.0,
+ })
+ })
+ .flatten(),
+ ),
+ );
- let stats = Mutex::new(Vec::new());
+ total_scheduled_urls += urls.len() as u64 - before as u64;
+ }
- let next_queue = AtomicUsize::new(0);
+ tracing::trace!(
+ "domain: {:#?} hosts: {:#?} urls: {:#?}",
+ domain,
+ hosts,
+ urls
+ );
+
+ let job = Job {
+ domain: domain.clone(),
+ urls: urls
+ .into_iter()
+ .sorted_by(|a, b| b.weight.total_cmp(&a.weight))
+ .collect(),
+ wandering_urls: total_wander_budget,
+ };
+
+ let domain_stats = DomainStats {
+ domain,
+ num_hosts: hosts.len(),
+ schedule_budget: total_schedule_budget,
+ wander_budget: total_wander_budget,
+ scheduled_urls: total_scheduled_urls,
+ known_urls: total_known_urls,
+ };
+
+ (job, domain_stats)
+ }
- let pool = ThreadPoolBuilder::new()
- .num_threads(num_cpus::get())
- .stack_size(80_000_000)
- .thread_name(move |num| format!("crawl-planner-{num}"))
- .build()?;
+ fn assign_host_budgets(&self, hosts: &[NodeID]) -> BTreeMap<NodeID, u64> {
+ let mut total_host_centrality = hosts
+ .iter()
+ .filter_map(|v| self.host_centrality.get(v))
+ .sum::<f64>();
- pool.install(|| {
let host_pages: BTreeMap<_, _> = hosts
.par_iter()
- .progress_count(num_hosts as u64)
+ .progress_count(hosts.len() as u64)
.map(|host| {
- let num_pages = page_graph.pages_by_host(host).len() as u64;
+ let num_pages = self.page_graph.pages_by_host(host).len() as u64;
(*host, num_pages)
})
.collect();
let mut host_budgets: BTreeMap<_, _> = hosts
.par_iter()
- .progress_count(num_hosts as u64)
+ .progress_count(hosts.len() as u64)
.map(|host| {
- let host_centrality = host_centrality.get(host).unwrap_or_default();
+ let host_centrality = self.host_centrality.get(host).unwrap_or_default();
- let host_budget = ((config.crawl_budget as f64 * host_centrality)
+ let host_budget = ((self.config.crawl_budget as f64 * host_centrality)
/ total_host_centrality)
.round()
.max(0.0) as u64;
let num_pages = host_pages.get(host).copied().unwrap_or_default();
- (*host, host_budget.min(num_pages as u64))
+ (*host, host_budget.min(num_pages))
})
.collect();
- let mut surplus_budget = config.crawl_budget as u64
+ let mut surplus_budget = self.config.crawl_budget as u64
- host_budgets
.values()
.sum::<u64>()
- .min(config.crawl_budget as u64);
+ .min(self.config.crawl_budget as u64);
+ // assign surplus budget
let mut has_updated = true;
let mut i = 0;
while surplus_budget > 0 && has_updated && i < MAX_SURPLUS_BUDGET_ITERATIONS {
@@ -176,7 +260,11 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
let mut new_surplus_budget = surplus_budget;
let mut new_total_host_centrality = 0.0;
- for host in hosts.iter().take(config.top_n_hosts_surplus).progress() {
+ for host in hosts
+ .iter()
+ .take(self.config.top_n_hosts_surplus)
+ .progress()
+ {
if new_surplus_budget == 0 {
break;
}
@@ -185,7 +273,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
let host_budget = host_budgets.get_mut(host).unwrap();
if num_pages > *host_budget {
- let centrality = host_centrality.get(host).unwrap_or_default();
+ let centrality = self.host_centrality.get(host).unwrap_or_default();
new_total_host_centrality += centrality;
let part = (centrality * surplus_budget as f64 / total_host_centrality)
@@ -199,7 +287,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
has_updated = true;
}
- let extra = ((extra as f64 / (1.0 - config.wander_fraction)) as u64)
+ let extra = ((extra as f64 / (1.0 - self.config.wander_fraction)) as u64)
.min(new_surplus_budget);
*host_budget += extra;
@@ -212,130 +300,108 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
}
tracing::info!("surplus done (remaining={})", surplus_budget);
+ host_budgets
+ }
- grouped
- .into_par_iter()
- .progress_count(num_groups as u64)
- .for_each(|(domain, hosts)| {
- if domain.as_str().is_empty() {
- return;
- }
+ pub fn build<P: AsRef<Path>>(&self, output: P) -> Result<()> {
+ if output.as_ref().exists() {
+ return Err(anyhow!("output path already exists"));
+ }
- let mut total_wander_budget = 0;
- let mut total_schedule_budget = 0;
- let mut total_scheduled_urls = 0;
- let mut total_known_urls = 0;
- let mut urls = VecDeque::new();
+ let queue_path = output.as_ref().join("job_queue");
+ std::fs::create_dir_all(&queue_path)?;
- for host in &hosts {
- let mut pages = all_pages(&page_centrality, &page_graph, *host);
- total_known_urls += pages.len();
+ let hosts = top_hosts(
+ &self.host_centrality,
+ TopHosts::Fraction(self.config.top_host_fraction),
+ );
+ tracing::info!("found {} hosts", hosts.len());
- if pages.is_empty() {
- continue;
- }
+ let grouped_hosts = self.group_domain(&hosts);
+ let num_groups = grouped_hosts.len();
- pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));
- tracing::debug!("num pages: {}", pages.len());
- let host_budget = host_budgets.get(host).copied().unwrap_or_default();
+ let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..self.config.num_job_queues)
+ .map(|i| {
+ let path = queue_path.join(format!("{}.queue", i));
+ FileQueueWriter::new(path)
+ })
+ .collect::<Result<Vec<_>>>()?
+ .into_iter()
+ .map(Mutex::new)
+ .collect();
- tracing::debug!("host_budget: {host_budget}");
- let schedule_budget = (host_budget as f64 * (1.0 - config.wander_fraction))
- .round()
- .max(0.0) as u64;
+ let stats = Mutex::new(Vec::new());
- tracing::debug!("schedule_budget: {schedule_budget}");
- let wander_budget = (host_budget as f64 * config.wander_fraction)
- .max(0.0)
- .round() as u64;
- tracing::debug!("wander_budget: {wander_budget}");
-
- total_wander_budget += wander_budget;
- total_schedule_budget += schedule_budget;
-
- let before = urls.len();
- urls.extend(
- pages
- .into_iter()
- .filter_map(|(id, score)| page_graph.id2node(&id).map(|n| (n, score)))
- .map(|(n, score)| (n.name, score))
- .filter_map(|(n, score)| {
- Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))
- })
- .map(|(url, score)| WeightedUrl { url, weight: score })
- .take(schedule_budget as usize),
- );
+ let next_queue = AtomicUsize::new(0);
- total_scheduled_urls += urls.len() as u64 - before as u64;
- }
+ let num_threads = self.config.num_threads.unwrap_or_else(num_cpus::get);
- tracing::trace!(
- "domain: {:#?} hosts: {:#?} urls: {:#?}",
- domain,
- hosts,
- urls
- );
-
- let job = Job {
- domain: domain.clone(),
- urls,
- wandering_urls: total_wander_budget,
- };
-
- let domain_stats = DomainStats {
- domain,
- num_hosts: hosts.len(),
- schedule_budget: total_schedule_budget,
- wander_budget: total_wander_budget,
- scheduled_urls: total_scheduled_urls,
- known_urls: total_known_urls,
- };
- stats
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .push(domain_stats);
-
- // `fetch_add` wraps around on overflow
- let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
- job_queues[queue_index % job_queues.len()]
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .push(job)
- .unwrap();
- })
- });
+ let pool = ThreadPoolBuilder::new()
+ .num_threads(num_threads)
+ .stack_size(80_000_000)
+ .thread_name(move |num| format!("crawl-planner-{num}"))
+ .build()?;
- for queue in job_queues {
- queue
- .into_inner()
- .unwrap_or_else(|e| e.into_inner())
- .finalize()?;
- }
+ pool.install(|| {
+ let host_budgets = self.assign_host_budgets(&hosts);
- let mut metadata = Metadata {
- stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),
- };
+ grouped_hosts
+ .into_par_iter()
+ .progress_count(num_groups as u64)
+ .for_each(|(domain, hosts)| {
+ if domain.as_str().is_empty() {
+ return;
+ }
+
+ let (job, domain_stats) = self.prepare_job(domain, &hosts, &host_budgets);
+
+ stats
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .push(domain_stats);
+
+ // `fetch_add` wraps around on overflow
+ let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ job_queues[queue_index % job_queues.len()]
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .push(job)
+ .unwrap();
+ })
+ });
+
+ for queue in job_queues {
+ queue
+ .into_inner()
+ .unwrap_or_else(|e| e.into_inner())
+ .finalize()?;
+ }
+
+ let mut metadata = Metadata {
+ stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),
+ };
- tracing::info!(
- "total scheduled urls: {}",
- metadata.stats.iter().map(|d| d.scheduled_urls).sum::<u64>()
- );
+ tracing::info!(
+ "total scheduled urls: {}",
+ metadata.stats.iter().map(|d| d.scheduled_urls).sum::<u64>()
+ );
- tracing::info!(
- "total wander budget: {}",
- metadata.stats.iter().map(|d| d.wander_budget).sum::<u64>()
- );
+ tracing::info!(
+ "total wander budget: {}",
+ metadata.stats.iter().map(|d| d.wander_budget).sum::<u64>()
+ );
- metadata
- .stats
- .sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));
+ metadata
+ .stats
+ .sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));
- let metadata_path = output.as_ref().join("metadata.json");
+ let metadata_path = output.as_ref().join("metadata.json");
- let metadata_file = std::fs::File::create(metadata_path)?;
- serde_json::to_writer_pretty(metadata_file, &metadata)?;
+ let metadata_file = std::fs::File::create(metadata_path)?;
+ serde_json::to_writer_pretty(metadata_file, &metadata)?;
- Ok(())
+ Ok(())
+ }
}
#[derive(serde::Serialize)]
@@ -352,3 +418,188 @@ struct DomainStats {
struct Metadata {
stats: Vec<DomainStats>,
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{
+ crawler::file_queue::FileQueue,
+ executor::Executor,
+ gen_temp_path,
+ webgraph::{centrality::harmonic::HarmonicCentrality, Compression, Node, WebgraphWriter},
+ };
+
+ fn host_test_edges() -> Vec<(Node, Node, String)> {
+ // ┌────┐
+ // │ │
+ // ┌───A◄─┐ │
+ // │ │ │
+ // ▼ │ │
+ // B─────►C◄┘
+ // ▲
+ // │
+ // │
+ // D
+
+ vec![
+ (
+ Node::from("http://a.com"),
+ Node::from("http://b.com"),
+ String::new(),
+ ),
+ (
+ Node::from("http://b.com"),
+ Node::from("http://c.com"),
+ String::new(),
+ ),
+ (
+ Node::from("http://a.com"),
+ Node::from("http://c.com"),
+ String::new(),
+ ),
+ (
+ Node::from("http://c.com"),
+ Node::from("http://a.com"),
+ String::new(),
+ ),
+ (
+ Node::from("http://d.com"),
+ Node::from("http://c.com"),
+ String::new(),
+ ),
+ ]
+ }
+
+ fn page_test_edges() -> Vec<(Node, Node, String)> {
+ vec![
+ (
+ Node::from("http://a.com/"),
+ Node::from("http://b.com/123"),
+ String::new(),
+ ),
+ (
+ Node::from("http://b.com/123"),
+ Node::from("http://c.com/page"),
+ String::new(),
+ ),
+ (
+ Node::from("http://a.com/whut"),
+ Node::from("http://c.com/page"),
+ String::new(),
+ ),
+ (
+ Node::from("http://c.com/"),
+ Node::from("http://a.com/"),
+ String::new(),
+ ),
+ (
+ Node::from("http://d.com/321"),
+ Node::from("http://c.com/page"),
+ String::new(),
+ ),
+ ]
+ }
+
+ fn test_graph(edges: Vec<(Node, Node, String)>) -> Webgraph {
+ let mut graph = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ );
+
+ for (from, to, label) in edges {
+ graph.insert(from, to, label);
+ }
+
+ graph.commit();
+
+ graph.finalize()
+ }
+
+ fn test_plan() -> Vec<Job> {
+ let host_graph = test_graph(host_test_edges());
+ let page_graph = test_graph(page_test_edges());
+
+ let centrality = HarmonicCentrality::calculate(&host_graph);
+ let host_centrality = RocksDbStore::open(crate::gen_temp_path());
+
+ for (node, score) in centrality.iter() {
+ host_centrality.insert(*node, score);
+ }
+ host_centrality.flush();
+
+ let centrality = HarmonicCentrality::calculate(&page_graph);
+ let page_centrality = RocksDbStore::open(crate::gen_temp_path());
+
+ for (node, score) in centrality.iter() {
+ page_centrality.insert(*node, score);
+ }
+ page_centrality.flush();
+
+ let planner = CrawlPlanner::new(
+ host_centrality,
+ page_centrality,
+ host_graph,
+ page_graph,
+ CrawlPlannerConfig {
+ crawl_budget: 100,
+ top_host_fraction: 1.0,
+ wander_fraction: 0.1,
+ top_n_hosts_surplus: 2,
+ num_job_queues: 1,
+ page_harmonic_path: String::new(),
+ host_harmonic_path: String::new(),
+ page_graph_path: String::new(),
+ host_graph_path: String::new(),
+ output_path: String::new(),
+ num_threads: Some(1),
+ },
+ )
+ .unwrap();
+
+ let planner_path = gen_temp_path();
+ planner.build(planner_path.clone()).unwrap();
+
+ let mut queue: FileQueue<Job> =
+ FileQueue::open(planner_path.join("job_queue").join("0.queue")).unwrap();
+
+ let mut jobs = Vec::new();
+
+ while let Some(job) = queue.pop().unwrap() {
+ jobs.push(job);
+ }
+
+ jobs
+ }
+
+ #[test]
+ fn test_ordered_by_centralities() {
+ let jobs = test_plan();
+
+ assert_eq!(jobs.len(), 3);
+
+ assert_eq!(jobs[0].domain.as_str(), "c.com");
+ assert_eq!(jobs[1].domain.as_str(), "a.com");
+ assert_eq!(jobs[2].domain.as_str(), "b.com");
+
+ for job in jobs {
+ let urls: Vec<_> = job.urls.iter().collect();
+
+ assert!(urls.windows(2).all(|w| w[0].weight >= w[1].weight));
+ }
+ }
+
+ #[test]
+ fn test_contains_frontpage() {
+ let jobs = test_plan();
+
+ for job in jobs {
+ let domain = job.domain.as_str();
+ assert!(job.urls.iter().any(|url| {
+ url.url.path() == "/"
+ && url.url.host_str() == Some(domain)
+ && url.url.query().is_none()
+ }))
+ }
+ }
+}
diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rs
index bd040fea..acb1d44f 100644
--- a/crates/core/src/entrypoint/centrality.rs
+++ b/crates/core/src/entrypoint/centrality.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
-use serde::{Deserialize, Serialize};
use std::{cmp::Reverse, fs::File, path::Path};
use crate::{
@@ -26,6 +25,7 @@ use crate::{
centrality::{approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality},
Node, WebgraphBuilder,
},
+ SortableFloat,
};
fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {
@@ -46,29 +46,6 @@ fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {
wtr.flush().unwrap();
}
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
-struct SortableFloat(f64);
-
-impl PartialEq for SortableFloat {
- fn eq(&self, other: &Self) -> bool {
- self.0 == other.0
- }
-}
-
-impl Eq for SortableFloat {}
-
-impl PartialOrd for SortableFloat {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl Ord for SortableFloat {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.0.total_cmp(&other.0)
- }
-}
-
pub struct Centrality;
impl Centrality {
diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rs
index ebba3103..d56266b1 100644
--- a/crates/core/src/entrypoint/crawler.rs
+++ b/crates/core/src/entrypoint/crawler.rs
@@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize};
use crate::{
config,
- crawler::{self, planner::make_crawl_plan, CrawlCoordinator, Crawler},
+ crawler::{self, planner::CrawlPlanner, CrawlCoordinator, Crawler},
distributed::sonic::{self, service::Message},
kv::rocksdb_store::RocksDbStore,
sonic_service,
@@ -75,15 +75,16 @@ pub fn planner(config: config::CrawlPlannerConfig) -> Result<()> {
let host_graph = WebgraphBuilder::new(&config.host_graph_path).open();
let output_path = config.output_path.clone();
- make_crawl_plan(
+ let planner = CrawlPlanner::new(
host_centrality,
page_centrality,
host_graph,
page_graph,
config,
- output_path,
)?;
+ planner.build(output_path)?;
+
Ok(())
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index e3fe7a44..1d32c8c0 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -23,6 +23,7 @@
#![allow(clippy::cast_precision_loss)]
#![allow(clippy::missing_errors_doc)]
+use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use thiserror::Error;
@@ -191,6 +192,41 @@ pub fn combine_u64s(nums: [u64; 2]) -> u128 {
((nums[0] as u128) << 64) | (nums[1] as u128)
}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct SortableFloat(f64);
+
+impl From<f64> for SortableFloat {
+ fn from(f: f64) -> Self {
+ SortableFloat(f)
+ }
+}
+
+impl From<SortableFloat> for f64 {
+ fn from(f: SortableFloat) -> Self {
+ f.0
+ }
+}
+
+impl PartialEq for SortableFloat {
+ fn eq(&self, other: &Self) -> bool {
+ self.0 == other.0
+ }
+}
+
+impl Eq for SortableFloat {}
+
+impl PartialOrd for SortableFloat {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl Ord for SortableFloat {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.0.total_cmp(&other.0)
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/core/src/rake.rs b/crates/core/src/rake.rs
index 616fbb94..bd35b18f 100644
--- a/crates/core/src/rake.rs
+++ b/crates/core/src/rake.rs
@@ -1,3 +1,19 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
//! Implementation of the RAKE keyword extraction algorithm.
//!
//! https://doi.org/10.1002/9780470689646.ch1
diff --git a/crates/core/src/stopwords.rs b/crates/core/src/stopwords.rs
index 8b13eb69..b7d44d15 100644
--- a/crates/core/src/stopwords.rs
+++ b/crates/core/src/stopwords.rs
@@ -1,3 +1,19 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
use hashbrown::{HashMap, HashSet};
use whatlang::Lang;
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index b23f6b78..125aefc5 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -16,6 +16,8 @@
use std::path::{Path, PathBuf};
+use itertools::Itertools;
+
use super::{
store::{EdgeStore, EdgeStoreWriter},
Compression, Edge, InnerEdge, NodeID,
@@ -137,7 +139,10 @@ impl Segment {
}
pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {
- self.reversed_adjacency.nodes_by_prefix(host_node)
+ let rev = self.reversed_adjacency.nodes_by_prefix(host_node);
+ let fwd = self.adjacency.nodes_by_prefix(host_node);
+
+ rev.into_iter().chain(fwd).sorted().dedup().collect()
}
pub fn id(&self) -> String {