Skip to content

web search done right

use remote webgraph in crawl planner

Mikkel Denker3c4a0c480e29
diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rs
index 73c28816..17e70b5f 100644
--- a/crates/core/src/ampc/dht/upsert.rs
+++ b/crates/core/src/ampc/dht/upsert.rs
@@ -45,6 +45,7 @@ pub enum UpsertEnum {
HyperLogLog64Upsert,
HyperLogLog128Upsert,
U64Add,
+ F32Add,
F64Add,
KahanSumAdd,
}
@@ -105,6 +106,24 @@ impl UpsertFn for U64Add {
}
}
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+pub struct F32Add;
+
+impl UpsertFn for F32Add {
+ fn upsert(&self, old: Value, new: Value) -> Value {
+ let (old, _) =
+ bincode::decode_from_slice::<f32, _>(old.as_bytes(), bincode::config::standard())
+ .unwrap();
+ let (new, _) =
+ bincode::decode_from_slice::<f32, _>(new.as_bytes(), bincode::config::standard())
+ .unwrap();
+
+ bincode::encode_to_vec(old + new, bincode::config::standard())
+ .unwrap()
+ .into()
+ }
+}
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct F64Add;
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 72fb9a4e..996c9d92 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -427,6 +427,12 @@ impl Default for WidgetsConfig {
}
}
+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
+pub struct TopHostsBudgetBoostConfig {
+ pub top_hosts: usize,
+ pub reserved_budget_fraction: f64,
+}
+
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlPlannerConfig {
pub host_harmonic_path: String,
@@ -437,6 +443,9 @@ pub struct CrawlPlannerConfig {
pub crawl_budget: usize,
pub wander_fraction: f64,
+ pub top_host_fraction: f64,
+
+ pub top_hosts_budget_boost: Option<TopHostsBudgetBoostConfig>,
pub gossip: GossipConfig,
}
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index 72d66223..b0a359ed 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -14,10 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::{anyhow, Result};
-use indicatif::ParallelProgressIterator;
+use futures::stream::FuturesOrdered;
+use futures::StreamExt;
+use indicatif::ProgressIterator;
use itertools::Itertools;
use rayon::{prelude::*, ThreadPoolBuilder};
+use std::cmp::Reverse;
use std::collections::BTreeMap;
+use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::{
path::Path,
@@ -28,10 +32,11 @@ use url::Url;
use crate::config::WebgraphGranularity;
use crate::crawler::WeightedUrl;
use crate::distributed::cluster::Cluster;
-use crate::webgraph::centrality::{top_nodes, TopNodes};
+use crate::external_sort::ExternalSorter;
use crate::webgraph::remote::RemoteWebgraph;
use crate::webgraph::Node;
use crate::webpage::url_ext::UrlExt;
+use crate::SortableFloat;
use crate::{
config::CrawlPlannerConfig,
crawler::{file_queue::FileQueueWriter, Job},
@@ -40,10 +45,111 @@ use crate::{
use super::Domain;
+const MAX_UNCOMMITTED_INSERTS_PER_GROUP: usize = 50_000;
+const NUM_GROUPS: usize = 1024;
+const CONCURRENCY_LIMIT: usize = 32;
+
+#[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, Hash)]
+struct StoredUrl(#[bincode(with_serde)] Url);
+
+impl StoredUrl {
+ pub fn icann_domain(&self) -> Option<&str> {
+ self.0.icann_domain()
+ }
+}
+
+impl From<Url> for StoredUrl {
+ fn from(url: Url) -> Self {
+ Self(url)
+ }
+}
+
+impl From<StoredUrl> for Url {
+ fn from(url: StoredUrl) -> Self {
+ url.0
+ }
+}
+
+struct UrlGrouper {
+ groups: Vec<speedy_kv::Db<StoredUrl, ()>>,
+ folder: std::path::PathBuf,
+}
+
+impl UrlGrouper {
+ pub fn new<P>(output_path: P, num_groups: usize) -> Self
+ where
+ P: AsRef<std::path::Path>,
+ {
+ let folder = output_path.as_ref().join("groups").to_path_buf();
+
+ Self {
+ groups: (0..num_groups)
+ .map(|g| {
+ speedy_kv::Db::open_or_create(folder.as_path().join(format!("{g}"))).unwrap()
+ })
+ .collect(),
+ folder,
+ }
+ }
+
+ fn group(&mut self, url: &StoredUrl) -> usize {
+ let domain = url.icann_domain().unwrap_or_default();
+
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ domain.hash(&mut hasher);
+
+ hasher.finish() as usize % self.groups.len()
+ }
+
+ pub fn insert(&mut self, url: Url) {
+ let url = StoredUrl::from(url);
+ let group = self.group(&url);
+ let group = &mut self.groups[group];
+ group.insert(url, ()).unwrap();
+
+ if group.uncommitted_inserts() > MAX_UNCOMMITTED_INSERTS_PER_GROUP {
+ group.commit().unwrap();
+ }
+ }
+
+ pub fn into_groups(mut self) -> Vec<speedy_kv::Db<StoredUrl, ()>> {
+ for group in &mut self.groups {
+ group.commit().unwrap();
+ }
+
+ self.groups
+ }
+}
+
+impl std::fmt::Debug for UrlGrouper {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("UrlGrouper")
+ .field("num_groups", &self.groups.len())
+ .field("folder", &self.folder)
+ .finish()
+ }
+}
+
+fn sorted_centralities(
+ it: impl Iterator<Item = (NodeID, f64)>,
+) -> impl Iterator<Item = (NodeID, f64)> {
+ ExternalSorter::new()
+ .with_chunk_size(100_000_000)
+ .sort(it.map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), node_id)))
+ .unwrap()
+ .map(|(Reverse(SortableFloat(centrality)), node_id)| (node_id, centrality))
+}
+
+#[derive(Debug, Clone)]
+struct Budget {
+ remaining_schedulable: u64,
+}
+
pub struct CrawlPlanner {
host_centrality: speedy_kv::Db<NodeID, f64>,
page_centrality: speedy_kv::Db<NodeID, f64>,
page_graph: RemoteWebgraph,
+ host_graph: RemoteWebgraph,
config: CrawlPlannerConfig,
}
@@ -56,12 +162,14 @@ impl CrawlPlanner {
) -> Result<Self> {
Self::check_config(&config)?;
- let page_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Page).await;
+ let page_graph = RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await;
+ let host_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Host).await;
Ok(Self {
host_centrality,
page_centrality,
page_graph,
+ host_graph,
config,
})
}
@@ -95,7 +203,7 @@ impl CrawlPlanner {
.map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())
.sum();
- let wander_budget = (wander_budget * (host_centrality / total_host_centralities))
+ let wander_budget = ((wander_budget * host_centrality) / total_host_centralities)
.max(1.0)
.round() as u64;
@@ -104,7 +212,7 @@ impl CrawlPlanner {
.chain(
hosts
.into_iter()
- .map(|host| Url::parse(&format!("https://{}", host.as_str())).unwrap()),
+ .filter_map(|host| Url::parse(&format!("https://{}", host.as_str())).ok()),
)
.unique()
.map(|url| WeightedUrl {
@@ -128,7 +236,6 @@ impl CrawlPlanner {
let domain_stats = DomainStats {
domain,
- known_urls: job.urls.len(),
num_hosts: job
.urls
.iter()
@@ -142,56 +249,164 @@ impl CrawlPlanner {
(job, domain_stats)
}
- async fn pages_to_crawl(&self) -> Vec<Url> {
- let page_ids = top_nodes(
- &self.page_centrality,
- TopNodes::Fraction(
- self.config.crawl_budget as f64 * (1.0 - self.config.wander_fraction),
- ),
- )
- .into_iter()
- .map(|(page, _)| page)
- .collect::<Vec<_>>();
+ fn assign_budgets(&self) -> BTreeMap<NodeID, Budget> {
+ let total_crawl_budget = self.config.crawl_budget as f64;
+ let num_hosts = self.host_centrality.len();
+ let num_hosts = ((num_hosts as f64) * self.config.top_host_fraction).ceil() as usize;
+
+ let crawl_budget_top_hosts_boost = total_crawl_budget
+ * self
+ .config
+ .top_hosts_budget_boost
+ .as_ref()
+ .map(|top_hosts_budget_boost| top_hosts_budget_boost.reserved_budget_fraction)
+ .unwrap_or_default();
+ let total_crawl_budget = total_crawl_budget - crawl_budget_top_hosts_boost;
+
+ let mut prev_sum = 0.0;
+ let total_host_centrality = sorted_centralities(self.host_centrality.iter())
+ .map(|(_, centrality)| centrality)
+ .take_while(|centrality| {
+ let sum = prev_sum + centrality;
+ prev_sum = sum;
+ // take as long as budget * c_i / S(c_i) > 0.5
+ // as this will include exactly the hosts where
+ // we crawl at least one page.
+ // S(c_i) denotes the sum of all centralities
+ // up to and including c_i
+ 2.0 * total_crawl_budget * centrality > sum
+ })
+ .take(num_hosts)
+ .sum::<f64>();
+
+ let top_hosts_total_centrality = self
+ .config
+ .top_hosts_budget_boost
+ .as_ref()
+ .map(|top_hosts_budget_boost| {
+ sorted_centralities(self.host_centrality.iter())
+ .take(top_hosts_budget_boost.top_hosts)
+ .map(|(_, centrality)| centrality)
+ .sum::<f64>()
+ })
+ .unwrap_or_default();
+
+ sorted_centralities(self.host_centrality.iter())
+ .take(num_hosts)
+ .enumerate()
+ .filter_map(|(rank, (id, centrality))| {
+ let mut total =
+ ((total_crawl_budget * centrality) / total_host_centrality).ceil() as u64;
+
+ if let Some(top_hosts_budget_boost) = &self.config.top_hosts_budget_boost {
+ if rank < top_hosts_budget_boost.top_hosts {
+ total += ((crawl_budget_top_hosts_boost * centrality)
+ / top_hosts_total_centrality)
+ .round() as u64;
+ }
+ }
+
+ let budget = Budget {
+ remaining_schedulable: (total as f64 * (1.0 - self.config.wander_fraction))
+ .ceil() as u64,
+ };
+
+ if budget.remaining_schedulable > 0 {
+ Some((id, budget))
+ } else {
+ None
+ }
+ })
+ .collect()
+ }
+
+ async fn pages_to_crawl(&self) -> (Vec<speedy_kv::Db<StoredUrl, ()>>, u64) {
+ let mut budgets = self.assign_budgets();
- let mut pages = Vec::with_capacity(page_ids.len());
+ let mut grouper = UrlGrouper::new(&self.config.output_path, NUM_GROUPS);
+ let mut futures = FuturesOrdered::new();
+
+ let num_pages = self.page_centrality.len();
+ for chunk in sorted_centralities(self.page_centrality.iter())
+ .map(|(id, _)| id)
+ .progress_count(num_pages as u64)
+ .chunks(128)
+ .into_iter()
+ {
+ while futures.len() >= CONCURRENCY_LIMIT {
+ let nodes: Vec<Option<Node>> = futures.next().await.unwrap();
+ add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);
+ }
+
+ if budgets.is_empty() {
+ break;
+ }
- for chunk in page_ids.into_iter().chunks(1024).into_iter() {
let nodes = chunk.collect::<Vec<_>>();
- let nodes = self.page_graph.batch_get_node(&nodes).await.unwrap();
- pages.extend(
- nodes
- .into_iter()
- .flatten()
- .map(|n| Url::parse(n.as_str()).unwrap()),
- );
+ let page_graph = self.page_graph.clone();
+
+ futures.push_back(async move { page_graph.batch_get_node(&nodes).await.unwrap() });
}
- pages
- }
+ while let Some(nodes) = futures.next().await {
+ add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);
+ }
- pub async fn build<P: AsRef<Path>>(&self, output: P) -> Result<()> {
- if output.as_ref().exists() {
- return Err(anyhow!("output path already exists"));
+ // make sure frontpage is added for all hosts
+ let mut futures = FuturesOrdered::new();
+ let num_missing_budgets = budgets.len();
+ let missing_budgets = budgets.clone();
+ for chunk in missing_budgets
+ .keys()
+ .copied()
+ .progress_count(num_missing_budgets as u64)
+ .chunks(128)
+ .into_iter()
+ {
+ while futures.len() >= CONCURRENCY_LIMIT {
+ let nodes: Vec<Option<Node>> = futures.next().await.unwrap();
+
+ let nodes = nodes.into_iter().flatten().filter_map(|node| {
+ if let Ok(url) = Url::parse(&format!("https://{}", node.as_str())) {
+ Some(Node::from(url))
+ } else {
+ None
+ }
+ });
+
+ add_nodes_to_grouper(nodes, &mut grouper, &mut budgets);
+ }
+
+ let nodes = chunk.collect::<Vec<_>>();
+ let host_graph = self.host_graph.clone();
+
+ futures.push_back(async move { host_graph.batch_get_node(&nodes).await.unwrap() });
}
- let queue_path = output.as_ref().join("job_queue");
- std::fs::create_dir_all(&queue_path)?;
+ while let Some(nodes) = futures.next().await {
+ add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);
+ }
- tracing::info!("getting pages to crawl");
+ let missing_budgets = budgets
+ .values()
+ .map(|budget| budget.remaining_schedulable)
+ .sum::<u64>();
- let pages = self.pages_to_crawl().await;
+ (grouper.into_groups(), missing_budgets)
+ }
- let mut grouped_pages = BTreeMap::new();
+ pub async fn build(self) -> Result<()> {
+ let output = Path::new(&self.config.output_path);
- for page in pages {
- if let Some(domain) = page.icann_domain() {
- grouped_pages
- .entry(Domain::from(domain.to_string()))
- .or_insert_with(Vec::new)
- .push(page);
- }
+ if output.exists() {
+ return Err(anyhow!("output path already exists"));
}
+ let queue_path = output.join("job_queue");
+ std::fs::create_dir_all(&queue_path)?;
+
+ tracing::info!("getting pages to crawl");
+
let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..self.config.num_job_queues)
.map(|i| {
let path = queue_path.join(format!("{}.queue", i));
@@ -210,29 +425,43 @@ impl CrawlPlanner {
.thread_name(move |num| format!("crawl-planner-{num}"))
.build()?;
- pool.install(|| {
- let total_host_centralities: f64 = grouped_pages
- .par_iter()
- .map(|(_, pages)| {
- pages
- .iter()
- .map(|url| Node::from(url).into_host().id())
- .unique()
- .map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())
- .sum::<f64>()
- })
- .sum();
+ let (groups, missing_budgets) = self.pages_to_crawl().await;
+
+ tracing::info!("grouping pages by domain");
+
+ for group in groups.into_iter().progress() {
+ let mut grouped_pages = BTreeMap::new();
+
+ for page in group.iter().map(|(url, _)| url).map(Url::from) {
+ if let Some(domain) = page.icann_domain() {
+ grouped_pages
+ .entry(Domain::from(domain.to_string()))
+ .or_insert_with(Vec::new)
+ .push(page);
+ }
+ }
+
+ pool.install(|| {
+ let total_host_centralities: f64 = grouped_pages
+ .par_iter()
+ .map(|(_, pages)| {
+ pages
+ .iter()
+ .map(|url| Node::from(url).into_host().id())
+ .unique()
+ .map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())
+ .sum::<f64>()
+ })
+ .sum();
- let wander_budget = self.config.crawl_budget as f64 * self.config.wander_fraction;
+ let total_host_centralities = total_host_centralities * NUM_GROUPS as f64;
- let next_queue = AtomicUsize::new(0);
+ let wander_budget = self.config.crawl_budget as f64 * self.config.wander_fraction
+ + missing_budgets as f64;
- let num_groups = grouped_pages.len();
+ let next_queue = AtomicUsize::new(0);
- grouped_pages
- .into_par_iter()
- .progress_count(num_groups as u64)
- .for_each(|(domain, pages)| {
+ grouped_pages.into_par_iter().for_each(|(domain, pages)| {
if domain.as_str().is_empty() {
return;
}
@@ -253,7 +482,8 @@ impl CrawlPlanner {
.push(job)
.unwrap();
})
- });
+ });
+ }
for queue in job_queues {
queue
@@ -278,19 +508,46 @@ impl CrawlPlanner {
tracing::info!("total wander budget: {}", metadata.total_wander_budget);
- let metadata_path = output.as_ref().join("metadata.json");
+ let metadata_path = output.join("metadata.json");
let metadata_file = std::fs::File::create(metadata_path)?;
serde_json::to_writer_pretty(metadata_file, &metadata)?;
+ std::fs::remove_dir_all(output.join("groups"))?;
+
Ok(())
}
}
+fn add_nodes_to_grouper(
+ nodes: impl Iterator<Item = Node>,
+ grouper: &mut UrlGrouper,
+ budgets: &mut BTreeMap<NodeID, Budget>,
+) {
+ for node in nodes {
+ let url = Url::parse(&format!("https://{}", node.as_str()));
+ let host = node.into_host();
+
+ if let Some(budget) = budgets.get_mut(&host.id()) {
+ if budget.remaining_schedulable == 0 {
+ continue;
+ }
+
+ if let Ok(url) = url {
+ budget.remaining_schedulable -= 1;
+ grouper.insert(url);
+
+ if budget.remaining_schedulable == 0 {
+ budgets.remove(&host.id());
+ }
+ }
+ }
+ }
+}
+
#[derive(serde::Serialize)]
struct DomainStats {
domain: Domain,
- known_urls: usize,
num_hosts: usize,
scheduled_urls: u64,
wander_budget: u64,
diff --git a/crates/core/src/distributed/cluster.rs b/crates/core/src/distributed/cluster.rs
index 7a9a4eed..2906b4e0 100644
--- a/crates/core/src/distributed/cluster.rs
+++ b/crates/core/src/distributed/cluster.rs
@@ -59,8 +59,6 @@ fn alive_nodes_updater(chitchat: Arc<Mutex<Chitchat>>) -> Arc<RwLock<HashSet<Mem
service,
id: member.id,
});
- } else {
- error!("failed to get service");
}
} else {
error!("no state found for node")
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
index 3a2009ec..b2c8e0de 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
@@ -144,6 +144,7 @@ pub fn build(
let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);
Coordinator::new(setup, workers.clone())
+ .with_mapper(ApproxCentralityMapper::InitCentrality)
.with_mapper(ApproxCentralityMapper::ApproximateCentrality)
}
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index f7d852d8..a8abaab4 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -21,8 +21,7 @@ use std::{cmp, collections::BTreeMap};
use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};
use crate::{
- ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,
- kahan_sum::KahanSum, webgraph,
+ ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable, webgraph,
};
use rayon::prelude::*;
@@ -30,6 +29,7 @@ const BATCH_SIZE: usize = 1024;
#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]
pub enum ApproxCentralityMapper {
+ InitCentrality,
ApproximateCentrality,
}
@@ -131,6 +131,18 @@ impl Mapper for ApproxCentralityMapper {
dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,
) {
match self {
+ ApproxCentralityMapper::InitCentrality => {
+ let num_nodes = worker.graph().estimate_num_nodes();
+ let nodes = worker.graph().nodes().collect::<Vec<_>>();
+ nodes
+ .par_chunks(BATCH_SIZE)
+ .progress_count(num_nodes as u64 / BATCH_SIZE as u64)
+ .for_each(|chunk| {
+ let pairs: Vec<_> = chunk.iter().map(|node| (*node, 0.0)).collect();
+
+ dht.next().centrality.batch_set(pairs)
+ });
+ }
ApproxCentralityMapper::ApproximateCentrality => {
let workers = Workers::new(worker.clone());
let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;
@@ -147,16 +159,14 @@ impl Mapper for ApproxCentralityMapper {
if d == 0 {
None
} else {
- Some((n, KahanSum::from((1.0 / d as f64) * job.norm)))
+ Some((n, (1.0 / d as f32) * (job.norm as f32)))
}
})
.chunks(BATCH_SIZE)
.into_iter()
{
let pairs: Vec<_> = chunk.collect();
- dht.next()
- .centrality
- .batch_upsert(upsert::KahanSumAdd, pairs);
+ dht.next().centrality.batch_upsert(upsert::F32Add, pairs);
}
});
}
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
index bcacd3e9..bbf62fed 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
@@ -23,7 +23,6 @@ use std::net::SocketAddr;
use crate::distributed::member::ShardId;
use crate::{
ampc::{prelude::*, DefaultDhtTable},
- kahan_sum::KahanSum,
webgraph,
};
@@ -38,7 +37,7 @@ pub struct Meta {
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct ApproxCentralityTables {
- centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,
+ centrality: DefaultDhtTable<webgraph::NodeID, f32>,
meta: DefaultDhtTable<(), Meta>,
}
diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rs
index 1807dfea..c6aa3136 100644
--- a/crates/core/src/entrypoint/crawler.rs
+++ b/crates/core/src/entrypoint/crawler.rs
@@ -75,11 +75,9 @@ pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {
.await?,
);
- let output_path = config.output_path.clone();
-
let planner = CrawlPlanner::new(host_centrality, page_centrality, cluster, config).await?;
- planner.build(output_path).await?;
+ planner.build().await?;
Ok(())
}
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 9645228b..6d6bd3e6 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -141,23 +141,23 @@ impl WebgraphWorker {
link.text = link.text.chars().take(128).collect();
- trace!("inserting link {:?}", link);
let mut source = Node::from(source);
let mut destination = Node::from(destination);
+ trace!("inserting link {:?}", link);
self.page_graph
.insert(source.clone(), destination.clone(), link.text.clone());
- source = source.into_host();
- destination = destination.into_host();
-
let dest_domain = link.destination.root_domain();
let source_domain = link.source.root_domain();
if dest_domain.is_some()
&& source_domain.is_some()
&& dest_domain != source_domain
{
+ source = source.into_host();
+ destination = destination.into_host();
+
self.host_graph.insert(source, destination, link.text);
}
}
diff --git a/crates/core/src/external_sort.rs b/crates/core/src/external_sort.rs
index e1164b44..9d619d3e 100644
--- a/crates/core/src/external_sort.rs
+++ b/crates/core/src/external_sort.rs
@@ -100,7 +100,9 @@ impl TempDir {
impl Drop for TempDir {
fn drop(&mut self) {
- std::fs::remove_dir_all(&self.path).unwrap();
+ if self.path.exists() {
+ std::fs::remove_dir_all(&self.path).unwrap();
+ }
}
}
@@ -127,7 +129,9 @@ impl TempFile {
impl Drop for TempFile {
fn drop(&mut self) {
- std::fs::remove_file(&self.path).unwrap();
+ if self.path.exists() {
+ std::fs::remove_file(&self.path).unwrap();
+ }
}
}
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index facadb11..aeb45518 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -89,8 +89,9 @@ impl sonic::replication::ReusableClientManager for WebgraphClientManager {
}
}
+#[derive(Clone)]
pub struct RemoteWebgraph {
- client: Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>,
+ client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>>,
}
impl RemoteWebgraph {
@@ -98,9 +99,9 @@ impl RemoteWebgraph {
let manager = WebgraphClientManager { granularity };
Self {
- client: Mutex::new(
+ client: Arc::new(Mutex::new(
sonic::replication::ReusableShardedClient::new(cluster, manager).await,
- ),
+ )),
}
}
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index f48e719c..32095524 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -185,7 +185,9 @@ impl EdgeStoreWriter {
}
}
- pub fn finalize(self) -> EdgeStore {
+ pub fn finalize(mut self) -> EdgeStore {
+ self.flush_to_file().unwrap();
+
let mut final_writer = FinalEdgeStoreWriter::open(
self.compression,
self.reversed,
diff --git a/crates/speedy-kv/src/lib.rs b/crates/speedy-kv/src/lib.rs
index 3f3d4afd..68cf8dd1 100644
--- a/crates/speedy-kv/src/lib.rs
+++ b/crates/speedy-kv/src/lib.rs
@@ -47,7 +47,8 @@ mod blob_store;
mod segment;
mod serialized;
-use file_store::ConstSerializable;
+use file_store::{ConstSerializable, Peekable};
+use segment::SortedSegments;
pub use serialized::{Serialized, SerializedRef};
struct BlobPointer {
@@ -129,6 +130,10 @@ impl<K, V> LiveSegment<K, V> {
}
}
+ fn len(&self) -> usize {
+ self.db.len()
+ }
+
fn insert_raw<SerializedKey, SerializedVal>(&mut self, key: SerializedKey, value: SerializedVal)
where
SerializedKey: Into<Serialized<K>>,
@@ -276,7 +281,7 @@ impl<K, V> Db<K, V> {
}
pub fn uncommitted_inserts(&self) -> usize {
- self.live_segment.db.len()
+ self.live_segment.len()
}
fn save_meta(&self) -> Result<()> {
@@ -411,6 +416,17 @@ where
) -> impl Iterator<Item = (SerializedRef<'_, K>, SerializedRef<'_, V>)> + '_ {
self.segments.iter().flat_map(|segment| segment.iter_raw())
}
+
+ pub fn sorted_iter_raw(
+ &self,
+ ) -> impl Iterator<Item = (SerializedRef<'_, K>, SerializedRef<'_, V>)> + '_ {
+ SortedSegments::new(
+ self.segments
+ .iter()
+ .map(|s| Peekable::new(s.iter_raw()))
+ .collect(),
+ )
+ }
}
impl<K, V> Db<K, V>
@@ -484,6 +500,17 @@ where
Some((k, v))
})
}
+
+ pub fn sorted_iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
+ self.sorted_iter_raw().filter_map(|(k, v)| {
+ let (k, _) =
+ bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;
+ let (v, _) =
+ bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;
+
+ Some((k, v))
+ })
+ }
}
#[cfg(test)]
diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rs
index e6758ca0..7bc7ede1 100644
--- a/crates/speedy-kv/src/segment.rs
+++ b/crates/speedy-kv/src/segment.rs
@@ -355,7 +355,7 @@ impl<'a, K, V, I> Eq for SortedPeekable<'a, K, V, I> where
{
}
-struct SortedSegments<'a, K, V, I>
+pub struct SortedSegments<'a, K, V, I>
where
I: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,
{
@@ -366,7 +366,7 @@ impl<'a, K, V, I> SortedSegments<'a, K, V, I>
where
I: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,
{
- fn new(segments: Vec<Peekable<I>>) -> Self {
+ pub fn new(segments: Vec<Peekable<I>>) -> Self {
Self {
segments: segments
.into_iter()