web search done right
use remote webgraph in crawl planner
Mikkel Denker3c4a0c480e29
diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rsindex 73c28816..17e70b5f 100644--- a/crates/core/src/ampc/dht/upsert.rs+++ b/crates/core/src/ampc/dht/upsert.rs@@ -45,6 +45,7 @@ pub enum UpsertEnum {HyperLogLog64Upsert,HyperLogLog128Upsert,U64Add,+ F32Add,F64Add,KahanSumAdd,}@@ -105,6 +106,24 @@ impl UpsertFn for U64Add {}}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct F32Add;++impl UpsertFn for F32Add {+ fn upsert(&self, old: Value, new: Value) -> Value {+ let (old, _) =+ bincode::decode_from_slice::<f32, _>(old.as_bytes(), bincode::config::standard())+ .unwrap();+ let (new, _) =+ bincode::decode_from_slice::<f32, _>(new.as_bytes(), bincode::config::standard())+ .unwrap();++ bincode::encode_to_vec(old + new, bincode::config::standard())+ .unwrap()+ .into()+ }+}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct F64Add;diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 72fb9a4e..996c9d92 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -427,6 +427,12 @@ impl Default for WidgetsConfig {}}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]+pub struct TopHostsBudgetBoostConfig {+ pub top_hosts: usize,+ pub reserved_budget_fraction: f64,+}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlPlannerConfig {pub host_harmonic_path: String,@@ -437,6 +443,9 @@ pub struct CrawlPlannerConfig {pub crawl_budget: usize,pub wander_fraction: f64,+ pub top_host_fraction: f64,++ pub top_hosts_budget_boost: Option<TopHostsBudgetBoostConfig>,pub gossip: GossipConfig,}diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 72d66223..b0a359ed 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -14,10 +14,14 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::{anyhow, Result};-use indicatif::ParallelProgressIterator;+use futures::stream::FuturesOrdered;+use futures::StreamExt;+use indicatif::ProgressIterator;use itertools::Itertools;use rayon::{prelude::*, ThreadPoolBuilder};+use std::cmp::Reverse;use std::collections::BTreeMap;+use std::hash::{Hash, Hasher};use std::sync::Arc;use std::{path::Path,@@ -28,10 +32,11 @@ use url::Url;use crate::config::WebgraphGranularity;use crate::crawler::WeightedUrl;use crate::distributed::cluster::Cluster;-use crate::webgraph::centrality::{top_nodes, TopNodes};+use crate::external_sort::ExternalSorter;use crate::webgraph::remote::RemoteWebgraph;use crate::webgraph::Node;use crate::webpage::url_ext::UrlExt;+use crate::SortableFloat;use crate::{config::CrawlPlannerConfig,crawler::{file_queue::FileQueueWriter, Job},@@ -40,10 +45,111 @@ use crate::{use super::Domain;+const MAX_UNCOMMITTED_INSERTS_PER_GROUP: usize = 50_000;+const NUM_GROUPS: usize = 1024;+const CONCURRENCY_LIMIT: usize = 32;++#[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, Hash)]+struct StoredUrl(#[bincode(with_serde)] Url);++impl StoredUrl {+ pub fn icann_domain(&self) -> Option<&str> {+ self.0.icann_domain()+ }+}++impl From<Url> for StoredUrl {+ fn from(url: Url) -> Self {+ Self(url)+ }+}++impl From<StoredUrl> for Url {+ fn from(url: StoredUrl) -> Self {+ url.0+ }+}++struct UrlGrouper {+ groups: Vec<speedy_kv::Db<StoredUrl, ()>>,+ folder: std::path::PathBuf,+}++impl UrlGrouper {+ pub fn new<P>(output_path: P, num_groups: usize) -> Self+ where+ P: AsRef<std::path::Path>,+ {+ let folder = output_path.as_ref().join("groups").to_path_buf();++ Self {+ groups: (0..num_groups)+ .map(|g| {+ speedy_kv::Db::open_or_create(folder.as_path().join(format!("{g}"))).unwrap()+ })+ .collect(),+ folder,+ }+ }++ fn group(&mut self, url: &StoredUrl) -> usize {+ let domain = url.icann_domain().unwrap_or_default();++ let mut hasher = std::collections::hash_map::DefaultHasher::new();+ domain.hash(&mut hasher);++ hasher.finish() as usize % self.groups.len()+ }++ pub fn insert(&mut self, url: Url) {+ let url = StoredUrl::from(url);+ let group = self.group(&url);+ let group = &mut self.groups[group];+ group.insert(url, ()).unwrap();++ if group.uncommitted_inserts() > MAX_UNCOMMITTED_INSERTS_PER_GROUP {+ group.commit().unwrap();+ }+ }++ pub fn into_groups(mut self) -> Vec<speedy_kv::Db<StoredUrl, ()>> {+ for group in &mut self.groups {+ group.commit().unwrap();+ }++ self.groups+ }+}++impl std::fmt::Debug for UrlGrouper {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("UrlGrouper")+ .field("num_groups", &self.groups.len())+ .field("folder", &self.folder)+ .finish()+ }+}++fn sorted_centralities(+ it: impl Iterator<Item = (NodeID, f64)>,+) -> impl Iterator<Item = (NodeID, f64)> {+ ExternalSorter::new()+ .with_chunk_size(100_000_000)+ .sort(it.map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), node_id)))+ .unwrap()+ .map(|(Reverse(SortableFloat(centrality)), node_id)| (node_id, centrality))+}++#[derive(Debug, Clone)]+struct Budget {+ remaining_schedulable: u64,+}+pub struct CrawlPlanner {host_centrality: speedy_kv::Db<NodeID, f64>,page_centrality: speedy_kv::Db<NodeID, f64>,page_graph: RemoteWebgraph,+ host_graph: RemoteWebgraph,config: CrawlPlannerConfig,}@@ -56,12 +162,14 @@ impl CrawlPlanner {) -> Result<Self> {Self::check_config(&config)?;- let page_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Page).await;+ let page_graph = RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await;+ let host_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Host).await;Ok(Self {host_centrality,page_centrality,page_graph,+ host_graph,config,})}@@ -95,7 +203,7 @@ impl CrawlPlanner {.map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default()).sum();- let wander_budget = (wander_budget * (host_centrality / total_host_centralities))+ let wander_budget = ((wander_budget * host_centrality) / total_host_centralities).max(1.0).round() as u64;@@ -104,7 +212,7 @@ impl CrawlPlanner {.chain(hosts.into_iter()- .map(|host| Url::parse(&format!("https://{}", host.as_str())).unwrap()),+ .filter_map(|host| Url::parse(&format!("https://{}", host.as_str())).ok()),).unique().map(|url| WeightedUrl {@@ -128,7 +236,6 @@ impl CrawlPlanner {let domain_stats = DomainStats {domain,- known_urls: job.urls.len(),num_hosts: job.urls.iter()@@ -142,56 +249,164 @@ impl CrawlPlanner {(job, domain_stats)}- async fn pages_to_crawl(&self) -> Vec<Url> {- let page_ids = top_nodes(- &self.page_centrality,- TopNodes::Fraction(- self.config.crawl_budget as f64 * (1.0 - self.config.wander_fraction),- ),- )- .into_iter()- .map(|(page, _)| page)- .collect::<Vec<_>>();+ fn assign_budgets(&self) -> BTreeMap<NodeID, Budget> {+ let total_crawl_budget = self.config.crawl_budget as f64;+ let num_hosts = self.host_centrality.len();+ let num_hosts = ((num_hosts as f64) * self.config.top_host_fraction).ceil() as usize;++ let crawl_budget_top_hosts_boost = total_crawl_budget+ * self+ .config+ .top_hosts_budget_boost+ .as_ref()+ .map(|top_hosts_budget_boost| top_hosts_budget_boost.reserved_budget_fraction)+ .unwrap_or_default();+ let total_crawl_budget = total_crawl_budget - crawl_budget_top_hosts_boost;++ let mut prev_sum = 0.0;+ let total_host_centrality = sorted_centralities(self.host_centrality.iter())+ .map(|(_, centrality)| centrality)+ .take_while(|centrality| {+ let sum = prev_sum + centrality;+ prev_sum = sum;+ // take as long as budget * c_i / S(c_i) > 0.5+ // as this will include exactly the hosts where+ // we crawl at least one page.+ // S(c_i) denotes the sum of all centralities+ // up to and including c_i+ 2.0 * total_crawl_budget * centrality > sum+ })+ .take(num_hosts)+ .sum::<f64>();++ let top_hosts_total_centrality = self+ .config+ .top_hosts_budget_boost+ .as_ref()+ .map(|top_hosts_budget_boost| {+ sorted_centralities(self.host_centrality.iter())+ .take(top_hosts_budget_boost.top_hosts)+ .map(|(_, centrality)| centrality)+ .sum::<f64>()+ })+ .unwrap_or_default();++ sorted_centralities(self.host_centrality.iter())+ .take(num_hosts)+ .enumerate()+ .filter_map(|(rank, (id, centrality))| {+ let mut total =+ ((total_crawl_budget * centrality) / total_host_centrality).ceil() as u64;++ if let Some(top_hosts_budget_boost) = &self.config.top_hosts_budget_boost {+ if rank < top_hosts_budget_boost.top_hosts {+ total += ((crawl_budget_top_hosts_boost * centrality)+ / top_hosts_total_centrality)+ .round() as u64;+ }+ }++ let budget = Budget {+ remaining_schedulable: (total as f64 * (1.0 - self.config.wander_fraction))+ .ceil() as u64,+ };++ if budget.remaining_schedulable > 0 {+ Some((id, budget))+ } else {+ None+ }+ })+ .collect()+ }++ async fn pages_to_crawl(&self) -> (Vec<speedy_kv::Db<StoredUrl, ()>>, u64) {+ let mut budgets = self.assign_budgets();- let mut pages = Vec::with_capacity(page_ids.len());+ let mut grouper = UrlGrouper::new(&self.config.output_path, NUM_GROUPS);+ let mut futures = FuturesOrdered::new();++ let num_pages = self.page_centrality.len();+ for chunk in sorted_centralities(self.page_centrality.iter())+ .map(|(id, _)| id)+ .progress_count(num_pages as u64)+ .chunks(128)+ .into_iter()+ {+ while futures.len() >= CONCURRENCY_LIMIT {+ let nodes: Vec<Option<Node>> = futures.next().await.unwrap();+ add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);+ }++ if budgets.is_empty() {+ break;+ }- for chunk in page_ids.into_iter().chunks(1024).into_iter() {let nodes = chunk.collect::<Vec<_>>();- let nodes = self.page_graph.batch_get_node(&nodes).await.unwrap();- pages.extend(- nodes- .into_iter()- .flatten()- .map(|n| Url::parse(n.as_str()).unwrap()),- );+ let page_graph = self.page_graph.clone();++ futures.push_back(async move { page_graph.batch_get_node(&nodes).await.unwrap() });}- pages- }+ while let Some(nodes) = futures.next().await {+ add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);+ }- pub async fn build<P: AsRef<Path>>(&self, output: P) -> Result<()> {- if output.as_ref().exists() {- return Err(anyhow!("output path already exists"));+ // make sure frontpage is added for all hosts+ let mut futures = FuturesOrdered::new();+ let num_missing_budgets = budgets.len();+ let missing_budgets = budgets.clone();+ for chunk in missing_budgets+ .keys()+ .copied()+ .progress_count(num_missing_budgets as u64)+ .chunks(128)+ .into_iter()+ {+ while futures.len() >= CONCURRENCY_LIMIT {+ let nodes: Vec<Option<Node>> = futures.next().await.unwrap();++ let nodes = nodes.into_iter().flatten().filter_map(|node| {+ if let Ok(url) = Url::parse(&format!("https://{}", node.as_str())) {+ Some(Node::from(url))+ } else {+ None+ }+ });++ add_nodes_to_grouper(nodes, &mut grouper, &mut budgets);+ }++ let nodes = chunk.collect::<Vec<_>>();+ let host_graph = self.host_graph.clone();++ futures.push_back(async move { host_graph.batch_get_node(&nodes).await.unwrap() });}- let queue_path = output.as_ref().join("job_queue");- std::fs::create_dir_all(&queue_path)?;+ while let Some(nodes) = futures.next().await {+ add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);+ }- tracing::info!("getting pages to crawl");+ let missing_budgets = budgets+ .values()+ .map(|budget| budget.remaining_schedulable)+ .sum::<u64>();- let pages = self.pages_to_crawl().await;+ (grouper.into_groups(), missing_budgets)+ }- let mut grouped_pages = BTreeMap::new();+ pub async fn build(self) -> Result<()> {+ let output = Path::new(&self.config.output_path);- for page in pages {- if let Some(domain) = page.icann_domain() {- grouped_pages- .entry(Domain::from(domain.to_string()))- .or_insert_with(Vec::new)- .push(page);- }+ if output.exists() {+ return Err(anyhow!("output path already exists"));}+ let queue_path = output.join("job_queue");+ std::fs::create_dir_all(&queue_path)?;++ tracing::info!("getting pages to crawl");+let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..self.config.num_job_queues).map(|i| {let path = queue_path.join(format!("{}.queue", i));@@ -210,29 +425,43 @@ impl CrawlPlanner {.thread_name(move |num| format!("crawl-planner-{num}")).build()?;- pool.install(|| {- let total_host_centralities: f64 = grouped_pages- .par_iter()- .map(|(_, pages)| {- pages- .iter()- .map(|url| Node::from(url).into_host().id())- .unique()- .map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())- .sum::<f64>()- })- .sum();+ let (groups, missing_budgets) = self.pages_to_crawl().await;++ tracing::info!("grouping pages by domain");++ for group in groups.into_iter().progress() {+ let mut grouped_pages = BTreeMap::new();++ for page in group.iter().map(|(url, _)| url).map(Url::from) {+ if let Some(domain) = page.icann_domain() {+ grouped_pages+ .entry(Domain::from(domain.to_string()))+ .or_insert_with(Vec::new)+ .push(page);+ }+ }++ pool.install(|| {+ let total_host_centralities: f64 = grouped_pages+ .par_iter()+ .map(|(_, pages)| {+ pages+ .iter()+ .map(|url| Node::from(url).into_host().id())+ .unique()+ .map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())+ .sum::<f64>()+ })+ .sum();- let wander_budget = self.config.crawl_budget as f64 * self.config.wander_fraction;+ let total_host_centralities = total_host_centralities * NUM_GROUPS as f64;- let next_queue = AtomicUsize::new(0);+ let wander_budget = self.config.crawl_budget as f64 * self.config.wander_fraction+ + missing_budgets as f64;- let num_groups = grouped_pages.len();+ let next_queue = AtomicUsize::new(0);- grouped_pages- .into_par_iter()- .progress_count(num_groups as u64)- .for_each(|(domain, pages)| {+ grouped_pages.into_par_iter().for_each(|(domain, pages)| {if domain.as_str().is_empty() {return;}@@ -253,7 +482,8 @@ impl CrawlPlanner {.push(job).unwrap();})- });+ });+ }for queue in job_queues {queue@@ -278,19 +508,46 @@ impl CrawlPlanner {tracing::info!("total wander budget: {}", metadata.total_wander_budget);- let metadata_path = output.as_ref().join("metadata.json");+ let metadata_path = output.join("metadata.json");let metadata_file = std::fs::File::create(metadata_path)?;serde_json::to_writer_pretty(metadata_file, &metadata)?;+ std::fs::remove_dir_all(output.join("groups"))?;+Ok(())}}+fn add_nodes_to_grouper(+ nodes: impl Iterator<Item = Node>,+ grouper: &mut UrlGrouper,+ budgets: &mut BTreeMap<NodeID, Budget>,+) {+ for node in nodes {+ let url = Url::parse(&format!("https://{}", node.as_str()));+ let host = node.into_host();++ if let Some(budget) = budgets.get_mut(&host.id()) {+ if budget.remaining_schedulable == 0 {+ continue;+ }++ if let Ok(url) = url {+ budget.remaining_schedulable -= 1;+ grouper.insert(url);++ if budget.remaining_schedulable == 0 {+ budgets.remove(&host.id());+ }+ }+ }+ }+}+#[derive(serde::Serialize)]struct DomainStats {domain: Domain,- known_urls: usize,num_hosts: usize,scheduled_urls: u64,wander_budget: u64,diff --git a/crates/core/src/distributed/cluster.rs b/crates/core/src/distributed/cluster.rsindex 7a9a4eed..2906b4e0 100644--- a/crates/core/src/distributed/cluster.rs+++ b/crates/core/src/distributed/cluster.rs@@ -59,8 +59,6 @@ fn alive_nodes_updater(chitchat: Arc<Mutex<Chitchat>>) -> Arc<RwLock<HashSet<Memservice,id: member.id,});- } else {- error!("failed to get service");}} else {error!("no state found for node")diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rsindex 3a2009ec..b2c8e0de 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs@@ -144,6 +144,7 @@ pub fn build(let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);Coordinator::new(setup, workers.clone())+ .with_mapper(ApproxCentralityMapper::InitCentrality).with_mapper(ApproxCentralityMapper::ApproximateCentrality)}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex f7d852d8..a8abaab4 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -21,8 +21,7 @@ use std::{cmp, collections::BTreeMap};use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};use crate::{- ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,- kahan_sum::KahanSum, webgraph,+ ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable, webgraph,};use rayon::prelude::*;@@ -30,6 +29,7 @@ const BATCH_SIZE: usize = 1024;#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]pub enum ApproxCentralityMapper {+ InitCentrality,ApproximateCentrality,}@@ -131,6 +131,18 @@ impl Mapper for ApproxCentralityMapper {dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,) {match self {+ ApproxCentralityMapper::InitCentrality => {+ let num_nodes = worker.graph().estimate_num_nodes();+ let nodes = worker.graph().nodes().collect::<Vec<_>>();+ nodes+ .par_chunks(BATCH_SIZE)+ .progress_count(num_nodes as u64 / BATCH_SIZE as u64)+ .for_each(|chunk| {+ let pairs: Vec<_> = chunk.iter().map(|node| (*node, 0.0)).collect();++ dht.next().centrality.batch_set(pairs)+ });+ }ApproxCentralityMapper::ApproximateCentrality => {let workers = Workers::new(worker.clone());let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;@@ -147,16 +159,14 @@ impl Mapper for ApproxCentralityMapper {if d == 0 {None} else {- Some((n, KahanSum::from((1.0 / d as f64) * job.norm)))+ Some((n, (1.0 / d as f32) * (job.norm as f32)))}}).chunks(BATCH_SIZE).into_iter(){let pairs: Vec<_> = chunk.collect();- dht.next()- .centrality- .batch_upsert(upsert::KahanSumAdd, pairs);+ dht.next().centrality.batch_upsert(upsert::F32Add, pairs);}});}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rsindex bcacd3e9..bbf62fed 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs@@ -23,7 +23,6 @@ use std::net::SocketAddr;use crate::distributed::member::ShardId;use crate::{ampc::{prelude::*, DefaultDhtTable},- kahan_sum::KahanSum,webgraph,};@@ -38,7 +37,7 @@ pub struct Meta {#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct ApproxCentralityTables {- centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,+ centrality: DefaultDhtTable<webgraph::NodeID, f32>,meta: DefaultDhtTable<(), Meta>,}diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rsindex 1807dfea..c6aa3136 100644--- a/crates/core/src/entrypoint/crawler.rs+++ b/crates/core/src/entrypoint/crawler.rs@@ -75,11 +75,9 @@ pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {.await?,);- let output_path = config.output_path.clone();-let planner = CrawlPlanner::new(host_centrality, page_centrality, cluster, config).await?;- planner.build(output_path).await?;+ planner.build().await?;Ok(())}diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 9645228b..6d6bd3e6 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -141,23 +141,23 @@ impl WebgraphWorker {link.text = link.text.chars().take(128).collect();- trace!("inserting link {:?}", link);let mut source = Node::from(source);let mut destination = Node::from(destination);+ trace!("inserting link {:?}", link);self.page_graph.insert(source.clone(), destination.clone(), link.text.clone());- source = source.into_host();- destination = destination.into_host();-let dest_domain = link.destination.root_domain();let source_domain = link.source.root_domain();if dest_domain.is_some()&& source_domain.is_some()&& dest_domain != source_domain{+ source = source.into_host();+ destination = destination.into_host();+self.host_graph.insert(source, destination, link.text);}}diff --git a/crates/core/src/external_sort.rs b/crates/core/src/external_sort.rsindex e1164b44..9d619d3e 100644--- a/crates/core/src/external_sort.rs+++ b/crates/core/src/external_sort.rs@@ -100,7 +100,9 @@ impl TempDir {impl Drop for TempDir {fn drop(&mut self) {- std::fs::remove_dir_all(&self.path).unwrap();+ if self.path.exists() {+ std::fs::remove_dir_all(&self.path).unwrap();+ }}}@@ -127,7 +129,9 @@ impl TempFile {impl Drop for TempFile {fn drop(&mut self) {- std::fs::remove_file(&self.path).unwrap();+ if self.path.exists() {+ std::fs::remove_file(&self.path).unwrap();+ }}}diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex facadb11..aeb45518 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -89,8 +89,9 @@ impl sonic::replication::ReusableClientManager for WebgraphClientManager {}}+#[derive(Clone)]pub struct RemoteWebgraph {- client: Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>,+ client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>>,}impl RemoteWebgraph {@@ -98,9 +99,9 @@ impl RemoteWebgraph {let manager = WebgraphClientManager { granularity };Self {- client: Mutex::new(+ client: Arc::new(Mutex::new(sonic::replication::ReusableShardedClient::new(cluster, manager).await,- ),+ )),}}diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex f48e719c..32095524 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -185,7 +185,9 @@ impl EdgeStoreWriter {}}- pub fn finalize(self) -> EdgeStore {+ pub fn finalize(mut self) -> EdgeStore {+ self.flush_to_file().unwrap();+let mut final_writer = FinalEdgeStoreWriter::open(self.compression,self.reversed,diff --git a/crates/speedy-kv/src/lib.rs b/crates/speedy-kv/src/lib.rsindex 3f3d4afd..68cf8dd1 100644--- a/crates/speedy-kv/src/lib.rs+++ b/crates/speedy-kv/src/lib.rs@@ -47,7 +47,8 @@ mod blob_store;mod segment;mod serialized;-use file_store::ConstSerializable;+use file_store::{ConstSerializable, Peekable};+use segment::SortedSegments;pub use serialized::{Serialized, SerializedRef};struct BlobPointer {@@ -129,6 +130,10 @@ impl<K, V> LiveSegment<K, V> {}}+ fn len(&self) -> usize {+ self.db.len()+ }+fn insert_raw<SerializedKey, SerializedVal>(&mut self, key: SerializedKey, value: SerializedVal)whereSerializedKey: Into<Serialized<K>>,@@ -276,7 +281,7 @@ impl<K, V> Db<K, V> {}pub fn uncommitted_inserts(&self) -> usize {- self.live_segment.db.len()+ self.live_segment.len()}fn save_meta(&self) -> Result<()> {@@ -411,6 +416,17 @@ where) -> impl Iterator<Item = (SerializedRef<'_, K>, SerializedRef<'_, V>)> + '_ {self.segments.iter().flat_map(|segment| segment.iter_raw())}++ pub fn sorted_iter_raw(+ &self,+ ) -> impl Iterator<Item = (SerializedRef<'_, K>, SerializedRef<'_, V>)> + '_ {+ SortedSegments::new(+ self.segments+ .iter()+ .map(|s| Peekable::new(s.iter_raw()))+ .collect(),+ )+ }}impl<K, V> Db<K, V>@@ -484,6 +500,17 @@ whereSome((k, v))})}++ pub fn sorted_iter(&self) -> impl Iterator<Item = (K, V)> + '_ {+ self.sorted_iter_raw().filter_map(|(k, v)| {+ let (k, _) =+ bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;+ let (v, _) =+ bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;++ Some((k, v))+ })+ }}#[cfg(test)]diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rsindex e6758ca0..7bc7ede1 100644--- a/crates/speedy-kv/src/segment.rs+++ b/crates/speedy-kv/src/segment.rs@@ -355,7 +355,7 @@ impl<'a, K, V, I> Eq for SortedPeekable<'a, K, V, I> where{}-struct SortedSegments<'a, K, V, I>+pub struct SortedSegments<'a, K, V, I>whereI: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,{@@ -366,7 +366,7 @@ impl<'a, K, V, I> SortedSegments<'a, K, V, I>whereI: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,{- fn new(segments: Vec<Peekable<I>>) -> Self {+ pub fn new(segments: Vec<Peekable<I>>) -> Self {Self {segments: segments.into_iter()