web search done right
Top k webgraph edges (#197)
Mikkel Denker9c983e5f967e
diff --git a/Cargo.lock b/Cargo.lockindex 465e44ec..9d548a3d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1544,6 +1544,8 @@ version = "0.1.0"dependencies = ["anyhow","bincode",+ "memmap2",+ "stable_deref_trait",][[package]]@@ -2799,16 +2801,6 @@ version = "2.7.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"-[[package]]-name = "memmap"-version = "0.7.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"-dependencies = [- "libc",- "winapi",-]-[[package]]name = "memmap2"version = "0.9.4"@@ -4631,7 +4623,7 @@ dependencies = ["bloom","file_store","fst",- "memmap",+ "memmap2","serde","serde_json","uuid",@@ -4725,7 +4717,6 @@ dependencies = ["lz4_flex","maplit","md5",- "memmap","memmap2","mime","min-max-heap",diff --git a/Cargo.toml b/Cargo.tomlindex bd673207..3ab388be 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -81,7 +81,6 @@ lz4_flex = "0.11.1"lzma = "0.2.2"maplit = "1.0.2"md5 = "0.7.0"-memmap = "0.7.0"memmap2 = "0.9.0"mime = "0.3.17"min-max-heap = "1.3.0"@@ -115,6 +114,7 @@ scylla = { version = "0.12.0", features = ["chrono"] }serde = { version = "1.0.137", features = ["rc", "derive"] }serde_json = "1.0.81"serde_urlencoded = "0.7.1"+stable_deref_trait = "1.2.0"strum = { version = "0.26.2", features = ["derive"] }tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "74940e9" }thiserror = "1.0.31"diff --git a/assets/licenses.html b/assets/licenses.htmlindex 517fc6fd..264820b9 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,7 +44,7 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (399)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (398)</li><li><a href="#MIT">MIT License</a> (192)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (7)</li>@@ -7028,7 +7028,6 @@ limitations under the License.<h4>Used by:</h4><ul class="license-used-by"><li><a href=" https://github.com/RazrFalcon/memmap2-rs ">memmap2 0.9.4</a></li>- <li><a href=" https://github.com/danburkert/memmap-rs ">memmap 0.7.0</a></li></ul><pre class="license-text"> Apache LicenseVersion 2.0, January 2004diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 48564c24..215c2bae 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -68,7 +68,6 @@ logos = { workspace = true }lz-str = { workspace = true }lz4_flex = { workspace = true }md5 = { workspace = true }-memmap = { workspace = true }memmap2 = { workspace = true }mime = { workspace = true }min-max-heap = { workspace = true }diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex 428afa73..94eaed07 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -22,7 +22,7 @@ use utoipa::{IntoParams, ToSchema};use crate::{config::WebgraphGranularity,- webgraph::{FullEdge, Node},+ webgraph::{EdgeLimit, FullEdge, Node},};use super::State;@@ -219,7 +219,7 @@ async fn ingoing_links(WebgraphGranularity::Page => &state.page_webgraph,};- graph.ingoing_edges(node).await+ graph.ingoing_edges(node, EdgeLimit::Limit(1024)).await}async fn outgoing_links(@@ -232,7 +232,7 @@ async fn outgoing_links(WebgraphGranularity::Page => &state.page_webgraph,};- graph.outgoing_edges(node).await+ graph.outgoing_edges(node, EdgeLimit::Limit(1024)).await}#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 89f341ab..d2de173b 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -73,6 +73,7 @@ pub struct WebgraphConstructConfig {pub skip_warc_files: Option<usize>,pub batch_size: Option<usize>,pub canonical_index_path: Option<String>,+ pub host_centrality_rank_store_path: Option<String>,}#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex bb090062..3146e510 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -532,6 +532,7 @@ mod tests {crate::gen_temp_path(),Executor::single_thread(),Compression::default(),+ None,);for (from, to, label) in edges {diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex c6fb7d92..9ff0f853 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -123,8 +123,8 @@ fn create_webgraph() -> Result<()> {};let mut worker = webgraph::WebgraphWorker {- host_graph: webgraph::open_host_graph_writer(&out_path_host),- page_graph: webgraph::open_page_graph_writer(&out_path_page),+ host_graph: webgraph::open_host_graph_writer(&out_path_host, None),+ page_graph: webgraph::open_page_graph_writer(&out_path_page, None),canonical_index: None,};diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex d8e7b2a6..d20558a3 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -34,7 +34,7 @@ use crate::human_website_annotations;use crate::index::Index;use crate::rake::RakeModel;use crate::ranking::SignalComputer;-use crate::webgraph::{self, Node, NodeID};+use crate::webgraph::{self, EdgeLimit, Node, NodeID};use crate::webpage::{safety_classifier, Html, Webpage};pub struct Config {@@ -83,17 +83,21 @@ pub(super) enum Webgraph {}impl Webgraph {- fn batch_raw_ingoing_edges_with_labels(&self, ids: Vec<NodeID>) -> Vec<Vec<String>> {+ fn batch_raw_ingoing_edges_with_labels(+ &self,+ ids: Vec<NodeID>,+ limit: EdgeLimit,+ ) -> Vec<Vec<String>> {let edges = match self {Self::Remote(webgraph) => {- crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids))+ crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit)).unwrap_or_default()}Self::Local(webgraph) => {let mut res = Vec::new();for id in ids {- res.push(webgraph.raw_ingoing_edges_with_labels(&id));+ res.push(webgraph.raw_ingoing_edges_with_labels(&id, limit));}res@@ -388,7 +392,7 @@ impl IndexingWorker {.map(|w| Node::from(w.html.url()).id()).collect::<Vec<_>>();- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids);+ let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(256));for (page, backlink) in pages.iter_mut().zip_eq(backlinks) {page.backlink_labels = backlink;diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 245c785f..6c1bfc86 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -17,7 +17,7 @@ use crate::{canon_index::CanonicalIndex,config::{self, WarcSource, WebgraphConstructConfig},entrypoint::download_all_warc_files,- webgraph::{self, Node, WebgraphWriter},+ webgraph::{self, Node, NodeID, WebgraphWriter},webpage::{url_ext::UrlExt, Html},Result,};@@ -66,19 +66,27 @@ pub struct Job {pub warc_paths: Vec<String>,}-pub fn open_host_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {+pub fn open_host_graph_writer<P: AsRef<Path>>(+ path: P,+ host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,+) -> webgraph::WebgraphWriter {WebgraphWriter::new(path,crate::executor::Executor::single_thread(),webgraph::Compression::Lz4,+ host_centrality_store,)}-pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {+pub fn open_page_graph_writer<P: AsRef<Path>>(+ path: P,+ host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,+) -> webgraph::WebgraphWriter {WebgraphWriter::new(path,crate::executor::Executor::single_thread(),webgraph::Compression::Lz4,+ host_centrality_store,)}@@ -189,6 +197,13 @@ impl Webgraph {None};+ let host_centrality_rank_store = if let Some(path) = &config.host_centrality_rank_store_path+ {+ Some(Arc::new(speedy_kv::Db::open_or_create(path)?))+ } else {+ None+ };+let num_workers = num_cpus::get();let mut handlers = Vec::new();@@ -212,8 +227,8 @@ impl Webgraph {let page_path = page_path.join(format!("worker_{i}"));let mut worker = WebgraphWorker {- host_graph: open_host_graph_writer(host_path),- page_graph: open_page_graph_writer(page_path),+ host_graph: open_host_graph_writer(host_path, host_centrality_rank_store.clone()),+ page_graph: open_page_graph_writer(page_path, host_centrality_rank_store.clone()),canonical_index: canonical_index.clone(),};diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex a41db1f9..26cdff16 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -26,8 +26,8 @@ use crate::distributed::member::Member;use crate::distributed::member::Service;use crate::distributed::sonic::service::sonic_service;use crate::distributed::sonic::service::Message;-use crate::webgraph::Compression;use crate::webgraph::Edge;+use crate::webgraph::EdgeLimit;use crate::webgraph::FullEdge;use crate::webgraph::Node;use crate::webgraph::NodeID;@@ -76,78 +76,88 @@ impl Message<WebGraphService> for GetNode {#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct IngoingEdges {pub node: Node,+ pub limit: EdgeLimit,}impl Message<WebGraphService> for IngoingEdges {type Response = Vec<FullEdge>;async fn handle(self, server: &WebGraphService) -> Self::Response {- server.graph.ingoing_edges(self.node)+ server.graph.ingoing_edges(self.node, self.limit)}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct OutgoingEdges {pub node: Node,+ pub limit: EdgeLimit,}impl Message<WebGraphService> for OutgoingEdges {type Response = Vec<FullEdge>;async fn handle(self, server: &WebGraphService) -> Self::Response {- server.graph.outgoing_edges(self.node)+ server.graph.outgoing_edges(self.node, self.limit)}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct RawIngoingEdges {pub node: NodeID,+ pub limit: EdgeLimit,}impl Message<WebGraphService> for RawIngoingEdges {type Response = Vec<Edge<()>>;async fn handle(self, server: &WebGraphService) -> Self::Response {- server.graph.raw_ingoing_edges(&self.node)+ server.graph.raw_ingoing_edges(&self.node, self.limit)}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct RawOutgoingEdges {pub node: NodeID,+ pub limit: EdgeLimit,}impl Message<WebGraphService> for RawOutgoingEdges {type Response = Vec<Edge<()>>;async fn handle(self, server: &WebGraphService) -> Self::Response {- server.graph.raw_outgoing_edges(&self.node)+ server.graph.raw_outgoing_edges(&self.node, self.limit)}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct RawIngoingEdgesWithLabels {pub node: NodeID,+ pub limit: EdgeLimit,}impl Message<WebGraphService> for RawIngoingEdgesWithLabels {type Response = Vec<Edge<String>>;async fn handle(self, server: &WebGraphService) -> Self::Response {- server.graph.raw_ingoing_edges_with_labels(&self.node)+ server+ .graph+ .raw_ingoing_edges_with_labels(&self.node, self.limit)}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct RawOutgoingEdgesWithLabels {pub node: NodeID,+ pub limit: EdgeLimit,}impl Message<WebGraphService> for RawOutgoingEdgesWithLabels {type Response = Vec<Edge<String>>;async fn handle(self, server: &WebGraphService) -> Self::Response {- server.graph.raw_outgoing_edges_with_labels(&self.node)+ server+ .graph+ .raw_outgoing_edges_with_labels(&self.node, self.limit)}}@@ -171,11 +181,7 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {.await?,);- let graph = Arc::new(- WebgraphBuilder::new(config.graph_path)- .compression(Compression::Lz4)- .open(),- );+ let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());let server = WebGraphService { graph }.bind(addr).await.unwrap();diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 68ca8a0b..2c7923ce 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -624,6 +624,7 @@ mod tests {gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);writer.insert(diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex ddbf63b1..87b80d27 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -149,7 +149,7 @@ mod tests {searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,},- webgraph::{Node, Webgraph, WebgraphWriter},+ webgraph::{EdgeLimit, Node, Webgraph, WebgraphWriter},webpage::{Html, Webpage},};@@ -158,7 +158,7 @@ mod tests {fn inbound(graph: &Webgraph, node: &NodeID) -> bitvec_similarity::BitVec {bitvec_similarity::BitVec::new(graph- .raw_ingoing_edges(node)+ .raw_ingoing_edges(node, EdgeLimit::Unlimited).into_iter().map(|e| e.from.as_u64()).collect(),@@ -171,6 +171,7 @@ mod tests {gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);wrt.insert(Node::from("a.com"), Node::from("b.com"), String::new());@@ -198,6 +199,7 @@ mod tests {crate::gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);wrt.insert(Node::from("b.com"), Node::from("a.com"), String::new());diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex c4b1f08f..33bffbed 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -41,6 +41,7 @@ mod tests {gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);wrt.insert(diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 976c4ee9..77a6c000 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -41,6 +41,7 @@ use crate::ranking::{use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};use crate::web_spell::SpellChecker;use crate::webgraph::remote::RemoteWebgraph;+use crate::webgraph::EdgeLimit;use crate::widgets::{Widget, Widgets};use crate::{bangs::Bangs,@@ -150,20 +151,32 @@ pub trait Graph {fn batch_raw_ingoing(&self,nodes: &[webgraph::NodeID],+ limit: EdgeLimit,) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>>;}impl Graph for RemoteWebgraph {- async fn batch_raw_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::Edge<()>>> {- self.batch_raw_ingoing_edges(nodes)+ async fn batch_raw_ingoing(+ &self,+ nodes: &[webgraph::NodeID],+ limit: EdgeLimit,+ ) -> Vec<Vec<webgraph::Edge<()>>> {+ self.batch_raw_ingoing_edges(nodes, limit).await.unwrap_or_default()}}impl Graph for webgraph::Webgraph {- async fn batch_raw_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::Edge<()>>> {- nodes.iter().map(|n| self.raw_ingoing_edges(n)).collect()+ async fn batch_raw_ingoing(+ &self,+ nodes: &[webgraph::NodeID],+ limit: EdgeLimit,+ ) -> Vec<Vec<webgraph::Edge<()>>> {+ nodes+ .iter()+ .map(|n| self.raw_ingoing_edges(n, limit))+ .collect()}}@@ -174,8 +187,9 @@ wherefn batch_raw_ingoing(&self,nodes: &[webgraph::NodeID],+ limit: EdgeLimit,) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>> {- self.as_ref().batch_raw_ingoing(nodes)+ self.as_ref().batch_raw_ingoing(nodes, limit)}}@@ -184,7 +198,7 @@ whereT: Graph,{async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {- self.batch_raw_ingoing(nodes)+ self.batch_raw_ingoing(nodes, EdgeLimit::Limit(128)).await.into_iter().map(|edges| edges.into_iter().map(|edge| edge.from).collect())diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex 2f72e254..31b91617 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -22,7 +22,7 @@ use url::Url;use crate::{ranking::{bitvec_similarity, inbound_similarity},- webgraph::{remote::RemoteWebgraph, Node, NodeID},+ webgraph::{remote::RemoteWebgraph, EdgeLimit, Node, NodeID},webpage::url_ext::UrlExt,SortableFloat,};@@ -75,7 +75,7 @@ impl SimilarHostsFinder {let in_edges = self.webgraph- .batch_raw_ingoing_edges(&nodes)+ .batch_raw_ingoing_edges(&nodes, EdgeLimit::Limit(128)).await.unwrap_or_default();@@ -88,7 +88,7 @@ impl SimilarHostsFinder {let outgoing_edges = self.webgraph- .batch_raw_outgoing_edges(&backlink_nodes)+ .batch_raw_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(4096)).await.unwrap_or_default();@@ -125,7 +125,7 @@ impl SimilarHostsFinder {// remove dead links (nodes without outgoing edges might be dead links)let known_nodes = self.webgraph- .batch_raw_ingoing_edges(&potential_nodes)+ .batch_raw_ingoing_edges(&potential_nodes, EdgeLimit::Limit(1)).await.unwrap_or_default();diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rsindex 23a4d6f9..998e2f46 100644--- a/crates/core/src/web_spell/stupid_backoff.rs+++ b/crates/core/src/web_spell/stupid_backoff.rs@@ -208,18 +208,18 @@ fn merge_streams(}pub struct StupidBackoff {- ngrams: fst::Map<memmap::Mmap>,- rotated_ngrams: fst::Map<memmap::Mmap>,+ ngrams: fst::Map<memmap2::Mmap>,+ rotated_ngrams: fst::Map<memmap2::Mmap>,n_counts: Vec<u64>,}impl StupidBackoff {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };let ngrams = fst::Map::new(mmap)?;let mmap =- unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };+ unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };let rotated_ngrams = fst::Map::new(mmap)?;let file = File::open(path.as_ref().join("n_counts.bin"))?;@@ -269,7 +269,7 @@ impl StupidBackoff {merge_streams(builder, streams)?;- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };let ngrams = fst::Map::new(mmap)?;let file = OpenOptions::new()@@ -285,7 +285,7 @@ impl StupidBackoff {merge_streams(builder, streams)?;- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };let rotated_ngrams = fst::Map::new(mmap)?;Ok(Self {diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rsindex 6d112145..0c663926 100644--- a/crates/core/src/web_spell/term_freqs.rs+++ b/crates/core/src/web_spell/term_freqs.rs@@ -71,13 +71,13 @@ impl DictBuilder {}struct StoredDict {- map: fst::Map<memmap::Mmap>,+ map: fst::Map<memmap2::Mmap>,path: PathBuf,}impl StoredDict {fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref())?)? };Ok(Self {map: fst::Map::new(mmap)?,@@ -147,7 +147,7 @@ impl StoredDict {builder.finish()?;- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref())?)? };Ok(StoredDict {map: fst::Map::new(mmap)?,diff --git a/crates/core/src/webgraph/builder.rs b/crates/core/src/webgraph/builder.rsindex c85239d6..4b3643b9 100644--- a/crates/core/src/webgraph/builder.rs+++ b/crates/core/src/webgraph/builder.rs@@ -18,12 +18,11 @@ use std::path::Path;use crate::executor::Executor;-use super::{Compression, Webgraph};+use super::Webgraph;pub struct WebgraphBuilder {path: Box<Path>,executor: Executor,- compression: Compression,}impl WebgraphBuilder {@@ -31,7 +30,6 @@ impl WebgraphBuilder {Self {path: path.as_ref().into(),executor: Executor::multi_thread("webgraph").unwrap(),- compression: Compression::default(),}}@@ -40,12 +38,7 @@ impl WebgraphBuilder {self}- pub fn compression(mut self, compression: Compression) -> Self {- self.compression = compression;- self- }-pub fn open(self) -> Webgraph {- Webgraph::open(self.path, self.executor, self.compression)+ Webgraph::open(self.path, self.executor)}}diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rsindex 7186eed2..44a2f060 100644--- a/crates/core/src/webgraph/centrality/betweenness.rs+++ b/crates/core/src/webgraph/centrality/betweenness.rs@@ -23,7 +23,7 @@ use indicatif::{ProgressBar, ProgressStyle};use crate::{intmap::IntMap,- webgraph::{Node, NodeID, Webgraph},+ webgraph::{EdgeLimit, Node, NodeID, Webgraph},};fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32) {@@ -70,7 +70,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)while let Some(v) = q.pop_front() {stack.push(v);- for edge in graph.raw_outgoing_edges(&v) {+ for edge in graph.raw_outgoing_edges(&v, EdgeLimit::Unlimited) {let w = edge.to;if !distances.contains_key(&w) {@@ -174,6 +174,7 @@ mod tests {crate::gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);for i in 0..n - 1 {diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rsindex b88e6a07..54548eba 100644--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs+++ b/crates/core/src/webgraph/centrality/derived_harmonic.rs@@ -23,7 +23,7 @@ use bloom::U64BloomFilter;use rayon::prelude::*;use std::{collections::BTreeMap, path::Path, sync::Mutex};-use crate::webgraph::{NodeID, Webgraph};+use crate::webgraph::{EdgeLimit, NodeID, Webgraph};struct BloomMap {map: Vec<Mutex<U64BloomFilter>>,@@ -101,7 +101,7 @@ impl DerivedCentrality {if let Some(harmonic) = host_harmonic.get(&host_node).unwrap() {let mut ingoing: Vec<_> = page_graph- .raw_ingoing_edges(&id)+ .raw_ingoing_edges(&id, EdgeLimit::Limit(128)).into_iter().filter_map(|e| page_graph.id2node(&e.from)).map(|n| n.into_host())diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex d800b10b..637bf89d 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -27,7 +27,7 @@ use tracing::info;use crate::{hyperloglog::HyperLogLog,kahan_sum::KahanSum,- webgraph::{NodeID, Webgraph},+ webgraph::{EdgeLimit, NodeID, Webgraph},};const HYPERLOGLOG_COUNTERS: usize = 64;@@ -66,7 +66,7 @@ fn update_changed_counters(let has_changes = AtomicBool::new(false);exact_changed_nodes.iter().for_each(|changed_node| {- for edge in graph.raw_outgoing_edges(changed_node) {+ for edge in graph.raw_outgoing_edges(changed_node, EdgeLimit::Unlimited) {if let (Some(counter_to), Some(counter_from)) =(counters.new.get_mut(&edge.to), counters.old.get(&edge.from)){@@ -316,6 +316,7 @@ mod tests {crate::gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);for (from, to, label) in test_edges() {@@ -331,6 +332,7 @@ mod tests {crate::gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);writer.insert(@@ -439,6 +441,7 @@ mod tests {crate::gen_temp_path(),crate::executor::Executor::single_thread(),crate::webgraph::Compression::default(),+ None,);for (from, to, label) in test_edges() {diff --git a/crates/core/src/webgraph/compression.rs b/crates/core/src/webgraph/compression.rsindex da89feb8..94a98ee8 100644--- a/crates/core/src/webgraph/compression.rs+++ b/crates/core/src/webgraph/compression.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-#[derive(Default, Debug, Clone, Copy)]+#[derive(Default, Debug, Clone, Copy, bincode::Encode, bincode::Decode)]pub enum Compression {None,#[default]diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 93caeabb..62a46071 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -86,13 +86,20 @@ impl Meta {}}+#[derive(+ Debug, Clone, Copy, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]+pub enum EdgeLimit {+ Unlimited,+ Limit(usize),+}+pub struct Webgraph {path: String,segments: Vec<Segment>,executor: Arc<Executor>,id2node: Id2NodeDb,meta: Meta,- compression: Compression,}impl Webgraph {@@ -110,7 +117,7 @@ impl Webgraph {self.meta.save(path);}- fn open<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {+ fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {fs::create_dir_all(&path).unwrap();let meta = Self::meta(&path);@@ -121,7 +128,6 @@ impl Webgraph {segments.push(Segment::open(path.as_ref().join("segments"),segment.clone(),- compression,));}@@ -131,7 +137,6 @@ impl Webgraph {executor: Arc::new(executor),id2node: Id2NodeDb::open(path.as_ref().join("id2node")),meta,- compression,}}@@ -147,8 +152,7 @@ impl Webgraph {self.meta.comitted_segments.push(segment.id());drop(segment);- self.segments- .push(Segment::open(new_path, id, self.compression));+ self.segments.push(Segment::open(new_path, id));}fs::remove_dir_all(other_folder)?;@@ -166,14 +170,14 @@ impl Webgraph {self.id2node.optimize_read();}- pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {+ pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {let dedup = |edges: &mut Vec<Edge<String>>| {edges.sort_by_key(|e| e.from);edges.dedup_by_key(|e| e.from);};self.inner_edges(- |segment| segment.ingoing_edges_with_label(&node.id()),+ |segment| segment.ingoing_edges_with_label(&node.id(), &limit),dedup,).into_iter()@@ -185,15 +189,6 @@ impl Webgraph {.collect()}- pub fn raw_ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {- let dedup = |edges: &mut Vec<Edge<()>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);- };-- self.inner_edges(|segment| segment.ingoing_edges_by_host(host_node), dedup)- }-pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {let mut pages: Vec<_> = self.executor@@ -212,41 +207,55 @@ impl Webgraph {pages}- pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {+ pub fn raw_ingoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {let dedup = |edges: &mut Vec<Edge<()>>| {edges.sort_by_key(|e| e.from);edges.dedup_by_key(|e| e.from);};- self.inner_edges(|segment| segment.ingoing_edges(node), dedup)+ self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup)}- pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {+ pub fn raw_ingoing_edges_with_labels(+ &self,+ node: &NodeID,+ limit: EdgeLimit,+ ) -> Vec<Edge<String>> {let dedup = |edges: &mut Vec<Edge<String>>| {edges.sort_by_key(|e| e.from);edges.dedup_by_key(|e| e.from);};- self.inner_edges(|segment| segment.ingoing_edges_with_label(node), dedup)+ self.inner_edges(+ |segment| segment.ingoing_edges_with_label(node, &limit),+ dedup,+ )}- pub fn raw_outgoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {+ pub fn raw_outgoing_edges_with_labels(+ &self,+ node: &NodeID,+ limit: EdgeLimit,+ ) -> Vec<Edge<String>> {let dedup = |edges: &mut Vec<Edge<String>>| {edges.sort_by_key(|e| e.from);edges.dedup_by_key(|e| e.from);};- self.inner_edges(|segment| segment.outgoing_edges_with_label(node), dedup)+ self.inner_edges(+ |segment| segment.outgoing_edges_with_label(node, &limit),+ dedup,+ )}- pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {+ pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {let dedup = |edges: &mut Vec<Edge<String>>| {edges.sort_by_key(|e| e.to);edges.dedup_by_key(|e| e.to);};self.inner_edges(- |segment| segment.outgoing_edges_with_label(&node.id()),+ |segment| segment.outgoing_edges_with_label(&node.id(), &limit),dedup,).into_iter()@@ -258,13 +267,13 @@ impl Webgraph {.collect()}- pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {+ pub fn raw_outgoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {let dedup = |edges: &mut Vec<Edge<()>>| {edges.sort_by_key(|e| e.to);edges.dedup_by_key(|e| e.to);};- self.inner_edges(|segment| segment.outgoing_edges(node), dedup)+ self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup)}fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>@@ -362,6 +371,7 @@ pub mod tests {crate::gen_temp_path(),Executor::single_thread(),Compression::default(),+ None,);for (from, to, label) in test_edges() {@@ -424,6 +434,7 @@ pub mod tests {crate::gen_temp_path(),Executor::single_thread(),Compression::default(),+ None,);wrt.insert(from.clone(), to.clone(), label.clone());graphs.push(wrt.finalize());@@ -453,6 +464,7 @@ pub mod tests {crate::gen_temp_path(),Executor::single_thread(),Compression::default(),+ None,);wrt.insert(from.clone(), to.clone(), label.clone());graphs.push(wrt.finalize());@@ -495,6 +507,7 @@ pub mod tests {crate::gen_temp_path(),Executor::single_thread(),Compression::default(),+ None,);writer.insert(@@ -507,46 +520,27 @@ pub mod tests {assert_eq!(graph.segments.len(), 1);assert_eq!(- graph.outgoing_edges(Node::from("A"))[0].label,+ graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)[0].label,"a".repeat(MAX_LABEL_LENGTH));}#[test]- fn edges_by_host() {- let mut writer = WebgraphWriter::new(- crate::gen_temp_path(),- Executor::single_thread(),- Compression::default(),- );+ fn test_edge_limits() {+ let graph = test_graph();- writer.insert(- Node::from("http://a.com/first"),- Node::from("http://b.com/first"),- String::new(),- );- writer.insert(- Node::from("http://c.com/first"),- Node::from("http://b.com/second"),- String::new(),+ assert_eq!(+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)+ .len(),+ 2);- let graph = writer.finalize();-- let mut res = graph- .raw_ingoing_edges_by_host(&Node::from("b.com").id())- .into_iter()- .map(|e| e.to)- .map(|id| graph.id2node(&id).unwrap())- .collect::<Vec<_>>();- res.sort();-assert_eq!(- res,- vec![- Node::from("http://b.com/first"),- Node::from("http://b.com/second")- ]+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))+ .len(),+ 1);}diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex fc08a11a..45b63127 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -66,16 +66,16 @@ impl From<u64> for NodeID {bincode::Decode,)]pub struct FullNodeID {- pub prefix: NodeID,+ pub host: NodeID,pub id: NodeID,}impl From<Node> for FullNodeID {fn from(value: Node) -> Self {let id = value.id();- let prefix = value.into_host().id();+ let host = value.into_host().id();- FullNodeID { prefix, id }+ FullNodeID { host, id }}}diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex efc53538..a363d794 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -38,7 +38,7 @@ use crate::{Result,};-use super::{Edge, FullEdge, Node, NodeID};+use super::{Edge, EdgeLimit, FullEdge, Node, NodeID};pub struct RemoteWebgraph {granularity: WebgraphGranularity,@@ -99,7 +99,7 @@ impl RemoteWebgraph {let url = Url::parse(&("http://".to_string() + host.as_str()))?;let node = Node::from(url).into_host();let id = node.id();- let edges = self.raw_ingoing_edges(id).await?;+ let edges = self.raw_ingoing_edges(id, EdgeLimit::Limit(1)).await?;if !edges.is_empty() {Ok(Some(node))@@ -153,12 +153,12 @@ impl RemoteWebgraph {Ok(nodes)}- pub async fn ingoing_edges(&self, node: Node) -> Result<Vec<FullEdge>> {+ pub async fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {let res = self.conn().await.send(- &IngoingEdges { node },+ &IngoingEdges { node, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -173,12 +173,12 @@ impl RemoteWebgraph {.collect())}- pub async fn raw_ingoing_edges(&self, id: NodeID) -> Result<Vec<Edge<()>>> {+ pub async fn raw_ingoing_edges(&self, id: NodeID, limit: EdgeLimit) -> Result<Vec<Edge<()>>> {let res = self.conn().await.send(- &RawIngoingEdges { node: id },+ &RawIngoingEdges { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -193,12 +193,16 @@ impl RemoteWebgraph {.collect())}- pub async fn raw_ingoing_edges_with_labels(&self, id: NodeID) -> Result<Vec<Edge<String>>> {+ pub async fn raw_ingoing_edges_with_labels(+ &self,+ id: NodeID,+ limit: EdgeLimit,+ ) -> Result<Vec<Edge<String>>> {let res = self.conn().await.send(- &RawIngoingEdgesWithLabels { node: id },+ &RawIngoingEdgesWithLabels { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -216,10 +220,11 @@ impl RemoteWebgraph {pub async fn batch_raw_ingoing_edges_with_labels(&self,ids: &[NodeID],+ limit: EdgeLimit,) -> Result<Vec<Vec<Edge<String>>>> {let reqs: Vec<_> = ids.iter()- .map(|id| RawIngoingEdgesWithLabels { node: *id })+ .map(|id| RawIngoingEdgesWithLabels { node: *id, limit }).collect();let res = self@@ -243,8 +248,15 @@ impl RemoteWebgraph {Ok(edges)}- pub async fn batch_raw_ingoing_edges(&self, ids: &[NodeID]) -> Result<Vec<Vec<Edge<()>>>> {- let reqs: Vec<_> = ids.iter().map(|id| RawIngoingEdges { node: *id }).collect();+ pub async fn batch_raw_ingoing_edges(+ &self,+ ids: &[NodeID],+ limit: EdgeLimit,+ ) -> Result<Vec<Vec<Edge<()>>>> {+ let reqs: Vec<_> = ids+ .iter()+ .map(|id| RawIngoingEdges { node: *id, limit })+ .collect();let res = self.conn()@@ -267,12 +279,12 @@ impl RemoteWebgraph {Ok(edges)}- pub async fn outgoing_edges(&self, node: Node) -> Result<Vec<FullEdge>> {+ pub async fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {let res = self.conn().await.send(- &OutgoingEdges { node },+ &OutgoingEdges { node, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -287,12 +299,12 @@ impl RemoteWebgraph {.collect())}- pub async fn raw_outgoing_edges(&self, id: NodeID) -> Result<Vec<Edge<()>>> {+ pub async fn raw_outgoing_edges(&self, id: NodeID, limit: EdgeLimit) -> Result<Vec<Edge<()>>> {let res = self.conn().await.send(- &RawOutgoingEdges { node: id },+ &RawOutgoingEdges { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -307,12 +319,16 @@ impl RemoteWebgraph {.collect())}- pub async fn raw_outgoing_edges_with_labels(&self, id: NodeID) -> Result<Vec<Edge<String>>> {+ pub async fn raw_outgoing_edges_with_labels(+ &self,+ id: NodeID,+ limit: EdgeLimit,+ ) -> Result<Vec<Edge<String>>> {let res = self.conn().await.send(- &RawOutgoingEdgesWithLabels { node: id },+ &RawOutgoingEdgesWithLabels { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -327,10 +343,14 @@ impl RemoteWebgraph {.collect())}- pub async fn batch_raw_outgoing_edges(&self, ids: &[NodeID]) -> Result<Vec<Vec<Edge<()>>>> {+ pub async fn batch_raw_outgoing_edges(+ &self,+ ids: &[NodeID],+ limit: EdgeLimit,+ ) -> Result<Vec<Vec<Edge<()>>>> {let reqs: Vec<_> = ids.iter()- .map(|id| RawOutgoingEdges { node: *id })+ .map(|id| RawOutgoingEdges { node: *id, limit }).collect();let res = selfdiff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex af257730..ee611ea5 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -14,10 +14,14 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::{Path, PathBuf};+use std::{+ path::{Path, PathBuf},+ sync::Arc,+};use super::{- store::EdgeStore, store_writer::EdgeStoreWriter, Compression, Edge, InnerEdge, NodeID,+ store::EdgeStore, store_writer::EdgeStoreWriter, Compression, Edge, EdgeLimit, InnerEdge,+ NodeID,};const ADJACENCY_STORE: &str = "adjacency";@@ -31,12 +35,18 @@ pub struct SegmentWriter {}impl SegmentWriter {- pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {+ pub fn open<P: AsRef<Path>>(+ folder_path: P,+ id: String,+ compression: Compression,+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,+ ) -> Self {SegmentWriter {adjacency: EdgeStoreWriter::new(folder_path.as_ref().join(&id).join(ADJACENCY_STORE),compression,false,+ host_centrality_rank_store.clone(),),reversed_adjacency: EdgeStoreWriter::new(folder_path@@ -45,6 +55,7 @@ impl SegmentWriter {.join(REVERSED_ADJACENCY_STORE),compression,true,+ host_centrality_rank_store.clone(),),folder_path: folder_path.as_ref()@@ -79,20 +90,15 @@ pub struct Segment {}impl Segment {- pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {Segment {- adjacency: EdgeStore::open(- folder_path.as_ref().join(&id).join(ADJACENCY_STORE),- false,- compression,- ),+ adjacency: EdgeStore::open(folder_path.as_ref().join(&id).join(ADJACENCY_STORE), false),reversed_adjacency: EdgeStore::open(folder_path.as_ref().join(&id).join(REVERSED_ADJACENCY_STORE),true,- compression,),folder_path: folder_path.as_ref()@@ -104,32 +110,24 @@ impl Segment {}}- pub fn outgoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {- self.adjacency.get_with_label(node)- }-- pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {- self.adjacency.get_without_label(node)+ pub fn outgoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {+ self.adjacency.get_with_label(node, limit)}- pub fn ingoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {- self.reversed_adjacency.get_with_label(node)+ pub fn outgoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {+ self.adjacency.get_without_label(node, limit)}- pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {- self.reversed_adjacency.get_without_label(node)+ pub fn ingoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {+ self.reversed_adjacency.get_with_label(node, limit)}- pub fn ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {- self.reversed_adjacency- .nodes_by_prefix(host_node)- .into_iter()- .flat_map(|node| self.ingoing_edges(&node))- .collect()+ pub fn ingoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {+ self.reversed_adjacency.get_without_label(node, limit)}pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {- self.reversed_adjacency.nodes_by_prefix(host_node)+ self.reversed_adjacency.nodes_by_host(host_node)}pub fn id(&self) -> String {@@ -169,21 +167,22 @@ mod test {crate::gen_temp_path(),"test".to_string(),Compression::default(),+ None,);let mut edges = Vec::new();let a = FullNodeID {id: NodeID::from(0 as u64),- prefix: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),};let b = FullNodeID {id: NodeID::from(1 as u64),- prefix: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),};let c = FullNodeID {id: NodeID::from(2 as u64),- prefix: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),};edges.push(InnerEdge {@@ -212,7 +211,7 @@ mod test {}let segment = writer.finalize();- let mut out: Vec<_> = segment.outgoing_edges(&a.id);+ let mut out: Vec<_> = segment.outgoing_edges(&a.id, &EdgeLimit::Unlimited);out.sort_by(|a, b| a.to.cmp(&b.to));@@ -232,7 +231,7 @@ mod test {]);- let mut out: Vec<_> = segment.outgoing_edges(&b.id);+ let mut out: Vec<_> = segment.outgoing_edges(&b.id, &EdgeLimit::Unlimited);out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,@@ -243,7 +242,7 @@ mod test {},]);- let mut out: Vec<_> = segment.outgoing_edges(&c.id);+ let mut out: Vec<_> = segment.outgoing_edges(&c.id, &EdgeLimit::Unlimited);out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,@@ -254,7 +253,7 @@ mod test {},]);- let out: Vec<_> = segment.ingoing_edges(&a.id);+ let out: Vec<_> = segment.ingoing_edges(&a.id, &EdgeLimit::Unlimited);assert_eq!(out,vec![Edge {@@ -264,7 +263,7 @@ mod test {},]);- let out: Vec<_> = segment.ingoing_edges(&b.id);+ let out: Vec<_> = segment.ingoing_edges(&b.id, &EdgeLimit::Unlimited);assert_eq!(out,vec![Edge {@@ -274,7 +273,7 @@ mod test {},]);- let mut out: Vec<_> = segment.ingoing_edges(&c.id);+ let mut out: Vec<_> = segment.ingoing_edges(&c.id, &EdgeLimit::Unlimited);out.sort_by(|a, b| a.from.cmp(&b.from));assert_eq!(out,diff --git a/crates/core/src/webgraph/shortest_path.rs b/crates/core/src/webgraph/shortest_path.rsindex a178d6c0..1915c54d 100644--- a/crates/core/src/webgraph/shortest_path.rs+++ b/crates/core/src/webgraph/shortest_path.rs@@ -89,7 +89,7 @@ impl ShortestPaths for Webgraph {fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8> {dijkstra_multi(&[source],- |node| self.raw_outgoing_edges(&node),+ |node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),|edge| edge.to,Some(max_dist),)@@ -98,7 +98,7 @@ impl ShortestPaths for Webgraph {fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {dijkstra_multi(&[source],- |node| self.raw_outgoing_edges(&node),+ |node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),|edge| edge.to,None,)@@ -107,7 +107,7 @@ impl ShortestPaths for Webgraph {fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {dijkstra_multi(&[source],- |node| self.raw_ingoing_edges(&node),+ |node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),|edge| edge.from,None,)diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 2e2d0a3d..a2fe9938 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -14,13 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{fs::File, io::Write, ops::Range, path::Path};+use std::{ops::Range, path::Path};+use file_store::{+ iterable::{ConstIterableStoreReader, IterableStoreReader},+ ConstSerializable,+};use fst::Automaton;use itertools::Itertools;-use memmap2::Mmap;-use super::{Compression, Edge, FullNodeID, InnerEdge, NodeID};+use super::{Compression, Edge, EdgeLimit, FullNodeID, NodeID};#[derive(Debug,@@ -34,17 +37,17 @@ use super::{Compression, Edge, FullNodeID, InnerEdge, NodeID};bincode::Decode,)]struct SerializedEdge {- from_prefix: NodeID,- to_prefix: NodeID,+ from_host: NodeID,+ to_host: NodeID,label: Vec<u8>,}-struct PrefixDb {+pub struct HostDb {db: speedy_kv::Db<Vec<u8>, Vec<u8>>,}-impl PrefixDb {- fn open<P: AsRef<Path>>(path: P) -> Self {+impl HostDb {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {let db = speedy_kv::Db::open_or_create(path).unwrap();Self { db }@@ -54,9 +57,9 @@ impl PrefixDb {self.db.merge_all_segments().unwrap();}- fn insert(&mut self, node: &FullNodeID) {+ pub fn insert(&mut self, node: &FullNodeID) {let key = [- node.prefix.as_u64().to_le_bytes(),+ node.host.as_u64().to_le_bytes(),node.id.as_u64().to_le_bytes(),].concat();@@ -65,10 +68,10 @@ impl PrefixDb {self.db.insert_raw(key, value);}- fn get(&self, prefix: &NodeID) -> Vec<NodeID> {- let prefix = prefix.as_u64().to_le_bytes().to_vec();+ fn get(&self, host: &NodeID) -> Vec<NodeID> {+ let host = host.as_u64().to_le_bytes().to_vec();- let query = speedy_kv::automaton::ExactMatch(&prefix).starts_with();+ let query = speedy_kv::automaton::ExactMatch(&host).starts_with();self.db.search_raw(query)@@ -81,19 +84,19 @@ impl PrefixDb {.collect()}- fn flush(&mut self) {+ pub fn flush(&mut self) {self.db.commit().unwrap();self.db.merge_all_segments().unwrap();}}-struct RangesDb {+pub struct RangesDb {nodes: speedy_kv::Db<NodeID, std::ops::Range<u64>>,labels: speedy_kv::Db<NodeID, std::ops::Range<u64>>,}impl RangesDb {- fn open<P: AsRef<Path>>(path: P) -> Self {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {let nodes = speedy_kv::Db::open_or_create(path.as_ref().join("nodes")).unwrap();let labels = speedy_kv::Db::open_or_create(path.as_ref().join("labels")).unwrap();@@ -104,202 +107,117 @@ impl RangesDb {self.nodes.merge_all_segments().unwrap();self.labels.merge_all_segments().unwrap();}-}-pub struct EdgeStore {- reversed: bool,- ranges: RangesDb,- prefixes: PrefixDb,+ pub fn commit(&mut self) {+ self.nodes.commit().unwrap();+ self.labels.commit().unwrap();+ }- edge_labels_file: File,- edge_labels_len: usize,- edge_labels: Mmap,+ pub fn nodes_get_raw<'a>(+ &'a self,+ key: &'a [u8],+ ) -> Option<speedy_kv::SerializedRef<'a, Range<u64>>> {+ self.nodes.get_raw(key)+ }- edge_nodes_file: File,- edge_nodes_len: usize,- edge_nodes: Mmap,+ pub fn labels_get_raw<'a>(+ &'a self,+ key: &'a [u8],+ ) -> Option<speedy_kv::SerializedRef<'a, Range<u64>>> {+ self.labels.get_raw(key)+ }- compression: Compression,-}+ pub fn insert_raw_node(&mut self, node: Vec<u8>, range: Vec<u8>) {+ self.nodes.insert_raw(node, range);+ }-impl EdgeStore {- pub fn open<P: AsRef<Path>>(path: P, reversed: bool, compression: Compression) -> Self {- let ranges = RangesDb::open(path.as_ref().join("ranges"));+ pub fn insert_raw_label(&mut self, node: Vec<u8>, range: Vec<u8>) {+ self.labels.insert_raw(node, range);+ }+}- let edge_labels_file = File::options()- .read(true)- .create(true)- .truncate(false)- .write(true)- .open(path.as_ref().join("labels"))- .unwrap();- let edge_labels = unsafe { Mmap::map(&edge_labels_file).unwrap() };- let edge_labels_len = edge_labels.len();-- let edge_nodes_file = File::options()- .read(true)- .create(true)- .truncate(false)- .write(true)- .open(path.as_ref().join("nodes"))- .unwrap();- let edge_nodes = unsafe { Mmap::map(&edge_nodes_file).unwrap() };- let edge_nodes_len = edge_nodes.len();+impl ConstSerializable for NodeID {+ const BYTES: usize = std::mem::size_of::<NodeID>();- Self {- reversed,- ranges,- prefixes: PrefixDb::open(path.as_ref().join("prefixes")),- edge_labels,- edge_labels_len,- edge_labels_file,- edge_nodes,- edge_nodes_file,- edge_nodes_len,- compression,- }+ fn serialize(&self, buf: &mut [u8]) {+ self.as_u64().serialize(buf);}- pub fn optimize_read(&mut self) {- self.ranges.optimize_read();- self.prefixes.optimize_read();+ fn deserialize(buf: &[u8]) -> Self {+ let id = u64::deserialize(buf);+ NodeID::from(id)}+}- /// Insert a batch of edges into the store.- /// The edges *must* have been de-duplicated by their from/to node.- /// I.e. if the store is not reversed, there should only ever be a single- /// put for each from node, and vice versa.- fn put(&mut self, edges: &[InnerEdge<String>]) {- if edges.is_empty() {- return;- }-- let node = if self.reversed {- edges[0].to.clone()- } else {- edges[0].from.clone()- };+pub const NUM_LABELS_PER_BLOCK: usize = 128;- self.prefixes.insert(&node);- let node_bytes = node.id.as_u64().to_le_bytes();+#[derive(bincode::Encode, bincode::Decode)]+pub struct LabelBlock {+ labels: Vec<String>,+}- debug_assert!(self.ranges.nodes.get_raw(&node_bytes).is_none());- debug_assert!(self.ranges.labels.get_raw(&node_bytes).is_none());+impl LabelBlock {+ pub fn new(labels: Vec<String>) -> Self {+ Self { labels }+ }- let mut edge_labels = Vec::new();- let mut edge_nodes = Vec::new();+ pub fn compress(&self, compression: Compression) -> CompressedLabelBlock {+ let bytes = bincode::encode_to_vec(self, bincode::config::standard()).unwrap();+ let compressed = compression.compress(&bytes);- for edge in edges {- edge_labels.push(edge.label.clone());- edge_nodes.push(if self.reversed {- edge.from.id- } else {- edge.to.id- });+ CompressedLabelBlock {+ compressions: compression,+ data: compressed,}+ }+}- let edge_labels_bytes =- bincode::encode_to_vec(&edge_labels, bincode::config::standard()).unwrap();- let edge_nodes_bytes =- bincode::encode_to_vec(&edge_nodes, bincode::config::standard()).unwrap();-- let edge_labels_bytes = self.compression.compress(&edge_labels_bytes);- let edge_nodes_bytes = self.compression.compress(&edge_nodes_bytes);-- let label_range = self.edge_labels_len..(self.edge_labels_len + edge_labels_bytes.len());- let node_range = self.edge_nodes_len..(self.edge_nodes_len + edge_nodes_bytes.len());+#[derive(bincode::Encode, bincode::Decode)]+pub struct CompressedLabelBlock {+ compressions: Compression,+ data: Vec<u8>,+}- self.edge_labels_len += edge_labels_bytes.len();- self.edge_nodes_len += edge_nodes_bytes.len();+impl CompressedLabelBlock {+ pub fn decompress(&self) -> LabelBlock {+ let bytes = self.compressions.decompress(&self.data);+ let (res, _) = bincode::decode_from_slice(&bytes, bincode::config::standard()).unwrap();+ res+ }+}- self.edge_labels_file.write_all(&edge_labels_bytes).unwrap();- self.edge_nodes_file.write_all(&edge_nodes_bytes).unwrap();+pub struct EdgeStore {+ reversed: bool,+ ranges: RangesDb,+ hosts: HostDb,- self.ranges.nodes.insert_raw(- node_bytes.to_vec(),- bincode::encode_to_vec(node_range, bincode::config::standard()).unwrap(),- );+ edge_labels: IterableStoreReader<CompressedLabelBlock>,+ edge_nodes: ConstIterableStoreReader<NodeID>,+}- self.ranges.labels.insert_raw(- node_bytes.to_vec(),- bincode::encode_to_vec(label_range, bincode::config::standard()).unwrap(),- );- }+impl EdgeStore {+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {+ let ranges = RangesDb::open(path.as_ref().join("ranges"));- /// Build a new edge store from a set of edges.- ///- /// **IMPORTANT** The edges must be sorted by- /// either the from or to node, depending on the value of `reversed`.- pub fn build<P: AsRef<Path>>(- path: P,- compression: Compression,- reversed: bool,- edges: impl Iterator<Item = InnerEdge<String>>,- ) -> Self {- let mut s = Self::open(path, reversed, compression);-- let mut inserts_since_last_flush = 0;-- // create batches of consecutive edges with the same from/to node- let mut batch = Vec::new();- let mut last_node = None;- for edge in edges {- if let Some(last_node) = last_node {- if (reversed && edge.to.id != last_node) || (!reversed && edge.from.id != last_node)- {- batch.sort_unstable_by_key(- |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },- );- batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });- let batch_len = batch.len();- s.put(&batch);- batch.clear();- inserts_since_last_flush += batch_len;-- if inserts_since_last_flush >= 1_000_000 {- s.flush();- inserts_since_last_flush = 0;- }- }- }+ let edge_labels = IterableStoreReader::open(path.as_ref().join("labels")).unwrap();- last_node = Some(if reversed { edge.to.id } else { edge.from.id });- batch.push(edge);- }+ let edge_nodes = ConstIterableStoreReader::open(path.as_ref().join("nodes")).unwrap();- if !batch.is_empty() {- batch.sort_unstable_by_key(- |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },- );- batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });- s.put(&batch);+ Self {+ ranges,+ hosts: HostDb::open(path.as_ref().join("hosts")),+ edge_labels,+ edge_nodes,+ reversed,}-- s.flush();-- s}- fn flush(&mut self) {- self.prefixes.flush();-- self.ranges.labels.commit().unwrap();- self.ranges.labels.merge_all_segments().unwrap();-- self.ranges.nodes.commit().unwrap();- self.ranges.nodes.merge_all_segments().unwrap();-- self.edge_nodes_file.flush().unwrap();- self.edge_labels_file.flush().unwrap();-- self.edge_nodes = unsafe { Mmap::map(&self.edge_nodes_file).unwrap() };- self.edge_labels = unsafe { Mmap::map(&self.edge_labels_file).unwrap() };-- self.edge_nodes_len = self.edge_nodes.len();- self.edge_labels_len = self.edge_labels.len();+ pub fn optimize_read(&mut self) {+ self.ranges.optimize_read();+ self.hosts.optimize_read();}- pub fn get_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {+ pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {let node_bytes = node.as_u64().to_le_bytes();match (@@ -318,15 +236,23 @@ impl EdgeStore {).unwrap();- let edge_labels = &self.edge_labels[edge_range];- let edge_labels = self.compression.decompress(edge_labels);- let (edge_labels, _): (Vec<_>, _) =- bincode::decode_from_slice(&edge_labels, bincode::config::standard()).unwrap();+ let edge_labels = self+ .edge_labels+ .slice(edge_range)+ .map(|r| r.unwrap().decompress())+ .flat_map(|block| block.labels.into_iter());++ let edge_labels: Vec<_> = match limit {+ EdgeLimit::Unlimited => edge_labels.collect(),+ EdgeLimit::Limit(limit) => edge_labels.take(*limit).collect(),+ };- let edge_nodes = &self.edge_nodes[node_range];- let edge_nodes = self.compression.decompress(edge_nodes);- let (edge_nodes, _): (Vec<_>, _) =- bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();+ let edge_nodes = self.edge_nodes.slice(node_range);++ let edge_nodes: Vec<_> = match limit {+ EdgeLimit::Unlimited => edge_nodes.collect(),+ EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),+ };edge_labels.into_iter()@@ -352,7 +278,7 @@ impl EdgeStore {}}- pub fn get_without_label(&self, node: &NodeID) -> Vec<Edge<()>> {+ pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {let node_bytes = node.as_u64().to_le_bytes();match self.ranges.nodes.get_raw(&node_bytes) {@@ -363,10 +289,12 @@ impl EdgeStore {).unwrap();- let edge_nodes = &self.edge_nodes[node_range];- let edge_nodes = self.compression.decompress(edge_nodes);- let (edge_nodes, _): (Vec<_>, _) =- bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();+ let edge_nodes = self.edge_nodes.slice(node_range);++ let edge_nodes: Vec<_> = match limit {+ EdgeLimit::Unlimited => edge_nodes.collect(),+ EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),+ };edge_nodes.into_iter()@@ -391,8 +319,8 @@ impl EdgeStore {}}- pub fn nodes_by_prefix(&self, prefix: &NodeID) -> Vec<NodeID> {- self.prefixes.get(prefix)+ pub fn nodes_by_host(&self, host: &NodeID) -> Vec<NodeID> {+ self.hosts.get(host)}pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {@@ -405,10 +333,8 @@ impl EdgeStore {bincode::config::standard(),).unwrap();- let edge_nodes = &self.edge_nodes[node_range];- let edge_nodes = self.compression.decompress(edge_nodes);- let (edge_nodes, _): (Vec<_>, _) =- bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();++ let edge_nodes = self.edge_nodes.slice(node_range).collect::<Vec<_>>();edge_nodes.into_iter().map(move |other| {if self.reversed {@@ -431,7 +357,9 @@ impl EdgeStore {#[cfg(test)]mod tests {- use crate::webgraph::store_writer::EdgeStoreWriter;+ use std::sync::Arc;++ use crate::webgraph::{store_writer::EdgeStoreWriter, InnerEdge};use super::*;@@ -441,16 +369,17 @@ mod tests {crate::gen_temp_path().join("test-segment"),Compression::default(),false,+ None,);let e = InnerEdge {from: FullNodeID {id: NodeID::from(0 as u64),- prefix: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),},to: FullNodeID {id: NodeID::from(1 as u64),- prefix: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),},label: "test".to_string(),};@@ -459,12 +388,12 @@ mod tests {let store = kv.finalize();- let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64));+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64), &EdgeLimit::Unlimited);assert_eq!(edges.len(), 1);assert_eq!(&edges[0], &Edge::from(e.clone()));- let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64));+ let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64), &EdgeLimit::Unlimited);assert_eq!(edges.len(), 0);}@@ -475,16 +404,17 @@ mod tests {crate::gen_temp_path().join("test-segment"),Compression::default(),true,+ None,);let e = InnerEdge {from: FullNodeID {id: NodeID::from(0 as u64),- prefix: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),},to: FullNodeID {id: NodeID::from(1 as u64),- prefix: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),},label: "test".to_string(),};@@ -493,11 +423,111 @@ mod tests {let store = kv.finalize();- let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64));+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64), &EdgeLimit::Unlimited);assert_eq!(edges.len(), 0);- let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64));+ let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64), &EdgeLimit::Unlimited);assert_eq!(edges.len(), 1);assert_eq!(&edges[0], &Edge::from(e.clone()));}++ #[test]+ fn test_limit() {+ let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(+ crate::gen_temp_path().join("test-segment"),+ Compression::default(),+ true,+ None,+ );++ for i in 0..10 {+ let e = InnerEdge {+ from: FullNodeID {+ id: NodeID::from(i as u64),+ host: NodeID::from(0 as u64),+ },+ to: FullNodeID {+ id: NodeID::from(1 as u64),+ host: NodeID::from(0 as u64),+ },+ label: "test".to_string(),+ };++ kv.put(e.clone());+ }++ let store = kv.finalize();++ let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64), &EdgeLimit::Limit(5));+ assert_eq!(edges.len(), 5);+ }++ #[test]+ fn test_edge_ordering() {+ let mut rank_store =+ speedy_kv::Db::open_or_create(crate::gen_temp_path().join("test-rank-store")).unwrap();++ rank_store.insert(NodeID::from(2 as u64), 1).unwrap();+ rank_store.insert(NodeID::from(3 as u64), 2).unwrap();+ rank_store.insert(NodeID::from(1 as u64), 3).unwrap();++ rank_store.commit().unwrap();++ let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(+ crate::gen_temp_path().join("test-segment"),+ Compression::default(),+ true,+ Some(Arc::new(rank_store)),+ );++ let e1 = InnerEdge {+ from: FullNodeID {+ id: NodeID::from(1 as u64),+ host: NodeID::from(1 as u64),+ },+ to: FullNodeID {+ id: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),+ },+ label: "1".to_string(),+ };++ let e2 = InnerEdge {+ from: FullNodeID {+ id: NodeID::from(2 as u64),+ host: NodeID::from(2 as u64),+ },+ to: FullNodeID {+ id: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),+ },+ label: "2".to_string(),+ };++ let e3 = InnerEdge {+ from: FullNodeID {+ id: NodeID::from(3 as u64),+ host: NodeID::from(3 as u64),+ },+ to: FullNodeID {+ id: NodeID::from(0 as u64),+ host: NodeID::from(0 as u64),+ },+ label: "3".to_string(),+ };++ kv.put(e1.clone());+ kv.put(e2.clone());+ kv.put(e3.clone());++ let store = kv.finalize();++ let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64), &EdgeLimit::Unlimited);++ assert_eq!(edges.len(), 3);++ assert_eq!(&edges[0], &Edge::from(e2.clone()));+ assert_eq!(&edges[1], &Edge::from(e3.clone()));+ assert_eq!(&edges[2], &Edge::from(e1.clone()));+ }}diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex 3adcca8a..b9cc1ed2 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -19,15 +19,22 @@ pub const MAX_BATCH_SIZE: usize = 3_000_000;use std::{collections::BTreeSet,fs::File,+ ops::Range,path::{Path, PathBuf},+ sync::Arc,};use itertools::Itertools;-use crate::Result;-use file_store::iterable::{IterableStoreReader, SortedIterableStoreReader};+use crate::{webgraph::store::NUM_LABELS_PER_BLOCK, Result};+use file_store::iterable::{+ ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter, SortedIterableStoreReader,+};-use super::{store::EdgeStore, Compression, EdgeLabel, InnerEdge, NodeID};+use super::{+ store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},+ Compression, EdgeLabel, InnerEdge, NodeID,+};#[derive(bincode::Encode, bincode::Decode)]struct SortableEdge<L: EdgeLabel> {@@ -97,28 +104,36 @@ pub struct EdgeStoreWriter {edges: BTreeSet<SortableEdge<String>>,stored_writers: Vec<PathBuf>,compression: Compression,+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,}impl EdgeStoreWriter {- pub fn new<P: AsRef<Path>>(path: P, compression: Compression, reversed: bool) -> Self {- let path = path.as_ref().join("writer");-- if !path.exists() {- std::fs::create_dir_all(&path).unwrap();+ pub fn new<P: AsRef<Path>>(+ path: P,+ compression: Compression,+ reversed: bool,+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,+ ) -> Self {+ let writer_path = path.as_ref().join("writer");++ if !writer_path.exists() {+ std::fs::create_dir_all(&writer_path).unwrap();}Self {edges: BTreeSet::new(),reversed,- path: path.to_path_buf(),+ path: path.as_ref().to_path_buf(),compression,stored_writers: Vec::new(),+ host_centrality_rank_store,}}fn flush_to_file(&mut self) -> Result<()> {let file_path = self.path+ .join("writer").join(format!("{}.store", self.stored_writers.len()));let file = File::create(&file_path)?;@@ -158,10 +173,7 @@ impl EdgeStoreWriter {let readers = self.stored_writers.iter()- .map(|p| {- let file = File::open(p).unwrap();- IterableStoreReader::new(file)- })+ .map(|p| IterableStoreReader::open(p).unwrap()).collect();let file_reader = SortedIterableStoreReader::new(readers).map(|r| r.unwrap());@@ -174,19 +186,234 @@ impl EdgeStoreWriter {}pub fn finalize(self) -> EdgeStore {- let p = self.path.parent().unwrap().to_path_buf();-- EdgeStore::build(- p,+ let mut final_writer = FinalEdgeStoreWriter::open(self.compression,self.reversed,- self.sorted_edges().dedup().map(|e| e.edge),- )+ self.host_centrality_rank_store.clone(),+ &self.path,+ );++ let mut store = final_writer.build_store(self.sorted_edges().dedup().map(|e| e.edge));+ store.optimize_read();++ store}}impl Drop for EdgeStoreWriter {fn drop(&mut self) {- std::fs::remove_dir_all(&self.path).unwrap();+ std::fs::remove_dir_all(self.path.join("writer")).unwrap();+ }+}++struct FinalEdgeStoreWriter {+ ranges: RangesDb,+ hosts: HostDb,++ edge_labels: IterableStoreWriter<CompressedLabelBlock, File>,+ edge_nodes: ConstIterableStoreWriter<NodeID, File>,++ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,++ compression: Compression,+ reversed: bool,++ path: PathBuf,+}++impl FinalEdgeStoreWriter {+ fn open<P: AsRef<Path>>(+ compression: Compression,+ reversed: bool,+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,+ path: P,+ ) -> Self {+ let ranges = RangesDb::open(path.as_ref().join("ranges"));++ let edge_labels_file = File::options()+ .read(true)+ .create(true)+ .truncate(false)+ .write(true)+ .open(path.as_ref().join("labels"))+ .unwrap();+ let edge_labels = IterableStoreWriter::new(edge_labels_file);++ let edge_nodes_file = File::options()+ .read(true)+ .create(true)+ .truncate(false)+ .write(true)+ .open(path.as_ref().join("nodes"))+ .unwrap();+ let edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);++ Self {+ ranges,+ hosts: HostDb::open(path.as_ref().join("hosts")),+ edge_labels,+ edge_nodes,+ reversed,+ compression,+ path: path.as_ref().to_path_buf(),+ host_centrality_rank_store,+ }+ }+ /// Insert a batch of edges into the store.+ /// The edges *must* have been de-duplicated by their from/to node.+ /// I.e. if the store is not reversed, there should only ever be a single+ /// put for each from node, and vice versa.+ fn put_store(&mut self, edges: &mut [InnerEdge<String>]) {+ if edges.is_empty() {+ return;+ }++ let node = if self.reversed {+ edges[0].to.clone()+ } else {+ edges[0].from.clone()+ };++ // order edges by centrality rank of the other node host+ if let Some(rank_store) = &self.host_centrality_rank_store {+ edges.sort_by_cached_key(|e| {+ if self.reversed {+ rank_store.get(&e.from.host).unwrap().unwrap_or(u64::MAX)+ } else {+ rank_store.get(&e.to.host).unwrap().unwrap_or(u64::MAX)+ }+ });+ }++ self.hosts.insert(&node);+ let node_bytes = node.id.as_u64().to_le_bytes();++ debug_assert!(self.ranges.nodes_get_raw(&node_bytes).is_none());+ debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());++ let mut edge_labels = Vec::new();+ let mut edge_nodes = Vec::new();++ for edge in edges {+ edge_labels.push(edge.label.clone());+ edge_nodes.push(if self.reversed {+ edge.from.id+ } else {+ edge.to.id+ });+ }++ let edge_labels: Vec<_> = edge_labels+ .into_iter()+ .chunks(NUM_LABELS_PER_BLOCK)+ .into_iter()+ .map(|chunk| LabelBlock::new(chunk.collect()).compress(self.compression))+ .collect();++ let mut first_label_offset = None;+ let mut last_label_offset = None;+ let mut first_node_offset = None;+ let mut last_node_offset = None;++ for block in &edge_labels {+ let offset = self.edge_labels.write(block).unwrap();++ if first_label_offset.is_none() {+ first_label_offset = Some(offset);+ }++ last_label_offset = Some(offset);+ }++ for node in &edge_nodes {+ let offset = self.edge_nodes.write(node).unwrap();++ if first_node_offset.is_none() {+ first_node_offset = Some(offset);+ }++ last_node_offset = Some(offset);+ }++ let label_range = Range {+ start: first_label_offset.unwrap().start,+ end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,+ };++ let node_range = Range {+ start: first_node_offset.unwrap().start,+ end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,+ };++ self.ranges.insert_raw_node(+ node_bytes.to_vec(),+ bincode::encode_to_vec(node_range, bincode::config::standard()).unwrap(),+ );++ self.ranges.insert_raw_label(+ node_bytes.to_vec(),+ bincode::encode_to_vec(label_range, bincode::config::standard()).unwrap(),+ );+ }++ /// Build a new edge store from a set of edges.+ ///+ /// **IMPORTANT** The edges must be sorted by+ /// either the from or to node, depending on the value of `reversed`.+ pub fn build_store(&mut self, edges: impl Iterator<Item = InnerEdge<String>>) -> EdgeStore {+ let mut inserts_since_last_flush = 0;++ // create batches of consecutive edges with the same from/to node+ let mut batch = Vec::new();+ let mut last_node = None;+ for edge in edges {+ if let Some(last_node) = last_node {+ if (self.reversed && edge.to.id != last_node)+ || (!self.reversed && edge.from.id != last_node)+ {+ batch.sort_unstable_by_key(+ |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },+ );+ batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });+ let batch_len = batch.len();+ self.put_store(&mut batch);+ batch.clear();+ inserts_since_last_flush += batch_len;++ if inserts_since_last_flush >= 1_000_000 {+ self.flush();+ inserts_since_last_flush = 0;+ }+ }+ }++ last_node = Some(if self.reversed {+ edge.to.id+ } else {+ edge.from.id+ });+ batch.push(edge);+ }++ if !batch.is_empty() {+ batch.sort_unstable_by_key(+ |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },+ );+ batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });+ self.put_store(&mut batch);+ }++ self.flush();++ EdgeStore::open(&self.path, self.reversed)+ }++ fn flush(&mut self) {+ self.hosts.flush();++ self.ranges.commit();++ self.edge_nodes.flush().unwrap();+ self.edge_labels.flush().unwrap();}}diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rsindex 2e619d14..ba8d2b5b 100644--- a/crates/core/src/webgraph/writer.rs+++ b/crates/core/src/webgraph/writer.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses-use std::{fs, path::Path};+use std::{fs, path::Path, sync::Arc};use crate::executor::Executor;@@ -29,7 +29,6 @@ pub struct WebgraphWriter {id2node: Id2NodeDb,executor: Executor,meta: Meta,- compression: Compression,}impl WebgraphWriter {@@ -43,7 +42,12 @@ impl WebgraphWriter {self.meta.save(path);}- pub fn new<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {+ pub fn new<P: AsRef<Path>>(+ path: P,+ executor: Executor,+ compression: Compression,+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,+ ) -> Self {fs::create_dir_all(&path).unwrap();let mut meta = Self::meta(&path);meta.comitted_segments.clear();@@ -51,7 +55,12 @@ impl WebgraphWriter {fs::create_dir_all(path.as_ref().join("segments")).unwrap();let id = uuid::Uuid::new_v4().to_string();- let segment = SegmentWriter::open(path.as_ref().join("segments"), id.clone(), compression);+ let segment = SegmentWriter::open(+ path.as_ref().join("segments"),+ id.clone(),+ compression,+ host_centrality_rank_store,+ );meta.comitted_segments.push(id);@@ -61,7 +70,6 @@ impl WebgraphWriter {id2node: Id2NodeDb::open(path.as_ref().join("id2node")),executor,meta,- compression,}}@@ -110,7 +118,6 @@ impl WebgraphWriter {executor: self.executor.into(),id2node: self.id2node,meta: self.meta,- compression: self.compression,}}}diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.tomlindex 7be7177e..8985e00d 100644--- a/crates/file-store/Cargo.toml+++ b/crates/file-store/Cargo.toml@@ -9,3 +9,5 @@ license = "AGPL-3.0"[dependencies]anyhow.workspace = truebincode.workspace = true+memmap2.workspace = true+stable_deref_trait.workspace = truediff --git a/crates/file-store/src/const_serializable.rs b/crates/file-store/src/const_serializable.rsnew file mode 100644index 00000000..1029a946--- /dev/null+++ b/crates/file-store/src/const_serializable.rs@@ -0,0 +1,64 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++// change ConstSerializable to something like the following+// once generic_const_exprs is stable. See https://github.com/rust-lang/rust/issues/60551+//+// pub trait ConstSerializable {+// const BYTES: usize;+// fn serialize(&self) -> [u8; Self::BYTES];+// fn deserialize(bytes: [u8; Self::BYTES]) -> Self;+// }++pub trait ConstSerializable {+ const BYTES: usize;++ fn serialize(&self, buf: &mut [u8]);+ fn deserialize(buf: &[u8]) -> Self;+}++macro_rules! impl_const_serializable_num {+ ($t:ty, $n:expr) => {+ impl ConstSerializable for $t {+ const BYTES: usize = $n;++ fn serialize(&self, buf: &mut [u8]) {+ buf.copy_from_slice(&self.to_le_bytes());+ }++ fn deserialize(buf: &[u8]) -> Self {+ let mut bytes = [0; $n];+ bytes.copy_from_slice(&buf[..$n]);+ <$t>::from_le_bytes(bytes)+ }+ }+ };+}++impl_const_serializable_num!(u8, 1);+impl_const_serializable_num!(u16, 2);+impl_const_serializable_num!(u32, 4);+impl_const_serializable_num!(u64, 8);+impl_const_serializable_num!(u128, 16);++impl_const_serializable_num!(i8, 1);+impl_const_serializable_num!(i16, 2);+impl_const_serializable_num!(i32, 4);+impl_const_serializable_num!(i64, 8);+impl_const_serializable_num!(i128, 16);++impl_const_serializable_num!(f32, 4);+impl_const_serializable_num!(f64, 8);diff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rsindex abf47791..e0afde60 100644--- a/crates/file-store/src/iterable.rs+++ b/crates/file-store/src/iterable.rs@@ -21,9 +21,16 @@//! 1. A 64-bit little-endian integer representing the number of bytes in the upcoming item.//! 2. The serialized item.//! 3. Repeat 1-2 until the end of the file.+//!+//! If the item type `T` implements `ConstSerializable`, then the `ConstIterableStoreWriter` can be+//! used to write items to the file without intermediate headers as the size of the serialied item is known upfront.-use crate::Result;-use std::io::{self, Read, Write};+use crate::{owned_bytes::OwnedBytes, ConstSerializable, Result};+use std::{+ io::{self, Write},+ ops::Range,+ path::Path,+};use super::Peekable;@@ -32,6 +39,11 @@ struct IterableHeader {}impl IterableHeader {+ #[inline]+ const fn serialized_size() -> usize {+ std::mem::size_of::<u64>()+ }+fn serialize<W>(&self, writer: &mut W) -> io::Result<()>whereW: io::Write,@@ -39,23 +51,38 @@ impl IterableHeader {writer.write_all(&self.num_upcoming_bytes.to_le_bytes())}- fn deserialize<R>(reader: &mut R) -> io::Result<Self>- where- R: io::Read,- {- let mut num_upcoming_bytes = [0; 8];- reader.read_exact(&mut num_upcoming_bytes)?;+ fn deserialize(bytes: &[u8]) -> io::Result<Self> {+ if bytes.len() != Self::serialized_size() {+ return Err(io::Error::new(+ io::ErrorKind::InvalidData,+ "Invalid number of bytes for IterableHeader",+ ));+ }+Ok(IterableHeader {- num_upcoming_bytes: u64::from_le_bytes(num_upcoming_bytes),+ num_upcoming_bytes: u64::from_le_bytes(bytes[..8].try_into().unwrap()),})}}+#[derive(Debug, Clone, Copy)]+pub struct WrittenOffset {+ pub start: u64,+ pub num_bytes: u64,+}++impl WrittenOffset {+ pub fn range(&self) -> Range<u64> {+ self.start..self.start + self.num_bytes+ }+}+pub struct IterableStoreWriter<T, W>whereW: io::Write,{writer: io::BufWriter<W>,+ next_start: u64,_marker: std::marker::PhantomData<T>,}@@ -68,17 +95,27 @@ whereSelf {writer: io::BufWriter::new(writer),_marker: std::marker::PhantomData,+ next_start: 0,}}- pub fn write(&mut self, item: &T) -> Result<()> {+ pub fn write(&mut self, item: &T) -> Result<WrittenOffset> {let serialized = bincode::encode_to_vec(item, bincode::config::standard())?;let header = IterableHeader {num_upcoming_bytes: serialized.len() as u64,};header.serialize(&mut self.writer)?;self.writer.write_all(&serialized)?;- Ok(())++ let start = self.next_start;+ let bytes_written = IterableHeader::serialized_size() as u64 + serialized.len() as u64;++ self.next_start += bytes_written;++ Ok(WrittenOffset {+ start,+ num_bytes: bytes_written,+ })}pub fn finalize(mut self) -> Result<W> {@@ -86,46 +123,71 @@ whereself.writer.into_inner().map_err(|e| anyhow::anyhow!("{e}"))}++ pub fn flush(&mut self) -> io::Result<()> {+ self.writer.flush()+ }}-pub struct IterableStoreReader<T, R> {- reader: io::BufReader<R>,+pub struct IterableStoreReader<T> {+ data: OwnedBytes,+ offset: usize,_marker: std::marker::PhantomData<T>,}-impl<T, R> IterableStoreReader<T, R>-where- R: io::Read,-{- pub fn new(reader: R) -> Self {+impl<T> IterableStoreReader<T> {+ pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {+ let data = OwnedBytes::mmap_from_path(path)?;++ Ok(Self {+ data,+ offset: 0,+ _marker: std::marker::PhantomData,+ })+ }++ pub fn from_bytes(data: Vec<u8>) -> Self {Self {- reader: io::BufReader::new(reader),+ data: OwnedBytes::new(data),+ offset: 0,+ _marker: std::marker::PhantomData,+ }+ }++ pub fn slice(&self, range: Range<usize>) -> IterableStoreReader<T> {+ IterableStoreReader {+ data: self.data.slice(range),+ offset: 0,_marker: std::marker::PhantomData,}}}-impl<T, R> Iterator for IterableStoreReader<T, R>+impl<T> Iterator for IterableStoreReader<T>whereT: bincode::Decode,- R: io::Read,{type Item = Result<T>;fn next(&mut self) -> Option<Self::Item> {- let header = match IterableHeader::deserialize(&mut self.reader) {+ if self.offset + IterableHeader::serialized_size() > self.data.len() {+ return None;+ }++ let header_bytes = &self.data[self.offset..self.offset + IterableHeader::serialized_size()];++ let header = match IterableHeader::deserialize(header_bytes) {Ok(header) => header,Err(_err) => return None,};- let mut serialized = vec![0; header.num_upcoming_bytes as usize];- match self.reader.read_exact(&mut serialized) {- Ok(()) => (),- Err(err) => return Some(Err(err.into())),- }+ self.offset += IterableHeader::serialized_size();+ let serialized = &self.data[self.offset..self.offset + header.num_upcoming_bytes as usize];++ self.offset += header.num_upcoming_bytes as usize;Some(- match bincode::decode_from_slice(&serialized, bincode::config::standard()) {+ match bincode::decode_from_slice(serialized, bincode::config::standard()) {Ok((item, _)) => Ok(item),Err(err) => Err(err.into()),},@@ -133,30 +195,45 @@ where}}-pub struct SortedIterableStoreReader<T, R>+impl<T> io::Seek for IterableStoreReader<T> {+ fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {+ match pos {+ io::SeekFrom::Start(offset) => {+ self.offset = offset as usize;+ }+ io::SeekFrom::End(offset) => {+ self.offset = self.data.len() - offset as usize;+ }+ io::SeekFrom::Current(offset) => {+ self.offset += offset as usize;+ }+ }++ Ok(self.offset as u64)+ }+}++pub struct SortedIterableStoreReader<T>where- R: io::Read,T: bincode::Decode,{- readers: Vec<Peekable<IterableStoreReader<T, R>>>,+ readers: Vec<Peekable<IterableStoreReader<T>>>,}-impl<T, R> SortedIterableStoreReader<T, R>+impl<T> SortedIterableStoreReader<T>whereT: Ord + bincode::Decode,- R: io::Read,{- pub fn new(readers: Vec<IterableStoreReader<T, R>>) -> Self {+ pub fn new(readers: Vec<IterableStoreReader<T>>) -> Self {let readers = readers.into_iter().map(Peekable::new).collect::<Vec<_>>();Self { readers }}}-impl<T, R> Iterator for SortedIterableStoreReader<T, R>+impl<T> Iterator for SortedIterableStoreReader<T>whereT: Ord + bincode::Decode,- R: io::Read,{type Item = Result<T>;@@ -170,7 +247,7 @@ wherematch item {Ok(item) => match min_index {Some(cur_min) => {- let cur_min_reader: &Peekable<IterableStoreReader<T, R>> =+ let cur_min_reader: &Peekable<IterableStoreReader<T>> =&self.readers[cur_min];match cur_min_reader.peek().unwrap().as_ref() {@@ -196,6 +273,131 @@ where}}+pub struct ConstIterableStoreWriter<T, W>+where+ W: io::Write,+{+ writer: io::BufWriter<W>,+ buf: Vec<u8>,+ next_start: u64,+ _marker: std::marker::PhantomData<T>,+}++impl<T, W> ConstIterableStoreWriter<T, W>+where+ T: ConstSerializable,+ W: io::Write,+{+ pub fn new(writer: W) -> Self {+ Self {+ writer: io::BufWriter::new(writer),+ _marker: std::marker::PhantomData,+ buf: Vec::new(),+ next_start: 0,+ }+ }++ pub fn write(&mut self, item: &T) -> Result<WrittenOffset> {+ self.buf.resize(T::BYTES, 0);+ item.serialize(&mut self.buf);++ assert_eq!(self.buf.len(), T::BYTES);++ self.writer.write_all(&self.buf)?;++ let start = self.next_start;+ let bytes_written = T::BYTES as u64;++ self.next_start += bytes_written;++ Ok(WrittenOffset {+ start,+ num_bytes: bytes_written,+ })+ }++ pub fn finalize(mut self) -> Result<W> {+ self.writer.flush()?;++ self.writer.into_inner().map_err(|e| anyhow::anyhow!("{e}"))+ }++ pub fn flush(&mut self) -> io::Result<()> {+ self.writer.flush()+ }+}++pub struct ConstIterableStoreReader<T> {+ data: OwnedBytes,+ offset: usize,+ _marker: std::marker::PhantomData<T>,+}++impl<T> ConstIterableStoreReader<T> {+ pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {+ let data = OwnedBytes::mmap_from_path(path)?;++ Ok(Self {+ data,+ offset: 0,+ _marker: std::marker::PhantomData,+ })+ }++ pub fn from_bytes(data: Vec<u8>) -> Self {+ Self {+ data: OwnedBytes::new(data),+ offset: 0,+ _marker: std::marker::PhantomData,+ }+ }++ pub fn slice(&self, range: Range<usize>) -> ConstIterableStoreReader<T> {+ ConstIterableStoreReader {+ data: self.data.slice(range),+ offset: 0,+ _marker: std::marker::PhantomData,+ }+ }+}++impl<T> Iterator for ConstIterableStoreReader<T>+where+ T: ConstSerializable,+{+ type Item = T;++ fn next(&mut self) -> Option<Self::Item> {+ if self.offset + T::BYTES > self.data.len() {+ return None;+ }++ let item_bytes = &self.data[self.offset..self.offset + T::BYTES];++ self.offset += T::BYTES;++ Some(T::deserialize(item_bytes))+ }+}++impl<T> io::Seek for ConstIterableStoreReader<T> {+ fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {+ match pos {+ io::SeekFrom::Start(offset) => {+ self.offset = offset as usize;+ }+ io::SeekFrom::End(offset) => {+ self.offset = self.data.len() - offset as usize;+ }+ io::SeekFrom::Current(offset) => {+ self.offset += offset as usize;+ }+ }++ Ok(self.offset as u64)+ }+}+#[cfg(test)]mod tests {use super::*;@@ -208,7 +410,7 @@ mod tests {writer.write(&3).unwrap();let writer = writer.finalize().unwrap();- let reader = IterableStoreReader::new(writer.as_slice());+ let reader = IterableStoreReader::from_bytes(writer);let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();assert_eq!(items, vec![1, 2, 3]);@@ -228,13 +430,27 @@ mod tests {writer2.write(&6).unwrap();let writer2 = writer2.finalize().unwrap();- let reader1 = IterableStoreReader::new(writer1.as_slice());+ let reader1 = IterableStoreReader::from_bytes(writer1);- let reader2 = IterableStoreReader::new(writer2.as_slice());+ let reader2 = IterableStoreReader::from_bytes(writer2);let reader = SortedIterableStoreReader::new(vec![reader1, reader2]);let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();assert_eq!(items, vec![1, 2, 3, 4, 5, 6]);}++ #[test]+ fn test_const_iterable_store() {+ let mut writer = ConstIterableStoreWriter::new(Vec::new());+ writer.write(&1).unwrap();+ writer.write(&2).unwrap();+ writer.write(&3).unwrap();+ let writer = writer.finalize().unwrap();++ let reader = ConstIterableStoreReader::<i32>::from_bytes(writer);++ let items: Vec<i32> = reader.collect();+ assert_eq!(items, vec![1, 2, 3]);+ }}diff --git a/crates/file-store/src/lib.rs b/crates/file-store/src/lib.rsindex 5a1157d1..e25172fc 100644--- a/crates/file-store/src/lib.rs+++ b/crates/file-store/src/lib.rs@@ -18,7 +18,11 @@pub type Result<T> = std::result::Result<T, anyhow::Error>;+pub mod const_serializable;pub mod iterable;+mod owned_bytes;pub mod peekable;+pub mod random_lookup;+pub use const_serializable::ConstSerializable;pub use peekable::Peekable;diff --git a/crates/file-store/src/owned_bytes.rs b/crates/file-store/src/owned_bytes.rsnew file mode 100644index 00000000..f5ff20ea--- /dev/null+++ b/crates/file-store/src/owned_bytes.rs@@ -0,0 +1,149 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++//! This is taken from https://docs.rs/ownedbytes/latest/ownedbytes/+//! to avoid having to pull in another dependency.++use stable_deref_trait::StableDeref;+use std::{+ fmt, io,+ ops::{Deref, Range},+ path::Path,+ sync::Arc,+};++pub struct OwnedBytes {+ data: &'static [u8],+ box_stable_deref: Arc<dyn Deref<Target = [u8]> + Sync + Send>,+}++impl OwnedBytes {+ pub fn mmap_from_path<P: AsRef<Path>>(path: P) -> io::Result<Self> {+ let path = path.as_ref();+ let mmap = unsafe { memmap2::Mmap::map(&std::fs::File::open(path)?)? };++ Ok(Self::new(mmap))+ }++ pub fn empty() -> Self {+ Self::new(&[][..])+ }++ pub fn new<T: StableDeref + Deref<Target = [u8]> + 'static + Send + Sync>(+ data_holder: T,+ ) -> Self {+ let box_stable_deref = Arc::new(data_holder);+ let bytes: &[u8] = box_stable_deref.deref();+ let data = unsafe { &*(bytes as *const [u8]) };+ Self {+ data,+ box_stable_deref,+ }+ }++ pub fn as_slice(&self) -> &[u8] {+ self.data+ }++ #[must_use]+ #[inline]+ pub fn slice(&self, range: Range<usize>) -> Self {+ Self {+ data: &self.data[range],+ box_stable_deref: self.box_stable_deref.clone(),+ }+ }+}++impl Deref for OwnedBytes {+ type Target = [u8];++ #[inline]+ fn deref(&self) -> &Self::Target {+ self.data+ }+}++impl AsRef<[u8]> for OwnedBytes {+ #[inline]+ fn as_ref(&self) -> &[u8] {+ self.data+ }+}++impl fmt::Debug for OwnedBytes {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ // We truncate the bytes in order to make sure the debug string+ // is not too long.+ let bytes_truncated: &[u8] = if self.len() > 8 {+ &self.as_slice()[..8]+ } else {+ self.as_slice()+ };++ write!(f, "OwnedBytes({bytes_truncated:?}, len={})", self.len())+ }+}++impl Clone for OwnedBytes {+ fn clone(&self) -> Self {+ OwnedBytes {+ data: self.data,+ box_stable_deref: self.box_stable_deref.clone(),+ }+ }+}++impl io::Read for OwnedBytes {+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {+ self.as_slice().read(buf)+ }+}++impl From<Vec<u8>> for OwnedBytes {+ fn from(vec: Vec<u8>) -> Self {+ Self::new(vec)+ }+}++#[cfg(test)]+mod tests {+ use std::io::Read;++ use super::*;++ #[test]+ fn test_owned_bytes() {+ let bytes = OwnedBytes::new(vec![1, 2, 3, 4, 5]);+ assert_eq!(bytes.len(), 5);+ assert_eq!(bytes.as_slice(), &[1, 2, 3, 4, 5]);+ }++ #[test]+ fn test_owned_bytes_empty() {+ let bytes = OwnedBytes::empty();+ assert_eq!(bytes.len(), 0);+ assert_eq!(bytes.as_slice(), &[]);+ }++ #[test]+ fn test_read() {+ let mut bytes = OwnedBytes::new(vec![1, 2, 3, 4, 5]);+ let mut buf = [0; 3];+ assert_eq!(bytes.read(&mut buf).unwrap(), 3);+ assert_eq!(&buf, &[1, 2, 3]);+ }+}diff --git a/crates/file-store/src/random_lookup.rs b/crates/file-store/src/random_lookup.rsnew file mode 100644index 00000000..53f32b6e--- /dev/null+++ b/crates/file-store/src/random_lookup.rs@@ -0,0 +1,196 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use std::{+ io::{self, Write},+ path::Path,+};++use crate::{owned_bytes::OwnedBytes, ConstSerializable};++#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]+pub struct ItemId(u64);++impl ItemId {+ pub fn into_inner(self) -> u64 {+ self.0+ }++ pub fn from_inner(inner: u64) -> Self {+ ItemId(inner)+ }+}++pub struct RandomLookupWriter<V, W>+where+ W: io::Write,+{+ next_id: u64,+ writer: io::BufWriter<W>,+ buf: Vec<u8>,+ _phantom: std::marker::PhantomData<V>,+}++impl<V, W> RandomLookupWriter<V, W>+where+ W: io::Write,+ V: ConstSerializable,+{+ pub fn new(writer: W) -> Self {+ RandomLookupWriter {+ next_id: 0,+ writer: io::BufWriter::new(writer),+ buf: Vec::new(),+ _phantom: std::marker::PhantomData,+ }+ }++ pub fn write(&mut self, item: &V) -> io::Result<ItemId> {+ self.buf.resize(V::BYTES, 0);+ item.serialize(&mut self.buf);++ assert_eq!(self.buf.len(), V::BYTES);++ self.writer.write_all(&self.buf)?;++ let id = ItemId(self.next_id);+ self.next_id += 1;++ Ok(id)+ }++ pub fn finish(mut self) -> io::Result<W> {+ self.writer.flush()?;++ Ok(self.writer.into_inner()?)+ }+}++pub struct RandomLookup<V> {+ data: OwnedBytes,+ _phantom: std::marker::PhantomData<V>,+}++impl<V> RandomLookup<V>+where+ V: ConstSerializable,+{+ pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {+ let data = OwnedBytes::mmap_from_path(path)?;++ Ok(RandomLookup {+ data,+ _phantom: std::marker::PhantomData,+ })+ }++ /// Returns the value at the given item id.+ /// Panics if the item id is not from this store.+ pub fn get(&self, id: ItemId) -> V {+ let item_size: usize = V::BYTES;++ let start = id.0 as usize * item_size;++ V::deserialize(&self.data[start..start + V::BYTES])+ }++ pub fn iter(&self) -> impl Iterator<Item = (ItemId, V)> + '_ {+ RandomLookupIter {+ data: &self.data,+ next_id: 0,+ _phantom: std::marker::PhantomData,+ }+ }+}++impl<V> From<OwnedBytes> for RandomLookup<V> {+ fn from(data: OwnedBytes) -> Self {+ RandomLookup {+ data,+ _phantom: std::marker::PhantomData,+ }+ }+}++pub struct RandomLookupIter<'a, V> {+ data: &'a OwnedBytes,+ next_id: u64,+ _phantom: std::marker::PhantomData<V>,+}++impl<'a, V> Iterator for RandomLookupIter<'a, V>+where+ V: ConstSerializable,+{+ type Item = (ItemId, V);++ fn next(&mut self) -> Option<Self::Item> {+ let item_size: usize = V::BYTES;+ let start = self.next_id as usize * item_size;++ if start >= self.data.len() {+ return None;+ }++ let id = ItemId::from_inner(self.next_id);++ self.next_id += 1;++ let val = V::deserialize(&self.data[start..start + V::BYTES]);++ Some((id, val))+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_simple() {+ let mut writer = RandomLookupWriter::new(Vec::new());++ let value = 42u64;++ let id = writer.write(&value).unwrap();+ let bytes = writer.finish().unwrap();++ let store = RandomLookup::<u64>::from(OwnedBytes::new(bytes));++ let value2 = store.get(id);++ assert_eq!(value, value2);+ }++ #[test]+ fn test_iter() {+ let mut writer = RandomLookupWriter::new(Vec::new());++ let items = vec![1u64, 2, 3, 4, 5];++ for item in &items {+ writer.write(item).unwrap();+ }++ let bytes = writer.finish().unwrap();++ let store = RandomLookup::<u64>::from(OwnedBytes::new(bytes));++ let res = store.iter().map(|(_, val)| val).collect::<Vec<_>>();++ assert_eq!(items, res);+ }+}diff --git a/crates/speedy-kv/Cargo.toml b/crates/speedy-kv/Cargo.tomlindex 35173333..364195ea 100644--- a/crates/speedy-kv/Cargo.toml+++ b/crates/speedy-kv/Cargo.toml@@ -12,7 +12,7 @@ bincode.workspace = truebloom = { path = "../bloom" }file_store = { path = "../file-store" }fst.workspace = true-memmap.workspace = true+memmap2.workspace = trueserde.workspace = trueserde_json.workspace = trueuuid.workspace = truediff --git a/crates/speedy-kv/src/blob_id_index.rs b/crates/speedy-kv/src/blob_id_index.rsindex 2f7e6143..480060b5 100644--- a/crates/speedy-kv/src/blob_id_index.rs+++ b/crates/speedy-kv/src/blob_id_index.rs@@ -27,7 +27,7 @@ use super::{BlobId, Serialized, SerializedRef};pub struct BlobIdIndex<K> {path: PathBuf,- fst: fst::Map<memmap::Mmap>,+ fst: fst::Map<memmap2::Mmap>,_marker: std::marker::PhantomData<K>,}@@ -36,7 +36,7 @@ impl<K> BlobIdIndex<K> {whereP: AsRef<Path>,{- let mmap = unsafe { memmap::Mmap::map(&std::fs::File::open(&path)?)? };+ let mmap = unsafe { memmap2::Mmap::map(&std::fs::File::open(&path)?)? };Ok(Self {path: path.as_ref().to_path_buf(),@@ -55,9 +55,9 @@ impl<K> BlobIdIndex<K> {.search(super::automaton::ExactMatch(key)).into_stream();- // let mut stream = self.fst.range().ge(key).le(key).into_stream();-- stream.next().map(|(_, v)| BlobId(v))+ stream+ .next()+ .map(|(_, v)| BlobId(file_store::random_lookup::ItemId::from_inner(v)))}pub fn search<'a, A>(@@ -131,9 +131,12 @@ wheretype Item = (Serialized<K>, BlobId);fn next(&mut self) -> Option<Self::Item> {- self.stream- .next()- .map(|(k, v)| (k.to_vec().into(), BlobId(v)))+ self.stream.next().map(|(k, v)| {+ (+ k.to_vec().into(),+ BlobId(file_store::random_lookup::ItemId::from_inner(v)),+ )+ })}}@@ -142,7 +145,6 @@ whereW: Write,{fst_builder: fst::MapBuilder<BufWriter<W>>,- next_blob_id: u64,_marker: std::marker::PhantomData<K>,}@@ -155,18 +157,14 @@ whereOk(Self {fst_builder,- next_blob_id: 0,_marker: std::marker::PhantomData,})}- pub fn insert(&mut self, key: &[u8]) -> Result<BlobId> {- let id = BlobId(self.next_blob_id);- self.next_blob_id += 1;-- self.fst_builder.insert(key, id.0)?;+ pub fn insert(&mut self, key: &[u8], id: &BlobId) -> Result<()> {+ self.fst_builder.insert(key, id.0.into_inner())?;- Ok(id)+ Ok(())}pub fn finish(self) -> Result<()> {diff --git a/crates/speedy-kv/src/blob_index.rs b/crates/speedy-kv/src/blob_index.rsindex dbc03026..10056b11 100644--- a/crates/speedy-kv/src/blob_index.rs+++ b/crates/speedy-kv/src/blob_index.rs@@ -16,7 +16,7 @@use crate::Result;use std::{- io::{BufWriter, Write},+ io::Write,path::{Path, PathBuf},};@@ -24,7 +24,7 @@ use super::{BlobId, BlobPointer};pub struct BlobIndex {path: PathBuf,- bytes: memmap::Mmap,+ data: file_store::random_lookup::RandomLookup<BlobPointer>,}impl BlobIndex {@@ -32,10 +32,10 @@ impl BlobIndex {whereP: AsRef<Path>,{- let bytes = unsafe { memmap::Mmap::map(&std::fs::File::open(&path)?)? };+ let data = file_store::random_lookup::RandomLookup::open(&path)?;Ok(Self {- bytes,+ data,path: path.as_ref().to_path_buf(),})}@@ -45,14 +45,11 @@ impl BlobIndex {}pub fn get(&self, id: BlobId) -> BlobPointer {- let offset = id.0 as usize * BlobPointer::size();- let bytes = self.bytes.as_ref();- let bytes = &bytes[offset..offset + BlobPointer::size()];- BlobPointer::from_bytes(bytes.try_into().unwrap())+ self.data.get(id.0)}pub fn iter(&self) -> impl Iterator<Item = BlobPointer> + '_ {- BlobIndexIter::new(self.bytes.as_ref())+ self.data.iter().map(|(_, v)| v)}pub fn path(&self) -> &Path {@@ -60,42 +57,11 @@ impl BlobIndex {}}-struct BlobIndexIter<'a> {- bytes: &'a [u8],- offset: usize,-}--impl<'a> BlobIndexIter<'a> {- fn new(bytes: &'a [u8]) -> Self {- Self { bytes, offset: 0 }- }-}--impl<'a> Iterator for BlobIndexIter<'a> {- type Item = BlobPointer;-- fn next(&mut self) -> Option<Self::Item> {- if self.offset >= self.bytes.len() {- return None;- }-- let ptr = BlobPointer::from_bytes(- self.bytes[self.offset..self.offset + BlobPointer::size()]- .try_into()- .unwrap(),- );- self.offset += BlobPointer::size();-- Some(ptr)- }-}-pub struct BlobIndexWriter<W>whereW: Write,{- wrt: BufWriter<W>,- _marker: std::marker::PhantomData<W>,+ wrt: file_store::random_lookup::RandomLookupWriter<BlobPointer, W>,}impl<W> BlobIndexWriter<W>@@ -104,18 +70,17 @@ where{pub fn new(wrt: W) -> Self {Self {- wrt: BufWriter::new(wrt),- _marker: std::marker::PhantomData,+ wrt: file_store::random_lookup::RandomLookupWriter::new(wrt),}}- pub fn write(&mut self, ptr: &BlobPointer) -> Result<()> {- self.wrt.write_all(&ptr.as_bytes())?;- Ok(())+ pub fn write(&mut self, ptr: &BlobPointer) -> Result<BlobId> {+ let id = self.wrt.write(ptr)?;+ Ok(BlobId(id))}- pub fn finish(mut self) -> Result<()> {- self.wrt.flush()?;+ pub fn finish(self) -> Result<()> {+ self.wrt.finish()?;Ok(())}}diff --git a/crates/speedy-kv/src/blob_store.rs b/crates/speedy-kv/src/blob_store.rsindex 852bfbe4..0c994581 100644--- a/crates/speedy-kv/src/blob_store.rs+++ b/crates/speedy-kv/src/blob_store.rs@@ -29,7 +29,7 @@ pub struct Blob<'a, K, V> {pub struct BlobStore<K, V> {path: PathBuf,- bytes: Option<memmap::Mmap>,+ bytes: Option<memmap2::Mmap>,_marker: std::marker::PhantomData<(K, V)>,}@@ -38,7 +38,7 @@ impl<K, V> BlobStore<K, V> {whereP: AsRef<Path>,{- let bytes = unsafe { memmap::Mmap::map(&std::fs::File::open(path.as_ref())?).ok() };+ let bytes = unsafe { memmap2::Mmap::map(&std::fs::File::open(path.as_ref())?).ok() };Ok(Self {path: path.as_ref().to_path_buf(),diff --git a/crates/speedy-kv/src/lib.rs b/crates/speedy-kv/src/lib.rsindex 9b15649d..5c9b2cee 100644--- a/crates/speedy-kv/src/lib.rs+++ b/crates/speedy-kv/src/lib.rs@@ -47,6 +47,7 @@ mod blob_store;mod segment;mod serialized;+use file_store::ConstSerializable;pub use serialized::{Serialized, SerializedRef};struct BlobPointer {@@ -93,8 +94,20 @@ impl BlobPointer {}}+impl ConstSerializable for BlobPointer {+ const BYTES: usize = Self::size();++ fn serialize(&self, buf: &mut [u8]) {+ buf.copy_from_slice(&self.as_bytes());+ }++ fn deserialize(buf: &[u8]) -> Self {+ Self::from_bytes(buf.try_into().unwrap())+ }+}+#[derive(Clone, Copy)]-struct BlobId(u64);+struct BlobId(file_store::random_lookup::ItemId);#[derive(Debug)]struct LiveSegment<K, V> {diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rsindex 47f820fc..34b7bffc 100644--- a/crates/speedy-kv/src/segment.rs+++ b/crates/speedy-kv/src/segment.rs@@ -63,9 +63,9 @@ whereW: Write,{fn insert(&mut self, key: SerializedRef<'_, K>, value: SerializedRef<'_, V>) -> Result<()> {- let _id = self.writers.id_index.insert(key.as_bytes())?;let ptr = self.writers.store.write(key, value)?;- self.writers.blob_index.write(&ptr)?;+ let id = self.writers.blob_index.write(&ptr)?;+ self.writers.id_index.insert(key.as_bytes(), &id)?;self.bloom.insert_raw(key.as_bytes());