Skip to content

web search done right

Top k webgraph edges (#197)

Mikkel Denker9c983e5f967e
diff --git a/Cargo.lock b/Cargo.lock
index 465e44ec..9d548a3d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1544,6 +1544,8 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bincode",
+ "memmap2",
+ "stable_deref_trait",
]
[[package]]
@@ -2799,16 +2801,6 @@ version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
-[[package]]
-name = "memmap"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
-dependencies = [
- "libc",
- "winapi",
-]
-
[[package]]
name = "memmap2"
version = "0.9.4"
@@ -4631,7 +4623,7 @@ dependencies = [
"bloom",
"file_store",
"fst",
- "memmap",
+ "memmap2",
"serde",
"serde_json",
"uuid",
@@ -4725,7 +4717,6 @@ dependencies = [
"lz4_flex",
"maplit",
"md5",
- "memmap",
"memmap2",
"mime",
"min-max-heap",
diff --git a/Cargo.toml b/Cargo.toml
index bd673207..3ab388be 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -81,7 +81,6 @@ lz4_flex = "0.11.1"
lzma = "0.2.2"
maplit = "1.0.2"
md5 = "0.7.0"
-memmap = "0.7.0"
memmap2 = "0.9.0"
mime = "0.3.17"
min-max-heap = "1.3.0"
@@ -115,6 +114,7 @@ scylla = { version = "0.12.0", features = ["chrono"] }
serde = { version = "1.0.137", features = ["rc", "derive"] }
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
+stable_deref_trait = "1.2.0"
strum = { version = "0.26.2", features = ["derive"] }
tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "74940e9" }
thiserror = "1.0.31"
diff --git a/assets/licenses.html b/assets/licenses.html
index 517fc6fd..264820b9 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,7 +44,7 @@
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
- <li><a href="#Apache-2.0">Apache License 2.0</a> (399)</li>
+ <li><a href="#Apache-2.0">Apache License 2.0</a> (398)</li>
<li><a href="#MIT">MIT License</a> (192)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (7)</li>
@@ -7028,7 +7028,6 @@ limitations under the License.
<h4>Used by:</h4>
<ul class="license-used-by">
<li><a href=" https://github.com/RazrFalcon/memmap2-rs ">memmap2 0.9.4</a></li>
- <li><a href=" https://github.com/danburkert/memmap-rs ">memmap 0.7.0</a></li>
</ul>
<pre class="license-text"> Apache License
Version 2.0, January 2004
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 48564c24..215c2bae 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -68,7 +68,6 @@ logos = { workspace = true }
lz-str = { workspace = true }
lz4_flex = { workspace = true }
md5 = { workspace = true }
-memmap = { workspace = true }
memmap2 = { workspace = true }
mime = { workspace = true }
min-max-heap = { workspace = true }
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rs
index 428afa73..94eaed07 100644
--- a/crates/core/src/api/webgraph.rs
+++ b/crates/core/src/api/webgraph.rs
@@ -22,7 +22,7 @@ use utoipa::{IntoParams, ToSchema};
use crate::{
config::WebgraphGranularity,
- webgraph::{FullEdge, Node},
+ webgraph::{EdgeLimit, FullEdge, Node},
};
use super::State;
@@ -219,7 +219,7 @@ async fn ingoing_links(
WebgraphGranularity::Page => &state.page_webgraph,
};
- graph.ingoing_edges(node).await
+ graph.ingoing_edges(node, EdgeLimit::Limit(1024)).await
}
async fn outgoing_links(
@@ -232,7 +232,7 @@ async fn outgoing_links(
WebgraphGranularity::Page => &state.page_webgraph,
};
- graph.outgoing_edges(node).await
+ graph.outgoing_edges(node, EdgeLimit::Limit(1024)).await
}
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 89f341ab..d2de173b 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -73,6 +73,7 @@ pub struct WebgraphConstructConfig {
pub skip_warc_files: Option<usize>,
pub batch_size: Option<usize>,
pub canonical_index_path: Option<String>,
+ pub host_centrality_rank_store_path: Option<String>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index bb090062..3146e510 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -532,6 +532,7 @@ mod tests {
crate::gen_temp_path(),
Executor::single_thread(),
Compression::default(),
+ None,
);
for (from, to, label) in edges {
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index c6fb7d92..9ff0f853 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -123,8 +123,8 @@ fn create_webgraph() -> Result<()> {
};
let mut worker = webgraph::WebgraphWorker {
- host_graph: webgraph::open_host_graph_writer(&out_path_host),
- page_graph: webgraph::open_page_graph_writer(&out_path_page),
+ host_graph: webgraph::open_host_graph_writer(&out_path_host, None),
+ page_graph: webgraph::open_page_graph_writer(&out_path_page, None),
canonical_index: None,
};
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index d8e7b2a6..d20558a3 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -34,7 +34,7 @@ use crate::human_website_annotations;
use crate::index::Index;
use crate::rake::RakeModel;
use crate::ranking::SignalComputer;
-use crate::webgraph::{self, Node, NodeID};
+use crate::webgraph::{self, EdgeLimit, Node, NodeID};
use crate::webpage::{safety_classifier, Html, Webpage};
pub struct Config {
@@ -83,17 +83,21 @@ pub(super) enum Webgraph {
}
impl Webgraph {
- fn batch_raw_ingoing_edges_with_labels(&self, ids: Vec<NodeID>) -> Vec<Vec<String>> {
+ fn batch_raw_ingoing_edges_with_labels(
+ &self,
+ ids: Vec<NodeID>,
+ limit: EdgeLimit,
+ ) -> Vec<Vec<String>> {
let edges = match self {
Self::Remote(webgraph) => {
- crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids))
+ crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit))
.unwrap_or_default()
}
Self::Local(webgraph) => {
let mut res = Vec::new();
for id in ids {
- res.push(webgraph.raw_ingoing_edges_with_labels(&id));
+ res.push(webgraph.raw_ingoing_edges_with_labels(&id, limit));
}
res
@@ -388,7 +392,7 @@ impl IndexingWorker {
.map(|w| Node::from(w.html.url()).id())
.collect::<Vec<_>>();
- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids);
+ let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(256));
for (page, backlink) in pages.iter_mut().zip_eq(backlinks) {
page.backlink_labels = backlink;
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 245c785f..6c1bfc86 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -17,7 +17,7 @@ use crate::{
canon_index::CanonicalIndex,
config::{self, WarcSource, WebgraphConstructConfig},
entrypoint::download_all_warc_files,
- webgraph::{self, Node, WebgraphWriter},
+ webgraph::{self, Node, NodeID, WebgraphWriter},
webpage::{url_ext::UrlExt, Html},
Result,
};
@@ -66,19 +66,27 @@ pub struct Job {
pub warc_paths: Vec<String>,
}
-pub fn open_host_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {
+pub fn open_host_graph_writer<P: AsRef<Path>>(
+ path: P,
+ host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
+) -> webgraph::WebgraphWriter {
WebgraphWriter::new(
path,
crate::executor::Executor::single_thread(),
webgraph::Compression::Lz4,
+ host_centrality_store,
)
}
-pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {
+pub fn open_page_graph_writer<P: AsRef<Path>>(
+ path: P,
+ host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
+) -> webgraph::WebgraphWriter {
WebgraphWriter::new(
path,
crate::executor::Executor::single_thread(),
webgraph::Compression::Lz4,
+ host_centrality_store,
)
}
@@ -189,6 +197,13 @@ impl Webgraph {
None
};
+ let host_centrality_rank_store = if let Some(path) = &config.host_centrality_rank_store_path
+ {
+ Some(Arc::new(speedy_kv::Db::open_or_create(path)?))
+ } else {
+ None
+ };
+
let num_workers = num_cpus::get();
let mut handlers = Vec::new();
@@ -212,8 +227,8 @@ impl Webgraph {
let page_path = page_path.join(format!("worker_{i}"));
let mut worker = WebgraphWorker {
- host_graph: open_host_graph_writer(host_path),
- page_graph: open_page_graph_writer(page_path),
+ host_graph: open_host_graph_writer(host_path, host_centrality_rank_store.clone()),
+ page_graph: open_page_graph_writer(page_path, host_centrality_rank_store.clone()),
canonical_index: canonical_index.clone(),
};
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index a41db1f9..26cdff16 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -26,8 +26,8 @@ use crate::distributed::member::Member;
use crate::distributed::member::Service;
use crate::distributed::sonic::service::sonic_service;
use crate::distributed::sonic::service::Message;
-use crate::webgraph::Compression;
use crate::webgraph::Edge;
+use crate::webgraph::EdgeLimit;
use crate::webgraph::FullEdge;
use crate::webgraph::Node;
use crate::webgraph::NodeID;
@@ -76,78 +76,88 @@ impl Message<WebGraphService> for GetNode {
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct IngoingEdges {
pub node: Node,
+ pub limit: EdgeLimit,
}
impl Message<WebGraphService> for IngoingEdges {
type Response = Vec<FullEdge>;
async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.ingoing_edges(self.node)
+ server.graph.ingoing_edges(self.node, self.limit)
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct OutgoingEdges {
pub node: Node,
+ pub limit: EdgeLimit,
}
impl Message<WebGraphService> for OutgoingEdges {
type Response = Vec<FullEdge>;
async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.outgoing_edges(self.node)
+ server.graph.outgoing_edges(self.node, self.limit)
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct RawIngoingEdges {
pub node: NodeID,
+ pub limit: EdgeLimit,
}
impl Message<WebGraphService> for RawIngoingEdges {
type Response = Vec<Edge<()>>;
async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.raw_ingoing_edges(&self.node)
+ server.graph.raw_ingoing_edges(&self.node, self.limit)
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct RawOutgoingEdges {
pub node: NodeID,
+ pub limit: EdgeLimit,
}
impl Message<WebGraphService> for RawOutgoingEdges {
type Response = Vec<Edge<()>>;
async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.raw_outgoing_edges(&self.node)
+ server.graph.raw_outgoing_edges(&self.node, self.limit)
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct RawIngoingEdgesWithLabels {
pub node: NodeID,
+ pub limit: EdgeLimit,
}
impl Message<WebGraphService> for RawIngoingEdgesWithLabels {
type Response = Vec<Edge<String>>;
async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.raw_ingoing_edges_with_labels(&self.node)
+ server
+ .graph
+ .raw_ingoing_edges_with_labels(&self.node, self.limit)
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct RawOutgoingEdgesWithLabels {
pub node: NodeID,
+ pub limit: EdgeLimit,
}
impl Message<WebGraphService> for RawOutgoingEdgesWithLabels {
type Response = Vec<Edge<String>>;
async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.raw_outgoing_edges_with_labels(&self.node)
+ server
+ .graph
+ .raw_outgoing_edges_with_labels(&self.node, self.limit)
}
}
@@ -171,11 +181,7 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
.await?,
);
- let graph = Arc::new(
- WebgraphBuilder::new(config.graph_path)
- .compression(Compression::Lz4)
- .open(),
- );
+ let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());
let server = WebGraphService { graph }.bind(addr).await.unwrap();
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 68ca8a0b..2c7923ce 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -624,6 +624,7 @@ mod tests {
gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
writer.insert(
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rs
index ddbf63b1..87b80d27 100644
--- a/crates/core/src/ranking/inbound_similarity.rs
+++ b/crates/core/src/ranking/inbound_similarity.rs
@@ -149,7 +149,7 @@ mod tests {
searcher::{
api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,
},
- webgraph::{Node, Webgraph, WebgraphWriter},
+ webgraph::{EdgeLimit, Node, Webgraph, WebgraphWriter},
webpage::{Html, Webpage},
};
@@ -158,7 +158,7 @@ mod tests {
fn inbound(graph: &Webgraph, node: &NodeID) -> bitvec_similarity::BitVec {
bitvec_similarity::BitVec::new(
graph
- .raw_ingoing_edges(node)
+ .raw_ingoing_edges(node, EdgeLimit::Unlimited)
.into_iter()
.map(|e| e.from.as_u64())
.collect(),
@@ -171,6 +171,7 @@ mod tests {
gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
wrt.insert(Node::from("a.com"), Node::from("b.com"), String::new());
@@ -198,6 +199,7 @@ mod tests {
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
wrt.insert(Node::from("b.com"), Node::from("a.com"), String::new());
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index c4b1f08f..33bffbed 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -41,6 +41,7 @@ mod tests {
gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
wrt.insert(
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 976c4ee9..77a6c000 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -41,6 +41,7 @@ use crate::ranking::{
use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};
use crate::web_spell::SpellChecker;
use crate::webgraph::remote::RemoteWebgraph;
+use crate::webgraph::EdgeLimit;
use crate::widgets::{Widget, Widgets};
use crate::{
bangs::Bangs,
@@ -150,20 +151,32 @@ pub trait Graph {
fn batch_raw_ingoing(
&self,
nodes: &[webgraph::NodeID],
+ limit: EdgeLimit,
) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>>;
}
impl Graph for RemoteWebgraph {
- async fn batch_raw_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::Edge<()>>> {
- self.batch_raw_ingoing_edges(nodes)
+ async fn batch_raw_ingoing(
+ &self,
+ nodes: &[webgraph::NodeID],
+ limit: EdgeLimit,
+ ) -> Vec<Vec<webgraph::Edge<()>>> {
+ self.batch_raw_ingoing_edges(nodes, limit)
.await
.unwrap_or_default()
}
}
impl Graph for webgraph::Webgraph {
- async fn batch_raw_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::Edge<()>>> {
- nodes.iter().map(|n| self.raw_ingoing_edges(n)).collect()
+ async fn batch_raw_ingoing(
+ &self,
+ nodes: &[webgraph::NodeID],
+ limit: EdgeLimit,
+ ) -> Vec<Vec<webgraph::Edge<()>>> {
+ nodes
+ .iter()
+ .map(|n| self.raw_ingoing_edges(n, limit))
+ .collect()
}
}
@@ -174,8 +187,9 @@ where
fn batch_raw_ingoing(
&self,
nodes: &[webgraph::NodeID],
+ limit: EdgeLimit,
) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>> {
- self.as_ref().batch_raw_ingoing(nodes)
+ self.as_ref().batch_raw_ingoing(nodes, limit)
}
}
@@ -184,7 +198,7 @@ where
T: Graph,
{
async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {
- self.batch_raw_ingoing(nodes)
+ self.batch_raw_ingoing(nodes, EdgeLimit::Limit(128))
.await
.into_iter()
.map(|edges| edges.into_iter().map(|edge| edge.from).collect())
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index 2f72e254..31b91617 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -22,7 +22,7 @@ use url::Url;
use crate::{
ranking::{bitvec_similarity, inbound_similarity},
- webgraph::{remote::RemoteWebgraph, Node, NodeID},
+ webgraph::{remote::RemoteWebgraph, EdgeLimit, Node, NodeID},
webpage::url_ext::UrlExt,
SortableFloat,
};
@@ -75,7 +75,7 @@ impl SimilarHostsFinder {
let in_edges = self
.webgraph
- .batch_raw_ingoing_edges(&nodes)
+ .batch_raw_ingoing_edges(&nodes, EdgeLimit::Limit(128))
.await
.unwrap_or_default();
@@ -88,7 +88,7 @@ impl SimilarHostsFinder {
let outgoing_edges = self
.webgraph
- .batch_raw_outgoing_edges(&backlink_nodes)
+ .batch_raw_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(4096))
.await
.unwrap_or_default();
@@ -125,7 +125,7 @@ impl SimilarHostsFinder {
// remove dead links (nodes without outgoing edges might be dead links)
let known_nodes = self
.webgraph
- .batch_raw_ingoing_edges(&potential_nodes)
+ .batch_raw_ingoing_edges(&potential_nodes, EdgeLimit::Limit(1))
.await
.unwrap_or_default();
diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rs
index 23a4d6f9..998e2f46 100644
--- a/crates/core/src/web_spell/stupid_backoff.rs
+++ b/crates/core/src/web_spell/stupid_backoff.rs
@@ -208,18 +208,18 @@ fn merge_streams(
}
pub struct StupidBackoff {
- ngrams: fst::Map<memmap::Mmap>,
- rotated_ngrams: fst::Map<memmap::Mmap>,
+ ngrams: fst::Map<memmap2::Mmap>,
+ rotated_ngrams: fst::Map<memmap2::Mmap>,
n_counts: Vec<u64>,
}
impl StupidBackoff {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
let ngrams = fst::Map::new(mmap)?;
let mmap =
- unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };
+ unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };
let rotated_ngrams = fst::Map::new(mmap)?;
let file = File::open(path.as_ref().join("n_counts.bin"))?;
@@ -269,7 +269,7 @@ impl StupidBackoff {
merge_streams(builder, streams)?;
- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
let ngrams = fst::Map::new(mmap)?;
let file = OpenOptions::new()
@@ -285,7 +285,7 @@ impl StupidBackoff {
merge_streams(builder, streams)?;
- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
let rotated_ngrams = fst::Map::new(mmap)?;
Ok(Self {
diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rs
index 6d112145..0c663926 100644
--- a/crates/core/src/web_spell/term_freqs.rs
+++ b/crates/core/src/web_spell/term_freqs.rs
@@ -71,13 +71,13 @@ impl DictBuilder {
}
struct StoredDict {
- map: fst::Map<memmap::Mmap>,
+ map: fst::Map<memmap2::Mmap>,
path: PathBuf,
}
impl StoredDict {
fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref())?)? };
Ok(Self {
map: fst::Map::new(mmap)?,
@@ -147,7 +147,7 @@ impl StoredDict {
builder.finish()?;
- let mmap = unsafe { memmap::Mmap::map(&File::open(path.as_ref())?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref())?)? };
Ok(StoredDict {
map: fst::Map::new(mmap)?,
diff --git a/crates/core/src/webgraph/builder.rs b/crates/core/src/webgraph/builder.rs
index c85239d6..4b3643b9 100644
--- a/crates/core/src/webgraph/builder.rs
+++ b/crates/core/src/webgraph/builder.rs
@@ -18,12 +18,11 @@ use std::path::Path;
use crate::executor::Executor;
-use super::{Compression, Webgraph};
+use super::Webgraph;
pub struct WebgraphBuilder {
path: Box<Path>,
executor: Executor,
- compression: Compression,
}
impl WebgraphBuilder {
@@ -31,7 +30,6 @@ impl WebgraphBuilder {
Self {
path: path.as_ref().into(),
executor: Executor::multi_thread("webgraph").unwrap(),
- compression: Compression::default(),
}
}
@@ -40,12 +38,7 @@ impl WebgraphBuilder {
self
}
- pub fn compression(mut self, compression: Compression) -> Self {
- self.compression = compression;
- self
- }
-
pub fn open(self) -> Webgraph {
- Webgraph::open(self.path, self.executor, self.compression)
+ Webgraph::open(self.path, self.executor)
}
}
diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rs
index 7186eed2..44a2f060 100644
--- a/crates/core/src/webgraph/centrality/betweenness.rs
+++ b/crates/core/src/webgraph/centrality/betweenness.rs
@@ -23,7 +23,7 @@ use indicatif::{ProgressBar, ProgressStyle};
use crate::{
intmap::IntMap,
- webgraph::{Node, NodeID, Webgraph},
+ webgraph::{EdgeLimit, Node, NodeID, Webgraph},
};
fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32) {
@@ -70,7 +70,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
while let Some(v) = q.pop_front() {
stack.push(v);
- for edge in graph.raw_outgoing_edges(&v) {
+ for edge in graph.raw_outgoing_edges(&v, EdgeLimit::Unlimited) {
let w = edge.to;
if !distances.contains_key(&w) {
@@ -174,6 +174,7 @@ mod tests {
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
for i in 0..n - 1 {
diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rs
index b88e6a07..54548eba 100644
--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs
+++ b/crates/core/src/webgraph/centrality/derived_harmonic.rs
@@ -23,7 +23,7 @@ use bloom::U64BloomFilter;
use rayon::prelude::*;
use std::{collections::BTreeMap, path::Path, sync::Mutex};
-use crate::webgraph::{NodeID, Webgraph};
+use crate::webgraph::{EdgeLimit, NodeID, Webgraph};
struct BloomMap {
map: Vec<Mutex<U64BloomFilter>>,
@@ -101,7 +101,7 @@ impl DerivedCentrality {
if let Some(harmonic) = host_harmonic.get(&host_node).unwrap() {
let mut ingoing: Vec<_> = page_graph
- .raw_ingoing_edges(&id)
+ .raw_ingoing_edges(&id, EdgeLimit::Limit(128))
.into_iter()
.filter_map(|e| page_graph.id2node(&e.from))
.map(|n| n.into_host())
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index d800b10b..637bf89d 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -27,7 +27,7 @@ use tracing::info;
use crate::{
hyperloglog::HyperLogLog,
kahan_sum::KahanSum,
- webgraph::{NodeID, Webgraph},
+ webgraph::{EdgeLimit, NodeID, Webgraph},
};
const HYPERLOGLOG_COUNTERS: usize = 64;
@@ -66,7 +66,7 @@ fn update_changed_counters(
let has_changes = AtomicBool::new(false);
exact_changed_nodes.iter().for_each(|changed_node| {
- for edge in graph.raw_outgoing_edges(changed_node) {
+ for edge in graph.raw_outgoing_edges(changed_node, EdgeLimit::Unlimited) {
if let (Some(counter_to), Some(counter_from)) =
(counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
{
@@ -316,6 +316,7 @@ mod tests {
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
for (from, to, label) in test_edges() {
@@ -331,6 +332,7 @@ mod tests {
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
writer.insert(
@@ -439,6 +441,7 @@ mod tests {
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
+ None,
);
for (from, to, label) in test_edges() {
diff --git a/crates/core/src/webgraph/compression.rs b/crates/core/src/webgraph/compression.rs
index da89feb8..94a98ee8 100644
--- a/crates/core/src/webgraph/compression.rs
+++ b/crates/core/src/webgraph/compression.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
-#[derive(Default, Debug, Clone, Copy)]
+#[derive(Default, Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
pub enum Compression {
None,
#[default]
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 93caeabb..62a46071 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -86,13 +86,20 @@ impl Meta {
}
}
+#[derive(
+ Debug, Clone, Copy, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,
+)]
+pub enum EdgeLimit {
+ Unlimited,
+ Limit(usize),
+}
+
pub struct Webgraph {
path: String,
segments: Vec<Segment>,
executor: Arc<Executor>,
id2node: Id2NodeDb,
meta: Meta,
- compression: Compression,
}
impl Webgraph {
@@ -110,7 +117,7 @@ impl Webgraph {
self.meta.save(path);
}
- fn open<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {
+ fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {
fs::create_dir_all(&path).unwrap();
let meta = Self::meta(&path);
@@ -121,7 +128,6 @@ impl Webgraph {
segments.push(Segment::open(
path.as_ref().join("segments"),
segment.clone(),
- compression,
));
}
@@ -131,7 +137,6 @@ impl Webgraph {
executor: Arc::new(executor),
id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
meta,
- compression,
}
}
@@ -147,8 +152,7 @@ impl Webgraph {
self.meta.comitted_segments.push(segment.id());
drop(segment);
- self.segments
- .push(Segment::open(new_path, id, self.compression));
+ self.segments.push(Segment::open(new_path, id));
}
fs::remove_dir_all(other_folder)?;
@@ -166,14 +170,14 @@ impl Webgraph {
self.id2node.optimize_read();
}
- pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
+ pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
let dedup = |edges: &mut Vec<Edge<String>>| {
edges.sort_by_key(|e| e.from);
edges.dedup_by_key(|e| e.from);
};
self.inner_edges(
- |segment| segment.ingoing_edges_with_label(&node.id()),
+ |segment| segment.ingoing_edges_with_label(&node.id(), &limit),
dedup,
)
.into_iter()
@@ -185,15 +189,6 @@ impl Webgraph {
.collect()
}
- pub fn raw_ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {
- let dedup = |edges: &mut Vec<Edge<()>>| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
- };
-
- self.inner_edges(|segment| segment.ingoing_edges_by_host(host_node), dedup)
- }
-
pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {
let mut pages: Vec<_> = self
.executor
@@ -212,41 +207,55 @@ impl Webgraph {
pages
}
- pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
+ pub fn raw_ingoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {
let dedup = |edges: &mut Vec<Edge<()>>| {
edges.sort_by_key(|e| e.from);
edges.dedup_by_key(|e| e.from);
};
- self.inner_edges(|segment| segment.ingoing_edges(node), dedup)
+ self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup)
}
- pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {
+ pub fn raw_ingoing_edges_with_labels(
+ &self,
+ node: &NodeID,
+ limit: EdgeLimit,
+ ) -> Vec<Edge<String>> {
let dedup = |edges: &mut Vec<Edge<String>>| {
edges.sort_by_key(|e| e.from);
edges.dedup_by_key(|e| e.from);
};
- self.inner_edges(|segment| segment.ingoing_edges_with_label(node), dedup)
+ self.inner_edges(
+ |segment| segment.ingoing_edges_with_label(node, &limit),
+ dedup,
+ )
}
- pub fn raw_outgoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {
+ pub fn raw_outgoing_edges_with_labels(
+ &self,
+ node: &NodeID,
+ limit: EdgeLimit,
+ ) -> Vec<Edge<String>> {
let dedup = |edges: &mut Vec<Edge<String>>| {
edges.sort_by_key(|e| e.from);
edges.dedup_by_key(|e| e.from);
};
- self.inner_edges(|segment| segment.outgoing_edges_with_label(node), dedup)
+ self.inner_edges(
+ |segment| segment.outgoing_edges_with_label(node, &limit),
+ dedup,
+ )
}
- pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {
+ pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
let dedup = |edges: &mut Vec<Edge<String>>| {
edges.sort_by_key(|e| e.to);
edges.dedup_by_key(|e| e.to);
};
self.inner_edges(
- |segment| segment.outgoing_edges_with_label(&node.id()),
+ |segment| segment.outgoing_edges_with_label(&node.id(), &limit),
dedup,
)
.into_iter()
@@ -258,13 +267,13 @@ impl Webgraph {
.collect()
}
- pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
+ pub fn raw_outgoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {
let dedup = |edges: &mut Vec<Edge<()>>| {
edges.sort_by_key(|e| e.to);
edges.dedup_by_key(|e| e.to);
};
- self.inner_edges(|segment| segment.outgoing_edges(node), dedup)
+ self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup)
}
fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>
@@ -362,6 +371,7 @@ pub mod tests {
crate::gen_temp_path(),
Executor::single_thread(),
Compression::default(),
+ None,
);
for (from, to, label) in test_edges() {
@@ -424,6 +434,7 @@ pub mod tests {
crate::gen_temp_path(),
Executor::single_thread(),
Compression::default(),
+ None,
);
wrt.insert(from.clone(), to.clone(), label.clone());
graphs.push(wrt.finalize());
@@ -453,6 +464,7 @@ pub mod tests {
crate::gen_temp_path(),
Executor::single_thread(),
Compression::default(),
+ None,
);
wrt.insert(from.clone(), to.clone(), label.clone());
graphs.push(wrt.finalize());
@@ -495,6 +507,7 @@ pub mod tests {
crate::gen_temp_path(),
Executor::single_thread(),
Compression::default(),
+ None,
);
writer.insert(
@@ -507,46 +520,27 @@ pub mod tests {
assert_eq!(graph.segments.len(), 1);
assert_eq!(
- graph.outgoing_edges(Node::from("A"))[0].label,
+ graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)[0].label,
"a".repeat(MAX_LABEL_LENGTH)
);
}
#[test]
- fn edges_by_host() {
- let mut writer = WebgraphWriter::new(
- crate::gen_temp_path(),
- Executor::single_thread(),
- Compression::default(),
- );
+ fn test_edge_limits() {
+ let graph = test_graph();
- writer.insert(
- Node::from("http://a.com/first"),
- Node::from("http://b.com/first"),
- String::new(),
- );
- writer.insert(
- Node::from("http://c.com/first"),
- Node::from("http://b.com/second"),
- String::new(),
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 2
);
- let graph = writer.finalize();
-
- let mut res = graph
- .raw_ingoing_edges_by_host(&Node::from("b.com").id())
- .into_iter()
- .map(|e| e.to)
- .map(|id| graph.id2node(&id).unwrap())
- .collect::<Vec<_>>();
- res.sort();
-
assert_eq!(
- res,
- vec![
- Node::from("http://b.com/first"),
- Node::from("http://b.com/second")
- ]
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
+ .len(),
+ 1
);
}
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index fc08a11a..45b63127 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -66,16 +66,16 @@ impl From<u64> for NodeID {
bincode::Decode,
)]
pub struct FullNodeID {
- pub prefix: NodeID,
+ pub host: NodeID,
pub id: NodeID,
}
impl From<Node> for FullNodeID {
fn from(value: Node) -> Self {
let id = value.id();
- let prefix = value.into_host().id();
+ let host = value.into_host().id();
- FullNodeID { prefix, id }
+ FullNodeID { host, id }
}
}
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index efc53538..a363d794 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -38,7 +38,7 @@ use crate::{
Result,
};
-use super::{Edge, FullEdge, Node, NodeID};
+use super::{Edge, EdgeLimit, FullEdge, Node, NodeID};
pub struct RemoteWebgraph {
granularity: WebgraphGranularity,
@@ -99,7 +99,7 @@ impl RemoteWebgraph {
let url = Url::parse(&("http://".to_string() + host.as_str()))?;
let node = Node::from(url).into_host();
let id = node.id();
- let edges = self.raw_ingoing_edges(id).await?;
+ let edges = self.raw_ingoing_edges(id, EdgeLimit::Limit(1)).await?;
if !edges.is_empty() {
Ok(Some(node))
@@ -153,12 +153,12 @@ impl RemoteWebgraph {
Ok(nodes)
}
- pub async fn ingoing_edges(&self, node: Node) -> Result<Vec<FullEdge>> {
+ pub async fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {
let res = self
.conn()
.await
.send(
- &IngoingEdges { node },
+ &IngoingEdges { node, limit },
&AllShardsSelector,
&RandomReplicaSelector,
)
@@ -173,12 +173,12 @@ impl RemoteWebgraph {
.collect())
}
- pub async fn raw_ingoing_edges(&self, id: NodeID) -> Result<Vec<Edge<()>>> {
+ pub async fn raw_ingoing_edges(&self, id: NodeID, limit: EdgeLimit) -> Result<Vec<Edge<()>>> {
let res = self
.conn()
.await
.send(
- &RawIngoingEdges { node: id },
+ &RawIngoingEdges { node: id, limit },
&AllShardsSelector,
&RandomReplicaSelector,
)
@@ -193,12 +193,16 @@ impl RemoteWebgraph {
.collect())
}
- pub async fn raw_ingoing_edges_with_labels(&self, id: NodeID) -> Result<Vec<Edge<String>>> {
+ pub async fn raw_ingoing_edges_with_labels(
+ &self,
+ id: NodeID,
+ limit: EdgeLimit,
+ ) -> Result<Vec<Edge<String>>> {
let res = self
.conn()
.await
.send(
- &RawIngoingEdgesWithLabels { node: id },
+ &RawIngoingEdgesWithLabels { node: id, limit },
&AllShardsSelector,
&RandomReplicaSelector,
)
@@ -216,10 +220,11 @@ impl RemoteWebgraph {
pub async fn batch_raw_ingoing_edges_with_labels(
&self,
ids: &[NodeID],
+ limit: EdgeLimit,
) -> Result<Vec<Vec<Edge<String>>>> {
let reqs: Vec<_> = ids
.iter()
- .map(|id| RawIngoingEdgesWithLabels { node: *id })
+ .map(|id| RawIngoingEdgesWithLabels { node: *id, limit })
.collect();
let res = self
@@ -243,8 +248,15 @@ impl RemoteWebgraph {
Ok(edges)
}
- pub async fn batch_raw_ingoing_edges(&self, ids: &[NodeID]) -> Result<Vec<Vec<Edge<()>>>> {
- let reqs: Vec<_> = ids.iter().map(|id| RawIngoingEdges { node: *id }).collect();
+ pub async fn batch_raw_ingoing_edges(
+ &self,
+ ids: &[NodeID],
+ limit: EdgeLimit,
+ ) -> Result<Vec<Vec<Edge<()>>>> {
+ let reqs: Vec<_> = ids
+ .iter()
+ .map(|id| RawIngoingEdges { node: *id, limit })
+ .collect();
let res = self
.conn()
@@ -267,12 +279,12 @@ impl RemoteWebgraph {
Ok(edges)
}
- pub async fn outgoing_edges(&self, node: Node) -> Result<Vec<FullEdge>> {
+ pub async fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {
let res = self
.conn()
.await
.send(
- &OutgoingEdges { node },
+ &OutgoingEdges { node, limit },
&AllShardsSelector,
&RandomReplicaSelector,
)
@@ -287,12 +299,12 @@ impl RemoteWebgraph {
.collect())
}
- pub async fn raw_outgoing_edges(&self, id: NodeID) -> Result<Vec<Edge<()>>> {
+ pub async fn raw_outgoing_edges(&self, id: NodeID, limit: EdgeLimit) -> Result<Vec<Edge<()>>> {
let res = self
.conn()
.await
.send(
- &RawOutgoingEdges { node: id },
+ &RawOutgoingEdges { node: id, limit },
&AllShardsSelector,
&RandomReplicaSelector,
)
@@ -307,12 +319,16 @@ impl RemoteWebgraph {
.collect())
}
- pub async fn raw_outgoing_edges_with_labels(&self, id: NodeID) -> Result<Vec<Edge<String>>> {
+ pub async fn raw_outgoing_edges_with_labels(
+ &self,
+ id: NodeID,
+ limit: EdgeLimit,
+ ) -> Result<Vec<Edge<String>>> {
let res = self
.conn()
.await
.send(
- &RawOutgoingEdgesWithLabels { node: id },
+ &RawOutgoingEdgesWithLabels { node: id, limit },
&AllShardsSelector,
&RandomReplicaSelector,
)
@@ -327,10 +343,14 @@ impl RemoteWebgraph {
.collect())
}
- pub async fn batch_raw_outgoing_edges(&self, ids: &[NodeID]) -> Result<Vec<Vec<Edge<()>>>> {
+ pub async fn batch_raw_outgoing_edges(
+ &self,
+ ids: &[NodeID],
+ limit: EdgeLimit,
+ ) -> Result<Vec<Vec<Edge<()>>>> {
let reqs: Vec<_> = ids
.iter()
- .map(|id| RawOutgoingEdges { node: *id })
+ .map(|id| RawOutgoingEdges { node: *id, limit })
.collect();
let res = self
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index af257730..ee611ea5 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -14,10 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::path::{Path, PathBuf};
+use std::{
+ path::{Path, PathBuf},
+ sync::Arc,
+};
use super::{
- store::EdgeStore, store_writer::EdgeStoreWriter, Compression, Edge, InnerEdge, NodeID,
+ store::EdgeStore, store_writer::EdgeStoreWriter, Compression, Edge, EdgeLimit, InnerEdge,
+ NodeID,
};
const ADJACENCY_STORE: &str = "adjacency";
@@ -31,12 +35,18 @@ pub struct SegmentWriter {
}
impl SegmentWriter {
- pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {
+ pub fn open<P: AsRef<Path>>(
+ folder_path: P,
+ id: String,
+ compression: Compression,
+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
+ ) -> Self {
SegmentWriter {
adjacency: EdgeStoreWriter::new(
folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
compression,
false,
+ host_centrality_rank_store.clone(),
),
reversed_adjacency: EdgeStoreWriter::new(
folder_path
@@ -45,6 +55,7 @@ impl SegmentWriter {
.join(REVERSED_ADJACENCY_STORE),
compression,
true,
+ host_centrality_rank_store.clone(),
),
folder_path: folder_path
.as_ref()
@@ -79,20 +90,15 @@ pub struct Segment {
}
impl Segment {
- pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {
+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
Segment {
- adjacency: EdgeStore::open(
- folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
- false,
- compression,
- ),
+ adjacency: EdgeStore::open(folder_path.as_ref().join(&id).join(ADJACENCY_STORE), false),
reversed_adjacency: EdgeStore::open(
folder_path
.as_ref()
.join(&id)
.join(REVERSED_ADJACENCY_STORE),
true,
- compression,
),
folder_path: folder_path
.as_ref()
@@ -104,32 +110,24 @@ impl Segment {
}
}
- pub fn outgoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {
- self.adjacency.get_with_label(node)
- }
-
- pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
- self.adjacency.get_without_label(node)
+ pub fn outgoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {
+ self.adjacency.get_with_label(node, limit)
}
- pub fn ingoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {
- self.reversed_adjacency.get_with_label(node)
+ pub fn outgoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {
+ self.adjacency.get_without_label(node, limit)
}
- pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
- self.reversed_adjacency.get_without_label(node)
+ pub fn ingoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {
+ self.reversed_adjacency.get_with_label(node, limit)
}
- pub fn ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {
- self.reversed_adjacency
- .nodes_by_prefix(host_node)
- .into_iter()
- .flat_map(|node| self.ingoing_edges(&node))
- .collect()
+ pub fn ingoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {
+ self.reversed_adjacency.get_without_label(node, limit)
}
pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {
- self.reversed_adjacency.nodes_by_prefix(host_node)
+ self.reversed_adjacency.nodes_by_host(host_node)
}
pub fn id(&self) -> String {
@@ -169,21 +167,22 @@ mod test {
crate::gen_temp_path(),
"test".to_string(),
Compression::default(),
+ None,
);
let mut edges = Vec::new();
let a = FullNodeID {
id: NodeID::from(0 as u64),
- prefix: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
};
let b = FullNodeID {
id: NodeID::from(1 as u64),
- prefix: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
};
let c = FullNodeID {
id: NodeID::from(2 as u64),
- prefix: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
};
edges.push(InnerEdge {
@@ -212,7 +211,7 @@ mod test {
}
let segment = writer.finalize();
- let mut out: Vec<_> = segment.outgoing_edges(&a.id);
+ let mut out: Vec<_> = segment.outgoing_edges(&a.id, &EdgeLimit::Unlimited);
out.sort_by(|a, b| a.to.cmp(&b.to));
@@ -232,7 +231,7 @@ mod test {
]
);
- let mut out: Vec<_> = segment.outgoing_edges(&b.id);
+ let mut out: Vec<_> = segment.outgoing_edges(&b.id, &EdgeLimit::Unlimited);
out.sort_by(|a, b| a.to.cmp(&b.to));
assert_eq!(
out,
@@ -243,7 +242,7 @@ mod test {
},]
);
- let mut out: Vec<_> = segment.outgoing_edges(&c.id);
+ let mut out: Vec<_> = segment.outgoing_edges(&c.id, &EdgeLimit::Unlimited);
out.sort_by(|a, b| a.to.cmp(&b.to));
assert_eq!(
out,
@@ -254,7 +253,7 @@ mod test {
},]
);
- let out: Vec<_> = segment.ingoing_edges(&a.id);
+ let out: Vec<_> = segment.ingoing_edges(&a.id, &EdgeLimit::Unlimited);
assert_eq!(
out,
vec![Edge {
@@ -264,7 +263,7 @@ mod test {
},]
);
- let out: Vec<_> = segment.ingoing_edges(&b.id);
+ let out: Vec<_> = segment.ingoing_edges(&b.id, &EdgeLimit::Unlimited);
assert_eq!(
out,
vec![Edge {
@@ -274,7 +273,7 @@ mod test {
},]
);
- let mut out: Vec<_> = segment.ingoing_edges(&c.id);
+ let mut out: Vec<_> = segment.ingoing_edges(&c.id, &EdgeLimit::Unlimited);
out.sort_by(|a, b| a.from.cmp(&b.from));
assert_eq!(
out,
diff --git a/crates/core/src/webgraph/shortest_path.rs b/crates/core/src/webgraph/shortest_path.rs
index a178d6c0..1915c54d 100644
--- a/crates/core/src/webgraph/shortest_path.rs
+++ b/crates/core/src/webgraph/shortest_path.rs
@@ -89,7 +89,7 @@ impl ShortestPaths for Webgraph {
fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
- |node| self.raw_outgoing_edges(&node),
+ |node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),
|edge| edge.to,
Some(max_dist),
)
@@ -98,7 +98,7 @@ impl ShortestPaths for Webgraph {
fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
- |node| self.raw_outgoing_edges(&node),
+ |node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),
|edge| edge.to,
None,
)
@@ -107,7 +107,7 @@ impl ShortestPaths for Webgraph {
fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
- |node| self.raw_ingoing_edges(&node),
+ |node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),
|edge| edge.from,
None,
)
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 2e2d0a3d..a2fe9938 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -14,13 +14,16 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{fs::File, io::Write, ops::Range, path::Path};
+use std::{ops::Range, path::Path};
+use file_store::{
+ iterable::{ConstIterableStoreReader, IterableStoreReader},
+ ConstSerializable,
+};
use fst::Automaton;
use itertools::Itertools;
-use memmap2::Mmap;
-use super::{Compression, Edge, FullNodeID, InnerEdge, NodeID};
+use super::{Compression, Edge, EdgeLimit, FullNodeID, NodeID};
#[derive(
Debug,
@@ -34,17 +37,17 @@ use super::{Compression, Edge, FullNodeID, InnerEdge, NodeID};
bincode::Decode,
)]
struct SerializedEdge {
- from_prefix: NodeID,
- to_prefix: NodeID,
+ from_host: NodeID,
+ to_host: NodeID,
label: Vec<u8>,
}
-struct PrefixDb {
+pub struct HostDb {
db: speedy_kv::Db<Vec<u8>, Vec<u8>>,
}
-impl PrefixDb {
- fn open<P: AsRef<Path>>(path: P) -> Self {
+impl HostDb {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
let db = speedy_kv::Db::open_or_create(path).unwrap();
Self { db }
@@ -54,9 +57,9 @@ impl PrefixDb {
self.db.merge_all_segments().unwrap();
}
- fn insert(&mut self, node: &FullNodeID) {
+ pub fn insert(&mut self, node: &FullNodeID) {
let key = [
- node.prefix.as_u64().to_le_bytes(),
+ node.host.as_u64().to_le_bytes(),
node.id.as_u64().to_le_bytes(),
]
.concat();
@@ -65,10 +68,10 @@ impl PrefixDb {
self.db.insert_raw(key, value);
}
- fn get(&self, prefix: &NodeID) -> Vec<NodeID> {
- let prefix = prefix.as_u64().to_le_bytes().to_vec();
+ fn get(&self, host: &NodeID) -> Vec<NodeID> {
+ let host = host.as_u64().to_le_bytes().to_vec();
- let query = speedy_kv::automaton::ExactMatch(&prefix).starts_with();
+ let query = speedy_kv::automaton::ExactMatch(&host).starts_with();
self.db
.search_raw(query)
@@ -81,19 +84,19 @@ impl PrefixDb {
.collect()
}
- fn flush(&mut self) {
+ pub fn flush(&mut self) {
self.db.commit().unwrap();
self.db.merge_all_segments().unwrap();
}
}
-struct RangesDb {
+pub struct RangesDb {
nodes: speedy_kv::Db<NodeID, std::ops::Range<u64>>,
labels: speedy_kv::Db<NodeID, std::ops::Range<u64>>,
}
impl RangesDb {
- fn open<P: AsRef<Path>>(path: P) -> Self {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
let nodes = speedy_kv::Db::open_or_create(path.as_ref().join("nodes")).unwrap();
let labels = speedy_kv::Db::open_or_create(path.as_ref().join("labels")).unwrap();
@@ -104,202 +107,117 @@ impl RangesDb {
self.nodes.merge_all_segments().unwrap();
self.labels.merge_all_segments().unwrap();
}
-}
-pub struct EdgeStore {
- reversed: bool,
- ranges: RangesDb,
- prefixes: PrefixDb,
+ pub fn commit(&mut self) {
+ self.nodes.commit().unwrap();
+ self.labels.commit().unwrap();
+ }
- edge_labels_file: File,
- edge_labels_len: usize,
- edge_labels: Mmap,
+ pub fn nodes_get_raw<'a>(
+ &'a self,
+ key: &'a [u8],
+ ) -> Option<speedy_kv::SerializedRef<'a, Range<u64>>> {
+ self.nodes.get_raw(key)
+ }
- edge_nodes_file: File,
- edge_nodes_len: usize,
- edge_nodes: Mmap,
+ pub fn labels_get_raw<'a>(
+ &'a self,
+ key: &'a [u8],
+ ) -> Option<speedy_kv::SerializedRef<'a, Range<u64>>> {
+ self.labels.get_raw(key)
+ }
- compression: Compression,
-}
+ pub fn insert_raw_node(&mut self, node: Vec<u8>, range: Vec<u8>) {
+ self.nodes.insert_raw(node, range);
+ }
-impl EdgeStore {
- pub fn open<P: AsRef<Path>>(path: P, reversed: bool, compression: Compression) -> Self {
- let ranges = RangesDb::open(path.as_ref().join("ranges"));
+ pub fn insert_raw_label(&mut self, node: Vec<u8>, range: Vec<u8>) {
+ self.labels.insert_raw(node, range);
+ }
+}
- let edge_labels_file = File::options()
- .read(true)
- .create(true)
- .truncate(false)
- .write(true)
- .open(path.as_ref().join("labels"))
- .unwrap();
- let edge_labels = unsafe { Mmap::map(&edge_labels_file).unwrap() };
- let edge_labels_len = edge_labels.len();
-
- let edge_nodes_file = File::options()
- .read(true)
- .create(true)
- .truncate(false)
- .write(true)
- .open(path.as_ref().join("nodes"))
- .unwrap();
- let edge_nodes = unsafe { Mmap::map(&edge_nodes_file).unwrap() };
- let edge_nodes_len = edge_nodes.len();
+impl ConstSerializable for NodeID {
+ const BYTES: usize = std::mem::size_of::<NodeID>();
- Self {
- reversed,
- ranges,
- prefixes: PrefixDb::open(path.as_ref().join("prefixes")),
- edge_labels,
- edge_labels_len,
- edge_labels_file,
- edge_nodes,
- edge_nodes_file,
- edge_nodes_len,
- compression,
- }
+ fn serialize(&self, buf: &mut [u8]) {
+ self.as_u64().serialize(buf);
}
- pub fn optimize_read(&mut self) {
- self.ranges.optimize_read();
- self.prefixes.optimize_read();
+ fn deserialize(buf: &[u8]) -> Self {
+ let id = u64::deserialize(buf);
+ NodeID::from(id)
}
+}
- /// Insert a batch of edges into the store.
- /// The edges *must* have been de-duplicated by their from/to node.
- /// I.e. if the store is not reversed, there should only ever be a single
- /// put for each from node, and vice versa.
- fn put(&mut self, edges: &[InnerEdge<String>]) {
- if edges.is_empty() {
- return;
- }
-
- let node = if self.reversed {
- edges[0].to.clone()
- } else {
- edges[0].from.clone()
- };
+pub const NUM_LABELS_PER_BLOCK: usize = 128;
- self.prefixes.insert(&node);
- let node_bytes = node.id.as_u64().to_le_bytes();
+#[derive(bincode::Encode, bincode::Decode)]
+pub struct LabelBlock {
+ labels: Vec<String>,
+}
- debug_assert!(self.ranges.nodes.get_raw(&node_bytes).is_none());
- debug_assert!(self.ranges.labels.get_raw(&node_bytes).is_none());
+impl LabelBlock {
+ pub fn new(labels: Vec<String>) -> Self {
+ Self { labels }
+ }
- let mut edge_labels = Vec::new();
- let mut edge_nodes = Vec::new();
+ pub fn compress(&self, compression: Compression) -> CompressedLabelBlock {
+ let bytes = bincode::encode_to_vec(self, bincode::config::standard()).unwrap();
+ let compressed = compression.compress(&bytes);
- for edge in edges {
- edge_labels.push(edge.label.clone());
- edge_nodes.push(if self.reversed {
- edge.from.id
- } else {
- edge.to.id
- });
+ CompressedLabelBlock {
+ compressions: compression,
+ data: compressed,
}
+ }
+}
- let edge_labels_bytes =
- bincode::encode_to_vec(&edge_labels, bincode::config::standard()).unwrap();
- let edge_nodes_bytes =
- bincode::encode_to_vec(&edge_nodes, bincode::config::standard()).unwrap();
-
- let edge_labels_bytes = self.compression.compress(&edge_labels_bytes);
- let edge_nodes_bytes = self.compression.compress(&edge_nodes_bytes);
-
- let label_range = self.edge_labels_len..(self.edge_labels_len + edge_labels_bytes.len());
- let node_range = self.edge_nodes_len..(self.edge_nodes_len + edge_nodes_bytes.len());
+#[derive(bincode::Encode, bincode::Decode)]
+pub struct CompressedLabelBlock {
+ compressions: Compression,
+ data: Vec<u8>,
+}
- self.edge_labels_len += edge_labels_bytes.len();
- self.edge_nodes_len += edge_nodes_bytes.len();
+impl CompressedLabelBlock {
+ pub fn decompress(&self) -> LabelBlock {
+ let bytes = self.compressions.decompress(&self.data);
+ let (res, _) = bincode::decode_from_slice(&bytes, bincode::config::standard()).unwrap();
+ res
+ }
+}
- self.edge_labels_file.write_all(&edge_labels_bytes).unwrap();
- self.edge_nodes_file.write_all(&edge_nodes_bytes).unwrap();
+pub struct EdgeStore {
+ reversed: bool,
+ ranges: RangesDb,
+ hosts: HostDb,
- self.ranges.nodes.insert_raw(
- node_bytes.to_vec(),
- bincode::encode_to_vec(node_range, bincode::config::standard()).unwrap(),
- );
+ edge_labels: IterableStoreReader<CompressedLabelBlock>,
+ edge_nodes: ConstIterableStoreReader<NodeID>,
+}
- self.ranges.labels.insert_raw(
- node_bytes.to_vec(),
- bincode::encode_to_vec(label_range, bincode::config::standard()).unwrap(),
- );
- }
+impl EdgeStore {
+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {
+ let ranges = RangesDb::open(path.as_ref().join("ranges"));
- /// Build a new edge store from a set of edges.
- ///
- /// **IMPORTANT** The edges must be sorted by
- /// either the from or to node, depending on the value of `reversed`.
- pub fn build<P: AsRef<Path>>(
- path: P,
- compression: Compression,
- reversed: bool,
- edges: impl Iterator<Item = InnerEdge<String>>,
- ) -> Self {
- let mut s = Self::open(path, reversed, compression);
-
- let mut inserts_since_last_flush = 0;
-
- // create batches of consecutive edges with the same from/to node
- let mut batch = Vec::new();
- let mut last_node = None;
- for edge in edges {
- if let Some(last_node) = last_node {
- if (reversed && edge.to.id != last_node) || (!reversed && edge.from.id != last_node)
- {
- batch.sort_unstable_by_key(
- |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },
- );
- batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });
- let batch_len = batch.len();
- s.put(&batch);
- batch.clear();
- inserts_since_last_flush += batch_len;
-
- if inserts_since_last_flush >= 1_000_000 {
- s.flush();
- inserts_since_last_flush = 0;
- }
- }
- }
+ let edge_labels = IterableStoreReader::open(path.as_ref().join("labels")).unwrap();
- last_node = Some(if reversed { edge.to.id } else { edge.from.id });
- batch.push(edge);
- }
+ let edge_nodes = ConstIterableStoreReader::open(path.as_ref().join("nodes")).unwrap();
- if !batch.is_empty() {
- batch.sort_unstable_by_key(
- |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },
- );
- batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });
- s.put(&batch);
+ Self {
+ ranges,
+ hosts: HostDb::open(path.as_ref().join("hosts")),
+ edge_labels,
+ edge_nodes,
+ reversed,
}
-
- s.flush();
-
- s
}
- fn flush(&mut self) {
- self.prefixes.flush();
-
- self.ranges.labels.commit().unwrap();
- self.ranges.labels.merge_all_segments().unwrap();
-
- self.ranges.nodes.commit().unwrap();
- self.ranges.nodes.merge_all_segments().unwrap();
-
- self.edge_nodes_file.flush().unwrap();
- self.edge_labels_file.flush().unwrap();
-
- self.edge_nodes = unsafe { Mmap::map(&self.edge_nodes_file).unwrap() };
- self.edge_labels = unsafe { Mmap::map(&self.edge_labels_file).unwrap() };
-
- self.edge_nodes_len = self.edge_nodes.len();
- self.edge_labels_len = self.edge_labels.len();
+ pub fn optimize_read(&mut self) {
+ self.ranges.optimize_read();
+ self.hosts.optimize_read();
}
- pub fn get_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {
+ pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {
let node_bytes = node.as_u64().to_le_bytes();
match (
@@ -318,15 +236,23 @@ impl EdgeStore {
)
.unwrap();
- let edge_labels = &self.edge_labels[edge_range];
- let edge_labels = self.compression.decompress(edge_labels);
- let (edge_labels, _): (Vec<_>, _) =
- bincode::decode_from_slice(&edge_labels, bincode::config::standard()).unwrap();
+ let edge_labels = self
+ .edge_labels
+ .slice(edge_range)
+ .map(|r| r.unwrap().decompress())
+ .flat_map(|block| block.labels.into_iter());
+
+ let edge_labels: Vec<_> = match limit {
+ EdgeLimit::Unlimited => edge_labels.collect(),
+ EdgeLimit::Limit(limit) => edge_labels.take(*limit).collect(),
+ };
- let edge_nodes = &self.edge_nodes[node_range];
- let edge_nodes = self.compression.decompress(edge_nodes);
- let (edge_nodes, _): (Vec<_>, _) =
- bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();
+ let edge_nodes = self.edge_nodes.slice(node_range);
+
+ let edge_nodes: Vec<_> = match limit {
+ EdgeLimit::Unlimited => edge_nodes.collect(),
+ EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),
+ };
edge_labels
.into_iter()
@@ -352,7 +278,7 @@ impl EdgeStore {
}
}
- pub fn get_without_label(&self, node: &NodeID) -> Vec<Edge<()>> {
+ pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {
let node_bytes = node.as_u64().to_le_bytes();
match self.ranges.nodes.get_raw(&node_bytes) {
@@ -363,10 +289,12 @@ impl EdgeStore {
)
.unwrap();
- let edge_nodes = &self.edge_nodes[node_range];
- let edge_nodes = self.compression.decompress(edge_nodes);
- let (edge_nodes, _): (Vec<_>, _) =
- bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();
+ let edge_nodes = self.edge_nodes.slice(node_range);
+
+ let edge_nodes: Vec<_> = match limit {
+ EdgeLimit::Unlimited => edge_nodes.collect(),
+ EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),
+ };
edge_nodes
.into_iter()
@@ -391,8 +319,8 @@ impl EdgeStore {
}
}
- pub fn nodes_by_prefix(&self, prefix: &NodeID) -> Vec<NodeID> {
- self.prefixes.get(prefix)
+ pub fn nodes_by_host(&self, host: &NodeID) -> Vec<NodeID> {
+ self.hosts.get(host)
}
pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {
@@ -405,10 +333,8 @@ impl EdgeStore {
bincode::config::standard(),
)
.unwrap();
- let edge_nodes = &self.edge_nodes[node_range];
- let edge_nodes = self.compression.decompress(edge_nodes);
- let (edge_nodes, _): (Vec<_>, _) =
- bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();
+
+ let edge_nodes = self.edge_nodes.slice(node_range).collect::<Vec<_>>();
edge_nodes.into_iter().map(move |other| {
if self.reversed {
@@ -431,7 +357,9 @@ impl EdgeStore {
#[cfg(test)]
mod tests {
- use crate::webgraph::store_writer::EdgeStoreWriter;
+ use std::sync::Arc;
+
+ use crate::webgraph::{store_writer::EdgeStoreWriter, InnerEdge};
use super::*;
@@ -441,16 +369,17 @@ mod tests {
crate::gen_temp_path().join("test-segment"),
Compression::default(),
false,
+ None,
);
let e = InnerEdge {
from: FullNodeID {
id: NodeID::from(0 as u64),
- prefix: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
},
to: FullNodeID {
id: NodeID::from(1 as u64),
- prefix: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
},
label: "test".to_string(),
};
@@ -459,12 +388,12 @@ mod tests {
let store = kv.finalize();
- let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64));
+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64), &EdgeLimit::Unlimited);
assert_eq!(edges.len(), 1);
assert_eq!(&edges[0], &Edge::from(e.clone()));
- let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64));
+ let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64), &EdgeLimit::Unlimited);
assert_eq!(edges.len(), 0);
}
@@ -475,16 +404,17 @@ mod tests {
crate::gen_temp_path().join("test-segment"),
Compression::default(),
true,
+ None,
);
let e = InnerEdge {
from: FullNodeID {
id: NodeID::from(0 as u64),
- prefix: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
},
to: FullNodeID {
id: NodeID::from(1 as u64),
- prefix: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
},
label: "test".to_string(),
};
@@ -493,11 +423,111 @@ mod tests {
let store = kv.finalize();
- let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64));
+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64), &EdgeLimit::Unlimited);
assert_eq!(edges.len(), 0);
- let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64));
+ let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64), &EdgeLimit::Unlimited);
assert_eq!(edges.len(), 1);
assert_eq!(&edges[0], &Edge::from(e.clone()));
}
+
+ #[test]
+ fn test_limit() {
+ let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(
+ crate::gen_temp_path().join("test-segment"),
+ Compression::default(),
+ true,
+ None,
+ );
+
+ for i in 0..10 {
+ let e = InnerEdge {
+ from: FullNodeID {
+ id: NodeID::from(i as u64),
+ host: NodeID::from(0 as u64),
+ },
+ to: FullNodeID {
+ id: NodeID::from(1 as u64),
+ host: NodeID::from(0 as u64),
+ },
+ label: "test".to_string(),
+ };
+
+ kv.put(e.clone());
+ }
+
+ let store = kv.finalize();
+
+ let edges: Vec<_> = store.get_with_label(&NodeID::from(1 as u64), &EdgeLimit::Limit(5));
+ assert_eq!(edges.len(), 5);
+ }
+
+ #[test]
+ fn test_edge_ordering() {
+ let mut rank_store =
+ speedy_kv::Db::open_or_create(crate::gen_temp_path().join("test-rank-store")).unwrap();
+
+ rank_store.insert(NodeID::from(2 as u64), 1).unwrap();
+ rank_store.insert(NodeID::from(3 as u64), 2).unwrap();
+ rank_store.insert(NodeID::from(1 as u64), 3).unwrap();
+
+ rank_store.commit().unwrap();
+
+ let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(
+ crate::gen_temp_path().join("test-segment"),
+ Compression::default(),
+ true,
+ Some(Arc::new(rank_store)),
+ );
+
+ let e1 = InnerEdge {
+ from: FullNodeID {
+ id: NodeID::from(1 as u64),
+ host: NodeID::from(1 as u64),
+ },
+ to: FullNodeID {
+ id: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
+ },
+ label: "1".to_string(),
+ };
+
+ let e2 = InnerEdge {
+ from: FullNodeID {
+ id: NodeID::from(2 as u64),
+ host: NodeID::from(2 as u64),
+ },
+ to: FullNodeID {
+ id: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
+ },
+ label: "2".to_string(),
+ };
+
+ let e3 = InnerEdge {
+ from: FullNodeID {
+ id: NodeID::from(3 as u64),
+ host: NodeID::from(3 as u64),
+ },
+ to: FullNodeID {
+ id: NodeID::from(0 as u64),
+ host: NodeID::from(0 as u64),
+ },
+ label: "3".to_string(),
+ };
+
+ kv.put(e1.clone());
+ kv.put(e2.clone());
+ kv.put(e3.clone());
+
+ let store = kv.finalize();
+
+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0 as u64), &EdgeLimit::Unlimited);
+
+ assert_eq!(edges.len(), 3);
+
+ assert_eq!(&edges[0], &Edge::from(e2.clone()));
+ assert_eq!(&edges[1], &Edge::from(e3.clone()));
+ assert_eq!(&edges[2], &Edge::from(e1.clone()));
+ }
}
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index 3adcca8a..b9cc1ed2 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -19,15 +19,22 @@ pub const MAX_BATCH_SIZE: usize = 3_000_000;
use std::{
collections::BTreeSet,
fs::File,
+ ops::Range,
path::{Path, PathBuf},
+ sync::Arc,
};
use itertools::Itertools;
-use crate::Result;
-use file_store::iterable::{IterableStoreReader, SortedIterableStoreReader};
+use crate::{webgraph::store::NUM_LABELS_PER_BLOCK, Result};
+use file_store::iterable::{
+ ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter, SortedIterableStoreReader,
+};
-use super::{store::EdgeStore, Compression, EdgeLabel, InnerEdge, NodeID};
+use super::{
+ store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},
+ Compression, EdgeLabel, InnerEdge, NodeID,
+};
#[derive(bincode::Encode, bincode::Decode)]
struct SortableEdge<L: EdgeLabel> {
@@ -97,28 +104,36 @@ pub struct EdgeStoreWriter {
edges: BTreeSet<SortableEdge<String>>,
stored_writers: Vec<PathBuf>,
compression: Compression,
+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
}
impl EdgeStoreWriter {
- pub fn new<P: AsRef<Path>>(path: P, compression: Compression, reversed: bool) -> Self {
- let path = path.as_ref().join("writer");
-
- if !path.exists() {
- std::fs::create_dir_all(&path).unwrap();
+ pub fn new<P: AsRef<Path>>(
+ path: P,
+ compression: Compression,
+ reversed: bool,
+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
+ ) -> Self {
+ let writer_path = path.as_ref().join("writer");
+
+ if !writer_path.exists() {
+ std::fs::create_dir_all(&writer_path).unwrap();
}
Self {
edges: BTreeSet::new(),
reversed,
- path: path.to_path_buf(),
+ path: path.as_ref().to_path_buf(),
compression,
stored_writers: Vec::new(),
+ host_centrality_rank_store,
}
}
fn flush_to_file(&mut self) -> Result<()> {
let file_path = self
.path
+ .join("writer")
.join(format!("{}.store", self.stored_writers.len()));
let file = File::create(&file_path)?;
@@ -158,10 +173,7 @@ impl EdgeStoreWriter {
let readers = self
.stored_writers
.iter()
- .map(|p| {
- let file = File::open(p).unwrap();
- IterableStoreReader::new(file)
- })
+ .map(|p| IterableStoreReader::open(p).unwrap())
.collect();
let file_reader = SortedIterableStoreReader::new(readers).map(|r| r.unwrap());
@@ -174,19 +186,234 @@ impl EdgeStoreWriter {
}
pub fn finalize(self) -> EdgeStore {
- let p = self.path.parent().unwrap().to_path_buf();
-
- EdgeStore::build(
- p,
+ let mut final_writer = FinalEdgeStoreWriter::open(
self.compression,
self.reversed,
- self.sorted_edges().dedup().map(|e| e.edge),
- )
+ self.host_centrality_rank_store.clone(),
+ &self.path,
+ );
+
+ let mut store = final_writer.build_store(self.sorted_edges().dedup().map(|e| e.edge));
+ store.optimize_read();
+
+ store
}
}
impl Drop for EdgeStoreWriter {
fn drop(&mut self) {
- std::fs::remove_dir_all(&self.path).unwrap();
+ std::fs::remove_dir_all(self.path.join("writer")).unwrap();
+ }
+}
+
+struct FinalEdgeStoreWriter {
+ ranges: RangesDb,
+ hosts: HostDb,
+
+ edge_labels: IterableStoreWriter<CompressedLabelBlock, File>,
+ edge_nodes: ConstIterableStoreWriter<NodeID, File>,
+
+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
+
+ compression: Compression,
+ reversed: bool,
+
+ path: PathBuf,
+}
+
+impl FinalEdgeStoreWriter {
+ fn open<P: AsRef<Path>>(
+ compression: Compression,
+ reversed: bool,
+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
+ path: P,
+ ) -> Self {
+ let ranges = RangesDb::open(path.as_ref().join("ranges"));
+
+ let edge_labels_file = File::options()
+ .read(true)
+ .create(true)
+ .truncate(false)
+ .write(true)
+ .open(path.as_ref().join("labels"))
+ .unwrap();
+ let edge_labels = IterableStoreWriter::new(edge_labels_file);
+
+ let edge_nodes_file = File::options()
+ .read(true)
+ .create(true)
+ .truncate(false)
+ .write(true)
+ .open(path.as_ref().join("nodes"))
+ .unwrap();
+ let edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);
+
+ Self {
+ ranges,
+ hosts: HostDb::open(path.as_ref().join("hosts")),
+ edge_labels,
+ edge_nodes,
+ reversed,
+ compression,
+ path: path.as_ref().to_path_buf(),
+ host_centrality_rank_store,
+ }
+ }
+ /// Insert a batch of edges into the store.
+ /// The edges *must* have been de-duplicated by their from/to node.
+ /// I.e. if the store is not reversed, there should only ever be a single
+ /// put for each from node, and vice versa.
+ fn put_store(&mut self, edges: &mut [InnerEdge<String>]) {
+ if edges.is_empty() {
+ return;
+ }
+
+ let node = if self.reversed {
+ edges[0].to.clone()
+ } else {
+ edges[0].from.clone()
+ };
+
+ // order edges by centrality rank of the other node host
+ if let Some(rank_store) = &self.host_centrality_rank_store {
+ edges.sort_by_cached_key(|e| {
+ if self.reversed {
+ rank_store.get(&e.from.host).unwrap().unwrap_or(u64::MAX)
+ } else {
+ rank_store.get(&e.to.host).unwrap().unwrap_or(u64::MAX)
+ }
+ });
+ }
+
+ self.hosts.insert(&node);
+ let node_bytes = node.id.as_u64().to_le_bytes();
+
+ debug_assert!(self.ranges.nodes_get_raw(&node_bytes).is_none());
+ debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());
+
+ let mut edge_labels = Vec::new();
+ let mut edge_nodes = Vec::new();
+
+ for edge in edges {
+ edge_labels.push(edge.label.clone());
+ edge_nodes.push(if self.reversed {
+ edge.from.id
+ } else {
+ edge.to.id
+ });
+ }
+
+ let edge_labels: Vec<_> = edge_labels
+ .into_iter()
+ .chunks(NUM_LABELS_PER_BLOCK)
+ .into_iter()
+ .map(|chunk| LabelBlock::new(chunk.collect()).compress(self.compression))
+ .collect();
+
+ let mut first_label_offset = None;
+ let mut last_label_offset = None;
+ let mut first_node_offset = None;
+ let mut last_node_offset = None;
+
+ for block in &edge_labels {
+ let offset = self.edge_labels.write(block).unwrap();
+
+ if first_label_offset.is_none() {
+ first_label_offset = Some(offset);
+ }
+
+ last_label_offset = Some(offset);
+ }
+
+ for node in &edge_nodes {
+ let offset = self.edge_nodes.write(node).unwrap();
+
+ if first_node_offset.is_none() {
+ first_node_offset = Some(offset);
+ }
+
+ last_node_offset = Some(offset);
+ }
+
+ let label_range = Range {
+ start: first_label_offset.unwrap().start,
+ end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,
+ };
+
+ let node_range = Range {
+ start: first_node_offset.unwrap().start,
+ end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,
+ };
+
+ self.ranges.insert_raw_node(
+ node_bytes.to_vec(),
+ bincode::encode_to_vec(node_range, bincode::config::standard()).unwrap(),
+ );
+
+ self.ranges.insert_raw_label(
+ node_bytes.to_vec(),
+ bincode::encode_to_vec(label_range, bincode::config::standard()).unwrap(),
+ );
+ }
+
+ /// Build a new edge store from a set of edges.
+ ///
+ /// **IMPORTANT** The edges must be sorted by
+ /// either the from or to node, depending on the value of `reversed`.
+ pub fn build_store(&mut self, edges: impl Iterator<Item = InnerEdge<String>>) -> EdgeStore {
+ let mut inserts_since_last_flush = 0;
+
+ // create batches of consecutive edges with the same from/to node
+ let mut batch = Vec::new();
+ let mut last_node = None;
+ for edge in edges {
+ if let Some(last_node) = last_node {
+ if (self.reversed && edge.to.id != last_node)
+ || (!self.reversed && edge.from.id != last_node)
+ {
+ batch.sort_unstable_by_key(
+ |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },
+ );
+ batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });
+ let batch_len = batch.len();
+ self.put_store(&mut batch);
+ batch.clear();
+ inserts_since_last_flush += batch_len;
+
+ if inserts_since_last_flush >= 1_000_000 {
+ self.flush();
+ inserts_since_last_flush = 0;
+ }
+ }
+ }
+
+ last_node = Some(if self.reversed {
+ edge.to.id
+ } else {
+ edge.from.id
+ });
+ batch.push(edge);
+ }
+
+ if !batch.is_empty() {
+ batch.sort_unstable_by_key(
+ |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },
+ );
+ batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });
+ self.put_store(&mut batch);
+ }
+
+ self.flush();
+
+ EdgeStore::open(&self.path, self.reversed)
+ }
+
+ fn flush(&mut self) {
+ self.hosts.flush();
+
+ self.ranges.commit();
+
+ self.edge_nodes.flush().unwrap();
+ self.edge_labels.flush().unwrap();
}
}
diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rs
index 2e619d14..ba8d2b5b 100644
--- a/crates/core/src/webgraph/writer.rs
+++ b/crates/core/src/webgraph/writer.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses
-use std::{fs, path::Path};
+use std::{fs, path::Path, sync::Arc};
use crate::executor::Executor;
@@ -29,7 +29,6 @@ pub struct WebgraphWriter {
id2node: Id2NodeDb,
executor: Executor,
meta: Meta,
- compression: Compression,
}
impl WebgraphWriter {
@@ -43,7 +42,12 @@ impl WebgraphWriter {
self.meta.save(path);
}
- pub fn new<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {
+ pub fn new<P: AsRef<Path>>(
+ path: P,
+ executor: Executor,
+ compression: Compression,
+ host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
+ ) -> Self {
fs::create_dir_all(&path).unwrap();
let mut meta = Self::meta(&path);
meta.comitted_segments.clear();
@@ -51,7 +55,12 @@ impl WebgraphWriter {
fs::create_dir_all(path.as_ref().join("segments")).unwrap();
let id = uuid::Uuid::new_v4().to_string();
- let segment = SegmentWriter::open(path.as_ref().join("segments"), id.clone(), compression);
+ let segment = SegmentWriter::open(
+ path.as_ref().join("segments"),
+ id.clone(),
+ compression,
+ host_centrality_rank_store,
+ );
meta.comitted_segments.push(id);
@@ -61,7 +70,6 @@ impl WebgraphWriter {
id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
executor,
meta,
- compression,
}
}
@@ -110,7 +118,6 @@ impl WebgraphWriter {
executor: self.executor.into(),
id2node: self.id2node,
meta: self.meta,
- compression: self.compression,
}
}
}
diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.toml
index 7be7177e..8985e00d 100644
--- a/crates/file-store/Cargo.toml
+++ b/crates/file-store/Cargo.toml
@@ -9,3 +9,5 @@ license = "AGPL-3.0"
[dependencies]
anyhow.workspace = true
bincode.workspace = true
+memmap2.workspace = true
+stable_deref_trait.workspace = true
diff --git a/crates/file-store/src/const_serializable.rs b/crates/file-store/src/const_serializable.rs
new file mode 100644
index 00000000..1029a946
--- /dev/null
+++ b/crates/file-store/src/const_serializable.rs
@@ -0,0 +1,64 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/
+
+// change ConstSerializable to something like the following
+// once generic_const_exprs is stable. See https://github.com/rust-lang/rust/issues/60551
+//
+// pub trait ConstSerializable {
+// const BYTES: usize;
+// fn serialize(&self) -> [u8; Self::BYTES];
+// fn deserialize(bytes: [u8; Self::BYTES]) -> Self;
+// }
+
+pub trait ConstSerializable {
+ const BYTES: usize;
+
+ fn serialize(&self, buf: &mut [u8]);
+ fn deserialize(buf: &[u8]) -> Self;
+}
+
+macro_rules! impl_const_serializable_num {
+ ($t:ty, $n:expr) => {
+ impl ConstSerializable for $t {
+ const BYTES: usize = $n;
+
+ fn serialize(&self, buf: &mut [u8]) {
+ buf.copy_from_slice(&self.to_le_bytes());
+ }
+
+ fn deserialize(buf: &[u8]) -> Self {
+ let mut bytes = [0; $n];
+ bytes.copy_from_slice(&buf[..$n]);
+ <$t>::from_le_bytes(bytes)
+ }
+ }
+ };
+}
+
+impl_const_serializable_num!(u8, 1);
+impl_const_serializable_num!(u16, 2);
+impl_const_serializable_num!(u32, 4);
+impl_const_serializable_num!(u64, 8);
+impl_const_serializable_num!(u128, 16);
+
+impl_const_serializable_num!(i8, 1);
+impl_const_serializable_num!(i16, 2);
+impl_const_serializable_num!(i32, 4);
+impl_const_serializable_num!(i64, 8);
+impl_const_serializable_num!(i128, 16);
+
+impl_const_serializable_num!(f32, 4);
+impl_const_serializable_num!(f64, 8);
diff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rs
index abf47791..e0afde60 100644
--- a/crates/file-store/src/iterable.rs
+++ b/crates/file-store/src/iterable.rs
@@ -21,9 +21,16 @@
//! 1. A 64-bit little-endian integer representing the number of bytes in the upcoming item.
//! 2. The serialized item.
//! 3. Repeat 1-2 until the end of the file.
+//!
+//! If the item type `T` implements `ConstSerializable`, then the `ConstIterableStoreWriter` can be
+//! used to write items to the file without intermediate headers as the size of the serialied item is known upfront.
-use crate::Result;
-use std::io::{self, Read, Write};
+use crate::{owned_bytes::OwnedBytes, ConstSerializable, Result};
+use std::{
+ io::{self, Write},
+ ops::Range,
+ path::Path,
+};
use super::Peekable;
@@ -32,6 +39,11 @@ struct IterableHeader {
}
impl IterableHeader {
+ #[inline]
+ const fn serialized_size() -> usize {
+ std::mem::size_of::<u64>()
+ }
+
fn serialize<W>(&self, writer: &mut W) -> io::Result<()>
where
W: io::Write,
@@ -39,23 +51,38 @@ impl IterableHeader {
writer.write_all(&self.num_upcoming_bytes.to_le_bytes())
}
- fn deserialize<R>(reader: &mut R) -> io::Result<Self>
- where
- R: io::Read,
- {
- let mut num_upcoming_bytes = [0; 8];
- reader.read_exact(&mut num_upcoming_bytes)?;
+ fn deserialize(bytes: &[u8]) -> io::Result<Self> {
+ if bytes.len() != Self::serialized_size() {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ "Invalid number of bytes for IterableHeader",
+ ));
+ }
+
Ok(IterableHeader {
- num_upcoming_bytes: u64::from_le_bytes(num_upcoming_bytes),
+ num_upcoming_bytes: u64::from_le_bytes(bytes[..8].try_into().unwrap()),
})
}
}
+#[derive(Debug, Clone, Copy)]
+pub struct WrittenOffset {
+ pub start: u64,
+ pub num_bytes: u64,
+}
+
+impl WrittenOffset {
+ pub fn range(&self) -> Range<u64> {
+ self.start..self.start + self.num_bytes
+ }
+}
+
pub struct IterableStoreWriter<T, W>
where
W: io::Write,
{
writer: io::BufWriter<W>,
+ next_start: u64,
_marker: std::marker::PhantomData<T>,
}
@@ -68,17 +95,27 @@ where
Self {
writer: io::BufWriter::new(writer),
_marker: std::marker::PhantomData,
+ next_start: 0,
}
}
- pub fn write(&mut self, item: &T) -> Result<()> {
+ pub fn write(&mut self, item: &T) -> Result<WrittenOffset> {
let serialized = bincode::encode_to_vec(item, bincode::config::standard())?;
let header = IterableHeader {
num_upcoming_bytes: serialized.len() as u64,
};
header.serialize(&mut self.writer)?;
self.writer.write_all(&serialized)?;
- Ok(())
+
+ let start = self.next_start;
+ let bytes_written = IterableHeader::serialized_size() as u64 + serialized.len() as u64;
+
+ self.next_start += bytes_written;
+
+ Ok(WrittenOffset {
+ start,
+ num_bytes: bytes_written,
+ })
}
pub fn finalize(mut self) -> Result<W> {
@@ -86,46 +123,71 @@ where
self.writer.into_inner().map_err(|e| anyhow::anyhow!("{e}"))
}
+
+ pub fn flush(&mut self) -> io::Result<()> {
+ self.writer.flush()
+ }
}
-pub struct IterableStoreReader<T, R> {
- reader: io::BufReader<R>,
+pub struct IterableStoreReader<T> {
+ data: OwnedBytes,
+ offset: usize,
_marker: std::marker::PhantomData<T>,
}
-impl<T, R> IterableStoreReader<T, R>
-where
- R: io::Read,
-{
- pub fn new(reader: R) -> Self {
+impl<T> IterableStoreReader<T> {
+ pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {
+ let data = OwnedBytes::mmap_from_path(path)?;
+
+ Ok(Self {
+ data,
+ offset: 0,
+ _marker: std::marker::PhantomData,
+ })
+ }
+
+ pub fn from_bytes(data: Vec<u8>) -> Self {
Self {
- reader: io::BufReader::new(reader),
+ data: OwnedBytes::new(data),
+ offset: 0,
+ _marker: std::marker::PhantomData,
+ }
+ }
+
+ pub fn slice(&self, range: Range<usize>) -> IterableStoreReader<T> {
+ IterableStoreReader {
+ data: self.data.slice(range),
+ offset: 0,
_marker: std::marker::PhantomData,
}
}
}
-impl<T, R> Iterator for IterableStoreReader<T, R>
+impl<T> Iterator for IterableStoreReader<T>
where
T: bincode::Decode,
- R: io::Read,
{
type Item = Result<T>;
fn next(&mut self) -> Option<Self::Item> {
- let header = match IterableHeader::deserialize(&mut self.reader) {
+ if self.offset + IterableHeader::serialized_size() > self.data.len() {
+ return None;
+ }
+
+ let header_bytes = &self.data[self.offset..self.offset + IterableHeader::serialized_size()];
+
+ let header = match IterableHeader::deserialize(header_bytes) {
Ok(header) => header,
Err(_err) => return None,
};
- let mut serialized = vec![0; header.num_upcoming_bytes as usize];
- match self.reader.read_exact(&mut serialized) {
- Ok(()) => (),
- Err(err) => return Some(Err(err.into())),
- }
+ self.offset += IterableHeader::serialized_size();
+ let serialized = &self.data[self.offset..self.offset + header.num_upcoming_bytes as usize];
+
+ self.offset += header.num_upcoming_bytes as usize;
Some(
- match bincode::decode_from_slice(&serialized, bincode::config::standard()) {
+ match bincode::decode_from_slice(serialized, bincode::config::standard()) {
Ok((item, _)) => Ok(item),
Err(err) => Err(err.into()),
},
@@ -133,30 +195,45 @@ where
}
}
-pub struct SortedIterableStoreReader<T, R>
+impl<T> io::Seek for IterableStoreReader<T> {
+ fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+ match pos {
+ io::SeekFrom::Start(offset) => {
+ self.offset = offset as usize;
+ }
+ io::SeekFrom::End(offset) => {
+ self.offset = self.data.len() - offset as usize;
+ }
+ io::SeekFrom::Current(offset) => {
+ self.offset += offset as usize;
+ }
+ }
+
+ Ok(self.offset as u64)
+ }
+}
+
+pub struct SortedIterableStoreReader<T>
where
- R: io::Read,
T: bincode::Decode,
{
- readers: Vec<Peekable<IterableStoreReader<T, R>>>,
+ readers: Vec<Peekable<IterableStoreReader<T>>>,
}
-impl<T, R> SortedIterableStoreReader<T, R>
+impl<T> SortedIterableStoreReader<T>
where
T: Ord + bincode::Decode,
- R: io::Read,
{
- pub fn new(readers: Vec<IterableStoreReader<T, R>>) -> Self {
+ pub fn new(readers: Vec<IterableStoreReader<T>>) -> Self {
let readers = readers.into_iter().map(Peekable::new).collect::<Vec<_>>();
Self { readers }
}
}
-impl<T, R> Iterator for SortedIterableStoreReader<T, R>
+impl<T> Iterator for SortedIterableStoreReader<T>
where
T: Ord + bincode::Decode,
- R: io::Read,
{
type Item = Result<T>;
@@ -170,7 +247,7 @@ where
match item {
Ok(item) => match min_index {
Some(cur_min) => {
- let cur_min_reader: &Peekable<IterableStoreReader<T, R>> =
+ let cur_min_reader: &Peekable<IterableStoreReader<T>> =
&self.readers[cur_min];
match cur_min_reader.peek().unwrap().as_ref() {
@@ -196,6 +273,131 @@ where
}
}
+pub struct ConstIterableStoreWriter<T, W>
+where
+ W: io::Write,
+{
+ writer: io::BufWriter<W>,
+ buf: Vec<u8>,
+ next_start: u64,
+ _marker: std::marker::PhantomData<T>,
+}
+
+impl<T, W> ConstIterableStoreWriter<T, W>
+where
+ T: ConstSerializable,
+ W: io::Write,
+{
+ pub fn new(writer: W) -> Self {
+ Self {
+ writer: io::BufWriter::new(writer),
+ _marker: std::marker::PhantomData,
+ buf: Vec::new(),
+ next_start: 0,
+ }
+ }
+
+ pub fn write(&mut self, item: &T) -> Result<WrittenOffset> {
+ self.buf.resize(T::BYTES, 0);
+ item.serialize(&mut self.buf);
+
+ assert_eq!(self.buf.len(), T::BYTES);
+
+ self.writer.write_all(&self.buf)?;
+
+ let start = self.next_start;
+ let bytes_written = T::BYTES as u64;
+
+ self.next_start += bytes_written;
+
+ Ok(WrittenOffset {
+ start,
+ num_bytes: bytes_written,
+ })
+ }
+
+ pub fn finalize(mut self) -> Result<W> {
+ self.writer.flush()?;
+
+ self.writer.into_inner().map_err(|e| anyhow::anyhow!("{e}"))
+ }
+
+ pub fn flush(&mut self) -> io::Result<()> {
+ self.writer.flush()
+ }
+}
+
+pub struct ConstIterableStoreReader<T> {
+ data: OwnedBytes,
+ offset: usize,
+ _marker: std::marker::PhantomData<T>,
+}
+
+impl<T> ConstIterableStoreReader<T> {
+ pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {
+ let data = OwnedBytes::mmap_from_path(path)?;
+
+ Ok(Self {
+ data,
+ offset: 0,
+ _marker: std::marker::PhantomData,
+ })
+ }
+
+ pub fn from_bytes(data: Vec<u8>) -> Self {
+ Self {
+ data: OwnedBytes::new(data),
+ offset: 0,
+ _marker: std::marker::PhantomData,
+ }
+ }
+
+ pub fn slice(&self, range: Range<usize>) -> ConstIterableStoreReader<T> {
+ ConstIterableStoreReader {
+ data: self.data.slice(range),
+ offset: 0,
+ _marker: std::marker::PhantomData,
+ }
+ }
+}
+
+impl<T> Iterator for ConstIterableStoreReader<T>
+where
+ T: ConstSerializable,
+{
+ type Item = T;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.offset + T::BYTES > self.data.len() {
+ return None;
+ }
+
+ let item_bytes = &self.data[self.offset..self.offset + T::BYTES];
+
+ self.offset += T::BYTES;
+
+ Some(T::deserialize(item_bytes))
+ }
+}
+
+impl<T> io::Seek for ConstIterableStoreReader<T> {
+ fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+ match pos {
+ io::SeekFrom::Start(offset) => {
+ self.offset = offset as usize;
+ }
+ io::SeekFrom::End(offset) => {
+ self.offset = self.data.len() - offset as usize;
+ }
+ io::SeekFrom::Current(offset) => {
+ self.offset += offset as usize;
+ }
+ }
+
+ Ok(self.offset as u64)
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -208,7 +410,7 @@ mod tests {
writer.write(&3).unwrap();
let writer = writer.finalize().unwrap();
- let reader = IterableStoreReader::new(writer.as_slice());
+ let reader = IterableStoreReader::from_bytes(writer);
let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();
assert_eq!(items, vec![1, 2, 3]);
@@ -228,13 +430,27 @@ mod tests {
writer2.write(&6).unwrap();
let writer2 = writer2.finalize().unwrap();
- let reader1 = IterableStoreReader::new(writer1.as_slice());
+ let reader1 = IterableStoreReader::from_bytes(writer1);
- let reader2 = IterableStoreReader::new(writer2.as_slice());
+ let reader2 = IterableStoreReader::from_bytes(writer2);
let reader = SortedIterableStoreReader::new(vec![reader1, reader2]);
let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();
assert_eq!(items, vec![1, 2, 3, 4, 5, 6]);
}
+
+ #[test]
+ fn test_const_iterable_store() {
+ let mut writer = ConstIterableStoreWriter::new(Vec::new());
+ writer.write(&1).unwrap();
+ writer.write(&2).unwrap();
+ writer.write(&3).unwrap();
+ let writer = writer.finalize().unwrap();
+
+ let reader = ConstIterableStoreReader::<i32>::from_bytes(writer);
+
+ let items: Vec<i32> = reader.collect();
+ assert_eq!(items, vec![1, 2, 3]);
+ }
}
diff --git a/crates/file-store/src/lib.rs b/crates/file-store/src/lib.rs
index 5a1157d1..e25172fc 100644
--- a/crates/file-store/src/lib.rs
+++ b/crates/file-store/src/lib.rs
@@ -18,7 +18,11 @@
pub type Result<T> = std::result::Result<T, anyhow::Error>;
+pub mod const_serializable;
pub mod iterable;
+mod owned_bytes;
pub mod peekable;
+pub mod random_lookup;
+pub use const_serializable::ConstSerializable;
pub use peekable::Peekable;
diff --git a/crates/file-store/src/owned_bytes.rs b/crates/file-store/src/owned_bytes.rs
new file mode 100644
index 00000000..f5ff20ea
--- /dev/null
+++ b/crates/file-store/src/owned_bytes.rs
@@ -0,0 +1,149 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/
+
+//! This is taken from https://docs.rs/ownedbytes/latest/ownedbytes/
+//! to avoid having to pull in another dependency.
+
+use stable_deref_trait::StableDeref;
+use std::{
+ fmt, io,
+ ops::{Deref, Range},
+ path::Path,
+ sync::Arc,
+};
+
+pub struct OwnedBytes {
+ data: &'static [u8],
+ box_stable_deref: Arc<dyn Deref<Target = [u8]> + Sync + Send>,
+}
+
+impl OwnedBytes {
+ pub fn mmap_from_path<P: AsRef<Path>>(path: P) -> io::Result<Self> {
+ let path = path.as_ref();
+ let mmap = unsafe { memmap2::Mmap::map(&std::fs::File::open(path)?)? };
+
+ Ok(Self::new(mmap))
+ }
+
+ pub fn empty() -> Self {
+ Self::new(&[][..])
+ }
+
+ pub fn new<T: StableDeref + Deref<Target = [u8]> + 'static + Send + Sync>(
+ data_holder: T,
+ ) -> Self {
+ let box_stable_deref = Arc::new(data_holder);
+ let bytes: &[u8] = box_stable_deref.deref();
+ let data = unsafe { &*(bytes as *const [u8]) };
+ Self {
+ data,
+ box_stable_deref,
+ }
+ }
+
+ pub fn as_slice(&self) -> &[u8] {
+ self.data
+ }
+
+ #[must_use]
+ #[inline]
+ pub fn slice(&self, range: Range<usize>) -> Self {
+ Self {
+ data: &self.data[range],
+ box_stable_deref: self.box_stable_deref.clone(),
+ }
+ }
+}
+
+impl Deref for OwnedBytes {
+ type Target = [u8];
+
+ #[inline]
+ fn deref(&self) -> &Self::Target {
+ self.data
+ }
+}
+
+impl AsRef<[u8]> for OwnedBytes {
+ #[inline]
+ fn as_ref(&self) -> &[u8] {
+ self.data
+ }
+}
+
+impl fmt::Debug for OwnedBytes {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ // We truncate the bytes in order to make sure the debug string
+ // is not too long.
+ let bytes_truncated: &[u8] = if self.len() > 8 {
+ &self.as_slice()[..8]
+ } else {
+ self.as_slice()
+ };
+
+ write!(f, "OwnedBytes({bytes_truncated:?}, len={})", self.len())
+ }
+}
+
+impl Clone for OwnedBytes {
+ fn clone(&self) -> Self {
+ OwnedBytes {
+ data: self.data,
+ box_stable_deref: self.box_stable_deref.clone(),
+ }
+ }
+}
+
+impl io::Read for OwnedBytes {
+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+ self.as_slice().read(buf)
+ }
+}
+
+impl From<Vec<u8>> for OwnedBytes {
+ fn from(vec: Vec<u8>) -> Self {
+ Self::new(vec)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::io::Read;
+
+ use super::*;
+
+ #[test]
+ fn test_owned_bytes() {
+ let bytes = OwnedBytes::new(vec![1, 2, 3, 4, 5]);
+ assert_eq!(bytes.len(), 5);
+ assert_eq!(bytes.as_slice(), &[1, 2, 3, 4, 5]);
+ }
+
+ #[test]
+ fn test_owned_bytes_empty() {
+ let bytes = OwnedBytes::empty();
+ assert_eq!(bytes.len(), 0);
+ assert_eq!(bytes.as_slice(), &[]);
+ }
+
+ #[test]
+ fn test_read() {
+ let mut bytes = OwnedBytes::new(vec![1, 2, 3, 4, 5]);
+ let mut buf = [0; 3];
+ assert_eq!(bytes.read(&mut buf).unwrap(), 3);
+ assert_eq!(&buf, &[1, 2, 3]);
+ }
+}
diff --git a/crates/file-store/src/random_lookup.rs b/crates/file-store/src/random_lookup.rs
new file mode 100644
index 00000000..53f32b6e
--- /dev/null
+++ b/crates/file-store/src/random_lookup.rs
@@ -0,0 +1,196 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/
+
+use std::{
+ io::{self, Write},
+ path::Path,
+};
+
+use crate::{owned_bytes::OwnedBytes, ConstSerializable};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ItemId(u64);
+
+impl ItemId {
+ pub fn into_inner(self) -> u64 {
+ self.0
+ }
+
+ pub fn from_inner(inner: u64) -> Self {
+ ItemId(inner)
+ }
+}
+
+pub struct RandomLookupWriter<V, W>
+where
+ W: io::Write,
+{
+ next_id: u64,
+ writer: io::BufWriter<W>,
+ buf: Vec<u8>,
+ _phantom: std::marker::PhantomData<V>,
+}
+
+impl<V, W> RandomLookupWriter<V, W>
+where
+ W: io::Write,
+ V: ConstSerializable,
+{
+ pub fn new(writer: W) -> Self {
+ RandomLookupWriter {
+ next_id: 0,
+ writer: io::BufWriter::new(writer),
+ buf: Vec::new(),
+ _phantom: std::marker::PhantomData,
+ }
+ }
+
+ pub fn write(&mut self, item: &V) -> io::Result<ItemId> {
+ self.buf.resize(V::BYTES, 0);
+ item.serialize(&mut self.buf);
+
+ assert_eq!(self.buf.len(), V::BYTES);
+
+ self.writer.write_all(&self.buf)?;
+
+ let id = ItemId(self.next_id);
+ self.next_id += 1;
+
+ Ok(id)
+ }
+
+ pub fn finish(mut self) -> io::Result<W> {
+ self.writer.flush()?;
+
+ Ok(self.writer.into_inner()?)
+ }
+}
+
+pub struct RandomLookup<V> {
+ data: OwnedBytes,
+ _phantom: std::marker::PhantomData<V>,
+}
+
+impl<V> RandomLookup<V>
+where
+ V: ConstSerializable,
+{
+ pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {
+ let data = OwnedBytes::mmap_from_path(path)?;
+
+ Ok(RandomLookup {
+ data,
+ _phantom: std::marker::PhantomData,
+ })
+ }
+
+ /// Returns the value at the given item id.
+ /// Panics if the item id is not from this store.
+ pub fn get(&self, id: ItemId) -> V {
+ let item_size: usize = V::BYTES;
+
+ let start = id.0 as usize * item_size;
+
+ V::deserialize(&self.data[start..start + V::BYTES])
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = (ItemId, V)> + '_ {
+ RandomLookupIter {
+ data: &self.data,
+ next_id: 0,
+ _phantom: std::marker::PhantomData,
+ }
+ }
+}
+
+impl<V> From<OwnedBytes> for RandomLookup<V> {
+ fn from(data: OwnedBytes) -> Self {
+ RandomLookup {
+ data,
+ _phantom: std::marker::PhantomData,
+ }
+ }
+}
+
+pub struct RandomLookupIter<'a, V> {
+ data: &'a OwnedBytes,
+ next_id: u64,
+ _phantom: std::marker::PhantomData<V>,
+}
+
+impl<'a, V> Iterator for RandomLookupIter<'a, V>
+where
+ V: ConstSerializable,
+{
+ type Item = (ItemId, V);
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let item_size: usize = V::BYTES;
+ let start = self.next_id as usize * item_size;
+
+ if start >= self.data.len() {
+ return None;
+ }
+
+ let id = ItemId::from_inner(self.next_id);
+
+ self.next_id += 1;
+
+ let val = V::deserialize(&self.data[start..start + V::BYTES]);
+
+ Some((id, val))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_simple() {
+ let mut writer = RandomLookupWriter::new(Vec::new());
+
+ let value = 42u64;
+
+ let id = writer.write(&value).unwrap();
+ let bytes = writer.finish().unwrap();
+
+ let store = RandomLookup::<u64>::from(OwnedBytes::new(bytes));
+
+ let value2 = store.get(id);
+
+ assert_eq!(value, value2);
+ }
+
+ #[test]
+ fn test_iter() {
+ let mut writer = RandomLookupWriter::new(Vec::new());
+
+ let items = vec![1u64, 2, 3, 4, 5];
+
+ for item in &items {
+ writer.write(item).unwrap();
+ }
+
+ let bytes = writer.finish().unwrap();
+
+ let store = RandomLookup::<u64>::from(OwnedBytes::new(bytes));
+
+ let res = store.iter().map(|(_, val)| val).collect::<Vec<_>>();
+
+ assert_eq!(items, res);
+ }
+}
diff --git a/crates/speedy-kv/Cargo.toml b/crates/speedy-kv/Cargo.toml
index 35173333..364195ea 100644
--- a/crates/speedy-kv/Cargo.toml
+++ b/crates/speedy-kv/Cargo.toml
@@ -12,7 +12,7 @@ bincode.workspace = true
bloom = { path = "../bloom" }
file_store = { path = "../file-store" }
fst.workspace = true
-memmap.workspace = true
+memmap2.workspace = true
serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
diff --git a/crates/speedy-kv/src/blob_id_index.rs b/crates/speedy-kv/src/blob_id_index.rs
index 2f7e6143..480060b5 100644
--- a/crates/speedy-kv/src/blob_id_index.rs
+++ b/crates/speedy-kv/src/blob_id_index.rs
@@ -27,7 +27,7 @@ use super::{BlobId, Serialized, SerializedRef};
pub struct BlobIdIndex<K> {
path: PathBuf,
- fst: fst::Map<memmap::Mmap>,
+ fst: fst::Map<memmap2::Mmap>,
_marker: std::marker::PhantomData<K>,
}
@@ -36,7 +36,7 @@ impl<K> BlobIdIndex<K> {
where
P: AsRef<Path>,
{
- let mmap = unsafe { memmap::Mmap::map(&std::fs::File::open(&path)?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&std::fs::File::open(&path)?)? };
Ok(Self {
path: path.as_ref().to_path_buf(),
@@ -55,9 +55,9 @@ impl<K> BlobIdIndex<K> {
.search(super::automaton::ExactMatch(key))
.into_stream();
- // let mut stream = self.fst.range().ge(key).le(key).into_stream();
-
- stream.next().map(|(_, v)| BlobId(v))
+ stream
+ .next()
+ .map(|(_, v)| BlobId(file_store::random_lookup::ItemId::from_inner(v)))
}
pub fn search<'a, A>(
@@ -131,9 +131,12 @@ where
type Item = (Serialized<K>, BlobId);
fn next(&mut self) -> Option<Self::Item> {
- self.stream
- .next()
- .map(|(k, v)| (k.to_vec().into(), BlobId(v)))
+ self.stream.next().map(|(k, v)| {
+ (
+ k.to_vec().into(),
+ BlobId(file_store::random_lookup::ItemId::from_inner(v)),
+ )
+ })
}
}
@@ -142,7 +145,6 @@ where
W: Write,
{
fst_builder: fst::MapBuilder<BufWriter<W>>,
- next_blob_id: u64,
_marker: std::marker::PhantomData<K>,
}
@@ -155,18 +157,14 @@ where
Ok(Self {
fst_builder,
- next_blob_id: 0,
_marker: std::marker::PhantomData,
})
}
- pub fn insert(&mut self, key: &[u8]) -> Result<BlobId> {
- let id = BlobId(self.next_blob_id);
- self.next_blob_id += 1;
-
- self.fst_builder.insert(key, id.0)?;
+ pub fn insert(&mut self, key: &[u8], id: &BlobId) -> Result<()> {
+ self.fst_builder.insert(key, id.0.into_inner())?;
- Ok(id)
+ Ok(())
}
pub fn finish(self) -> Result<()> {
diff --git a/crates/speedy-kv/src/blob_index.rs b/crates/speedy-kv/src/blob_index.rs
index dbc03026..10056b11 100644
--- a/crates/speedy-kv/src/blob_index.rs
+++ b/crates/speedy-kv/src/blob_index.rs
@@ -16,7 +16,7 @@
use crate::Result;
use std::{
- io::{BufWriter, Write},
+ io::Write,
path::{Path, PathBuf},
};
@@ -24,7 +24,7 @@ use super::{BlobId, BlobPointer};
pub struct BlobIndex {
path: PathBuf,
- bytes: memmap::Mmap,
+ data: file_store::random_lookup::RandomLookup<BlobPointer>,
}
impl BlobIndex {
@@ -32,10 +32,10 @@ impl BlobIndex {
where
P: AsRef<Path>,
{
- let bytes = unsafe { memmap::Mmap::map(&std::fs::File::open(&path)?)? };
+ let data = file_store::random_lookup::RandomLookup::open(&path)?;
Ok(Self {
- bytes,
+ data,
path: path.as_ref().to_path_buf(),
})
}
@@ -45,14 +45,11 @@ impl BlobIndex {
}
pub fn get(&self, id: BlobId) -> BlobPointer {
- let offset = id.0 as usize * BlobPointer::size();
- let bytes = self.bytes.as_ref();
- let bytes = &bytes[offset..offset + BlobPointer::size()];
- BlobPointer::from_bytes(bytes.try_into().unwrap())
+ self.data.get(id.0)
}
pub fn iter(&self) -> impl Iterator<Item = BlobPointer> + '_ {
- BlobIndexIter::new(self.bytes.as_ref())
+ self.data.iter().map(|(_, v)| v)
}
pub fn path(&self) -> &Path {
@@ -60,42 +57,11 @@ impl BlobIndex {
}
}
-struct BlobIndexIter<'a> {
- bytes: &'a [u8],
- offset: usize,
-}
-
-impl<'a> BlobIndexIter<'a> {
- fn new(bytes: &'a [u8]) -> Self {
- Self { bytes, offset: 0 }
- }
-}
-
-impl<'a> Iterator for BlobIndexIter<'a> {
- type Item = BlobPointer;
-
- fn next(&mut self) -> Option<Self::Item> {
- if self.offset >= self.bytes.len() {
- return None;
- }
-
- let ptr = BlobPointer::from_bytes(
- self.bytes[self.offset..self.offset + BlobPointer::size()]
- .try_into()
- .unwrap(),
- );
- self.offset += BlobPointer::size();
-
- Some(ptr)
- }
-}
-
pub struct BlobIndexWriter<W>
where
W: Write,
{
- wrt: BufWriter<W>,
- _marker: std::marker::PhantomData<W>,
+ wrt: file_store::random_lookup::RandomLookupWriter<BlobPointer, W>,
}
impl<W> BlobIndexWriter<W>
@@ -104,18 +70,17 @@ where
{
pub fn new(wrt: W) -> Self {
Self {
- wrt: BufWriter::new(wrt),
- _marker: std::marker::PhantomData,
+ wrt: file_store::random_lookup::RandomLookupWriter::new(wrt),
}
}
- pub fn write(&mut self, ptr: &BlobPointer) -> Result<()> {
- self.wrt.write_all(&ptr.as_bytes())?;
- Ok(())
+ pub fn write(&mut self, ptr: &BlobPointer) -> Result<BlobId> {
+ let id = self.wrt.write(ptr)?;
+ Ok(BlobId(id))
}
- pub fn finish(mut self) -> Result<()> {
- self.wrt.flush()?;
+ pub fn finish(self) -> Result<()> {
+ self.wrt.finish()?;
Ok(())
}
}
diff --git a/crates/speedy-kv/src/blob_store.rs b/crates/speedy-kv/src/blob_store.rs
index 852bfbe4..0c994581 100644
--- a/crates/speedy-kv/src/blob_store.rs
+++ b/crates/speedy-kv/src/blob_store.rs
@@ -29,7 +29,7 @@ pub struct Blob<'a, K, V> {
pub struct BlobStore<K, V> {
path: PathBuf,
- bytes: Option<memmap::Mmap>,
+ bytes: Option<memmap2::Mmap>,
_marker: std::marker::PhantomData<(K, V)>,
}
@@ -38,7 +38,7 @@ impl<K, V> BlobStore<K, V> {
where
P: AsRef<Path>,
{
- let bytes = unsafe { memmap::Mmap::map(&std::fs::File::open(path.as_ref())?).ok() };
+ let bytes = unsafe { memmap2::Mmap::map(&std::fs::File::open(path.as_ref())?).ok() };
Ok(Self {
path: path.as_ref().to_path_buf(),
diff --git a/crates/speedy-kv/src/lib.rs b/crates/speedy-kv/src/lib.rs
index 9b15649d..5c9b2cee 100644
--- a/crates/speedy-kv/src/lib.rs
+++ b/crates/speedy-kv/src/lib.rs
@@ -47,6 +47,7 @@ mod blob_store;
mod segment;
mod serialized;
+use file_store::ConstSerializable;
pub use serialized::{Serialized, SerializedRef};
struct BlobPointer {
@@ -93,8 +94,20 @@ impl BlobPointer {
}
}
+impl ConstSerializable for BlobPointer {
+ const BYTES: usize = Self::size();
+
+ fn serialize(&self, buf: &mut [u8]) {
+ buf.copy_from_slice(&self.as_bytes());
+ }
+
+ fn deserialize(buf: &[u8]) -> Self {
+ Self::from_bytes(buf.try_into().unwrap())
+ }
+}
+
#[derive(Clone, Copy)]
-struct BlobId(u64);
+struct BlobId(file_store::random_lookup::ItemId);
#[derive(Debug)]
struct LiveSegment<K, V> {
diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rs
index 47f820fc..34b7bffc 100644
--- a/crates/speedy-kv/src/segment.rs
+++ b/crates/speedy-kv/src/segment.rs
@@ -63,9 +63,9 @@ where
W: Write,
{
fn insert(&mut self, key: SerializedRef<'_, K>, value: SerializedRef<'_, V>) -> Result<()> {
- let _id = self.writers.id_index.insert(key.as_bytes())?;
let ptr = self.writers.store.write(key, value)?;
- self.writers.blob_index.write(&ptr)?;
+ let id = self.writers.blob_index.write(&ptr)?;
+ self.writers.id_index.insert(key.as_bytes(), &id)?;
self.bloom.insert_raw(key.as_bytes());