Skip to content

web search done right

Re-write webgraph to be based on rocksdb. This allows us to create really big graphs (needed for page-level webgraphs).

Mikkel Denker3a87834547d4
diff --git a/Cargo.toml b/Cargo.toml
index c53e50a8..89aae053 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -78,7 +78,7 @@ tokio-stream = "0.1.11"
scylla = "0.7.0"
chitchat = "0.5.0"
memmap2 = "0.5.10"
-rkyv = "0.7.41"
+rkyv = "0.7.41"
dashmap = "5.4.0"
tch = "0.13.0"
torch-sys = "0.13.0"
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rs
index f984edc0..06b7ab74 100644
--- a/core/examples/print_inbound_links.rs
+++ b/core/examples/print_inbound_links.rs
@@ -28,7 +28,7 @@ pub fn main() {
let node = Node::from(host).into_host();
let node_id = graph.node2id(&node).unwrap();
- let inbound_vec = inbound.get(node_id).unwrap();
+ let inbound_vec = inbound.get(&node_id).unwrap();
println!("{:?}", inbound_vec);
for edge in graph.ingoing_edges(node) {
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 05814790..6dd0a80b 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -132,7 +132,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
);
let host_centrality = node_id
- .and_then(|node_id| worker.centrality_store.harmonic.host.get(node_id))
+ .and_then(|node_id| worker.centrality_store.harmonic.host.get(&node_id))
.unwrap_or_default();
if let Some(host_centrality_threshold) = job.host_centrality_threshold {
@@ -176,7 +176,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
.get(&html.url().without_protocol().to_lowercase().into());
let page_centrality = node_id
- .and_then(|node_id| worker.centrality_store.harmonic.full.get(node_id))
+ .and_then(|node_id| worker.centrality_store.harmonic.full.get(&node_id))
.unwrap_or_default();
let fetch_time_ms = record.metadata.fetch_time_ms as u64;
@@ -189,8 +189,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let node_id = worker
.centrality_store
.node2id
- .get(&Node::from_url(html.url()).into_host())
- .cloned();
+ .get(&Node::from_url(html.url()).into_host());
let mut dmoz_description = None;
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index f6d2cd2a..68b0fb4c 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -101,6 +101,10 @@ pub fn process_job(job: &Job) -> webgraph::Webgraph {
.filter(|link| link.source.domain() != link.destination.domain())
.filter(|link| link.matches_url_regex())
{
+ if link.source.domain() == link.destination.domain() {
+ continue;
+ }
+
trace!("inserting link {:?}", link);
let mut source = Node::from(link.source);
source.remove_protocol();
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 9caf1374..ee057897 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -655,7 +655,7 @@ mod tests {
pre_computed_score: 0.0,
dmoz_description: None,
- node_id: Some(*graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),
+ node_id: Some(graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),
})
.expect("failed to insert webpage");
index
@@ -683,7 +683,7 @@ mod tests {
pre_computed_score: 0.0,
fetch_time_ms: 500,
dmoz_description: None,
- node_id: Some(*graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),
+ node_id: Some(graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),
})
.expect("failed to insert webpage");
index
@@ -711,7 +711,7 @@ mod tests {
pre_computed_score: 0.0,
fetch_time_ms: 500,
dmoz_description: None,
- node_id: Some(*graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),
+ node_id: Some(graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),
})
.expect("failed to insert webpage");
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 7cc995a9..af99a2e5 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -14,13 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::io::{BufReader, BufWriter, Read};
-use std::{collections::BTreeMap, fs::File, path::Path};
+use std::{fs::File, path::Path};
+use rocksdb::BlockBasedOptions;
use tracing::debug;
use tracing::log::trace;
-use crate::Result;
use crate::{
kv::{rocksdb_store::RocksDbStore, Kv},
webgraph::{centrality::harmonic::HarmonicCentrality, Node, NodeID, Webgraph},
@@ -28,6 +27,113 @@ use crate::{
use super::inbound_similarity::InboundSimilarity;
+pub struct Node2Id {
+ db: rocksdb::DB,
+}
+
+impl Node2Id {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ let mut options = rocksdb::Options::default();
+ options.create_if_missing(true);
+ options.increase_parallelism(8);
+ options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable
+ options.set_max_write_buffer_number(8);
+
+ let mut block_options = BlockBasedOptions::default();
+ block_options.set_bloom_filter(64.0, true);
+
+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cache
+ block_options.set_block_cache(&cache);
+
+ options.set_block_based_table_factory(&block_options);
+
+ let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();
+
+ Self { db }
+ }
+
+ pub fn get(&self, key: &Node) -> Option<NodeID> {
+ let bytes = bincode::serialize(key).unwrap();
+
+ self.db
+ .get(bytes)
+ .unwrap()
+ .map(|bytes| bincode::deserialize(&bytes).unwrap())
+ }
+
+ pub fn put(&self, key: &Node, value: &NodeID) {
+ let key_bytes = bincode::serialize(key).unwrap();
+ let value_bytes = bincode::serialize(value).unwrap();
+
+ self.db.put(key_bytes, value_bytes).unwrap();
+ }
+
+ pub fn batch_put(&self, it: impl Iterator<Item = (Node, NodeID)>) {
+ let mut batch = rocksdb::WriteBatch::default();
+
+ for (key, value) in it {
+ let key_bytes = bincode::serialize(&key).unwrap();
+ let value_bytes = bincode::serialize(&value).unwrap();
+
+ batch.put(key_bytes, value_bytes);
+ }
+
+ self.db.write(batch).unwrap();
+ }
+
+ pub fn contains(&self, key: &Node) -> bool {
+ let bytes = bincode::serialize(key).unwrap();
+
+ self.db.get(bytes).unwrap().is_some()
+ }
+
+ pub fn nodes(&self) -> impl Iterator<Item = Node> + '_ {
+ let mut read_opts = rocksdb::ReadOptions::default();
+
+ read_opts.set_readahead_size(4_194_304); // 4 MB
+
+ self.db
+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)
+ .map(|res| {
+ let (key, _) = res.unwrap();
+ bincode::deserialize(&key).unwrap()
+ })
+ }
+
+ pub fn ids(&self) -> impl Iterator<Item = NodeID> + '_ {
+ let mut read_opts = rocksdb::ReadOptions::default();
+
+ read_opts.set_readahead_size(4_194_304); // 4 MB
+
+ self.db
+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)
+ .map(|res| {
+ let (_, val) = res.unwrap();
+ bincode::deserialize(&val).unwrap()
+ })
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {
+ let mut read_opts = rocksdb::ReadOptions::default();
+
+ read_opts.set_readahead_size(4_194_304); // 4 MB
+
+ self.db
+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)
+ .map(|res| {
+ let (key, val) = res.unwrap();
+ (
+ bincode::deserialize(&key).unwrap(),
+ bincode::deserialize(&val).unwrap(),
+ )
+ })
+ }
+
+ pub fn flush(&self) {
+ self.db.flush().unwrap();
+ }
+}
+
pub struct HarmonicCentralityStore {
pub host: Box<dyn Kv<NodeID, f64>>,
pub full: Box<dyn Kv<NodeID, f64>>,
@@ -49,14 +155,14 @@ impl HarmonicCentralityStore {
pub struct IndexerCentralityStore {
pub harmonic: HarmonicCentralityStore,
- pub node2id: BTreeMap<Node, NodeID>,
+ pub node2id: Node2Id,
}
impl IndexerCentralityStore {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),
- node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),
+ node2id: Node2Id::open(path.as_ref().join("node2id")),
}
}
}
@@ -72,7 +178,7 @@ impl From<CentralityStore> for IndexerCentralityStore {
pub struct SearchCentralityStore {
pub inbound_similarity: InboundSimilarity,
- pub node2id: BTreeMap<Node, NodeID>,
+ pub node2id: Node2Id,
}
impl SearchCentralityStore {
@@ -80,7 +186,7 @@ impl SearchCentralityStore {
Self {
inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
.unwrap(),
- node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),
+ node2id: Node2Id::open(path.as_ref().join("node2id")),
}
}
}
@@ -97,20 +203,10 @@ impl From<CentralityStore> for SearchCentralityStore {
pub struct CentralityStore {
pub harmonic: HarmonicCentralityStore,
pub inbound_similarity: InboundSimilarity,
- pub node2id: BTreeMap<Node, NodeID>,
+ pub node2id: Node2Id,
pub base_path: String,
}
-fn open_node2id<P: AsRef<Path>>(path: P) -> Result<BTreeMap<Node, NodeID>> {
- let file = File::open(path)?;
- let mut reader = BufReader::new(file);
-
- let mut buf = Vec::new();
- reader.read_to_end(&mut buf)?;
-
- Ok(bincode::deserialize(&buf)?)
-}
-
impl CentralityStore {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
@@ -118,9 +214,7 @@ impl CentralityStore {
inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
.ok()
.unwrap_or_default(),
- node2id: open_node2id(path.as_ref().join("node2id"))
- .ok()
- .unwrap_or_default(),
+ node2id: Node2Id::open(path.as_ref().join("node2id")),
base_path: path.as_ref().to_str().unwrap().to_string(),
}
}
@@ -134,7 +228,7 @@ impl CentralityStore {
.write(true)
.create(true)
.truncate(true)
- .open(output_path.as_ref().join("harmonic_host.csv"))
+ .open(output_path.as_ref().join("harmonic.csv"))
.unwrap();
let mut host: Vec<_> = harmonic_centrality.host.into_iter().collect();
@@ -142,7 +236,7 @@ impl CentralityStore {
let mut wtr = csv::Writer::from_writer(csv_file);
for (node, centrality) in host {
let node_id = store.node2id.get(&node).unwrap();
- store.harmonic.host.insert(*node_id, centrality);
+ store.harmonic.host.insert(node_id, centrality);
wtr.write_record(&[node.name, centrality.to_string()])
.unwrap();
}
@@ -158,10 +252,7 @@ impl CentralityStore {
pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
let mut store = CentralityStore::open(output_path.as_ref());
- store.node2id = graph
- .node_ids()
- .map(|(node, node_id)| (node.clone(), *node_id))
- .collect();
+ store.node2id.batch_put(graph.node_ids());
let harmonic_centrality = HarmonicCentrality::calculate(graph);
Self::store_host(&output_path, &mut store, harmonic_centrality);
@@ -189,14 +280,6 @@ impl CentralityStore {
.unwrap();
trace!("saving node2id");
- let mut file = BufWriter::new(
- File::options()
- .create(true)
- .truncate(true)
- .write(true)
- .open(Path::new(&self.base_path).join("node2id"))
- .unwrap(),
- );
- bincode::serialize_into(&mut file, &self.node2id).unwrap();
+ self.node2id.flush();
}
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 9cdb3516..bd83cfab 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -285,18 +285,18 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(*graph.node2id(&node).unwrap(), centrality);
+ .insert(graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
let inbound =
InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);
- let scorer = inbound.scorer(&[*graph.node2id(&Node::from("b.com")).unwrap()], &[]);
+ let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);
let e = graph.node2id(&Node::from("e.com")).unwrap();
let d = graph.node2id(&Node::from("d.com")).unwrap();
- assert!(scorer.score(e) > scorer.score(d));
+ assert!(scorer.score(&e) > scorer.score(&d));
}
#[test]
@@ -315,7 +315,7 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(*graph.node2id(&node).unwrap(), centrality);
+ .insert(graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -351,7 +351,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),
+ node_id: Some(graph.node2id(&Node::from("e.com")).unwrap()),
dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -379,7 +379,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),
+ node_id: Some(graph.node2id(&Node::from("d.com")).unwrap()),
dmoz_description: None,
})
.expect("failed to insert webpage");
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 1976ca3f..d5b05e11 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -95,7 +95,6 @@ mod tests {
node_id: Some(
graph
.node2id(&Node::from("https://www.first.com").into_host())
- .copied()
.unwrap(),
),
})
@@ -128,7 +127,6 @@ mod tests {
node_id: Some(
graph
.node2id(&Node::from("https://www.second.com").into_host())
- .copied()
.unwrap(),
),
})
@@ -159,7 +157,7 @@ mod tests {
dmoz_description: None,
node_id: Some(
- *graph
+ graph
.node2id(&Node::from("https://www.third.com").into_host())
.unwrap(),
),
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 40608036..3c89e904 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -167,7 +167,7 @@ impl LocalSearcher {
.liked
.iter()
.map(|site| Node::from(site.clone()).into_host())
- .filter_map(|node| store.node2id.get(&node).copied())
+ .filter_map(|node| store.node2id.get(&node))
.collect();
let disliked_sites: Vec<_> = parsed_query
@@ -175,7 +175,7 @@ impl LocalSearcher {
.disliked
.iter()
.map(|site| Node::from(site.clone()).into_host())
- .filter_map(|node| store.node2id.get(&node).copied())
+ .filter_map(|node| store.node2id.get(&node))
.collect();
aggregator.set_inbound_similarity(
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index 363c8adc..dc30b08e 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -74,7 +74,7 @@ impl SimilarSitesFinder {
let nodes: Vec<_> = nodes
.iter()
.map(|url| Node::from(url.to_string()).into_host())
- .filter_map(|node| self.webgraph.node2id(&node).copied())
+ .filter_map(|node| self.webgraph.node2id(&node))
.collect();
let scorer = self.inbound_similarity.scorer(&nodes, &[]);
@@ -134,7 +134,7 @@ impl SimilarSitesFinder {
scored_nodes
.into_iter()
.map(|ScoredNodeID { node_id, score }| {
- let node = self.webgraph.id2node(&node_id).unwrap().clone();
+ let node = self.webgraph.id2node(&node_id).unwrap();
ScoredNode { node, score }
})
.collect()
@@ -142,7 +142,7 @@ impl SimilarSitesFinder {
pub fn knows_about(&self, node: &Node) -> bool {
match self.webgraph.node2id(node) {
- Some(node_id) => self.inbound_similarity.knows_about(*node_id),
+ Some(node_id) => self.inbound_similarity.knows_about(node_id),
None => false,
}
}
diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rs
index 742e115d..3bed8968 100644
--- a/core/src/webgraph/centrality/betweenness.rs
+++ b/core/src/webgraph/centrality/betweenness.rs
@@ -32,7 +32,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
let mut n = 0;
let mut max_dist = 0;
- let nodes: Vec<_> = graph.nodes().cloned().take(100_000).collect();
+ let nodes: Vec<_> = graph.nodes().take(100_000).collect();
let pb =
if with_progress {
@@ -128,7 +128,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
(
centrality
.into_iter()
- .map(|(id, centrality)| (graph.id2node(&id).unwrap().clone(), centrality / norm))
+ .map(|(id, centrality)| (graph.id2node(&id).unwrap(), centrality / norm))
.collect(),
max_dist,
)
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index e8d3376f..37300a60 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -207,7 +207,7 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {
.filter(|(_, centrality)| *centrality > 0.0)
.map(|(node_id, centrality)| {
(
- graph.id2node(&NodeID::from(node_id)).unwrap().clone(),
+ graph.id2node(&NodeID::from(node_id)).unwrap(),
centrality / norm_factor,
)
})
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 4674442c..16473596 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod segment;
-use rkyv::Archive;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BinaryHeap, HashSet};
@@ -32,6 +31,7 @@ use crate::webpage::Url;
pub mod centrality;
mod store;
use self::segment::{LiveSegment, SegmentNodeID, StoredSegment};
+use self::store::Store;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NodeID(pub u64);
@@ -42,8 +42,7 @@ impl From<u64> for NodeID {
}
}
-#[derive(Debug, Clone, Archive, rkyv::Serialize, rkyv::Deserialize, PartialEq, Eq, Hash)]
-#[archive_attr(derive(Eq, Hash, PartialEq, Debug))]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
pub(crate) struct StoredEdge {
other: SegmentNodeID,
label: String,
@@ -182,8 +181,8 @@ where
let mut queue = BinaryHeap::new();
- queue.push(cmp::Reverse((0, *source_id)));
- distances.insert(*source_id, 0);
+ queue.push(cmp::Reverse((0, source_id)));
+ distances.insert(source_id, 0);
while let Some(state) = queue.pop() {
let (cost, v) = state.0;
@@ -212,7 +211,7 @@ impl ShortestPaths for Webgraph {
fn distances(&self, source: Node) -> BTreeMap<Node, u8> {
self.raw_distances(source)
.into_iter()
- .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist))
+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
.collect()
}
@@ -237,7 +236,7 @@ impl ShortestPaths for Webgraph {
fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {
self.raw_reversed_distances(source)
.into_iter()
- .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist))
+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
.collect()
}
}
@@ -291,8 +290,8 @@ pub struct Webgraph {
live_segment: LiveSegment,
segments: Vec<StoredSegment>,
executor: Arc<Executor>,
- node2id: BTreeMap<Node, NodeID>,
- id2node: BTreeMap<NodeID, Node>,
+ node2id: Store<Node, NodeID>,
+ id2node: Store<NodeID, Node>,
meta: Meta,
}
@@ -346,8 +345,8 @@ impl Webgraph {
live_segment: LiveSegment::default(),
segments,
executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
- node2id: open_bin(path.as_ref().join("node2id.bin")),
- id2node: open_bin(path.as_ref().join("id2node.bin")),
+ node2id: Store::open(path.as_ref().join("node2id")),
+ id2node: Store::open(path.as_ref().join("id2node")),
meta,
}
}
@@ -360,12 +359,12 @@ impl Webgraph {
fn id_or_assign(&mut self, node: &Node) -> NodeID {
match self.node2id(node) {
- Some(id) => *id,
+ Some(id) => id,
None => {
let id = self.id_and_increment();
- self.node2id.insert(node.clone(), id);
- self.id2node.insert(id, node.clone());
+ self.node2id.put(node, &id);
+ self.id2node.put(&id, node);
id
}
@@ -382,11 +381,11 @@ impl Webgraph {
let mut mapping = Vec::new();
for (node, other_id) in other.node2id.iter() {
- match self.node2id(node) {
- Some(this_id) => mapping.push((*other_id, *this_id)),
+ match self.node2id(&node) {
+ Some(this_id) => mapping.push((other_id, this_id)),
None => {
- let new_id = self.id_or_assign(node);
- mapping.push((*other_id, new_id));
+ let new_id = self.id_or_assign(&node);
+ mapping.push((other_id, new_id));
}
}
}
@@ -421,8 +420,8 @@ impl Webgraph {
}
self.save_metadata();
- save_bin(&self.node2id, Path::new(&self.path).join("node2id.bin"));
- save_bin(&self.id2node, Path::new(&self.path).join("id2node.bin"));
+ self.node2id.flush();
+ self.id2node.flush();
if self.segments.len() > 2 * num_cpus::get() {
self.merge_segments(num_cpus::get());
@@ -431,11 +430,11 @@ impl Webgraph {
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
if let Some(node_id) = self.node2id(&node) {
- self.inner_ingoing_edges(node_id, true)
+ self.inner_ingoing_edges(&node_id, true)
.into_iter()
.map(|edge| FullEdge {
- from: self.id2node(&edge.from).unwrap().clone(),
- to: self.id2node(&edge.to).unwrap().clone(),
+ from: self.id2node(&edge.from).unwrap(),
+ to: self.id2node(&edge.to).unwrap(),
label: edge.label.loaded().unwrap(),
})
.collect()
@@ -478,11 +477,11 @@ impl Webgraph {
pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {
if let Some(node_id) = self.node2id(&node) {
- self.inner_outgoing_edges(node_id, true)
+ self.inner_outgoing_edges(&node_id, true)
.into_iter()
.map(|edge| FullEdge {
- from: self.id2node(&edge.from).unwrap().clone(),
- to: self.id2node(&edge.to).unwrap().clone(),
+ from: self.id2node(&edge.from).unwrap(),
+ to: self.id2node(&edge.to).unwrap(),
label: edge.label.loaded().unwrap(),
})
.collect()
@@ -491,19 +490,19 @@ impl Webgraph {
}
}
- pub fn node2id(&self, node: &Node) -> Option<&NodeID> {
+ pub fn node2id(&self, node: &Node) -> Option<NodeID> {
self.node2id.get(node)
}
- pub fn id2node(&self, id: &NodeID) -> Option<&Node> {
+ pub fn id2node(&self, id: &NodeID) -> Option<Node> {
self.id2node.get(id)
}
- pub fn nodes(&self) -> impl Iterator<Item = &NodeID> + '_ {
+ pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {
self.node2id.values()
}
- pub fn node_ids(&self) -> impl Iterator<Item = (&Node, &NodeID)> + '_ {
+ pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {
self.node2id.iter()
}
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 1c248e28..ac74f098 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -49,16 +49,16 @@ impl From<u64> for SegmentNodeID {
struct MergePacket {
next_segment_node_id: SegmentNodeID,
- new_id_mapping: BTreeMap<NodeID, SegmentNodeID>,
- new_rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,
+ new_id_mapping: Store<NodeID, SegmentNodeID>,
+ new_rev_id_mapping: Store<SegmentNodeID, NodeID>,
new_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,
}
pub struct StoredSegment {
adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,
reversed_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,
- id_mapping: BTreeMap<NodeID, SegmentNodeID>,
- rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,
+ id_mapping: Store<NodeID, SegmentNodeID>,
+ rev_id_mapping: Store<SegmentNodeID, NodeID>,
meta: Meta,
id: String,
folder_path: String,
@@ -67,11 +67,10 @@ pub struct StoredSegment {
impl StoredSegment {
pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
StoredSegment {
- adjacency: Store::open(folder_path.as_ref().join("adjacency"), &id).unwrap(),
- reversed_adjacency: Store::open(folder_path.as_ref().join("reversed_adjacency"), &id)
- .unwrap(),
- id_mapping: open_bin(folder_path.as_ref().join("id_mapping.bin")),
- rev_id_mapping: open_bin(folder_path.as_ref().join("rev_id_mapping.bin")),
+ adjacency: Store::open(folder_path.as_ref().join("adjacency")),
+ reversed_adjacency: Store::open(folder_path.as_ref().join("reversed_adjacency")),
+ id_mapping: Store::open(folder_path.as_ref().join("id_mapping")),
+ rev_id_mapping: Store::open(folder_path.as_ref().join("rev_id_mapping")),
meta: open_bin(folder_path.as_ref().join("meta.bin")),
folder_path: folder_path
.as_ref()
@@ -88,11 +87,11 @@ impl StoredSegment {
}
fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {
- self.id_mapping.get(node).copied()
+ self.id_mapping.get(node)
}
fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {
- self.rev_id_mapping.get(node).copied()
+ self.rev_id_mapping.get(node)
}
pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {
@@ -154,10 +153,11 @@ impl StoredSegment {
}
fn flush(&mut self) {
- self.adjacency.flush().unwrap();
- self.reversed_adjacency.flush().unwrap();
- save_bin(&self.id_mapping, self.path().join("id_mapping.bin"));
- save_bin(&self.rev_id_mapping, self.path().join("rev_id_mapping.bin"));
+ self.adjacency.flush();
+ self.reversed_adjacency.flush();
+ self.id_mapping.flush();
+ self.rev_id_mapping.flush();
+
save_bin(&self.meta, self.path().join("meta.bin"));
}
@@ -171,7 +171,7 @@ impl StoredSegment {
pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
self.adjacency.iter().flat_map(move |(node_id, edges)| {
- let from = self.rev_id_mapping(node_id).unwrap();
+ let from = self.rev_id_mapping(&node_id).unwrap();
edges.into_iter().map(move |stored_edge| Edge {
from,
@@ -200,7 +200,7 @@ impl StoredSegment {
let mut node_seen_before = false;
let mut j = 0;
while j < i {
- if segments[j].id_mapping.contains_key(node_id) {
+ if segments[j].id_mapping.contains_key(&node_id) {
node_seen_before = true;
break;
}
@@ -208,15 +208,13 @@ impl StoredSegment {
}
if !node_seen_before {
- if !packet.new_id_mapping.contains_key(node_id) {
- let r = packet
+ if !packet.new_id_mapping.contains_key(&node_id) {
+ packet
.new_id_mapping
- .insert(*node_id, packet.next_segment_node_id);
- debug_assert!(r.is_none());
- let r = packet
+ .put(&node_id, &packet.next_segment_node_id);
+ packet
.new_rev_id_mapping
- .insert(packet.next_segment_node_id, *node_id);
- debug_assert!(r.is_none());
+ .put(&packet.next_segment_node_id, &node_id);
packet.next_segment_node_id.0 += 1;
}
@@ -226,23 +224,21 @@ impl StoredSegment {
while j < segments.len() {
let segment = &segments[j];
- if let Some(edges_j) = edges_fn(segment, node_id) {
+ if let Some(edges_j) = edges_fn(segment, &node_id) {
for edge in edges_j {
let node = segment.rev_id_mapping(&edge.other).unwrap();
- if let std::collections::btree_map::Entry::Vacant(e) =
- packet.new_id_mapping.entry(node)
- {
- e.insert(packet.next_segment_node_id);
- let r = packet
+ if !packet.new_id_mapping.contains_key(&node) {
+ packet
+ .new_id_mapping
+ .put(&node, &packet.next_segment_node_id);
+ packet
.new_rev_id_mapping
- .insert(packet.next_segment_node_id, node);
- debug_assert!(r.is_none());
+ .put(&packet.next_segment_node_id, &node);
packet.next_segment_node_id.0 += 1;
}
- let new_segment_node_id =
- packet.new_id_mapping.get(&node).copied().unwrap();
+ let new_segment_node_id = packet.new_id_mapping.get(&node).unwrap();
let new_edge = StoredEdge {
other: new_segment_node_id,
@@ -257,11 +253,8 @@ impl StoredSegment {
}
if !edges.is_empty() {
- let segment_node_id = packet.new_id_mapping.get(node_id).copied().unwrap();
- packet
- .new_adjacency
- .insert(segment_node_id, &edges)
- .unwrap();
+ let segment_node_id = packet.new_id_mapping.get(&node_id).unwrap();
+ packet.new_adjacency.put(&segment_node_id, &edges);
}
}
}
@@ -276,12 +269,13 @@ impl StoredSegment {
let new_path = segments[0].path().parent().unwrap().join(&new_segment_id);
- let new_adjacency = Store::open(new_path.join("adjacency"), &new_segment_id).unwrap();
+ let new_adjacency = Store::open(new_path.join("adjacency"));
let next_segment_node_id = SegmentNodeID(0);
- let new_id_mapping = BTreeMap::new();
- let new_rev_id_mapping = BTreeMap::new();
+ let new_id_mapping = Store::open(new_path.join("id_mapping"));
+ let new_rev_id_mapping = Store::open(new_path.join("rev_id_mapping"));
+
let mut packet = MergePacket {
next_segment_node_id,
new_id_mapping,
@@ -292,7 +286,7 @@ impl StoredSegment {
Self::merge_adjacency(
|segment: &StoredSegment, node_id: &NodeID| {
segment.id_mapping.get(node_id).map(|segment_node_id| {
- segment.adjacency.get(segment_node_id).unwrap_or_default()
+ segment.adjacency.get(&segment_node_id).unwrap_or_default()
})
},
&mut packet,
@@ -300,8 +294,7 @@ impl StoredSegment {
);
let new_adjacency = packet.new_adjacency;
- let new_reversed_adjacency =
- Store::open(new_path.join("reversed_adjacency"), &new_segment_id).unwrap();
+ let new_reversed_adjacency = Store::open(new_path.join("reversed_adjacency"));
let mut packet = MergePacket {
next_segment_node_id: packet.next_segment_node_id,
@@ -315,7 +308,7 @@ impl StoredSegment {
segment.id_mapping.get(node_id).map(|segment_node_id| {
segment
.reversed_adjacency
- .get(segment_node_id)
+ .get(&segment_node_id)
.unwrap_or_default()
})
},
@@ -352,10 +345,16 @@ impl StoredSegment {
}
}
- for (segment_id, new_id) in new_mappings {
- self.id_mapping.insert(new_id, segment_id);
- self.rev_id_mapping.insert(segment_id, new_id);
- }
+ self.id_mapping.batch_put(
+ new_mappings
+ .iter()
+ .map(|(segment_id, new_id)| (new_id, segment_id)),
+ );
+ self.rev_id_mapping.batch_put(
+ new_mappings
+ .iter()
+ .map(|(segment_id, new_id)| (segment_id, new_id)),
+ );
self.flush();
}
@@ -409,24 +408,24 @@ impl LiveSegment {
let segment_id = uuid::Uuid::new_v4().to_string();
let path = folder_path.as_ref().join(&segment_id);
- let mut adjacency = Store::open(path.join("adjacency"), &segment_id).unwrap();
+ let adjacency = Store::open(path.join("adjacency"));
+ adjacency.batch_put(self.adjacency.iter());
- for (from, edges) in self.adjacency {
- adjacency.insert(from, &edges).unwrap();
- }
+ let reversed_adjacency = Store::open(path.join("reversed_adjacency"));
- let mut reversed_adjacency =
- Store::open(path.join("reversed_adjacency"), &segment_id).unwrap();
+ reversed_adjacency.batch_put(self.reversed_adjacency.iter());
- for (to, edges) in self.reversed_adjacency {
- reversed_adjacency.insert(to, &edges).unwrap();
- }
+ let id_mapping = Store::open(path.join("id_mapping"));
+ id_mapping.batch_put(self.id_mapping.iter());
+
+ let rev_id_mapping = Store::open(path.join("rev_id_mapping"));
+ rev_id_mapping.batch_put(self.rev_id_mapping.iter());
let mut stored_segment = StoredSegment {
adjacency,
reversed_adjacency,
- id_mapping: self.id_mapping,
- rev_id_mapping: self.rev_id_mapping,
+ id_mapping,
+ rev_id_mapping,
meta: Meta {
num_nodes: self.next_id,
},
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index e7394953..e2dbe043 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -14,228 +14,135 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{
- collections::BTreeMap,
- fs::OpenOptions,
- io::BufReader,
- ops::Range,
- path::{Path, PathBuf},
-};
+use std::path::Path;
-use rkyv::{ser::serializers::AllocSerializer, AlignedVec};
-
-type Result<T> = std::result::Result<T, Error>;
-
-const SCRATCH_SIZE: usize = 4096;
-const LOCATIONS_NAME: &str = "locations.bin";
-const MMAP_NAME: &str = "data.bin";
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
- #[error("key already exists")]
- KeyExists,
-
- #[error("io error: {0}")]
- Io(std::io::Error),
-
- #[error("bincode error: {0}")]
- Bincode(bincode::Error),
-}
+use rocksdb::BlockBasedOptions;
pub struct Store<K, V> {
- locations: BTreeMap<K, Range<usize>>,
- folder_path: PathBuf,
- cur_end_range: usize,
- map: memmap2::MmapMut,
- has_new_inserts: bool,
- segment_name: String,
- _values: std::marker::PhantomData<V>,
+ db: rocksdb::DB,
+ _phantom: std::marker::PhantomData<(K, V)>,
}
impl<K, V> Store<K, V>
where
- K: serde::de::DeserializeOwned + serde::Serialize + Ord,
- V: rkyv::Archive + 'static,
- V::Archived: rkyv::Deserialize<V, rkyv::de::deserializers::SharedDeserializeMap> + 'static,
+ K: serde::de::DeserializeOwned + serde::Serialize,
+ V: serde::de::DeserializeOwned + serde::Serialize,
{
- pub fn open<P: AsRef<Path>>(folder_path: P, segment_name: &str) -> Result<Self> {
- if folder_path.as_ref().is_file() {
- return Err(Error::Io(std::io::Error::new(
- std::io::ErrorKind::NotFound,
- format!("{} is not a directory", folder_path.as_ref().display()),
- )));
- }
-
- std::fs::create_dir_all(folder_path.as_ref()).map_err(Error::Io)?;
-
- let locations_path = folder_path
- .as_ref()
- .join(format!("{segment_name}-{LOCATIONS_NAME}"));
- let f = OpenOptions::new()
- .read(true)
- .write(true)
- .create(true)
- .open(locations_path)
- .map_err(Error::Io)?;
-
- let mut reader = BufReader::new(f);
- let locations: BTreeMap<K, Range<usize>> =
- bincode::deserialize_from(&mut reader).unwrap_or_default();
-
- let cur_end_range = locations.values().map(|range| range.end).max().unwrap_or(0);
-
- let mmap_path = folder_path
- .as_ref()
- .join(format!("{segment_name}-{MMAP_NAME}"));
- let map = unsafe {
- let file = OpenOptions::new()
- .read(true)
- .write(true)
- .create(true)
- .open(mmap_path)
- .map_err(Error::Io)?;
-
- memmap2::MmapOptions::new()
- .map_mut(&file)
- .map_err(Error::Io)?
- };
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ let mut options = rocksdb::Options::default();
+ options.create_if_missing(true);
+ options.increase_parallelism(8);
+ options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable
+ options.set_max_write_buffer_number(8);
- Ok(Self {
- locations,
- cur_end_range,
- map,
- folder_path: folder_path.as_ref().to_path_buf(),
- has_new_inserts: false,
- _values: std::marker::PhantomData,
- segment_name: segment_name.to_string(),
- })
- }
+ let mut block_options = BlockBasedOptions::default();
+ block_options.set_bloom_filter(64.0, true);
- pub fn get(&self, key: &K) -> Option<V> {
- let range = self.locations.get(key)?;
- let bytes = &self.map[range.clone()];
+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cache
+ block_options.set_block_cache(&cache);
- let mut aligned = AlignedVec::with_capacity(bytes.len());
- aligned.extend_from_slice(bytes);
+ options.set_block_based_table_factory(&block_options);
- // SAFETY:
- // We ensure the target `V` is `'static` and contains only owned data so it's safe to
- // temporarily extend the lifetime so we can allocate the type entirely.
- let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };
+ let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();
- Some(v)
+ Self {
+ db,
+ _phantom: std::marker::PhantomData,
+ }
}
- pub fn iter(&self) -> impl Iterator<Item = (&K, V)> + '_ {
- self.locations.iter().map(move |(key, range)| {
- let bytes = &self.map[range.clone()];
+ pub fn get(&self, key: &K) -> Option<V> {
+ let bytes = bincode::serialize(key).unwrap();
- let mut aligned = AlignedVec::with_capacity(bytes.len());
- aligned.extend_from_slice(bytes);
+ self.db
+ .get(bytes)
+ .unwrap()
+ .map(|bytes| bincode::deserialize(&bytes).unwrap())
+ }
- // SAFETY:
- // We ensure the target `V` is `'static` and contains only owned data so it's safe to
- // temporarily extend the lifetime so we can allocate the type entirely.
- let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };
+ pub fn put(&self, key: &K, value: &V) {
+ let key_bytes = bincode::serialize(key).unwrap();
+ let value_bytes = bincode::serialize(value).unwrap();
- (key, v)
- })
+ self.db.put(key_bytes, value_bytes).unwrap();
}
- pub fn insert(&mut self, key: K, value: &V) -> Result<()>
- where
- V: rkyv::Serialize<AllocSerializer<SCRATCH_SIZE>>,
- {
- if self.locations.contains_key(&key) {
- return Err(Error::KeyExists);
+ pub fn batch_put<'a>(&'a self, it: impl Iterator<Item = (&'a K, &'a V)>) {
+ let mut batch = rocksdb::WriteBatch::default();
+
+ for (key, value) in it {
+ let key_bytes = bincode::serialize(&key).unwrap();
+ let value_bytes = bincode::serialize(&value).unwrap();
+
+ batch.put(key_bytes, value_bytes);
}
- let bytes = rkyv::to_bytes::<_, SCRATCH_SIZE>(value).unwrap();
- self.insert_raw(key, &bytes[..])
+ self.db.write(batch).unwrap();
}
- fn insert_raw(&mut self, key: K, bytes: &[u8]) -> Result<()> {
- let old_end = self.cur_end_range;
+ pub fn contains_key(&self, key: &K) -> bool {
+ let bytes = bincode::serialize(key).unwrap();
- let new_end = bytes.len() + self.cur_end_range;
+ self.db.get(bytes).unwrap().is_some()
+ }
- if new_end > self.map.len() {
- self.resize(new_end)?;
- }
+ pub fn keys(&self) -> impl Iterator<Item = K> + '_ {
+ let mut read_opts = rocksdb::ReadOptions::default();
- self.map[old_end..new_end].copy_from_slice(bytes);
- self.locations.insert(key, old_end..new_end);
- self.cur_end_range = new_end;
- self.has_new_inserts = true;
+ read_opts.set_readahead_size(4_194_304); // 4 MB
- Ok(())
+ self.db
+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)
+ .map(|res| {
+ let (key, _) = res.unwrap();
+ bincode::deserialize(&key).unwrap()
+ })
}
- fn resize(&mut self, size: usize) -> Result<()> {
- if size > self.map.len() {
- self.flush_map()?;
-
- let file = OpenOptions::new()
- .read(true)
- .write(true)
- .create(true)
- .open(
- self.folder_path
- .join(format!("{}-{MMAP_NAME}", self.segment_name)),
- )
- .map_err(Error::Io)?;
-
- file.set_len(size as u64).map_err(Error::Io)?;
+ pub fn values(&self) -> impl Iterator<Item = V> + '_ {
+ let mut read_opts = rocksdb::ReadOptions::default();
- self.map = unsafe {
- memmap2::MmapOptions::new()
- .map_mut(&file)
- .map_err(Error::Io)?
- };
- }
+ read_opts.set_readahead_size(4_194_304); // 4 MB
- Ok(())
+ self.db
+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)
+ .map(|res| {
+ let (_, val) = res.unwrap();
+ bincode::deserialize(&val).unwrap()
+ })
}
- fn flush_map(&mut self) -> Result<()> {
- if self.has_new_inserts {
- self.map.flush().map_err(Error::Io)?;
- }
+ pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
+ let mut read_opts = rocksdb::ReadOptions::default();
- Ok(())
- }
+ read_opts.set_readahead_size(4_194_304); // 4 MB
- pub fn flush(&mut self) -> Result<()> {
- if self.has_new_inserts {
- self.flush_map()?;
-
- let f = OpenOptions::new()
- .read(true)
- .write(true)
- .create(true)
- .open(
- self.folder_path
- .join(format!("{}-{}", self.segment_name, LOCATIONS_NAME)),
+ self.db
+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)
+ .map(|res| {
+ let (key, val) = res.unwrap();
+ (
+ bincode::deserialize(&key).unwrap(),
+ bincode::deserialize(&val).unwrap(),
)
- .map_err(Error::Io)?;
+ })
+ }
- bincode::serialize_into(f, &self.locations).map_err(Error::Bincode)?;
- }
- self.has_new_inserts = false;
+ pub fn flush(&self) {
+ self.db.flush().unwrap();
+ }
- Ok(())
+ pub fn remove(&self, key: &K) {
+ let bytes = bincode::serialize(key).unwrap();
+ self.db.delete(bytes).unwrap();
}
}
#[cfg(test)]
mod tests {
- use rkyv::Archive;
-
use super::*;
- #[derive(Debug, PartialEq, Eq, Clone, Archive, rkyv::Serialize, rkyv::Deserialize)]
- #[archive(compare(PartialEq))]
- #[archive_attr(derive(Debug))]
+ #[derive(Debug, PartialEq, Eq, Clone, serde::Serialize, serde::Deserialize)]
struct TestStruct {
a: String,
b: i32,
@@ -243,8 +150,7 @@ mod tests {
#[test]
fn test_insert() {
- let mut kv =
- Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();
+ let kv = Store::<String, TestStruct>::open(crate::gen_temp_path().join("test-segment"));
assert!(kv.get(&"test".to_string()).is_none());
@@ -253,8 +159,8 @@ mod tests {
b: 5,
};
- kv.insert("test".to_string(), &test_struct).unwrap();
- kv.flush().unwrap();
+ kv.put(&"test".to_string(), &test_struct);
+ kv.flush();
let archived = kv.get(&"test".to_string()).unwrap();
assert_eq!(test_struct, archived);
@@ -270,43 +176,17 @@ mod tests {
};
{
- let mut kv = Store::<String, TestStruct>::open(path.clone(), segment_name).unwrap();
+ let kv = Store::<String, TestStruct>::open(path.join(segment_name));
assert!(kv.get(&"test".to_string()).is_none());
- kv.insert("test".to_string(), &test_struct).unwrap();
- kv.flush().unwrap();
+ kv.put(&"test".to_string(), &test_struct);
+ kv.flush();
}
- let kv = Store::<String, TestStruct>::open(path, segment_name).unwrap();
+ let kv = Store::<String, TestStruct>::open(path.join(segment_name));
let archived = kv.get(&"test".to_string()).unwrap();
assert_eq!(test_struct, archived);
}
-
- #[test]
- fn test_insert_existing_key() {
- let mut kv =
- Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();
-
- assert!(kv.get(&"test".to_string()).is_none());
-
- let test_struct = TestStruct {
- a: "test".to_string(),
- b: 5,
- };
-
- kv.insert("test".to_string(), &test_struct).unwrap();
- kv.flush().unwrap();
-
- let archived = kv.get(&"test".to_string()).unwrap();
- assert_eq!(test_struct, archived);
-
- let test_struct = TestStruct {
- a: "test".to_string(),
- b: 6,
- };
-
- assert!(kv.insert("test".to_string(), &test_struct).is_err());
- }
}