Skip to content

web search done right

better caching for node ids in webgraph

Mikkel Denker405e59335ee4
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 7d718651..4d2df7b9 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -29,6 +29,9 @@ use super::inbound_similarity::InboundSimilarity;
pub struct Node2Id {
db: rocksdb::DB,
+ // from rocksdb docs: `Cache must outlive DB instance which uses it.`
+ #[allow(dead_code)]
+ cache: rocksdb::Cache,
}
impl Node2Id {
@@ -49,7 +52,7 @@ impl Node2Id {
let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();
- Self { db }
+ Self { db, cache }
}
pub fn get(&self, key: &Node) -> Option<NodeID> {
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 9dac1fc3..454c2f69 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -14,11 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use lru::LruCache;
use rkyv::Archive;
use std::{
collections::{BTreeMap, HashSet},
fmt::Debug,
path::Path,
+ sync::Mutex,
};
use super::{
@@ -72,6 +74,7 @@ pub struct StoredSegment {
small_reversed_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,
id_mapping: Store<NodeID, SegmentNodeID>,
rev_id_mapping: Store<SegmentNodeID, NodeID>,
+ rev_id_mapping_cache: Mutex<LruCache<SegmentNodeID, NodeID>>,
meta: Meta,
id: String,
folder_path: String,
@@ -98,6 +101,7 @@ impl StoredSegment {
.unwrap()
.to_string(),
id,
+ rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),
}
}
@@ -110,7 +114,19 @@ impl StoredSegment {
}
fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {
- self.rev_id_mapping.get(node)
+ // check cache
+ let mut guard = self.rev_id_mapping_cache.lock().unwrap();
+ if let Some(node_id) = guard.get(node) {
+ return Some(*node_id);
+ }
+
+ let node_id = self.rev_id_mapping.get(node);
+
+ if let Some(node_id) = node_id {
+ guard.put(*node, node_id);
+ }
+
+ node_id
}
pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {
@@ -368,6 +384,7 @@ impl StoredSegment {
},
id: new_segment_id,
folder_path: new_path.to_str().unwrap().to_string(),
+ rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),
};
res.flush();
@@ -511,6 +528,7 @@ impl LiveSegment {
},
folder_path: path.as_os_str().to_str().unwrap().to_string(),
id: segment_id,
+ rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),
};
stored_segment.flush();
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index ea4e9911..103dd094 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -20,6 +20,9 @@ use rocksdb::BlockBasedOptions;
pub struct Store<K, V> {
db: rocksdb::DB,
+ // from rocksdb docs: `Cache must outlive DB instance which uses it.`
+ #[allow(dead_code)]
+ cache: rocksdb::Cache,
_phantom: std::marker::PhantomData<(K, V)>,
}
@@ -38,7 +41,7 @@ where
let mut block_options = BlockBasedOptions::default();
block_options.set_bloom_filter(128.0, true);
- let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024).unwrap(); // 1024 MB cache
+ let cache = rocksdb::Cache::new_lru_cache(512 * 1024 * 1024).unwrap(); // 512 MB cache
block_options.set_block_cache(&cache);
block_options.set_block_size(128 * 1024); // 128 KB block size
@@ -49,6 +52,7 @@ where
Self {
db,
+ cache,
_phantom: std::marker::PhantomData,
}
}