web search done right
better caching for node ids in webgraph
Mikkel Denker405e59335ee4
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 7d718651..4d2df7b9 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -29,6 +29,9 @@ use super::inbound_similarity::InboundSimilarity;pub struct Node2Id {db: rocksdb::DB,+ // from rocksdb docs: `Cache must outlive DB instance which uses it.`+ #[allow(dead_code)]+ cache: rocksdb::Cache,}impl Node2Id {@@ -49,7 +52,7 @@ impl Node2Id {let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();- Self { db }+ Self { db, cache }}pub fn get(&self, key: &Node) -> Option<NodeID> {diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 9dac1fc3..454c2f69 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -14,11 +14,13 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use lru::LruCache;use rkyv::Archive;use std::{collections::{BTreeMap, HashSet},fmt::Debug,path::Path,+ sync::Mutex,};use super::{@@ -72,6 +74,7 @@ pub struct StoredSegment {small_reversed_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,id_mapping: Store<NodeID, SegmentNodeID>,rev_id_mapping: Store<SegmentNodeID, NodeID>,+ rev_id_mapping_cache: Mutex<LruCache<SegmentNodeID, NodeID>>,meta: Meta,id: String,folder_path: String,@@ -98,6 +101,7 @@ impl StoredSegment {.unwrap().to_string(),id,+ rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),}}@@ -110,7 +114,19 @@ impl StoredSegment {}fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {- self.rev_id_mapping.get(node)+ // check cache+ let mut guard = self.rev_id_mapping_cache.lock().unwrap();+ if let Some(node_id) = guard.get(node) {+ return Some(*node_id);+ }++ let node_id = self.rev_id_mapping.get(node);++ if let Some(node_id) = node_id {+ guard.put(*node, node_id);+ }++ node_id}pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {@@ -368,6 +384,7 @@ impl StoredSegment {},id: new_segment_id,folder_path: new_path.to_str().unwrap().to_string(),+ rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),};res.flush();@@ -511,6 +528,7 @@ impl LiveSegment {},folder_path: path.as_os_str().to_str().unwrap().to_string(),id: segment_id,+ rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),};stored_segment.flush();diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex ea4e9911..103dd094 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -20,6 +20,9 @@ use rocksdb::BlockBasedOptions;pub struct Store<K, V> {db: rocksdb::DB,+ // from rocksdb docs: `Cache must outlive DB instance which uses it.`+ #[allow(dead_code)]+ cache: rocksdb::Cache,_phantom: std::marker::PhantomData<(K, V)>,}@@ -38,7 +41,7 @@ wherelet mut block_options = BlockBasedOptions::default();block_options.set_bloom_filter(128.0, true);- let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024).unwrap(); // 1024 MB cache+ let cache = rocksdb::Cache::new_lru_cache(512 * 1024 * 1024).unwrap(); // 512 MB cacheblock_options.set_block_cache(&cache);block_options.set_block_size(128 * 1024); // 128 KB block size@@ -49,6 +52,7 @@ whereSelf {db,+ cache,_phantom: std::marker::PhantomData,}}