web search done right
Disable webgraph checksum (#173)
Mikkel Denker13b8e7bdec1b
diff --git a/crates/core/src/kv/rocksdb_store.rs b/crates/core/src/kv/rocksdb_store.rsindex ecdcf697..4c306199 100644--- a/crates/core/src/kv/rocksdb_store.rs+++ b/crates/core/src/kv/rocksdb_store.rs@@ -51,7 +51,7 @@ whereoptions.set_level_compaction_dynamic_level_bytes(true);options.set_bytes_per_sync(1048576);let mut block_options = BlockBasedOptions::default();- block_options.set_block_size(16 * 1024);+ block_options.set_block_size(1024 * 1024 * 1024); // 1 GBblock_options.set_format_version(5);block_options.set_cache_index_and_filter_blocks(true);block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);@@ -117,7 +117,10 @@ whereV: Serialize + DeserializeOwned + 'static + Send + Sync,{fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {- self.db.get(key).expect("failed to retrieve key")+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);++ self.db.get_opt(key, &opts).expect("failed to retrieve key")}fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {@@ -139,7 +142,11 @@ where}fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {- let iter = self.db.iterator(IteratorMode::Start);+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);+ opts.set_async_io(true);++ let iter = self.db.iterator_opt(IteratorMode::Start, opts);Box::new(IntoIter {inner: iter,diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex c816dac7..6fa51926 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -470,6 +470,7 @@ impl Meta {struct Id2NodeDb {db: rocksdb::DB,+ _cache: rocksdb::Cache, // needs to be kept alive for as long as the db is alive}impl Id2NodeDb {@@ -484,22 +485,25 @@ impl Id2NodeDb {opts.set_target_file_size_base(512 * 1024 * 1024); // 512 MBopts.set_target_file_size_multiplier(10);+ opts.set_compression_type(rocksdb::DBCompressionType::Lz4);+let mut block_opts = rocksdb::BlockBasedOptions::default();+ let cache = rocksdb::Cache::new_lru_cache(8 * 1024 * 1024 * 1024); // 8 gb+ opts.set_block_based_table_factory(&block_opts);// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)opts.set_level_compaction_dynamic_level_bytes(true);opts.set_bytes_per_sync(1048576);+block_opts.set_block_size(16 * 1024);block_opts.set_format_version(5);block_opts.set_cache_index_and_filter_blocks(true);block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);-- opts.set_block_based_table_factory(&block_opts);- opts.set_compression_type(rocksdb::DBCompressionType::Lz4);+ block_opts.set_block_cache(&cache);let db = rocksdb::DB::open(&opts, path).unwrap();- Self { db }+ Self { db, _cache: cache }}fn put(&mut self, id: &NodeID, node: &Node) {@@ -516,15 +520,22 @@ impl Id2NodeDb {}fn get(&self, id: &NodeID) -> Option<Node> {+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);+self.db- .get(id.as_u64().to_le_bytes())+ .get_opt(id.as_u64().to_le_bytes(), &opts).unwrap().map(|bytes| bincode::deserialize(&bytes).unwrap())}fn keys(&self) -> impl Iterator<Item = NodeID> + '_ {+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);+ opts.set_async_io(true);+self.db- .iterator(rocksdb::IteratorMode::Start)+ .iterator_opt(rocksdb::IteratorMode::Start, opts).filter_map(|r| {let (key, _) = r.ok()?;Some(NodeID(u64::from_le_bytes((*key).try_into().unwrap())))@@ -540,8 +551,11 @@ impl Id2NodeDb {}fn iter(&self) -> impl Iterator<Item = (NodeID, Node)> + '_ {+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);+self.db- .iterator(rocksdb::IteratorMode::Start)+ .iterator_opt(rocksdb::IteratorMode::Start, opts).filter_map(|r| {let (key, value) = r.ok()?;diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex d407886d..55333416 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -120,7 +120,9 @@ impl EdgeStoreWriter {}pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = InnerEdge<L>> + '_ + Send + Sync {- let read_opts = rocksdb::ReadOptions::default();+ let mut read_opts = rocksdb::ReadOptions::default();+ read_opts.set_verify_checksums(false);+ read_opts.set_async_io(true);self.db.iterator_opt(rocksdb::IteratorMode::Start, read_opts)@@ -200,6 +202,7 @@ impl PrefixDb {options.set_level_zero_slowdown_writes_trigger(-1);options.set_level_zero_stop_writes_trigger(-1);+ options.set_compression_type(rocksdb::DBCompressionType::None);// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)options.set_level_compaction_dynamic_level_bytes(true);@@ -211,7 +214,6 @@ impl PrefixDb {block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);options.set_block_based_table_factory(&block_options);- options.set_compression_type(rocksdb::DBCompressionType::Lz4);let db = rocksdb::DB::open(&options, path).unwrap();@@ -241,10 +243,14 @@ impl PrefixDb {let mut res = Vec::new();- let iter = self.db.iterator(rocksdb::IteratorMode::From(- &start,- rocksdb::Direction::Forward,- ));+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);+ opts.set_async_io(true);++ let iter = self.db.iterator_opt(+ rocksdb::IteratorMode::From(&start, rocksdb::Direction::Forward),+ opts,+ );for item in iter {let (key, _) = item.unwrap();@@ -514,9 +520,12 @@ impl EdgeStore {let node_cf = self.ranges.cf_handle("nodes").unwrap();let edge_cf = self.ranges.cf_handle("labels").unwrap();+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);+match (- self.ranges.get_cf(node_cf, node_bytes).unwrap(),- self.ranges.get_cf(edge_cf, node_bytes).unwrap(),+ self.ranges.get_cf_opt(node_cf, node_bytes, &opts).unwrap(),+ self.ranges.get_cf_opt(edge_cf, node_bytes, &opts).unwrap(),) {(Some(node_range_bytes), Some(edge_range_bytes)) => {let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();@@ -559,7 +568,10 @@ impl EdgeStore {let node_cf = self.ranges.cf_handle("nodes").unwrap();- match self.ranges.get_cf(node_cf, node_bytes).unwrap() {+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);++ match self.ranges.get_cf_opt(node_cf, node_bytes, &opts).unwrap() {Some(node_range_bytes) => {let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();@@ -597,8 +609,12 @@ impl EdgeStore {pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {let node_cf = self.ranges.cf_handle("nodes").unwrap();+ let mut opts = rocksdb::ReadOptions::default();+ opts.set_verify_checksums(false);+ opts.set_async_io(true);+self.ranges- .iterator_cf(node_cf, rocksdb::IteratorMode::Start)+ .iterator_cf_opt(node_cf, opts, rocksdb::IteratorMode::Start).flat_map(move |res| {let (key, val) = res.unwrap();