web search done right
open entity attribute occurences as read-only rocksdb to allow multiple readers for same db
Mikkel Denker991b341e7386
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rsindex e18bc3d7..55742f07 100644--- a/core/src/entity_index/mod.rs+++ b/core/src/entity_index/mod.rs@@ -160,7 +160,8 @@ pub struct EntityIndex {tv_index: tantivy::Index,schema: Arc<Schema>,stopwords: HashSet<String>,- attribute_occurrences: Box<dyn Kv<String, u32>>,+ attribute_occurrences: RocksDbStore<String, u32>,+ path: std::path::PathBuf,}impl EntityIndex {@@ -178,9 +179,8 @@ impl EntityIndex {tantivy::Index::create_in_dir(&tv_path, schema.clone())?};- let attribute_occurrences = Box::new(RocksDbStore::open(- path.as_ref().join("attribute_occurrences"),- ));+ let attribute_occurrences =+ RocksDbStore::open_read_only(path.as_ref().join("attribute_occurrences"));let stopwords: HashSet<String> = include_str!("../../stopwords/English.txt").lines()@@ -206,11 +206,13 @@ impl EntityIndex {schema: Arc::new(schema),stopwords,attribute_occurrences,+ path: path.as_ref().to_path_buf(),})}pub fn prepare_writer(&mut self) {self.writer = Some(self.tv_index.writer(10_000_000_000).unwrap());+ self.attribute_occurrences = RocksDbStore::open(self.path.join("attribute_occurrences"));}fn best_info(&self, info: BTreeMap<String, Span>) -> Vec<(String, Span)> {diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rsindex a825799c..117fd132 100644--- a/core/src/kv/rocksdb_store.rs+++ b/core/src/kv/rocksdb_store.rs@@ -39,14 +39,7 @@ whereK: Serialize + DeserializeOwned,V: Serialize + DeserializeOwned,{- pub fn open<P>(path: P) -> Self- where- P: AsRef<std::path::Path>,- {- if !path.as_ref().exists() {- fs::create_dir_all(path.as_ref()).expect("faild to create dir");- }-+ fn options(cache: &rocksdb::Cache) -> Options {let mut options = Options::default();options.create_if_missing(true);@@ -54,8 +47,6 @@ whereoptions.increase_parallelism(8);options.set_max_subcompactions(8);- let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb-// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)options.set_level_compaction_dynamic_level_bytes(true);options.set_bytes_per_sync(1048576);@@ -68,11 +59,48 @@ whereblock_options.set_ribbon_filter(10.0);options.set_block_based_table_factory(&block_options);- options.set_row_cache(&cache);+ options.set_row_cache(cache);options.set_compression_type(rocksdb::DBCompressionType::Lz4);options.optimize_for_point_lookup(512); // 512 MB+ options+ }++ pub fn open_read_only<P>(path: P) -> Self+ where+ P: AsRef<std::path::Path>,+ {+ if !path.as_ref().exists() {+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");+ }++ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb+ let options = Self::options(&cache);++ // create db to ensure it exists+ DB::open(&options, &path).expect("unable to open rocks db");++ let db = DB::open_for_read_only(&options, path, false).expect("unable to open rocks db");++ Self {+ db,+ _cache: cache,+ _phantom: PhantomData,+ }+ }++ pub fn open<P>(path: P) -> Self+ where+ P: AsRef<std::path::Path>,+ {+ if !path.as_ref().exists() {+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");+ }++ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb+ let options = Self::options(&cache);+let db = DB::open(&options, path).expect("unable to open rocks db");Self {