Skip to content

web search done right

open entity attribute occurences as read-only rocksdb to allow multiple readers for same db

Mikkel Denker991b341e7386
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index e18bc3d7..55742f07 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -160,7 +160,8 @@ pub struct EntityIndex {
tv_index: tantivy::Index,
schema: Arc<Schema>,
stopwords: HashSet<String>,
- attribute_occurrences: Box<dyn Kv<String, u32>>,
+ attribute_occurrences: RocksDbStore<String, u32>,
+ path: std::path::PathBuf,
}
impl EntityIndex {
@@ -178,9 +179,8 @@ impl EntityIndex {
tantivy::Index::create_in_dir(&tv_path, schema.clone())?
};
- let attribute_occurrences = Box::new(RocksDbStore::open(
- path.as_ref().join("attribute_occurrences"),
- ));
+ let attribute_occurrences =
+ RocksDbStore::open_read_only(path.as_ref().join("attribute_occurrences"));
let stopwords: HashSet<String> = include_str!("../../stopwords/English.txt")
.lines()
@@ -206,11 +206,13 @@ impl EntityIndex {
schema: Arc::new(schema),
stopwords,
attribute_occurrences,
+ path: path.as_ref().to_path_buf(),
})
}
pub fn prepare_writer(&mut self) {
self.writer = Some(self.tv_index.writer(10_000_000_000).unwrap());
+ self.attribute_occurrences = RocksDbStore::open(self.path.join("attribute_occurrences"));
}
fn best_info(&self, info: BTreeMap<String, Span>) -> Vec<(String, Span)> {
diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rs
index a825799c..117fd132 100644
--- a/core/src/kv/rocksdb_store.rs
+++ b/core/src/kv/rocksdb_store.rs
@@ -39,14 +39,7 @@ where
K: Serialize + DeserializeOwned,
V: Serialize + DeserializeOwned,
{
- pub fn open<P>(path: P) -> Self
- where
- P: AsRef<std::path::Path>,
- {
- if !path.as_ref().exists() {
- fs::create_dir_all(path.as_ref()).expect("faild to create dir");
- }
-
+ fn options(cache: &rocksdb::Cache) -> Options {
let mut options = Options::default();
options.create_if_missing(true);
@@ -54,8 +47,6 @@ where
options.increase_parallelism(8);
options.set_max_subcompactions(8);
- let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb
-
// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
options.set_level_compaction_dynamic_level_bytes(true);
options.set_bytes_per_sync(1048576);
@@ -68,11 +59,48 @@ where
block_options.set_ribbon_filter(10.0);
options.set_block_based_table_factory(&block_options);
- options.set_row_cache(&cache);
+ options.set_row_cache(cache);
options.set_compression_type(rocksdb::DBCompressionType::Lz4);
options.optimize_for_point_lookup(512); // 512 MB
+ options
+ }
+
+ pub fn open_read_only<P>(path: P) -> Self
+ where
+ P: AsRef<std::path::Path>,
+ {
+ if !path.as_ref().exists() {
+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");
+ }
+
+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb
+ let options = Self::options(&cache);
+
+ // create db to ensure it exists
+ DB::open(&options, &path).expect("unable to open rocks db");
+
+ let db = DB::open_for_read_only(&options, path, false).expect("unable to open rocks db");
+
+ Self {
+ db,
+ _cache: cache,
+ _phantom: PhantomData,
+ }
+ }
+
+ pub fn open<P>(path: P) -> Self
+ where
+ P: AsRef<std::path::Path>,
+ {
+ if !path.as_ref().exists() {
+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");
+ }
+
+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb
+ let options = Self::options(&cache);
+
let db = DB::open(&options, path).expect("unable to open rocks db");
Self {