Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Ensure that all RocksDB iterators use upper bound.
Kishore Nallanf9598dfd559b
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex 3d132eac..c9af312f 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -67,6 +67,9 @@ private:rocksdb::Iterator* skip_index_iter = nullptr;static constexpr const char* SKIP_INDICES_PREFIX = "$XP";+ std::string skip_index_upper_bound_key = std::string(SKIP_INDICES_PREFIX) + "`"; // cannot inline this+ rocksdb::Slice* skip_index_iter_upper_bound = nullptr;+// When set, all writes (both live and log serialized) are skipped with 422 responseconst std::atomic<bool>& skip_writes;diff --git a/include/core_api_utils.h b/include/core_api_utils.hindex b571bdd4..eca4d6af 100644--- a/include/core_api_utils.h+++ b/include/core_api_utils.h@@ -29,12 +29,15 @@ struct export_state_t: public req_state_t {bool filtered_export = false;rocksdb::Iterator* it = nullptr;+ std::string iter_upper_bound_key;+ rocksdb::Slice* iter_upper_bound = nullptr;~export_state_t() override {for(auto& kv: index_ids) {delete [] kv.second;}+ delete iter_upper_bound;delete it;}};diff --git a/include/store.h b/include/store.hindex a35532b7..84cbf77c 100644--- a/include/store.h+++ b/include/store.h@@ -168,7 +168,7 @@ public:return status.ok();}- rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound = nullptr) {+ rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound) {std::shared_lock lock(mutex);rocksdb::ReadOptions read_opts;if(iterate_upper_bound) {@@ -185,10 +185,14 @@ public:return it;};- void scan_fill(const std::string & prefix, std::vector<std::string> & values) {+ void scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values) {+ rocksdb::ReadOptions read_opts;+ rocksdb::Slice upper_bound(prefix_end);+ read_opts.iterate_upper_bound = &upper_bound;+std::shared_lock lock(mutex);- rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());- for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) {+ rocksdb::Iterator *iter = db->NewIterator(read_opts);+ for (iter->Seek(prefix_start); iter->Valid() && iter->key().starts_with(prefix_start); iter->Next()) {values.push_back(iter->value().ToString());}diff --git a/src/auth_manager.cpp b/src/auth_manager.cppindex cb7a3231..ee3eed7b 100644--- a/src/auth_manager.cpp+++ b/src/auth_manager.cpp@@ -27,7 +27,9 @@ Option<bool> AuthManager::init(Store* store, const std::string& bootstrap_auth_k}std::vector<std::string> api_key_json_strs;- store->scan_fill(API_KEYS_PREFIX, api_key_json_strs);+ store->scan_fill(std::string(API_KEYS_PREFIX) + "_",+ std::string(API_KEYS_PREFIX) + "`",+ api_key_json_strs);LOG(INFO) << "Indexing " << api_key_json_strs.size() << " API key(s) found on disk.";@@ -48,7 +50,8 @@ Option<std::vector<api_key_t>> AuthManager::list_keys() const {std::shared_lock lock(mutex);std::vector<std::string> api_key_json_strs;- store->scan_fill(API_KEYS_PREFIX, api_key_json_strs);+ store->scan_fill(std::string(API_KEYS_PREFIX) + "_",+ std::string(API_KEYS_PREFIX) + "`", api_key_json_strs);std::vector<api_key_t> stored_api_keys;diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex 248c31b5..ba82b1a9 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -9,6 +9,7 @@ BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_stoskip_writes(skip_writes) {queues.resize(num_threads);qmutuxes = new await_t[num_threads];+ skip_index_iter_upper_bound = new rocksdb::Slice(skip_index_upper_bound_key);}void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {@@ -121,8 +122,7 @@ std::string BatchedIndexer::get_collection_name(const std::shared_ptr<http_req>&void BatchedIndexer::run() {LOG(INFO) << "Starting batch indexer with " << num_threads << " threads.";ThreadPool* thread_pool = new ThreadPool(num_threads);-- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX, skip_index_iter_upper_bound);populate_skip_index();LOG(INFO) << "BatchedIndexer skip_index: " << skip_index;@@ -163,8 +163,8 @@ void BatchedIndexer::run() {const std::string& req_key_start_prefix = req_key_prefix + StringUtils::serialize_uint32_t(orig_req_res.next_chunk_index);- const std::string& req_key_upper_bound = get_req_suffix_key(req_id);- rocksdb::Slice upper_bound(req_key_upper_bound); // cannot inline req_key_upper_bound+ const std::string& req_key_upper_bound = get_req_suffix_key(req_id); // cannot inline this+ rocksdb::Slice upper_bound(req_key_upper_bound);rocksdb::Iterator* iter = store->scan(req_key_start_prefix, &upper_bound);// used to handle partial JSON documents caused by chunking@@ -318,6 +318,7 @@ std::string BatchedIndexer::get_req_suffix_key(uint64_t req_id) {BatchedIndexer::~BatchedIndexer() {delete [] qmutuxes;+ delete skip_index_iter_upper_bound;delete skip_index_iter;}@@ -431,7 +432,7 @@ std::shared_mutex& BatchedIndexer::get_pause_mutex() {void BatchedIndexer::clear_skip_indices() {delete skip_index_iter;- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX, skip_index_iter_upper_bound);while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {meta_store->remove(skip_index_iter->key().ToString());diff --git a/src/collection.cpp b/src/collection.cppindex 49df2df8..d39a8e0e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3266,7 +3266,10 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields// Now, we can index existing data onto the updated schemaconst std::string seq_id_prefix = get_seq_id_collection_prefix();- rocksdb::Iterator* iter = store->scan(seq_id_prefix);+ std::string upper_bound_key = get_seq_id_collection_prefix() + "`"; // cannot inline this+ rocksdb::Slice upper_bound(upper_bound_key);++ rocksdb::Iterator* iter = store->scan(seq_id_prefix, &upper_bound);std::unique_ptr<rocksdb::Iterator> iter_guard(iter);size_t num_found_docs = 0;@@ -3662,8 +3665,10 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,// data validations: here we ensure that already stored data is compatible with requested schema changesconst std::string seq_id_prefix = get_seq_id_collection_prefix();+ std::string upper_bound_key = get_seq_id_collection_prefix() + "`"; // cannot inline this+ rocksdb::Slice upper_bound(upper_bound_key);- rocksdb::Iterator* iter = store->scan(seq_id_prefix);+ rocksdb::Iterator* iter = store->scan(seq_id_prefix, &upper_bound);std::unique_ptr<rocksdb::Iterator> iter_guard(iter);size_t num_found_docs = 0;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex a27f46bd..68c6a6c2 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -180,7 +180,9 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s<< document_batch_size << " documents at a time.";std::vector<std::string> collection_meta_jsons;- store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);+ store->scan_fill(std::string(Collection::COLLECTION_META_PREFIX) + "_",+ std::string(Collection::COLLECTION_META_PREFIX) + "`",+ collection_meta_jsons);const size_t num_collections = collection_meta_jsons.size();LOG(INFO) << "Found " << num_collections << " collection(s) on disk.";@@ -237,7 +239,10 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s// load aliasesstd::string symlink_prefix_key = std::string(SYMLINK_PREFIX) + "_";- rocksdb::Iterator* iter = store->scan(symlink_prefix_key);+ std::string upper_bound_key = std::string(SYMLINK_PREFIX) + "`"; // cannot inline this+ rocksdb::Slice upper_bound(upper_bound_key);++ rocksdb::Iterator* iter = store->scan(symlink_prefix_key, &upper_bound);while(iter->Valid() && iter->key().starts_with(symlink_prefix_key)) {std::vector<std::string> parts;StringUtils::split(iter->key().ToString(), parts, symlink_prefix_key);@@ -250,7 +255,10 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s// load presetsstd::string preset_prefix_key = std::string(PRESET_PREFIX) + "_";- iter = store->scan(preset_prefix_key);+ std::string preset_upper_bound_key = std::string(PRESET_PREFIX) + "`"; // cannot inline this+ rocksdb::Slice preset_upper_bound(preset_upper_bound_key);++ iter = store->scan(preset_prefix_key, &preset_upper_bound);while(iter->Valid() && iter->key().starts_with(preset_prefix_key)) {std::vector<std::string> parts;StringUtils::split(iter->key().ToString(), parts, preset_prefix_key);@@ -447,7 +455,11 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col// delete overridesconst std::string& del_override_prefix =std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" + actual_coll_name + "_";- rocksdb::Iterator* iter = store->scan(del_override_prefix);+ std::string upper_bound_key = std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" ++ actual_coll_name + "`"; // cannot inline this+ rocksdb::Slice upper_bound(upper_bound_key);++ rocksdb::Iterator* iter = store->scan(del_override_prefix, &upper_bound);while(iter->Valid() && iter->key().starts_with(del_override_prefix)) {store->remove(iter->key().ToString());iter->Next();@@ -457,7 +469,12 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col// delete synonymsconst std::string& del_synonym_prefix =std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "_" + actual_coll_name + "_";- iter = store->scan(del_synonym_prefix);++ std::string syn_upper_bound_key = std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "_" ++ actual_coll_name + "`"; // cannot inline this+ rocksdb::Slice syn_upper_bound(syn_upper_bound_key);++ iter = store->scan(del_synonym_prefix, &syn_upper_bound);while(iter->Valid() && iter->key().starts_with(del_synonym_prefix)) {store->remove(iter->key().ToString());iter->Next();@@ -1166,7 +1183,9 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection// initialize overridesstd::vector<std::string> collection_override_jsons;- cm.store->scan_fill(Collection::get_override_key(this_collection_name, ""), collection_override_jsons);+ cm.store->scan_fill(Collection::get_override_key(this_collection_name, ""),+ std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" + this_collection_name + "`",+ collection_override_jsons);for(const auto & collection_override_json: collection_override_jsons) {nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);@@ -1181,7 +1200,9 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection// initialize synonymsstd::vector<std::string> collection_synonym_jsons;- cm.store->scan_fill(SynonymIndex::get_synonym_key(this_collection_name, ""), collection_synonym_jsons);+ cm.store->scan_fill(SynonymIndex::get_synonym_key(this_collection_name, ""),+ std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "_" + this_collection_name + "`",+ collection_synonym_jsons);for(const auto & collection_synonym_json: collection_synonym_jsons) {nlohmann::json collection_synonym = nlohmann::json::parse(collection_synonym_json);@@ -1190,8 +1211,10 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection// Fetch records from the store and re-create memory indexconst std::string seq_id_prefix = collection->get_seq_id_collection_prefix();+ std::string upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline this+ rocksdb::Slice upper_bound(upper_bound_key);- rocksdb::Iterator* iter = cm.store->scan(seq_id_prefix);+ rocksdb::Iterator* iter = cm.store->scan(seq_id_prefix, &upper_bound);std::unique_ptr<rocksdb::Iterator> iter_guard(iter);std::vector<index_record> index_records;diff --git a/src/core_api.cpp b/src/core_api.cppindex a44181f2..7086ffed 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -618,7 +618,9 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share}if(simple_filter_query.empty()) {- export_state->it = collectionManager.get_store()->scan(seq_id_prefix);+ export_state->iter_upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline this+ export_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);+ export_state->it = collectionManager.get_store()->scan(seq_id_prefix, export_state->iter_upper_bound);} else {auto filter_ids_op = collection->get_filter_ids(simple_filter_query, export_state->index_ids);diff --git a/src/ratelimit_manager.cpp b/src/ratelimit_manager.cppindex c5b43662..01634d30 100644--- a/src/ratelimit_manager.cpp+++ b/src/ratelimit_manager.cpp@@ -436,8 +436,10 @@ Option<bool> RateLimitManager::init(Store *store) {else {last_rule_id = 0;}+std::vector<std::string> rule_json_strs;- store->scan_fill(RULES_PREFIX, rule_json_strs);+ store->scan_fill(std::string(RULES_PREFIX) + "_", std::string(RULES_PREFIX) + "`", rule_json_strs);+for(const auto& rule_json_str: rule_json_strs) {nlohmann::json rule_json = nlohmann::json::parse(rule_json_str);Option<rate_limit_rule_t> rule_option = parse_rule(rule_json);@@ -460,8 +462,10 @@ Option<bool> RateLimitManager::init(Store *store) {else {last_ban_id = 0;}+std::vector<std::string> ban_json_strs;- store->scan_fill(BANS_PREFIX, ban_json_strs);+ store->scan_fill(std::string(BANS_PREFIX) + "_", std::string(BANS_PREFIX) + "`", ban_json_strs);+for(const auto& ban_json_str: ban_json_strs) {nlohmann::json ban_json = nlohmann::json::parse(ban_json_str);rate_limit_status_t ban_status;diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 91c257a0..8ad0cbc0 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -663,11 +663,13 @@ TEST_F(CollectionSpecificTest, DeleteOverridesAndSynonymsOnDiskDuringCollDrop) {// overrides should have been deleted from the storestd::vector<std::string> stored_values;- store->scan_fill(Collection::COLLECTION_OVERRIDE_PREFIX, stored_values);+ store->scan_fill(Collection::COLLECTION_OVERRIDE_PREFIX, std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "`",+ stored_values);ASSERT_TRUE(stored_values.empty());// synonyms should also have been deleted from the store- store->scan_fill(SynonymIndex::COLLECTION_SYNONYM_PREFIX, stored_values);+ store->scan_fill(SynonymIndex::COLLECTION_SYNONYM_PREFIX, std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "`",+ stored_values);ASSERT_TRUE(stored_values.empty());}