Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Ensure that all RocksDB iterators use upper bound.

Kishore Nallanf9598dfd559b
diff --git a/include/batched_indexer.h b/include/batched_indexer.h
index 3d132eac..c9af312f 100644
--- a/include/batched_indexer.h
+++ b/include/batched_indexer.h
@@ -67,6 +67,9 @@ private:
rocksdb::Iterator* skip_index_iter = nullptr;
static constexpr const char* SKIP_INDICES_PREFIX = "$XP";
+ std::string skip_index_upper_bound_key = std::string(SKIP_INDICES_PREFIX) + "`"; // cannot inline this
+ rocksdb::Slice* skip_index_iter_upper_bound = nullptr;
+
// When set, all writes (both live and log serialized) are skipped with 422 response
const std::atomic<bool>& skip_writes;
diff --git a/include/core_api_utils.h b/include/core_api_utils.h
index b571bdd4..eca4d6af 100644
--- a/include/core_api_utils.h
+++ b/include/core_api_utils.h
@@ -29,12 +29,15 @@ struct export_state_t: public req_state_t {
bool filtered_export = false;
rocksdb::Iterator* it = nullptr;
+ std::string iter_upper_bound_key;
+ rocksdb::Slice* iter_upper_bound = nullptr;
~export_state_t() override {
for(auto& kv: index_ids) {
delete [] kv.second;
}
+ delete iter_upper_bound;
delete it;
}
};
diff --git a/include/store.h b/include/store.h
index a35532b7..84cbf77c 100644
--- a/include/store.h
+++ b/include/store.h
@@ -168,7 +168,7 @@ public:
return status.ok();
}
- rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound = nullptr) {
+ rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound) {
std::shared_lock lock(mutex);
rocksdb::ReadOptions read_opts;
if(iterate_upper_bound) {
@@ -185,10 +185,14 @@ public:
return it;
};
- void scan_fill(const std::string & prefix, std::vector<std::string> & values) {
+ void scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values) {
+ rocksdb::ReadOptions read_opts;
+ rocksdb::Slice upper_bound(prefix_end);
+ read_opts.iterate_upper_bound = &upper_bound;
+
std::shared_lock lock(mutex);
- rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());
- for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) {
+ rocksdb::Iterator *iter = db->NewIterator(read_opts);
+ for (iter->Seek(prefix_start); iter->Valid() && iter->key().starts_with(prefix_start); iter->Next()) {
values.push_back(iter->value().ToString());
}
diff --git a/src/auth_manager.cpp b/src/auth_manager.cpp
index cb7a3231..ee3eed7b 100644
--- a/src/auth_manager.cpp
+++ b/src/auth_manager.cpp
@@ -27,7 +27,9 @@ Option<bool> AuthManager::init(Store* store, const std::string& bootstrap_auth_k
}
std::vector<std::string> api_key_json_strs;
- store->scan_fill(API_KEYS_PREFIX, api_key_json_strs);
+ store->scan_fill(std::string(API_KEYS_PREFIX) + "_",
+ std::string(API_KEYS_PREFIX) + "`",
+ api_key_json_strs);
LOG(INFO) << "Indexing " << api_key_json_strs.size() << " API key(s) found on disk.";
@@ -48,7 +50,8 @@ Option<std::vector<api_key_t>> AuthManager::list_keys() const {
std::shared_lock lock(mutex);
std::vector<std::string> api_key_json_strs;
- store->scan_fill(API_KEYS_PREFIX, api_key_json_strs);
+ store->scan_fill(std::string(API_KEYS_PREFIX) + "_",
+ std::string(API_KEYS_PREFIX) + "`", api_key_json_strs);
std::vector<api_key_t> stored_api_keys;
diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cpp
index 248c31b5..ba82b1a9 100644
--- a/src/batched_indexer.cpp
+++ b/src/batched_indexer.cpp
@@ -9,6 +9,7 @@ BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_sto
skip_writes(skip_writes) {
queues.resize(num_threads);
qmutuxes = new await_t[num_threads];
+ skip_index_iter_upper_bound = new rocksdb::Slice(skip_index_upper_bound_key);
}
void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
@@ -121,8 +122,7 @@ std::string BatchedIndexer::get_collection_name(const std::shared_ptr<http_req>&
void BatchedIndexer::run() {
LOG(INFO) << "Starting batch indexer with " << num_threads << " threads.";
ThreadPool* thread_pool = new ThreadPool(num_threads);
-
- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX, skip_index_iter_upper_bound);
populate_skip_index();
LOG(INFO) << "BatchedIndexer skip_index: " << skip_index;
@@ -163,8 +163,8 @@ void BatchedIndexer::run() {
const std::string& req_key_start_prefix = req_key_prefix + StringUtils::serialize_uint32_t(
orig_req_res.next_chunk_index);
- const std::string& req_key_upper_bound = get_req_suffix_key(req_id);
- rocksdb::Slice upper_bound(req_key_upper_bound); // cannot inline req_key_upper_bound
+ const std::string& req_key_upper_bound = get_req_suffix_key(req_id); // cannot inline this
+ rocksdb::Slice upper_bound(req_key_upper_bound);
rocksdb::Iterator* iter = store->scan(req_key_start_prefix, &upper_bound);
// used to handle partial JSON documents caused by chunking
@@ -318,6 +318,7 @@ std::string BatchedIndexer::get_req_suffix_key(uint64_t req_id) {
BatchedIndexer::~BatchedIndexer() {
delete [] qmutuxes;
+ delete skip_index_iter_upper_bound;
delete skip_index_iter;
}
@@ -431,7 +432,7 @@ std::shared_mutex& BatchedIndexer::get_pause_mutex() {
void BatchedIndexer::clear_skip_indices() {
delete skip_index_iter;
- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX, skip_index_iter_upper_bound);
while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {
meta_store->remove(skip_index_iter->key().ToString());
diff --git a/src/collection.cpp b/src/collection.cpp
index 49df2df8..d39a8e0e 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3266,7 +3266,10 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
// Now, we can index existing data onto the updated schema
const std::string seq_id_prefix = get_seq_id_collection_prefix();
- rocksdb::Iterator* iter = store->scan(seq_id_prefix);
+ std::string upper_bound_key = get_seq_id_collection_prefix() + "`"; // cannot inline this
+ rocksdb::Slice upper_bound(upper_bound_key);
+
+ rocksdb::Iterator* iter = store->scan(seq_id_prefix, &upper_bound);
std::unique_ptr<rocksdb::Iterator> iter_guard(iter);
size_t num_found_docs = 0;
@@ -3662,8 +3665,10 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
// data validations: here we ensure that already stored data is compatible with requested schema changes
const std::string seq_id_prefix = get_seq_id_collection_prefix();
+ std::string upper_bound_key = get_seq_id_collection_prefix() + "`"; // cannot inline this
+ rocksdb::Slice upper_bound(upper_bound_key);
- rocksdb::Iterator* iter = store->scan(seq_id_prefix);
+ rocksdb::Iterator* iter = store->scan(seq_id_prefix, &upper_bound);
std::unique_ptr<rocksdb::Iterator> iter_guard(iter);
size_t num_found_docs = 0;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index a27f46bd..68c6a6c2 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -180,7 +180,9 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
<< document_batch_size << " documents at a time.";
std::vector<std::string> collection_meta_jsons;
- store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);
+ store->scan_fill(std::string(Collection::COLLECTION_META_PREFIX) + "_",
+ std::string(Collection::COLLECTION_META_PREFIX) + "`",
+ collection_meta_jsons);
const size_t num_collections = collection_meta_jsons.size();
LOG(INFO) << "Found " << num_collections << " collection(s) on disk.";
@@ -237,7 +239,10 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
// load aliases
std::string symlink_prefix_key = std::string(SYMLINK_PREFIX) + "_";
- rocksdb::Iterator* iter = store->scan(symlink_prefix_key);
+ std::string upper_bound_key = std::string(SYMLINK_PREFIX) + "`"; // cannot inline this
+ rocksdb::Slice upper_bound(upper_bound_key);
+
+ rocksdb::Iterator* iter = store->scan(symlink_prefix_key, &upper_bound);
while(iter->Valid() && iter->key().starts_with(symlink_prefix_key)) {
std::vector<std::string> parts;
StringUtils::split(iter->key().ToString(), parts, symlink_prefix_key);
@@ -250,7 +255,10 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
// load presets
std::string preset_prefix_key = std::string(PRESET_PREFIX) + "_";
- iter = store->scan(preset_prefix_key);
+ std::string preset_upper_bound_key = std::string(PRESET_PREFIX) + "`"; // cannot inline this
+ rocksdb::Slice preset_upper_bound(preset_upper_bound_key);
+
+ iter = store->scan(preset_prefix_key, &preset_upper_bound);
while(iter->Valid() && iter->key().starts_with(preset_prefix_key)) {
std::vector<std::string> parts;
StringUtils::split(iter->key().ToString(), parts, preset_prefix_key);
@@ -447,7 +455,11 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col
// delete overrides
const std::string& del_override_prefix =
std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" + actual_coll_name + "_";
- rocksdb::Iterator* iter = store->scan(del_override_prefix);
+ std::string upper_bound_key = std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" +
+ actual_coll_name + "`"; // cannot inline this
+ rocksdb::Slice upper_bound(upper_bound_key);
+
+ rocksdb::Iterator* iter = store->scan(del_override_prefix, &upper_bound);
while(iter->Valid() && iter->key().starts_with(del_override_prefix)) {
store->remove(iter->key().ToString());
iter->Next();
@@ -457,7 +469,12 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col
// delete synonyms
const std::string& del_synonym_prefix =
std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "_" + actual_coll_name + "_";
- iter = store->scan(del_synonym_prefix);
+
+ std::string syn_upper_bound_key = std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "_" +
+ actual_coll_name + "`"; // cannot inline this
+ rocksdb::Slice syn_upper_bound(syn_upper_bound_key);
+
+ iter = store->scan(del_synonym_prefix, &syn_upper_bound);
while(iter->Valid() && iter->key().starts_with(del_synonym_prefix)) {
store->remove(iter->key().ToString());
iter->Next();
@@ -1166,7 +1183,9 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
// initialize overrides
std::vector<std::string> collection_override_jsons;
- cm.store->scan_fill(Collection::get_override_key(this_collection_name, ""), collection_override_jsons);
+ cm.store->scan_fill(Collection::get_override_key(this_collection_name, ""),
+ std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" + this_collection_name + "`",
+ collection_override_jsons);
for(const auto & collection_override_json: collection_override_jsons) {
nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);
@@ -1181,7 +1200,9 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
// initialize synonyms
std::vector<std::string> collection_synonym_jsons;
- cm.store->scan_fill(SynonymIndex::get_synonym_key(this_collection_name, ""), collection_synonym_jsons);
+ cm.store->scan_fill(SynonymIndex::get_synonym_key(this_collection_name, ""),
+ std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "_" + this_collection_name + "`",
+ collection_synonym_jsons);
for(const auto & collection_synonym_json: collection_synonym_jsons) {
nlohmann::json collection_synonym = nlohmann::json::parse(collection_synonym_json);
@@ -1190,8 +1211,10 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
// Fetch records from the store and re-create memory index
const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();
+ std::string upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline this
+ rocksdb::Slice upper_bound(upper_bound_key);
- rocksdb::Iterator* iter = cm.store->scan(seq_id_prefix);
+ rocksdb::Iterator* iter = cm.store->scan(seq_id_prefix, &upper_bound);
std::unique_ptr<rocksdb::Iterator> iter_guard(iter);
std::vector<index_record> index_records;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index a44181f2..7086ffed 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -618,7 +618,9 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
}
if(simple_filter_query.empty()) {
- export_state->it = collectionManager.get_store()->scan(seq_id_prefix);
+ export_state->iter_upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline this
+ export_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);
+ export_state->it = collectionManager.get_store()->scan(seq_id_prefix, export_state->iter_upper_bound);
} else {
auto filter_ids_op = collection->get_filter_ids(simple_filter_query, export_state->index_ids);
diff --git a/src/ratelimit_manager.cpp b/src/ratelimit_manager.cpp
index c5b43662..01634d30 100644
--- a/src/ratelimit_manager.cpp
+++ b/src/ratelimit_manager.cpp
@@ -436,8 +436,10 @@ Option<bool> RateLimitManager::init(Store *store) {
else {
last_rule_id = 0;
}
+
std::vector<std::string> rule_json_strs;
- store->scan_fill(RULES_PREFIX, rule_json_strs);
+ store->scan_fill(std::string(RULES_PREFIX) + "_", std::string(RULES_PREFIX) + "`", rule_json_strs);
+
for(const auto& rule_json_str: rule_json_strs) {
nlohmann::json rule_json = nlohmann::json::parse(rule_json_str);
Option<rate_limit_rule_t> rule_option = parse_rule(rule_json);
@@ -460,8 +462,10 @@ Option<bool> RateLimitManager::init(Store *store) {
else {
last_ban_id = 0;
}
+
std::vector<std::string> ban_json_strs;
- store->scan_fill(BANS_PREFIX, ban_json_strs);
+ store->scan_fill(std::string(BANS_PREFIX) + "_", std::string(BANS_PREFIX) + "`", ban_json_strs);
+
for(const auto& ban_json_str: ban_json_strs) {
nlohmann::json ban_json = nlohmann::json::parse(ban_json_str);
rate_limit_status_t ban_status;
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 91c257a0..8ad0cbc0 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -663,11 +663,13 @@ TEST_F(CollectionSpecificTest, DeleteOverridesAndSynonymsOnDiskDuringCollDrop) {
// overrides should have been deleted from the store
std::vector<std::string> stored_values;
- store->scan_fill(Collection::COLLECTION_OVERRIDE_PREFIX, stored_values);
+ store->scan_fill(Collection::COLLECTION_OVERRIDE_PREFIX, std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "`",
+ stored_values);
ASSERT_TRUE(stored_values.empty());
// synonyms should also have been deleted from the store
- store->scan_fill(SynonymIndex::COLLECTION_SYNONYM_PREFIX, stored_values);
+ store->scan_fill(SynonymIndex::COLLECTION_SYNONYM_PREFIX, std::string(SynonymIndex::COLLECTION_SYNONYM_PREFIX) + "`",
+ stored_values);
ASSERT_TRUE(stored_values.empty());
}