Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix collection drop induced write latency.

Kishore Nallandfa7191f3290
diff --git a/include/batched_indexer.h b/include/batched_indexer.h
index 60e162d4..e75a0cba 100644
--- a/include/batched_indexer.h
+++ b/include/batched_indexer.h
@@ -72,6 +72,8 @@ private:
static std::string get_req_prefix_key(uint64_t req_id);
+ static std::string get_req_suffix_key(uint64_t req_id);
+
public:
static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";
diff --git a/include/store.h b/include/store.h
index d695c9e7..b94420a7 100644
--- a/include/store.h
+++ b/include/store.h
@@ -15,6 +15,7 @@
#include <rocksdb/transaction_log.h>
#include <butil/file_util.h>
#include <rocksdb/utilities/checkpoint.h>
+#include <rocksdb/utilities/table_properties_collectors.h>
#include "string_utils.h"
#include "logger.h"
#include "file_utils.h"
@@ -91,7 +92,9 @@ public:
options.max_write_buffer_number = 2;
options.merge_operator.reset(new UInt64AddOperator);
options.compression = rocksdb::CompressionType::kSnappyCompression;
- //options.bottommost_compression = rocksdb::CompressionType::kSnappyCompression;
+
+ /*options.table_properties_collector_factories.emplace_back(
+ rocksdb::NewCompactOnDeletionCollectorFactory(10000, 7500, 0.5));*/
// these need to be high for replication scenarios
options.WAL_ttl_seconds = wal_ttl_secs;
@@ -164,9 +167,13 @@ public:
return status.ok();
}
- rocksdb::Iterator* scan(const std::string & prefix) {
+ rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound = nullptr) {
std::shared_lock lock(mutex);
- rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());
+ rocksdb::ReadOptions read_opts;
+ if(iterate_upper_bound) {
+ read_opts.iterate_upper_bound = iterate_upper_bound;
+ }
+ rocksdb::Iterator *iter = db->NewIterator(read_opts);
iter->Seek(prefix);
return iter;
}
diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cpp
index f1857814..99e27a71 100644
--- a/src/batched_indexer.cpp
+++ b/src/batched_indexer.cpp
@@ -60,7 +60,7 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh
}
{
- std::unique_lock lk1(qmutuxes[queue_id].mcv);
+ std::lock_guard lk1(qmutuxes[queue_id].mcv);
queues[queue_id].emplace_back(req->start_ts);
}
@@ -155,10 +155,15 @@ void BatchedIndexer::run() {
// scan db for all logs associated with request
const std::string& req_key_prefix = get_req_prefix_key(req_id);
- const std::string& req_key_start_prefix = get_req_prefix_key(req_id) +
- StringUtils::serialize_uint32_t(orig_req_res.next_chunk_index);
+ /* Format of the key: $RL_reqId_chunkId
+ NOTE: we use an explicit `next_chunk_index` so that the reads can resume from a partially request.
+ */
+ const std::string& req_key_start_prefix = req_key_prefix + StringUtils::serialize_uint32_t(
+ orig_req_res.next_chunk_index);
- rocksdb::Iterator* iter = store->scan(req_key_start_prefix);
+ const std::string& req_key_upper_bound = get_req_suffix_key(req_id);
+ rocksdb::Slice upper_bound(req_key_upper_bound); // cannot inline req_key_upper_bound
+ rocksdb::Iterator* iter = store->scan(req_key_start_prefix, &upper_bound);
// used to handle partial JSON documents caused by chunking
std::string& prev_body = orig_req_res.prev_req_body;
@@ -293,9 +298,12 @@ void BatchedIndexer::run() {
}
std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {
- const std::string& req_key_prefix =
- RAFT_REQ_LOG_PREFIX + StringUtils::serialize_uint64_t(req_id) + "_";
+ const std::string& req_key_prefix = RAFT_REQ_LOG_PREFIX + StringUtils::serialize_uint64_t(req_id) + "_";
+ return req_key_prefix;
+}
+std::string BatchedIndexer::get_req_suffix_key(uint64_t req_id) {
+ const std::string& req_key_prefix = RAFT_REQ_LOG_PREFIX + StringUtils::serialize_uint64_t(req_id) + "`";
return req_key_prefix;
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index bb3b7801..131d3c30 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -411,18 +411,14 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col
if(remove_from_store) {
const std::string& del_key_prefix = std::to_string(collection->get_collection_id()) + "_";
-
- rocksdb::Iterator* iter = store->scan(del_key_prefix);
- while(iter->Valid() && iter->key().starts_with(del_key_prefix)) {
- store->remove(iter->key().ToString());
- iter->Next();
- }
- delete iter;
+ const std::string& del_end_prefix = std::to_string(collection->get_collection_id()) + "`";
+ store->delete_range(del_key_prefix, del_end_prefix);
+ store->flush();
// delete overrides
const std::string& del_override_prefix =
std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" + actual_coll_name + "_";
- iter = store->scan(del_override_prefix);
+ rocksdb::Iterator* iter = store->scan(del_override_prefix);
while(iter->Valid() && iter->key().starts_with(del_override_prefix)) {
store->remove(iter->key().ToString());
iter->Next();