Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix collection drop induced write latency.
Kishore Nallandfa7191f3290
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex 60e162d4..e75a0cba 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -72,6 +72,8 @@ private:static std::string get_req_prefix_key(uint64_t req_id);+ static std::string get_req_suffix_key(uint64_t req_id);+public:static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";diff --git a/include/store.h b/include/store.hindex d695c9e7..b94420a7 100644--- a/include/store.h+++ b/include/store.h@@ -15,6 +15,7 @@#include <rocksdb/transaction_log.h>#include <butil/file_util.h>#include <rocksdb/utilities/checkpoint.h>+#include <rocksdb/utilities/table_properties_collectors.h>#include "string_utils.h"#include "logger.h"#include "file_utils.h"@@ -91,7 +92,9 @@ public:options.max_write_buffer_number = 2;options.merge_operator.reset(new UInt64AddOperator);options.compression = rocksdb::CompressionType::kSnappyCompression;- //options.bottommost_compression = rocksdb::CompressionType::kSnappyCompression;++ /*options.table_properties_collector_factories.emplace_back(+ rocksdb::NewCompactOnDeletionCollectorFactory(10000, 7500, 0.5));*/// these need to be high for replication scenariosoptions.WAL_ttl_seconds = wal_ttl_secs;@@ -164,9 +167,13 @@ public:return status.ok();}- rocksdb::Iterator* scan(const std::string & prefix) {+ rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound = nullptr) {std::shared_lock lock(mutex);- rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());+ rocksdb::ReadOptions read_opts;+ if(iterate_upper_bound) {+ read_opts.iterate_upper_bound = iterate_upper_bound;+ }+ rocksdb::Iterator *iter = db->NewIterator(read_opts);iter->Seek(prefix);return iter;}diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex f1857814..99e27a71 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -60,7 +60,7 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh}{- std::unique_lock lk1(qmutuxes[queue_id].mcv);+ std::lock_guard lk1(qmutuxes[queue_id].mcv);queues[queue_id].emplace_back(req->start_ts);}@@ -155,10 +155,15 @@ void BatchedIndexer::run() {// scan db for all logs associated with requestconst std::string& req_key_prefix = get_req_prefix_key(req_id);- const std::string& req_key_start_prefix = get_req_prefix_key(req_id) +- StringUtils::serialize_uint32_t(orig_req_res.next_chunk_index);+ /* Format of the key: $RL_reqId_chunkId+ NOTE: we use an explicit `next_chunk_index` so that the reads can resume from a partially request.+ */+ const std::string& req_key_start_prefix = req_key_prefix + StringUtils::serialize_uint32_t(+ orig_req_res.next_chunk_index);- rocksdb::Iterator* iter = store->scan(req_key_start_prefix);+ const std::string& req_key_upper_bound = get_req_suffix_key(req_id);+ rocksdb::Slice upper_bound(req_key_upper_bound); // cannot inline req_key_upper_bound+ rocksdb::Iterator* iter = store->scan(req_key_start_prefix, &upper_bound);// used to handle partial JSON documents caused by chunkingstd::string& prev_body = orig_req_res.prev_req_body;@@ -293,9 +298,12 @@ void BatchedIndexer::run() {}std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {- const std::string& req_key_prefix =- RAFT_REQ_LOG_PREFIX + StringUtils::serialize_uint64_t(req_id) + "_";+ const std::string& req_key_prefix = RAFT_REQ_LOG_PREFIX + StringUtils::serialize_uint64_t(req_id) + "_";+ return req_key_prefix;+}+std::string BatchedIndexer::get_req_suffix_key(uint64_t req_id) {+ const std::string& req_key_prefix = RAFT_REQ_LOG_PREFIX + StringUtils::serialize_uint64_t(req_id) + "`";return req_key_prefix;}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex bb3b7801..131d3c30 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -411,18 +411,14 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& colif(remove_from_store) {const std::string& del_key_prefix = std::to_string(collection->get_collection_id()) + "_";-- rocksdb::Iterator* iter = store->scan(del_key_prefix);- while(iter->Valid() && iter->key().starts_with(del_key_prefix)) {- store->remove(iter->key().ToString());- iter->Next();- }- delete iter;+ const std::string& del_end_prefix = std::to_string(collection->get_collection_id()) + "`";+ store->delete_range(del_key_prefix, del_end_prefix);+ store->flush();// delete overridesconst std::string& del_override_prefix =std::string(Collection::COLLECTION_OVERRIDE_PREFIX) + "_" + actual_coll_name + "_";- iter = store->scan(del_override_prefix);+ rocksdb::Iterator* iter = store->scan(del_override_prefix);while(iter->Valid() && iter->key().starts_with(del_override_prefix)) {store->remove(iter->key().ToString());iter->Next();