Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Address valgrind warnings.
Kishore Nallan27ebae1f88c0
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex 93b6ee50..feddcd55 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -33,7 +33,7 @@ private:ThreadPool* thread_pool;const size_t num_threads;- std::vector<std::deque<req_res_t>> queues;+ std::vector<std::deque<uint64_t>> queues;std::mutex* qmutuxes;std::mutex mutex;diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex cdf8ed04..a6d6cc74 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -28,19 +28,15 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::shstore->insert(request_chunk_key, req->serialize());req->body = "";- uint64_t batch_begin_ts;-{std::unique_lock lk(mutex);auto req_res_map_it = req_res_map.find(req->start_ts);if(req_res_map_it == req_res_map.end()) {- batch_begin_ts = std::chrono::duration_cast<std::chrono::seconds>(+ uint64_t batch_begin_ts = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();req_res_t req_res{req, res, batch_begin_ts};req_res_map[req->start_ts] = req_res;- } else {- batch_begin_ts = req_res_map_it->second.batch_begin_ts;}}@@ -49,15 +45,14 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh{std::unique_lock lk(qmutuxes[queue_id]);- req_res_t req_res(req, res, batch_begin_ts);- queues[queue_id].emplace_back(req_res);+ queues[queue_id].emplace_back(req->start_ts);}std::unique_lock lk(mutex);request_to_chunk.erase(req->start_ts);}- if(req->_req != nullptr) {+ if(req->_req != nullptr && req->_req->proceed_req) {deferred_req_res_t* req_res = new deferred_req_res_t(req, res, server, true);server->get_message_dispatcher()->send_message(HttpServer::REQUEST_PROCEED_MESSAGE, req_res);}@@ -67,7 +62,7 @@ void BatchedIndexer::run() {LOG(INFO) << "Starting batch indexer with " << num_threads << " threads.";for(size_t i = 0; i < num_threads; i++) {- std::deque<req_res_t>& queue = queues[i];+ std::deque<uint64_t>& queue = queues[i];std::mutex& queue_mutex = qmutuxes[i];thread_pool->enqueue([&queue, &queue_mutex, this, i]() {@@ -77,42 +72,45 @@ void BatchedIndexer::run() {if(queue.empty()) {qlk.unlock();} else {- size_t qs = queue.size();- req_res_t req_res = queue.front();+ uint64_t req_id = queue.front();queue.pop_front();qlk.unlock();std::unique_lock mlk(mutex);- req_res_t orig_req_res = req_res_map[req_res.req->start_ts];+ req_res_t orig_req_res = req_res_map[req_id];mlk.unlock();// scan db for all logs associated with request- const std::string& req_key_prefix = get_req_prefix_key(req_res.req->start_ts);+ const std::string& req_key_prefix = get_req_prefix_key(req_id);rocksdb::Iterator* iter = store->scan(req_key_prefix);std::string prev_body = ""; // used to handle partial JSON documents caused by chunkingwhile(iter->Valid() && iter->key().starts_with(req_key_prefix)) {- std::shared_ptr<http_req> saved_req = std::make_shared<http_req>();- saved_req->body = prev_body;- saved_req->deserialize(iter->value().ToString());- saved_req->_req = orig_req_res.req->_req;+ std::shared_ptr<http_req>& orig_req = orig_req_res.req;+ auto _req = orig_req->_req;+ orig_req->body = prev_body;+ orig_req->deserialize(iter->value().ToString());+ orig_req->_req = _req;++ //LOG(INFO) << "original request: " << orig_req_res.req << ", _req: " << orig_req_res.req->_req;route_path* found_rpath = nullptr;- bool route_found = server->get_route(saved_req->route_hash, &found_rpath);+ bool route_found = server->get_route(orig_req->route_hash, &found_rpath);bool async_res = false;if(route_found) {async_res = found_rpath->async_res;- found_rpath->handler(saved_req, req_res.res);- prev_body = saved_req->body;+ found_rpath->handler(orig_req, orig_req_res.res);+ prev_body = orig_req->body;} else {- req_res.res->set_404();+ orig_req_res.res->set_404();prev_body = "";}if(!async_res && orig_req_res.req->_req != nullptr) {- deferred_req_res_t* deferred_req_res = new deferred_req_res_t(saved_req, req_res.res,+ deferred_req_res_t* deferred_req_res = new deferred_req_res_t(orig_req_res.req,+ orig_req_res.res,server, true);server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE,deferred_req_res);@@ -121,13 +119,15 @@ void BatchedIndexer::run() {iter->Next();}+ delete iter;+//LOG(INFO) << "Erasing request data from disk and memory for request " << req_res.req->start_ts;// we can delete the buffered request contentstore->delete_range(req_key_prefix, req_key_prefix + StringUtils::serialize_uint32_t(UINT32_MAX));std::unique_lock lk(mutex);- req_res_map.erase(req_res.req->start_ts);+ req_res_map.erase(req_id);}std::this_thread::sleep_for(std::chrono::milliseconds (10));@@ -184,6 +184,7 @@ std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {BatchedIndexer::~BatchedIndexer() {delete [] qmutuxes;+ delete thread_pool;}void BatchedIndexer::stop() {diff --git a/src/http_server.cpp b/src/http_server.cppindex bad28b36..17eab04b 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -899,6 +899,10 @@ bool HttpServer::on_request_proceed_message(void *data) {req_res->req->_req->proceed_req(req_res->req->_req, written, stream_state);}+ if(req_res->destroy_after_stream_response) {+ delete req_res;+ }+return true;}