Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.25-join
Ozan Armağan73791a057b2b
diff --cc src/analytics_manager.cppindex 9c5fe583,26d58ac6..1721fcae--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@@ -162,68 -209,81 +209,69 @@@ void AnalyticsManager::run(ReplicationSbreak;}- for(const auto& suggestion_config: suggestion_configs) {- const std::string& sink_name = suggestion_config.first;- const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;+ persist_suggestions(raft_server, prev_persistence_s);- auto popular_queries_it = popular_queries.find(suggestion_coll);- if(popular_queries_it == popular_queries.end()) {- continue;- }+ lk.unlock();+ }- // need to prepare the counts as JSON docs for import into the suggestion collection- // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}+ dispose();+ }- PopularQueries* popularQueries = popular_queries_it->second;+ void AnalyticsManager::persist_suggestions(ReplicationState *raft_server, uint64_t prev_persistence_s) {+ // lock is held by caller+ for(const auto& suggestion_config: suggestion_configs) {+ const std::string& sink_name = suggestion_config.first;+ const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;- // aggregate prefix queries to their final form- auto now = std::chrono::system_clock::now().time_since_epoch();- auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();- popularQueries->compact_user_queries(now_ts_us);+ auto popular_queries_it = popular_queries.find(suggestion_coll);+ if(popular_queries_it == popular_queries.end()) {+ continue;+ }- auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(now).count();+ // need to prepare the counts as JSON docs for import into the suggestion collection+ // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}- if(now_ts_seconds - prev_persistence_s < Config::get_instance().get_analytics_flush_interval()) {- // we will persist aggregation every hour- continue;- }+ PopularQueries* popularQueries = popular_queries_it->second;- prev_persistence_s = now_ts_seconds;+ // aggregate prefix queries to their final form+ auto now = std::chrono::system_clock::now().time_since_epoch();+ auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();+ popularQueries->compact_user_queries(now_ts_us);- std::string import_payload;- popularQueries->serialize_as_docs(import_payload);-- if(import_payload.empty()) {- continue;- }+ auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(now).count();+ if(now_ts_seconds - prev_persistence_s < Config::get_instance().get_analytics_flush_interval()) {+ // we will persist aggregation every hour+ continue;+ }+ // send http request+ std::string leader_url = raft_server->get_leader_url();+ if(!leader_url.empty()) {+ const std::string& resource_url = leader_url + "collections/" + suggestion_coll ++ "/documents/import?action=emplace";+ std::string res;+ std::map<std::string, std::string> res_headers;+ std::unordered_map<std::string, std::string> headers;+ long status_code = HttpClient::post_response(resource_url, import_payload,+ res, res_headers, headers, 10*1000);++ if(status_code != 200) {+ LOG(ERROR) << "Error while sending query suggestions events to leader. "+ << "Status code: " << status_code << ", response: " << res;+ } else {+ LOG(INFO) << "Sent query suggestions to leader for aggregation.";+ popularQueries->reset_local_counts();++ if(raft_server->is_leader()) {+ // try to run top-K compaction of suggestion collection+ auto coll = CollectionManager::get_instance().get_collection(suggestion_coll);+ if (coll == nullptr) {+ LOG(ERROR) << "No collection found for suggestions aggregation: " + suggestion_coll;+ continue;+ }++ coll->truncate_after_top_k("count", popularQueries->get_k());+- prev_persistence_s = now_ts_seconds;-- std::string import_payload;- popularQueries->serialize_as_docs(import_payload);-- if(import_payload.empty()) {- continue;- }-- // send http request- std::string leader_url = raft_server->get_leader_url();- if(!leader_url.empty()) {- const std::string& base_url = leader_url + "collections/" + suggestion_coll;- std::string res;-- const std::string& update_url = base_url + "/documents/import?action=emplace";- std::map<std::string, std::string> res_headers;- long status_code = HttpClient::post_response(update_url, import_payload,- res, res_headers, 10*1000);-- if(status_code != 200) {- LOG(ERROR) << "Error while sending query suggestions events to leader. "- << "Status code: " << status_code << ", response: " << res;- } else {- LOG(INFO) << "Query aggregation for collection: " + suggestion_coll;- popularQueries->reset_local_counts();-- if(raft_server->is_leader()) {- // try to run top-K compaction of suggestion collection- const std::string top_k_param = "count:" + std::to_string(popularQueries->get_k());- const std::string& truncate_topk_url = base_url + "/documents?top_k_by=" + top_k_param;- res.clear();- res_headers.clear();- status_code = HttpClient::delete_response(truncate_topk_url, res, res_headers, 10*1000);- if(status_code != 200) {- LOG(ERROR) << "Error while running top K for query suggestions collection. "- << "Status code: " << status_code << ", response: " << res;- } else {- LOG(INFO) << "Top K aggregation for collection: " + suggestion_coll;}}}