Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.25-join

Ozan Armağan73791a057b2b
diff --cc src/analytics_manager.cpp
index 9c5fe583,26d58ac6..1721fcae
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@@ -162,68 -209,81 +209,69 @@@ void AnalyticsManager::run(ReplicationS
break;
}
- for(const auto& suggestion_config: suggestion_configs) {
- const std::string& sink_name = suggestion_config.first;
- const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;
+ persist_suggestions(raft_server, prev_persistence_s);
- auto popular_queries_it = popular_queries.find(suggestion_coll);
- if(popular_queries_it == popular_queries.end()) {
- continue;
- }
+ lk.unlock();
+ }
- // need to prepare the counts as JSON docs for import into the suggestion collection
- // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}
+ dispose();
+ }
- PopularQueries* popularQueries = popular_queries_it->second;
+ void AnalyticsManager::persist_suggestions(ReplicationState *raft_server, uint64_t prev_persistence_s) {
+ // lock is held by caller
+ for(const auto& suggestion_config: suggestion_configs) {
+ const std::string& sink_name = suggestion_config.first;
+ const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;
- // aggregate prefix queries to their final form
- auto now = std::chrono::system_clock::now().time_since_epoch();
- auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();
- popularQueries->compact_user_queries(now_ts_us);
+ auto popular_queries_it = popular_queries.find(suggestion_coll);
+ if(popular_queries_it == popular_queries.end()) {
+ continue;
+ }
- auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(now).count();
+ // need to prepare the counts as JSON docs for import into the suggestion collection
+ // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}
- if(now_ts_seconds - prev_persistence_s < Config::get_instance().get_analytics_flush_interval()) {
- // we will persist aggregation every hour
- continue;
- }
+ PopularQueries* popularQueries = popular_queries_it->second;
- prev_persistence_s = now_ts_seconds;
+ // aggregate prefix queries to their final form
+ auto now = std::chrono::system_clock::now().time_since_epoch();
+ auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();
+ popularQueries->compact_user_queries(now_ts_us);
- std::string import_payload;
- popularQueries->serialize_as_docs(import_payload);
-
- if(import_payload.empty()) {
- continue;
- }
+ auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(now).count();
+ if(now_ts_seconds - prev_persistence_s < Config::get_instance().get_analytics_flush_interval()) {
+ // we will persist aggregation every hour
+ continue;
+ }
+ // send http request
+ std::string leader_url = raft_server->get_leader_url();
+ if(!leader_url.empty()) {
+ const std::string& resource_url = leader_url + "collections/" + suggestion_coll +
+ "/documents/import?action=emplace";
+ std::string res;
+ std::map<std::string, std::string> res_headers;
+ std::unordered_map<std::string, std::string> headers;
+ long status_code = HttpClient::post_response(resource_url, import_payload,
+ res, res_headers, headers, 10*1000);
+
+ if(status_code != 200) {
+ LOG(ERROR) << "Error while sending query suggestions events to leader. "
+ << "Status code: " << status_code << ", response: " << res;
+ } else {
+ LOG(INFO) << "Sent query suggestions to leader for aggregation.";
+ popularQueries->reset_local_counts();
+
+ if(raft_server->is_leader()) {
+ // try to run top-K compaction of suggestion collection
+ auto coll = CollectionManager::get_instance().get_collection(suggestion_coll);
+ if (coll == nullptr) {
+ LOG(ERROR) << "No collection found for suggestions aggregation: " + suggestion_coll;
+ continue;
+ }
+
+ coll->truncate_after_top_k("count", popularQueries->get_k());
+
- prev_persistence_s = now_ts_seconds;
-
- std::string import_payload;
- popularQueries->serialize_as_docs(import_payload);
-
- if(import_payload.empty()) {
- continue;
- }
-
- // send http request
- std::string leader_url = raft_server->get_leader_url();
- if(!leader_url.empty()) {
- const std::string& base_url = leader_url + "collections/" + suggestion_coll;
- std::string res;
-
- const std::string& update_url = base_url + "/documents/import?action=emplace";
- std::map<std::string, std::string> res_headers;
- long status_code = HttpClient::post_response(update_url, import_payload,
- res, res_headers, 10*1000);
-
- if(status_code != 200) {
- LOG(ERROR) << "Error while sending query suggestions events to leader. "
- << "Status code: " << status_code << ", response: " << res;
- } else {
- LOG(INFO) << "Query aggregation for collection: " + suggestion_coll;
- popularQueries->reset_local_counts();
-
- if(raft_server->is_leader()) {
- // try to run top-K compaction of suggestion collection
- const std::string top_k_param = "count:" + std::to_string(popularQueries->get_k());
- const std::string& truncate_topk_url = base_url + "/documents?top_k_by=" + top_k_param;
- res.clear();
- res_headers.clear();
- status_code = HttpClient::delete_response(truncate_topk_url, res, res_headers, 10*1000);
- if(status_code != 200) {
- LOG(ERROR) << "Error while running top K for query suggestions collection. "
- << "Status code: " << status_code << ", response: " << res;
- } else {
- LOG(INFO) << "Top K aggregation for collection: " + suggestion_coll;
}
}
}