Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

merge v0.26-facets

krunal61c71bc451dc
diff --cc src/core_api.cpp
index 5688ef1b,0adbe342..1e1f8302
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@@ -531,6 -534,70 +534,70 @@@ bool post_multi_search(const std::share
}
}
+ bool conversation = orig_req_params["conversation"] == "true";
+ bool conversation_history = orig_req_params.find("conversation_id") != orig_req_params.end();
+ std::string common_query;
+
+ if(!conversation && conversation_history) {
+ res->set_400("`conversation_id` can only be used if `conversation` is enabled.");
+ return false;
+ }
-
++
+ if(conversation) {
+ if(orig_req_params.find("q") == orig_req_params.end()) {
+ res->set_400("`q` parameter has to be common for all searches if conversation is enabled. Please set `q` as a query parameter in the request, instead of inside the POST body");
+ return false;
+ }
+
+ if(orig_req_params.find("conversation_model_id") == orig_req_params.end()) {
+ res->set_400("`conversation_model_id` is needed if conversation is enabled.");
+ return false;
+ }
+
+ if(StringUtils::is_uint32_t(orig_req_params["conversation_model_id"])) {
+ uint32_t conversation_model_id = std::stoul(orig_req_params["conversation_model_id"]);
+ auto conversation_model = ConversationModelManager::get_model(conversation_model_id);
+
+ if(!conversation_model.ok()) {
+ res->set_400("`conversation_model_id` is invalid.");
+ return false;
+ }
+ } else {
+ res->set_400("`conversation_model_id` is invalid.");
+ return false;
+ }
+
+ if(conversation_history) {
+ std::string conversation_id = orig_req_params["conversation_id"];
+
+ auto conversation_history = ConversationManager::get_conversation(conversation_id);
+
+ if(!conversation_history.ok()) {
+ res->set_400("`conversation_id` is invalid.");
+ return false;
+ }
+ }
+
+ common_query = orig_req_params["q"];
+
+ if(conversation_history) {
+ auto conversation_model_id = std::stoul(orig_req_params["conversation_model_id"]);
+ auto conversation_id = orig_req_params["conversation_id"];
+ auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();
+ auto conversation_history = ConversationManager::get_conversation(conversation_id).get();
+ auto generate_standalone_q = ConversationModel::get_standalone_question(conversation_history, common_query, conversation_model);
+
+ if(!generate_standalone_q.ok()) {
+ res->set_400(generate_standalone_q.error());
+ return false;
+ }
+
+ orig_req_params["q"] = generate_standalone_q.get();
+ }
+ }
+
+
+
for(size_t i = 0; i < searches.size(); i++) {
auto& search_params = searches[i];
@@@ -574,6 -658,146 +657,146 @@@
}
}
+ if(conversation) {
+ nlohmann::json result_docs_arr = nlohmann::json::array();
+ int res_index = 0;
+ for(const auto& result : response["results"]) {
+ nlohmann::json result_docs = nlohmann::json::array();
+
+ std::vector<std::string> vector_fields;
+
+ auto collection = CollectionManager::get_instance().get_collection(searches[res_index]["collection"].get<std::string>());
+ auto search_schema = collection->get_schema();
+
+ for(const auto& field : search_schema) {
+ if(field.type == field_types::FLOAT_ARRAY) {
+ vector_fields.push_back(field.name);
+ }
+ }
+
+ if(result.contains("grouped_hits")) {
+ for(const auto& grouped_hit : result["grouped_hits"]) {
+ for(const auto& hit : grouped_hit["hits"]) {
+ auto doc = hit["document"];
+ for(const auto& vector_field : vector_fields) {
+ if(doc.contains(vector_field)) {
+ doc.erase(vector_field);
+ }
+ }
+ result_docs.push_back(doc);
+ }
+ }
+ }
+ else {
+ for(const auto& hit : result["hits"]) {
+ auto doc = hit["document"];
+ for(const auto& vector_field : vector_fields) {
+ if(doc.contains(vector_field)) {
+ doc.erase(vector_field);
+ }
+ }
+ result_docs.push_back(doc);
+ }
+ }
+
+ result_docs_arr.push_back(result_docs);
+ }
+
+ // We have to pop a document from the search result with max size
+ // Until we do not exceed MAX_TOKENS limit
+ while(ConversationManager::get_token_count(result_docs_arr) > ConversationManager::MAX_TOKENS) {
+ // sort the result_docs_arr by size descending
+ std::sort(result_docs_arr.begin(), result_docs_arr.end(), [](const auto& a, const auto& b) {
+ return a.size() > b.size();
+ });
+
+ // pop the last element from first array
+ if(result_docs_arr.size() > 0 && result_docs_arr[0].size() > 0) {
+ result_docs_arr[0].erase(result_docs_arr[0].size() - 1);
+ }
+ }
+
+
+ // Make result_docs_arr 1D
+ nlohmann::json result_docs = nlohmann::json::array();
+ for(const auto& result_doc : result_docs_arr) {
+ for(const auto& doc : result_doc) {
+ result_docs.push_back(doc);
+ }
+ }
+
+ auto conversation_model_id = std::stoul(orig_req_params["conversation_model_id"]);
+ auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();
+
+ auto prompt = req->params["q"];
+
+ auto answer_op = ConversationModel::get_answer(result_docs.dump(0), prompt, conversation_model);
+
+ if(!answer_op.ok()) {
+ res->set_400(answer_op.error());
+ return false;
+ }
+
+ response["conversation"] = nlohmann::json::object();
+ response["conversation"]["query"] = common_query;
+ response["conversation"]["answer"] = answer_op.get();
+
+ auto formatted_question_op = ConversationModel::format_question(common_query, conversation_model);
+ if(!formatted_question_op.ok()) {
+ res->set_400(formatted_question_op.error());
+ return false;
+ }
-
++
+
+ auto formatted_answer_op = ConversationModel::format_answer(answer_op.get(), conversation_model);
+ if(!formatted_answer_op.ok()) {
+ res->set_400(formatted_answer_op.error());
+ return false;
+ }
+
+ std::vector<std::string> exclude_fields;
- StringUtils::split(req->params["exclude_fields"], exclude_fields, ",");
++ StringUtils::split(req->params["exclude_fields"], exclude_fields, ",");
+ bool exclude_conversation_history = std::find(exclude_fields.begin(), exclude_fields.end(), "conversation_history") != exclude_fields.end();
+
+ if(conversation_history) {
+ std::string conversation_id = orig_req_params["conversation_id"];
+ ConversationManager::append_conversation(conversation_id, formatted_question_op.get());
+ ConversationManager::append_conversation(conversation_id, formatted_answer_op.get());
+ auto get_conversation_op = ConversationManager::get_conversation(conversation_id);
+ if(!get_conversation_op.ok()) {
+ res->set_400(get_conversation_op.error());
+ return false;
+ }
+
+ auto conversation_history = get_conversation_op.get();
+ if(!exclude_conversation_history) {
+ response["conversation"]["conversation_history"] = conversation_history;
+ }
+ response["conversation"]["conversation_id"] = conversation_id;
+ } else {
+ nlohmann::json conversation_history = nlohmann::json::array();
+ conversation_history.push_back(formatted_question_op.get());
+ conversation_history.push_back(formatted_answer_op.get());
+
+ auto create_conversation_op = ConversationManager::create_conversation(conversation_history);
+ if(!create_conversation_op.ok()) {
+ res->set_400(create_conversation_op.error());
+ return false;
+ }
+
+ auto get_conversation_op = ConversationManager::get_conversation(create_conversation_op.get());
+ if(!get_conversation_op.ok()) {
+ res->set_400(get_conversation_op.error());
+ return false;
+ }
+ if(!exclude_conversation_history) {
+ response["conversation"]["conversation_history"] = get_conversation_op.get();
+ }
+ response["conversation"]["conversation_id"] = create_conversation_op.get();
+ }
+
+ }
+
res->set_200(response.dump());
// we will cache only successful requests
@@@ -2380,34 -2604,199 +2603,230 @@@ bool post_proxy(const std::shared_ptr<h
return true;
}
+
+ bool get_conversation(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ std::string conversation_id = req->params["id"];
+
+ auto conversation_op = ConversationManager::get_conversation(conversation_id);
+
+ if(!conversation_op.ok()) {
+ res->set(conversation_op.code(), conversation_op.error());
+ return false;
+ }
+
+ res->set_200(conversation_op.get().dump());
- return true;
++ return true;
+ }
+
+
+ bool del_conversation(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ std::string conversation_id = req->params["id"];
+
+ auto conversation_op = ConversationManager::delete_conversation(conversation_id);
+
+ if(!conversation_op.ok()) {
+ res->set(conversation_op.code(), conversation_op.error());
+ return false;
+ }
+
+ res->set_200(conversation_op.get().dump());
- return true;
++ return true;
+ }
+
+ bool get_conversations(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ auto conversations_op = ConversationManager::get_all_conversations();
+
+ if(!conversations_op.ok()) {
+ res->set(conversations_op.code(), conversations_op.error());
+ return false;
+ }
+
+ res->set_200(conversations_op.get().dump());
- return true;
++ return true;
+ }
+
+ bool put_conversation(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ std::string conversation_id = req->params["id"];
+
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const nlohmann::json::parse_error& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ req_json["id"] = conversation_id;
+
+ auto conversation_op = ConversationManager::update_conversation(req_json);
+
+ if(!conversation_op.ok()) {
+ res->set(conversation_op.code(), conversation_op.error());
+ return false;
+ }
+
+ res->set_200(conversation_op.get().dump());
+ return true;
+ }
+
+ bool post_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const nlohmann::json::parse_error& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ if(!req_json.is_object()) {
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ auto add_model_op = ConversationModelManager::add_model(req_json);
+
+ if(!add_model_op.ok()) {
+ res->set(add_model_op.code(), add_model_op.error());
+ return false;
+ }
+
+ auto model = add_model_op.get();
+ Collection::hide_credential(model, "api_key");
+
+ res->set_200(model.dump());
+ return true;
+ }
+
+ bool get_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ if(!StringUtils::is_uint32_t(req->params["id"])) {
+ res->set_400("Invalid ID.");
+ return false;
+ }
+ const int model_id = std::stoi(req->params["id"]);
+
+ auto model_op = ConversationModelManager::get_model(model_id);
+
+ if(!model_op.ok()) {
+ res->set(model_op.code(), model_op.error());
+ return false;
+ }
+
+ auto model = model_op.get();
+ Collection::hide_credential(model, "api_key");
+
+ res->set_200(model.dump());
- return true;
++ return true;
+ }
+
+ bool get_conversation_models(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ auto models_op = ConversationModelManager::get_all_models();
+
+ if(!models_op.ok()) {
+ res->set(models_op.code(), models_op.error());
+ return false;
+ }
+
+ auto models = models_op.get();
+
+ for(auto& model: models) {
+ Collection::hide_credential(model, "api_key");
+ }
+
+ res->set_200(models.dump());
- return true;
++ return true;
+ }
+
+
+ bool del_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ if(!StringUtils::is_uint32_t(req->params["id"])) {
+ res->set_400("Invalid ID.");
+ return false;
+ }
+ const int model_id = std::stoi(req->params["id"]);
+
+ auto model_op = ConversationModelManager::delete_model(model_id);
+
+ if(!model_op.ok()) {
+ res->set(model_op.code(), model_op.error());
+ return false;
+ }
+
+ auto model = model_op.get();
+
+ Collection::hide_credential(model, "api_key");
+
+ res->set_200(model.dump());
- return true;
++ return true;
+ }
+
+ bool put_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ if(!StringUtils::is_uint32_t(req->params["id"])) {
+ res->set_400("Invalid ID.");
+ return false;
+ }
+ const int model_id = std::stoi(req->params["id"]);
+
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const nlohmann::json::parse_error& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ if(!req_json.is_object()) {
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ auto model_op = ConversationModelManager::update_model(model_id, req_json);
+
+ if(!model_op.ok()) {
+ res->set(model_op.code(), model_op.error());
+ return false;
+ }
+
+ auto model = model_op.get();
+
+ Collection::hide_credential(model, "api_key");
+
+ res->set_200(model.dump());
- return true;
++ return true;
++}
+bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ auto click_events_op = AnalyticsManager::get_instance().get_click_events();
+
+ if(!click_events_op.ok()) {
+ res->set(click_events_op.code(), click_events_op.error());
+ }
+
+ res->set_200(click_events_op.get().dump());
+ return true;
+}
+
+bool post_replicate_click_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const std::exception& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ auto op = AnalyticsManager::get_instance().write_click_event_to_store(req_json);
+ if(!op.ok()) {
+ res->set_body(op.code(), op.error());
+ return false;
+ }
+
+ res->set_200("ClickEvent wrote to DB.");
+ return true;
}