Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use collections to store conversations (#1723)
Ozan Armağan1b4e08b29def
diff --git a/include/conversation_manager.h b/include/conversation_manager.hindex 3cc5e74f..ef565901 100644--- a/include/conversation_manager.h+++ b/include/conversation_manager.h@@ -2,12 +2,14 @@#include <string>#include <unordered_map>+#include <unordered_set>#include <shared_mutex>#include <mutex>#include <json.hpp>#include "option.h"#include "store.h"#include "sole.hpp"+#include "collection_manager.h"class ConversationManager {@@ -21,35 +23,39 @@ class ConversationManager {static ConversationManager instance;return instance;}- Option<std::string> create_conversation(const nlohmann::json& conversation);+ Option<std::string> add_conversation(const nlohmann::json& conversation, const std::string& conversation_collection, const std::string& id = "");Option<nlohmann::json> get_conversation(const std::string& conversation_id);- Option<bool> append_conversation(const std::string& conversation_id, const nlohmann::json& message);- Option<nlohmann::json> truncate_conversation(nlohmann::json conversation, size_t limit);+ static Option<nlohmann::json> truncate_conversation(nlohmann::json conversation, size_t limit);Option<nlohmann::json> update_conversation(nlohmann::json conversation);Option<nlohmann::json> delete_conversation(const std::string& conversation_id);+ Option<bool> check_conversation_exists(const std::string& conversation_id);+ Option<std::unordered_set<std::string>> get_conversation_ids();Option<nlohmann::json> get_all_conversations();static constexpr size_t MAX_TOKENS = 3000;- Option<int> init(Store* store);+ Option<bool> init(ReplicationState* raft_server);void clear_expired_conversations();void run();void stop();void _set_ttl_offset(size_t offset) {TTL_OFFSET = offset;}++ Option<bool> validate_conversation_store_schema(Collection* collection);+ Option<bool> validate_conversation_store_collection(const std::string& collection);+ Option<bool> add_conversation_collection(const std::string& collection);+ Option<bool> remove_conversation_collection(const std::string& collection);+ Option<Collection*> get_conversation_collection(const std::string& conversation_id);private:ConversationManager() {}- std::unordered_map<std::string, nlohmann::json> conversations;std::mutex conversations_mutex;-- static constexpr char* CONVERSATION_RPEFIX = "$CNVP";- Store* store;-- static const std::string get_conversation_key(const std::string& conversation_id);-+ ReplicationState* raft_server;static constexpr size_t CONVERSATION_TTL = 60 * 60 * 24;size_t TTL_OFFSET = 0;+ size_t MAX_CONVERSATIONS_TO_DELETE_ONCE = 5;std::atomic<bool> quit = false;std::condition_variable cv;+ std::unordered_map<std::string, uint32_t> conversation_collection_map;+ std::unordered_map<std::string, std::string> conversation_mapper;};\ No newline at end of filediff --git a/include/conversation_model_manager.h b/include/conversation_model_manager.hindex 5782e884..2e43ee16 100644--- a/include/conversation_model_manager.h+++ b/include/conversation_model_manager.h@@ -7,6 +7,7 @@#include <option.h>#include "store.h"#include "sole.hpp"+#include "collection.h"class ConversationModelManager{@@ -21,16 +22,17 @@ class ConversationModelManagerstatic Option<nlohmann::json> delete_model(const std::string& model_id);static Option<nlohmann::json> update_model(const std::string& model_id, nlohmann::json model);static Option<nlohmann::json> get_all_models();--static Option<int> init(Store* store);+ static Option<nlohmann::json> migrate_model(nlohmann::json model);private:static inline std::unordered_map<std::string, nlohmann::json> models;static inline std::shared_mutex models_mutex;static constexpr char* MODEL_NEXT_ID = "$CVMN";static constexpr char* MODEL_KEY_PREFIX = "$CVMP";-+ static inline int64_t DEFAULT_CONVERSATION_COLLECTION_SUFFIX = 0;static inline Store* store;static const std::string get_model_key(const std::string& model_id);+ static Option<Collection*> get_default_conversation_collection();+};\ No newline at end of filediff --git a/include/core_api.h b/include/core_api.hindex 780f7b3a..ebdf1062 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -213,4 +213,4 @@ bool del_conversation_model(const std::shared_ptr<http_req>& req, const std::shabool get_conversation_models(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);-bool put_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);\ No newline at end of file+bool put_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);diff --git a/src/collection.cpp b/src/collection.cppindex 43ddd06d..6d306b26 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -77,6 +77,8 @@ Collection::~Collection() {VQModelManager::get_instance().delete_model(vq_model->get_model_name());}}++ ConversationManager::get_instance().remove_conversation_collection(name);}uint32_t Collection::get_next_seq_id() {@@ -2791,39 +2793,24 @@ Option<nlohmann::json> Collection::search(std::string raw_query,return Option<nlohmann::json>(formatted_answer_op.code(), formatted_answer_op.error());}- if(has_conversation_history) {- ConversationManager::get_instance().append_conversation(conversation_id, formatted_question_op.get());- ConversationManager::get_instance().append_conversation(conversation_id, formatted_answer_op.get());- auto get_conversation_op = ConversationManager::get_instance().get_conversation(conversation_id);- if(!get_conversation_op.ok()) {- return Option<nlohmann::json>(get_conversation_op.code(), get_conversation_op.error());- }-- auto conversation_history = get_conversation_op.get();- if(exclude_fields.count("conversation_history") == 0) {- result["conversation"]["conversation_history"] = conversation_history;- }- result["conversation"]["conversation_id"] = conversation_id;- } else {- nlohmann::json conversation_history = nlohmann::json::array();- conversation_history.push_back(formatted_question_op.get());- conversation_history.push_back(formatted_answer_op.get());+ nlohmann::json conversation_history = nlohmann::json::array();+ conversation_history.push_back(formatted_question_op.get());+ conversation_history.push_back(formatted_answer_op.get());- auto create_conversation_op = ConversationManager::get_instance().create_conversation(conversation_history);- if(!create_conversation_op.ok()) {- return Option<nlohmann::json>(create_conversation_op.code(), create_conversation_op.error());- }+ auto add_conversation_op = ConversationManager::get_instance().add_conversation(conversation_history, conversation_model["conversation_collection"].get<std::string>(), conversation_id);+ if(!add_conversation_op.ok()) {+ return Option<nlohmann::json>(add_conversation_op.code(), add_conversation_op.error());+ }- auto get_conversation_op = ConversationManager::get_instance().get_conversation(create_conversation_op.get());- if(!get_conversation_op.ok()) {- return Option<nlohmann::json>(get_conversation_op.code(), get_conversation_op.error());- }+ auto get_conversation_op = ConversationManager::get_instance().get_conversation(add_conversation_op.get());+ if(!get_conversation_op.ok()) {+ return Option<nlohmann::json>(get_conversation_op.code(), get_conversation_op.error());+ }- if(exclude_fields.count("conversation_history") == 0) {- result["conversation"]["conversation_history"] = get_conversation_op.get();- }- result["conversation"]["conversation_id"] = create_conversation_op.get();+ if(exclude_fields.count("conversation_history") == 0) {+ result["conversation"]["conversation_history"] = get_conversation_op.get();}+ result["conversation"]["conversation_id"] = add_conversation_op.get();}result["facet_counts"] = nlohmann::json::array();diff --git a/src/conversation_manager.cpp b/src/conversation_manager.cppindex bf71618b..9e5ed11e 100644--- a/src/conversation_manager.cpp+++ b/src/conversation_manager.cpp@@ -1,75 +1,146 @@#include "conversation_manager.h"#include "logger.h"#include <chrono>+#include "http_client.h"+#include "core_api.h"-Option<std::string> ConversationManager::create_conversation(const nlohmann::json& conversation) {+Option<std::string> ConversationManager::add_conversation(const nlohmann::json& conversation, const std::string& conversation_collection, const std::string& id) {std::unique_lock lock(conversations_mutex);if(!conversation.is_array()) {return Option<std::string>(400, "Conversation is not an array");}- std::string conversation_id = sole::uuid4().str();- auto conversation_key = get_conversation_key(conversation_id);- nlohmann::json conversation_store_json;- conversation_store_json["id"] = conversation_id;- conversation_store_json["conversation"] = conversation;- conversation_store_json["last_updated"] = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();- conversation_store_json["ttl"] = CONVERSATION_TTL;- bool insert_op = store->insert(conversation_key, conversation_store_json.dump(0));- if(!insert_op) {- return Option<std::string>(500, "Error while inserting conversation into the store");+ if(!id.empty()) {+ auto conversation_exists = check_conversation_exists(id);+ if(!conversation_exists.ok()) {+ return Option<std::string>(conversation_exists.code(), conversation_exists.error());+ }}- conversations[conversation_id] = conversation_store_json;-- return Option<std::string>(conversation_id);-}+ std::string conversation_id = id.empty() ? sole::uuid4().str() : id;-Option<nlohmann::json> ConversationManager::get_conversation(const std::string& conversation_id) {- std::unique_lock lock(conversations_mutex);- auto conversation = conversations.find(conversation_id);- if (conversation == conversations.end()) {- return Option<nlohmann::json>(404, "Conversation not found");+ auto collection = CollectionManager::get_instance().get_collection(conversation_collection).get();+ if(!collection) {+ return Option<std::string>(404, "Conversation store collection not found");}- return Option<nlohmann::json>(conversation->second);-}+ std::string body;++ for(const auto& message : conversation) {+ if(!message.is_object()) {+ return Option<std::string>(400, "Message is not an object");+ }++ // key is role, value is message+ const auto& message_it = message.items().begin();+ if(message_it == message.items().end()) {+ return Option<std::string>(400, "Message is empty");+ }-Option<bool> ConversationManager::append_conversation(const std::string& conversation_id, const nlohmann::json& message) {- std::unique_lock lock(conversations_mutex);- auto conversation_it = conversations.find(conversation_id);- if (conversation_it == conversations.end()) {- return Option<bool>(404, "Conversation not found");+ if(!message_it.value().is_string()) {+ return Option<std::string>(400, "Role and message must be strings");+ }++ nlohmann::json message_json;+ message_json["conversation_id"] = conversation_id;+ message_json["role"] = message_it.key();+ message_json["message"] = message_it.value();+ message_json["timestamp"] = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();+ body += message_json.dump(-1) + "\n";}- if(!message.is_object() && !message.is_array()) {- return Option<bool>(400, "Message is not an object or array");+ if(!raft_server) {+ auto req = std::make_shared<http_req>();+ auto resp = std::make_shared<http_res>(nullptr);++ req->params["action"] = "emplace";+ req->params["collection"] = conversation_collection;+ req->body = body;++ auto api_res = post_import_documents(req, resp);+ if(!api_res) {+ return Option<std::string>(resp->status_code, resp->body);+ }++ conversation_mapper[conversation_id] = conversation_collection;+ return Option<std::string>(conversation_id);}- nlohmann::json conversation = conversation_it->second;- if(!message.is_array()) {- conversation["conversation"].push_back(message);- } else {- for(auto& m : message) {- conversation["conversation"].push_back(m);+ std::string leader_url = raft_server->get_leader_url();++ if(!leader_url.empty()) {+ std::string base_url = leader_url + "collections/" + conversation_collection;+ std::string res;+ std::string url = base_url + "/documents/import?action=emplace";+ std::map<std::string, std::string> res_headers;++ long status = HttpClient::post_response(url, body, res, res_headers, {}, 10*1000, true);++ if(status != 200) {+ LOG(ERROR) << "Error while creating conversation: " << res;+ LOG(ERROR) << "Status: " << status;+ return Option<std::string>(400, "Error while creating conversation");+ } else {+ conversation_mapper[conversation_id] = conversation_collection;+ return Option<std::string>(conversation_id);}+ } else {+ return Option<std::string>(500, "Leader URL is empty");}+}- conversation["last_updated"] = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();+Option<nlohmann::json> ConversationManager::get_conversation(const std::string& conversation_id) {- auto conversation_key = get_conversation_key(conversation_id);- bool insert_op = store->insert(conversation_key, conversation.dump(0));- if(!insert_op) {- return Option<bool>(500, "Error while inserting conversation into the store");+ auto collection_op = get_conversation_collection(conversation_id);+ if(!collection_op.ok()) {+ return Option<nlohmann::json>(collection_op.code(), collection_op.error());+ }+ auto collection = collection_op.get();++ nlohmann::json res;+ size_t total = 0;+ std::vector<sort_by> sort_by_vec = {{"timestamp", sort_field_const::asc}};+ auto search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, sort_by_vec, {}, 250);+ if(!search_res.ok()) {+ return Option<nlohmann::json>(400, "Error while searching conversation store: " + search_res.error());+ }+ auto search_res_json = search_res.get();+ total = search_res_json["found"].get<uint32_t>();+ if(total == 0) {+ return Option<nlohmann::json>(404, "Conversation not found");}- conversations[conversation_id] = conversation;+ res["conversation"] = nlohmann::json::array();+ for(auto& hit : search_res_json["hits"]) {+ nlohmann::json message;+ message[hit["document"]["role"]] = hit["document"]["message"];+ res["conversation"].push_back(message);+ }+ res["id"] = conversation_id;+ res["ttl"] = CONVERSATION_TTL;+ res["last_updated"] = (search_res_json["hits"].size() > 0) ? search_res_json["hits"][search_res_json["hits"].size() - 1]["document"]["timestamp"].get<uint32_t>() : 0;++ if(total > 250) {+ while(total > 0) {+ search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, sort_by_vec, {}, 250, search_res_json["page"].get<uint32_t>() + 1);+ if(!search_res.ok()) {+ return Option<nlohmann::json>(400, "Error while searching conversation store: " + search_res.error());+ }+ search_res_json = search_res.get();+ for(auto& hit : search_res_json["hits"]) {+ nlohmann::json message;+ message[hit["document"]["role"]] = hit["document"]["message"];+ res["conversation"].push_back(message);+ }+ res["last_updated"] = search_res_json["hits"][search_res_json["hits"].size() - 1]["document"]["timestamp"];+ total -= search_res_json["hits"].size();+ }+ }- return Option<bool>(true);+ return Option<nlohmann::json>(res);}-// pop front elements until the conversation is less than MAX_TOKENSOption<nlohmann::json> ConversationManager::truncate_conversation(nlohmann::json conversation, size_t limit) {if(!conversation.is_array()) {@@ -92,96 +163,148 @@ Option<nlohmann::json> ConversationManager::truncate_conversation(nlohmann::jsonOption<nlohmann::json> ConversationManager::delete_conversation(const std::string& conversation_id) {std::unique_lock lock(conversations_mutex);- auto conversation = conversations.find(conversation_id);- if (conversation == conversations.end()) {- return Option<nlohmann::json>(404, "Conversation not found");++ auto conversation_exists = check_conversation_exists(conversation_id);+ if(!conversation_exists.ok()) {+ return Option<nlohmann::json>(conversation_exists.code(), conversation_exists.error());+ }++ auto conversation_collection_op = get_conversation_collection(conversation_id);+ if(!conversation_collection_op.ok()) {+ return Option<nlohmann::json>(conversation_collection_op.code(), conversation_collection_op.error());}+ auto conversation_collection = conversation_collection_op.get()->get_name();+ if(!raft_server) {+ auto req = std::make_shared<http_req>();+ auto resp = std::make_shared<http_res>(nullptr);- Option<nlohmann::json> conversation_res(conversation->second);+ req->params["filter_by"] = "conversation_id:" + conversation_id;+ req->params["collection"] = conversation_collection;- auto conversation_key = get_conversation_key(conversation_id);- bool delete_op = store->remove(conversation_key);- if(!delete_op) {- return Option<nlohmann::json>(500, "Error while deleting conversation from the store");+ auto api_res = del_remove_documents(req, resp);+ if(!api_res) {+ return Option<nlohmann::json>(resp->status_code, resp->body);+ }++ nlohmann::json res_json;+ res_json["id"] = conversation_id;+ conversation_mapper.erase(conversation_id);+ return Option<nlohmann::json>(res_json);}- conversations.erase(conversation);- return conversation_res;-}+ auto leader_url = raft_server->get_leader_url();-Option<nlohmann::json> ConversationManager::get_all_conversations() {- std::unique_lock lock(conversations_mutex);- nlohmann::json all_conversations = nlohmann::json::array();- for(auto& conversation : conversations) {- all_conversations.push_back(conversation.second);+ if(leader_url.empty()) {+ return Option<nlohmann::json>(500, "Leader URL is empty");}- return Option<nlohmann::json>(all_conversations);-}+ std::string base_url = leader_url + "collections/" + conversation_collection;+ std::string res;+ std::string url = base_url + "/documents?filter_by=conversation_id:" + conversation_id;+ std::map<std::string, std::string> res_headers;-Option<int> ConversationManager::init(Store* store) {+ long status = HttpClient::delete_response(url, res, res_headers, 10*1000, true);++ if(status != 200) {+ LOG(ERROR) << "Error while deleting conversation: " << res;+ LOG(ERROR) << "Status: " << status;+ return Option<nlohmann::json>(400, "Error while deleting conversation");+ } else {+ nlohmann::json res_json;+ res_json["conversation_id"] = conversation_id;+ conversation_mapper.erase(conversation_id);+ return Option<nlohmann::json>(res_json);+ }+}- if(store == nullptr) {- return Option<int>(500, "Store is null");+Option<nlohmann::json> ConversationManager::get_all_conversations() {+ auto conversation_ids_op = get_conversation_ids();+ if(!conversation_ids_op.ok()) {+ return Option<nlohmann::json>(conversation_ids_op.code(), conversation_ids_op.error());+ }+ auto conversation_ids = conversation_ids_op.get();+ nlohmann::json res;+ res = nlohmann::json::array();++ for(auto& conversation_id : conversation_ids) {+ auto conversation_op = get_conversation(conversation_id);+ if(!conversation_op.ok()) {+ return Option<nlohmann::json>(conversation_op.code(), conversation_op.error());+ }+ res.push_back(conversation_op.get());}- std::unique_lock lock(conversations_mutex);- ConversationManager::store = store;-+ return Option<nlohmann::json>(res);+}- std::vector<std::string> conversation_strs;- store->scan_fill(std::string(CONVERSATION_RPEFIX) + "_", std::string(CONVERSATION_RPEFIX) + "`", conversation_strs);-- size_t loaded_conversations = 0;+Option<bool> ConversationManager::init(ReplicationState* raft_server) {- for(auto& conversation_str : conversation_strs) {- nlohmann::json conversation_json = nlohmann::json::parse(conversation_str);- if(conversation_json.count("id") == 0) {- LOG(INFO) << "key: " << conversation_str << " is missing id";- continue;- }- conversations[conversation_json["id"]] = conversation_json;- loaded_conversations++;+ if(raft_server == nullptr) {+ return Option<bool>(400, "Raft server is null");}- return Option<int>(loaded_conversations);+ this->raft_server = raft_server;++ return Option<bool>(true);}-const std::string ConversationManager::get_conversation_key(const std::string& conversation_id) {- return std::string(CONVERSATION_RPEFIX) + "_" + conversation_id;-}+void ConversationManager::clear_expired_conversations() {std::unique_lock lock(conversations_mutex);int cleared_conversations = 0;+ auto conversation_ids_op = get_conversation_ids();+ if(!conversation_ids_op.ok()) {+ LOG(ERROR) << conversation_ids_op.error();+ return;+ }- for(auto it = conversations.begin(); it != conversations.end();) {- const std::string& conversation_id = it->first;- nlohmann::json conversation_json = it->second;- if(conversation_json.count("last_updated") == 0 || conversation_json.count("ttl") == 0) {- bool delete_op = store->remove(get_conversation_key(conversation_id));- if(!delete_op) {- LOG(ERROR) << "Error while deleting conversation from the store";- }- it = conversations.erase(it);- cleared_conversations++;- continue;- }- int last_updated = conversation_json["last_updated"];- int ttl = conversation_json["ttl"];- if(last_updated + ttl < TTL_OFFSET + std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()) {- auto delete_op = store->remove(get_conversation_key(conversation_id));- if(!delete_op) {- LOG(ERROR) << "Error while deleting conversation from the store";- }- it = conversations.erase(it);- cleared_conversations++;- } else {- it++;+ auto conversation_ids = conversation_ids_op.get();+ std::vector<std::string> conversation_ids_to_delete;+ std::vector<sort_by> sort_by_vec = {{"timestamp", sort_field_const::desc}};++ for(auto& conversation_id : conversation_ids) {+ auto collection_op = get_conversation_collection(conversation_id);+ if(!collection_op.ok()) {+ LOG(ERROR) << collection_op.error();+ continue;+ }+ auto collection = collection_op.get();+ auto search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, sort_by_vec, {}, 1);+ if(!search_res.ok()) {+ LOG(ERROR) << "Error while searching conversation store: " << search_res.error();+ continue;}+ auto search_res_json = search_res.get();+ if(search_res_json["hits"].size() == 0) {+ continue;+ }+ auto last_updated = search_res_json["hits"][0]["document"]["timestamp"].get<uint32_t>();+ if(std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count() - last_updated + TTL_OFFSET > CONVERSATION_TTL) {+ conversation_ids_to_delete.push_back(conversation_id);+ }++ if(conversation_ids_to_delete.size() >= MAX_CONVERSATIONS_TO_DELETE_ONCE) {+ break;+ }+ }++ if(conversation_ids_to_delete.empty()) {+ return;+ }++ for(auto& conversation_id : conversation_ids_to_delete) {+ lock.unlock();+ auto delete_op = delete_conversation(conversation_id);+ lock.lock();+ if(!delete_op.ok()) {+ LOG(ERROR) << delete_op.error();+ continue;+ }+ cleared_conversations++;}LOG(INFO) << "Cleared " << cleared_conversations << " expired conversations";@@ -197,31 +320,56 @@ Option<nlohmann::json> ConversationManager::update_conversation(nlohmann::json creturn Option<nlohmann::json>(400, "Conversation is missing id");}- const std::string& conversation_id = conversation["id"];+ if(!conversation["id"].is_string()) {+ return Option<nlohmann::json>(400, "Conversation id must be a string");+ }- auto conversation_it = conversations.find(conversation_id);- if (conversation_it == conversations.end()) {- return Option<nlohmann::json>(404, "Conversation not found");+ if(conversation.count("conversation") == 0) {+ return Option<nlohmann::json>(400, "Conversation is missing conversation");}- auto conversation_key = get_conversation_key(conversation_id);+ if(!conversation["conversation"].is_array()) {+ return Option<nlohmann::json>(400, "Conversation history is not an array");+ }- auto actual_conversation = conversation_it->second;+ for(auto& message : conversation["conversation"]) {+ if(!message.is_object()) {+ return Option<nlohmann::json>(400, "Message is not an object");+ }- if(conversation.count("ttl") == 0) {- return Option<nlohmann::json>(400, "Only `ttl` can be updated");+ const auto& message_it = message.items().begin();+ if(message_it == message.items().end()) {+ return Option<nlohmann::json>(400, "Message is empty");+ }++ if(!message_it.value().is_string()) {+ return Option<nlohmann::json>(400, "Role and message must be strings");+ }+ }++ const std::string& conversation_id = conversation["id"];++ auto conversation_exists = check_conversation_exists(conversation_id);+ if(!conversation_exists.ok()) {+ return Option<nlohmann::json>(conversation_exists.code(), conversation_exists.error());}-- actual_conversation["ttl"] = conversation["ttl"];- bool insert_op = store->insert(conversation_key, actual_conversation.dump(0));- if(!insert_op) {- return Option<nlohmann::json>(500, "Error while inserting conversation into the store");+ auto delete_op = delete_conversation(conversation_id);+ if(!delete_op.ok()) {+ return Option<nlohmann::json>(delete_op.code(), delete_op.error());}- conversations[conversation_id] = actual_conversation;+ auto create_op = add_conversation(conversation["conversation"], conversation_id);+ if(!create_op.ok()) {+ return Option<nlohmann::json>(create_op.code(), create_op.error());+ }++ auto actual_conversation_op = get_conversation(conversation_id);+ if(!actual_conversation_op.ok()) {+ return Option<nlohmann::json>(actual_conversation_op.code(), actual_conversation_op.error());+ }- return Option<nlohmann::json>(actual_conversation);+ return Option<nlohmann::json>(actual_conversation_op.get());}void ConversationManager::run() {@@ -241,4 +389,174 @@ void ConversationManager::run() {void ConversationManager::stop() {quit = true;cv.notify_all();+}++++Option<bool> ConversationManager::validate_conversation_store_schema(Collection* collection) {+ const auto& schema = collection->get_schema();++ if(schema.count("conversation_id") == 0) {+ return Option<bool>(400, "Schema is missing `conversation_id` field");+ }++ if(schema.count("role") == 0) {+ return Option<bool>(400, "Schema is missing `role` field");+ }++ if(schema.count("message") == 0) {+ return Option<bool>(400, "Schema is missing `message` field");+ }++ if(schema.count("timestamp") == 0) {+ return Option<bool>(400, "Schema is missing `timestamp` field");+ }++ if(schema.at("conversation_id").type != field_types::STRING) {+ return Option<bool>(400, "`conversation_id` field must be a string");+ }++ if(!schema.at("conversation_id").facet) {+ return Option<bool>(400, "`conversation_id` field must be a facet");+ }++ if(schema.at("role").type != field_types::STRING) {+ return Option<bool>(400, "`role` field must be a string");+ }++ if(schema.at("message").type != field_types::STRING) {+ return Option<bool>(400, "`message` field must be a string");+ }++ if(schema.at("timestamp").type != field_types::INT32) {+ return Option<bool>(400, "`timestamp` field must be an integer");+ }++ if(!schema.at("timestamp").sort) {+ return Option<bool>(400, "`timestamp` field must be a sort field");+ }++ return Option<bool>(true);+}+++Option<bool> ConversationManager::check_conversation_exists(const std::string& conversation_id) {+ auto collection_op = get_conversation_collection(conversation_id);+ if(!collection_op.ok()) {+ return Option<bool>(collection_op.code(), collection_op.error());+ }++ auto collection = collection_op.get();++ nlohmann::json res;+ size_t total = 0;+ auto search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, {}, {}, 250);+ if(!search_res.ok()) {+ return Option<bool>(400, "Error while searching conversation store: " + search_res.error());+ }+ auto search_res_json = search_res.get();+ total = search_res_json["found"].get<uint32_t>();+ if(total == 0) {+ return Option<bool>(404, "Conversation not found");+ }++ return Option<bool>(true);+}++Option<std::unordered_set<std::string>> ConversationManager::get_conversation_ids() {+ std::unordered_set<std::string> conversation_ids;+ for(const auto& conversation_id : conversation_mapper) {+ conversation_ids.insert(conversation_id.first);+ }++ return Option<std::unordered_set<std::string>>(conversation_ids);+}++Option<bool> ConversationManager::add_conversation_collection(const std::string& collection) {+ std::unique_lock lock(conversations_mutex);+ if(conversation_collection_map.count(collection) > 0) {+ conversation_collection_map[collection]++;+ } else {+ auto collection_ptr = CollectionManager::get_instance().get_collection(collection).get();+ if(!collection_ptr) {+ return Option<bool>(404, "Collection not found");+ }++ auto validate_op = validate_conversation_store_schema(collection_ptr);+ if(!validate_op.ok()) {+ return Option<bool>(validate_op.code(), validate_op.error());+ }++ conversation_collection_map[collection] = 1;+ }++ return Option<bool>(true);+}++Option<bool> ConversationManager::remove_conversation_collection(const std::string& collection) {+ std::unique_lock lock(conversations_mutex);+ if(conversation_collection_map.count(collection) == 0) {+ return Option<bool>(404, "Collection not found");+ }++ conversation_collection_map[collection]--;++ if(conversation_collection_map[collection] == 0) {+ std::vector<std::string> conversations_to_delete;+ for(auto& conversation : conversation_mapper) {+ if(conversation.second == collection) {+ conversations_to_delete.push_back(conversation.first);+ }+ }+ for(auto conversation_id : conversations_to_delete) {+ conversation_mapper.erase(conversation_id);+ }+ conversation_collection_map.erase(collection);+ }++ return Option<bool>(true);+}++Option<Collection*> ConversationManager::get_conversation_collection(const std::string& conversation_id) {++ if(conversation_mapper.count(conversation_id) > 0) {+ auto collection = CollectionManager::get_instance().get_collection(conversation_mapper[conversation_id]).get();+ if(collection) {+ return Option<Collection*>(collection);+ }+ }++ for(auto& collection : conversation_collection_map) {+ auto collection_ptr = CollectionManager::get_instance().get_collection(collection.first).get();+ if(!collection_ptr) {+ continue;+ }++ auto search_res = collection_ptr->search("*", {}, "conversation_id:" + conversation_id, {}, {}, {}, 1);+ if(!search_res.ok()) {+ continue;+ }++ auto search_res_json = search_res.get();+ if(search_res_json["found"].get<uint32_t>() > 0) {+ conversation_mapper[conversation_id] = collection.first;+ return Option<Collection*>(collection_ptr);+ }+ }++ return Option<Collection*>(404, "Conversation not found");+}++Option<bool> ConversationManager::validate_conversation_store_collection(const std::string& collection) {+ auto collection_ptr = CollectionManager::get_instance().get_collection(collection).get();+ if(!collection_ptr) {+ return Option<bool>(404, "Collection not found");+ }++ auto validate_op = validate_conversation_store_schema(collection_ptr);+ if(!validate_op.ok()) {+ return Option<bool>(validate_op.code(), validate_op.error());+ }++ return Option<bool>(true);}\ No newline at end of filediff --git a/src/conversation_model.cpp b/src/conversation_model.cppindex b5a2b678..068deadf 100644--- a/src/conversation_model.cpp+++ b/src/conversation_model.cpp@@ -28,6 +28,15 @@ Option<bool> ConversationModel::validate_model(const nlohmann::json& model_confireturn Option<bool>(400, "Property `max_bytes` is not provided or not a positive integer.");}+ if(model_config.count("conversation_collection") == 0 || !model_config["conversation_collection"].is_string()) {+ return Option<bool>(400, "Property `conversation_collection` is not provided or not a string.");+ }++ auto validate_converson_collection_op = ConversationManager::get_instance().validate_conversation_store_collection(model_config["conversation_collection"].get<std::string>());+ if(!validate_converson_collection_op.ok()) {+ return Option<bool>(400, validate_converson_collection_op.error());+ }+const std::string model_namespace = get_model_namespace(model_config["model_name"].get<std::string>());if(model_namespace == "openai") {return OpenAIConversationModel::validate_model(model_config);diff --git a/src/conversation_model_manager.cpp b/src/conversation_model_manager.cppindex 4cd81511..57aef3c9 100644--- a/src/conversation_model_manager.cpp+++ b/src/conversation_model_manager.cpp@@ -1,5 +1,6 @@#include "conversation_model_manager.h"#include "conversation_model.h"+#include "conversation_manager.h"Option<nlohmann::json> ConversationModelManager::get_model(const std::string& model_id) {std::shared_lock lock(models_mutex);@@ -29,6 +30,8 @@ Option<nlohmann::json> ConversationModelManager::add_model(nlohmann::json model,models[model_id] = model;++ ConversationManager::get_instance().add_conversation_collection(model["conversation_collection"]);return Option<nlohmann::json>(model);}@@ -43,7 +46,8 @@ Option<nlohmann::json> ConversationModelManager::delete_model(const std::string&auto model_key = get_model_key(model_id);bool delete_op = store->remove(model_key);-++ ConversationManager::get_instance().remove_conversation_collection(model["conversation_collection"]);models.erase(it);return Option<nlohmann::json>(model);}@@ -78,6 +82,11 @@ Option<nlohmann::json> ConversationModelManager::update_model(const std::string&return Option<nlohmann::json>(500, "Error while inserting model into the store");}+ if(it->second["conversation_collection"] != model["conversation_collection"]) {+ ConversationManager::get_instance().remove_conversation_collection(it->second["conversation_collection"]);+ ConversationManager::get_instance().add_conversation_collection(model["conversation_collection"]);+ }+models[model_id] = model;return Option<nlohmann::json>(model);@@ -95,6 +104,18 @@ Option<int> ConversationModelManager::init(Store* store) {nlohmann::json model_json = nlohmann::json::parse(model_str);std::string model_id = model_json["id"];models[model_id] = model_json;+ if(model_json.count("conversation_collection") == 0) {+ auto delete_op = delete_model(model_id);+ if(!delete_op.ok()) {+ return Option<int>(delete_op.code(), delete_op.error());+ }+ auto migrate_op = migrate_model(model_json);+ if(!migrate_op.ok()) {+ return Option<int>(migrate_op.code(), migrate_op.error());+ }+ model_json = migrate_op.get();+ }+ ConversationManager::get_instance().add_conversation_collection(model_json["conversation_collection"]);loaded_models++;}@@ -103,4 +124,65 @@ Option<int> ConversationModelManager::init(Store* store) {const std::string ConversationModelManager::get_model_key(const std::string& model_id) {return std::string(MODEL_KEY_PREFIX) + "_" + model_id;+}+++Option<Collection*> ConversationModelManager::get_default_conversation_collection() {+ int64_t time_epoch;+ if(DEFAULT_CONVERSATION_COLLECTION_SUFFIX == 0) {+ time_epoch = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();+ DEFAULT_CONVERSATION_COLLECTION_SUFFIX = time_epoch;+ } else {+ time_epoch = DEFAULT_CONVERSATION_COLLECTION_SUFFIX;+ }++ std::string collection_id = "default_conversation_history_" + std::to_string(time_epoch);+ nlohmann::json schema_json = R"({+ "fields": [+ {+ "name": "conversation_id",+ "type": "string",+ "facet": true+ },+ {+ "name": "role",+ "type": "string"+ },+ {+ "name": "message",+ "type": "string"+ },+ {+ "name": "timestamp",+ "type": "int32",+ "sort": true+ }+ ]+ })"_json;+ schema_json["name"] = collection_id;++ auto get_res = CollectionManager::get_instance().get_collection(collection_id).get();+ if(get_res) {+ return Option<Collection*>(get_res);+ }++ auto create_res = CollectionManager::get_instance().create_collection(schema_json);+ if(!create_res.ok()) {+ return Option<Collection*>(create_res.code(), create_res.error());+ }+ return Option<Collection*>(create_res.get());+}++Option<nlohmann::json> ConversationModelManager::migrate_model(nlohmann::json model) {+ auto model_id = model["id"];+ auto default_collection = get_default_conversation_collection();+ if(!default_collection.ok()) {+ return Option<nlohmann::json>(default_collection.code(), default_collection.error());+ }+ model["conversation_collection"] = default_collection.get()->get_name();+ auto add_res = add_model(model, model_id);+ if(!add_res.ok()) {+ return Option<nlohmann::json>(add_res.code(), add_res.error());+ }+ return Option<nlohmann::json>(model);}\ No newline at end of filediff --git a/src/core_api.cpp b/src/core_api.cppindex 3a9dac74..61920ff0 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -866,43 +866,26 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_pStringUtils::split(req->params["exclude_fields"], exclude_fields, ",");bool exclude_conversation_history = std::find(exclude_fields.begin(), exclude_fields.end(), "conversation_history") != exclude_fields.end();- if(conversation_history) {- std::string conversation_id = orig_req_params["conversation_id"];- ConversationManager::get_instance().append_conversation(conversation_id, formatted_question_op.get());- ConversationManager::get_instance().append_conversation(conversation_id, formatted_answer_op.get());- auto get_conversation_op = ConversationManager::get_instance().get_conversation(conversation_id);- if(!get_conversation_op.ok()) {- res->set_400(get_conversation_op.error());- return false;- }-- auto conversation_history = get_conversation_op.get();- if(!exclude_conversation_history) {- response["conversation"]["conversation_history"] = conversation_history;- } else {- response["conversation"]["conversation_id"] = conversation_id;- }- } else {- nlohmann::json conversation_history = nlohmann::json::array();- conversation_history.push_back(formatted_question_op.get());- conversation_history.push_back(formatted_answer_op.get());-- auto create_conversation_op = ConversationManager::get_instance().create_conversation(conversation_history);- if(!create_conversation_op.ok()) {- res->set_400(create_conversation_op.error());- return false;- }+ nlohmann::json conversation_history = nlohmann::json::array();+ conversation_history.push_back(formatted_question_op.get());+ conversation_history.push_back(formatted_answer_op.get());+ std::string conversation_id = conversation_history ? orig_req_params["conversation_id"] : "";++ auto add_conversation_op = ConversationManager::get_instance().add_conversation(conversation_history, conversation_model["conversation_collection"], conversation_id);+ if(!add_conversation_op.ok()) {+ res->set_400(add_conversation_op.error());+ return false;+ }- auto get_conversation_op = ConversationManager::get_instance().get_conversation(create_conversation_op.get());- if(!get_conversation_op.ok()) {- res->set_400(get_conversation_op.error());- return false;- }- if(!exclude_conversation_history) {- response["conversation"]["conversation_history"] = get_conversation_op.get();- }- response["conversation"]["conversation_id"] = create_conversation_op.get();+ auto get_conversation_op = ConversationManager::get_instance().get_conversation(add_conversation_op.get());+ if(!get_conversation_op.ok()) {+ res->set_400(get_conversation_op.error());+ return false;+ }+ if(!exclude_conversation_history) {+ response["conversation"]["conversation_history"] = get_conversation_op.get();}+ response["conversation"]["conversation_id"] = add_conversation_op.get();}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex c0379461..dd3fd032 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -450,13 +450,6 @@ int run_server(const Config & config, const std::string & version, void (*master}EmbedderManager::set_model_dir(config.get_data_dir() + "/models");- auto conversations_init = ConversationManager::get_instance().init(&store);-- if(!conversations_init.ok()) {- LOG(INFO) << "Failed to initialize conversation manager: " << conversations_init.error();- } else {- LOG(INFO) << "Loaded " << conversations_init.get() << "(s) conversations.";- }auto conversation_models_init = ConversationModelManager::init(&store);@@ -475,6 +468,12 @@ int run_server(const Config & config, const std::string & version, void (*masternum_collections_parallel_load,config.get_num_documents_parallel_load());+ auto conversations_init = ConversationManager::get_instance().init(&replication_state);++ if(!conversations_init.ok()) {+ LOG(INFO) << "Failed to initialize conversation manager: " << conversations_init.error();+ }+std::thread raft_thread([&replication_state, &config, &state_dir,&app_thread_pool, &server_thread_pool, &replication_thread_pool, batch_indexer]() {diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex c4a0301b..5613aeeb 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -31,7 +31,31 @@ protected:collectionManager.load(8, 1000);ConversationModelManager::init(store);- ConversationManager::get_instance().init(store);+ nlohmann::json schema_json = R"({+ "name": "conversation_store",+ "fields": [+ {+ "name": "conversation_id",+ "type": "string",+ "facet": true+ },+ {+ "name": "role",+ "type": "string"+ },+ {+ "name": "message",+ "type": "string"+ },+ {+ "name": "timestamp",+ "type": "int32",+ "sort": true+ }+ ]+ })"_json;++ collectionManager.create_collection(schema_json);}virtual void SetUp() {@@ -2873,7 +2897,8 @@ TEST_F(CollectionVectorTest, TestQAConversation) {auto conversation_model_config = R"({"model_name": "openai/gpt-3.5-turbo",- "max_bytes: 1000+ "max_bytes: 1000,+ "conversation_collection": "conversation_store",})"_json;conversation_model_config["api_key"] = api_key;@@ -3320,7 +3345,8 @@ TEST_F(CollectionVectorTest, InvalidMultiSearchConversation) {auto conversation_model_config = R"({"model_name": "openai/gpt-3.5-turbo",- "max_bytes": 1000+ "max_bytes": 1000,+ "conversation_collection": "conversation_store"})"_json;conversation_model_config["api_key"] = api_key;@@ -3396,6 +3422,28 @@ TEST_F(CollectionVectorTest, InvalidMultiSearchConversation) {ASSERT_EQ(res_json["message"], "`conversation` cannot be used in POST body. Please set `conversation` as a query parameter in the request, instead of inside the POST body");}+TEST_F(CollectionVectorTest, TestMigratingConversationModel) {+ auto conversation_model_config = R"({+ "model_name": "openai/gpt-3.5-turbo",+ "max_bytes": 1000,+ "conversation_collection": "conversation_store"+ })"_json;++ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";+ return;+ }++ auto api_key = std::string(std::getenv("api_key"));++ auto migrate_res = ConversationModelManager::migrate_model(conversation_model_config);+ ASSERT_TRUE(migrate_res.ok());+ auto migrated_model = migrate_res.get();+ ASSERT_TRUE(migrated_model.count("conversation_collection") == 1);++ auto collection = CollectionManager::get_instance().get_collection("conversation_store").get();+ ASSERT_TRUE(collection != nullptr);+}TEST_F(CollectionVectorTest, TestVectorQueryQs) {diff --git a/test/conversation_test.cpp b/test/conversation_test.cppindex d8546945..3f6c2400 100644--- a/test/conversation_test.cpp+++ b/test/conversation_test.cpp@@ -4,34 +4,62 @@class ConversationTest : public ::testing::Test {protected:+ CollectionManager & collectionManager = CollectionManager::get_instance();+ Store* store;+ std::atomic<bool> quit = false;void SetUp() override {std::string state_dir_path = "/tmp/typesense_test/conversation_test";system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());+store = new Store(state_dir_path);- ConversationManager::get_instance().init(store);+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);++ nlohmann::json schema_json = R"({+ "name": "conversation_store",+ "fields": [+ {+ "name": "conversation_id",+ "type": "string",+ "facet": true+ },+ {+ "name": "role",+ "type": "string"+ },+ {+ "name": "message",+ "type": "string"+ },+ {+ "name": "timestamp",+ "type": "int32",+ "sort": true+ }+ ]+ })"_json;++ collectionManager.create_collection(schema_json);+ ConversationManager::get_instance().add_conversation_collection("conversation_store");}void TearDown() override {- auto conversations = ConversationManager::get_instance().get_all_conversations();- for (auto& conversation : conversations.get()) {- ConversationManager::get_instance().delete_conversation(conversation["id"]);- }+ ConversationManager::get_instance().remove_conversation_collection("conversation_store");+ collectionManager.dispose();delete store;}-- Store* store;};TEST_F(ConversationTest, CreateConversation) {nlohmann::json conversation = nlohmann::json::array();- auto create_res = ConversationManager::get_instance().create_conversation(conversation);+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");ASSERT_TRUE(create_res.ok());}TEST_F(ConversationTest, CreateConversationInvalidType) {nlohmann::json conversation = nlohmann::json::object();- auto create_res = ConversationManager::get_instance().create_conversation(conversation);+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");ASSERT_FALSE(create_res.ok());ASSERT_EQ(create_res.code(), 400);ASSERT_EQ(create_res.error(), "Conversation is not an array");@@ -49,14 +77,14 @@ TEST_F(ConversationTest, AppendConversation) {nlohmann::json message = nlohmann::json::object();message["user"] = "Hello";conversation.push_back(message);- auto create_res = ConversationManager::get_instance().create_conversation(conversation);+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");ASSERT_TRUE(create_res.ok());std::string conversation_id = create_res.get();- auto append_res = ConversationManager::get_instance().append_conversation(conversation_id, message);+ auto append_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store", conversation_id);ASSERT_TRUE(append_res.ok());- ASSERT_EQ(append_res.get(), true);+ ASSERT_EQ(append_res.get(), conversation_id);auto get_res = ConversationManager::get_instance().get_conversation(conversation_id);@@ -73,32 +101,36 @@ TEST_F(ConversationTest, AppendInvalidConversation) {nlohmann::json conversation = nlohmann::json::array();nlohmann::json message = nlohmann::json::object();message["user"] = "Hello";- auto create_res = ConversationManager::get_instance().create_conversation(conversation);+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");ASSERT_TRUE(create_res.ok());std::string conversation_id = create_res.get();message = "invalid";- auto append_res = ConversationManager::get_instance().append_conversation(conversation_id, message);+ auto append_res = ConversationManager::get_instance().add_conversation(message, "conversation_store", conversation_id);ASSERT_FALSE(append_res.ok());ASSERT_EQ(append_res.code(), 400);- ASSERT_EQ(append_res.error(), "Message is not an object or array");+ ASSERT_EQ(append_res.error(), "Conversation is not an array");}TEST_F(ConversationTest, DeleteConversation) {nlohmann::json conversation = nlohmann::json::array();- auto create_res = ConversationManager::get_instance().create_conversation(conversation);+ nlohmann::json message = nlohmann::json::object();+ message["user"] = "Hello";+ conversation.push_back(message);+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");ASSERT_TRUE(create_res.ok());std::string conversation_id = create_res.get();+ LOG(INFO) << conversation_id;auto delete_res = ConversationManager::get_instance().delete_conversation(conversation_id);+ LOG(INFO) << delete_res.error();ASSERT_TRUE(delete_res.ok());auto delete_res_json = delete_res.get();ASSERT_EQ(delete_res_json["id"], conversation_id);- ASSERT_TRUE(delete_res_json["conversation"].is_array());auto get_res = ConversationManager::get_instance().get_conversation(conversation_id);ASSERT_FALSE(get_res.ok());@@ -155,7 +187,7 @@ TEST_F(ConversationTest, TestConversationExpire) {nlohmann::json message = nlohmann::json::object();message["user"] = "Hello";conversation.push_back(message);- auto create_res = ConversationManager::get_instance().create_conversation(conversation);+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");ASSERT_TRUE(create_res.ok());std::string conversation_id = create_res.get();@@ -169,7 +201,9 @@ TEST_F(ConversationTest, TestConversationExpire) {ASSERT_EQ(get_res.get()["conversation"].size(), 1);ConversationManager::get_instance()._set_ttl_offset(24 * 60 * 60 * 2);+ LOG(INFO) << "Clearing expired conversations";ConversationManager::get_instance().clear_expired_conversations();+ LOG(INFO) << "Cleared expired conversations";get_res = ConversationManager::get_instance().get_conversation(conversation_id);ASSERT_FALSE(get_res.ok());@@ -179,4 +213,143 @@ TEST_F(ConversationTest, TestConversationExpire) {ConversationManager::get_instance()._set_ttl_offset(0);}+TEST_F(ConversationTest, TestRemoveConversationCollection) {+ nlohmann::json schema_json = R"({+ "name": "conversation_store2",+ "fields": [+ {+ "name": "conversation_id",+ "type": "string",+ "facet": true+ },+ {+ "name": "role",+ "type": "string"+ },+ {+ "name": "message",+ "type": "string"+ },+ {+ "name": "timestamp",+ "type": "int32",+ "sort": true+ }+ ]+ })"_json;+ LOG(INFO) << "Creating collection";+ collectionManager.create_collection(schema_json);+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");+ LOG(INFO) << "Collection created";+ nlohmann::json conversation = nlohmann::json::array();+ nlohmann::json message = nlohmann::json::object();+ message["user"] = "Hello";+ conversation.push_back(message);++ LOG(INFO) << "Adding conversation";+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store2");+ ASSERT_TRUE(create_res.ok());++ LOG(INFO) << "Getting conversation";+ auto get_res = ConversationManager::get_instance().get_conversation(create_res.get());+ ASSERT_TRUE(get_res.ok());+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);+++ LOG(INFO) << "Removing collection";+ auto remove_res = ConversationManager::get_instance().remove_conversation_collection("conversation_store2");+ ASSERT_TRUE(remove_res.ok());++ LOG(INFO) << "Getting conversation";+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());+ ASSERT_FALSE(get_res.ok());+ ASSERT_EQ(get_res.code(), 404);+ ASSERT_EQ(get_res.error(), "Conversation not found");++ LOG(INFO) << "Adding collection again";+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");+++ LOG(INFO) << "Adding conversation";+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());+ ASSERT_TRUE(get_res.ok());+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);++ LOG(INFO) << "Dropping collection";+ collectionManager.drop_collection("conversation_store2");++ LOG(INFO) << "Getting conversation";+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());+ ASSERT_FALSE(get_res.ok());+ ASSERT_EQ(get_res.code(), 404);+ ASSERT_EQ(get_res.error(), "Conversation not found");++ LOG(INFO) << "TestRemoveConversationCollection done";+}++TEST_F(ConversationTest, TestMultipleRefConversationCollection) {+ nlohmann::json schema_json = R"({+ "name": "conversation_store2",+ "fields": [+ {+ "name": "conversation_id",+ "type": "string",+ "facet": true+ },+ {+ "name": "role",+ "type": "string"+ },+ {+ "name": "message",+ "type": "string"+ },+ {+ "name": "timestamp",+ "type": "int32",+ "sort": true+ }+ ]+ })"_json;++ collectionManager.create_collection(schema_json);+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");++ nlohmann::json conversation = nlohmann::json::array();+ nlohmann::json message = nlohmann::json::object();+ message["user"] = "Hello";+ conversation.push_back(message);++ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store2");+ ASSERT_TRUE(create_res.ok());++ auto get_res = ConversationManager::get_instance().get_conversation(create_res.get());+ ASSERT_TRUE(get_res.ok());+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);++ auto remove_res = ConversationManager::get_instance().remove_conversation_collection("conversation_store2");+ ASSERT_TRUE(remove_res.ok());+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());+ ASSERT_TRUE(get_res.ok());+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);+}++TEST_F(ConversationTest, TestInvalidConversationCollection) {+ nlohmann::json schema_json = R"({+ "name": "conversation_store2",+ "fields": [+ {+ "name": "lorem",+ "type": "string"+ }+ ]+ })"_json;++ collectionManager.create_collection(schema_json);+ auto res = ConversationManager::get_instance().add_conversation_collection("conversation_store2");+ ASSERT_FALSE(res.ok());+ ASSERT_EQ(res.code(), 400);+ ASSERT_EQ(res.error(), "Schema is missing `conversation_id` field");+}diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 87f884e2..abae043f 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -29,7 +29,32 @@ protected:collectionManager.load(8, 1000);ConversationModelManager::init(store);- ConversationManager::get_instance().init(store);+ nlohmann::json schema_json = R"({+ "name": "conversation_store",+ "fields": [+ {+ "name": "conversation_id",+ "type": "string",+ "facet": true+ },+ {+ "name": "role",+ "type": "string"+ },+ {+ "name": "message",+ "type": "string"+ },+ {+ "name": "timestamp",+ "type": "int32",+ "sort": true+ }+ ]+ })"_json;++ collectionManager.create_collection(schema_json);+ ConversationManager::get_instance().add_conversation_collection("conversation_store");}virtual void SetUp() {@@ -1389,7 +1414,8 @@ TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) {TEST_F(CoreAPIUtilsTest, TestConversationModels) {nlohmann::json model_config = R"({"model_name": "openai/gpt-3.5-turbo",- "max_bytes": 10000+ "max_bytes": 10000,+ "conversation_collection": "conversation_store"})"_json;EmbedderManager::set_model_dir("/tmp/typesense_test/models");@@ -1430,6 +1456,7 @@ TEST_F(CoreAPIUtilsTest, TestConversationModels) {TEST_F(CoreAPIUtilsTest, TestInvalidConversationModels) {// test with no model_namenlohmann::json model_config = R"({+ "conversation_collection": "conversation_store"})"_json;if (std::getenv("api_key") == nullptr) {@@ -1517,6 +1544,25 @@ TEST_F(CoreAPIUtilsTest, TestInvalidConversationModels) {ASSERT_EQ(400, resp->status_code);ASSERT_EQ("Property `max_bytes` must be a positive number.", nlohmann::json::parse(resp->body)["message"]);++ model_config["max_bytes"] = 10000;+ model_config["conversation_collection"] = 123;++ // test with conversation_collection as integer+ req->body = model_config.dump();+ post_conversation_model(req, resp);++ ASSERT_EQ(400, resp->status_code);+ ASSERT_EQ("Property `conversation_collection` is not provided or not a string.", nlohmann::json::parse(resp->body)["message"]);++ // test with conversation_collection as empty string+ model_config["conversation_collection"] = "";++ req->body = model_config.dump();+ post_conversation_model(req, resp);++ ASSERT_EQ(400, resp->status_code);+ ASSERT_EQ("Collection not found", nlohmann::json::parse(resp->body)["message"]);}TEST_F(CoreAPIUtilsTest, DeleteNonExistingDoc) {