Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Use collections to store conversations (#1723)

Ozan Armağan1b4e08b29def
diff --git a/include/conversation_manager.h b/include/conversation_manager.h
index 3cc5e74f..ef565901 100644
--- a/include/conversation_manager.h
+++ b/include/conversation_manager.h
@@ -2,12 +2,14 @@
#include <string>
#include <unordered_map>
+#include <unordered_set>
#include <shared_mutex>
#include <mutex>
#include <json.hpp>
#include "option.h"
#include "store.h"
#include "sole.hpp"
+#include "collection_manager.h"
class ConversationManager {
@@ -21,35 +23,39 @@ class ConversationManager {
static ConversationManager instance;
return instance;
}
- Option<std::string> create_conversation(const nlohmann::json& conversation);
+ Option<std::string> add_conversation(const nlohmann::json& conversation, const std::string& conversation_collection, const std::string& id = "");
Option<nlohmann::json> get_conversation(const std::string& conversation_id);
- Option<bool> append_conversation(const std::string& conversation_id, const nlohmann::json& message);
- Option<nlohmann::json> truncate_conversation(nlohmann::json conversation, size_t limit);
+ static Option<nlohmann::json> truncate_conversation(nlohmann::json conversation, size_t limit);
Option<nlohmann::json> update_conversation(nlohmann::json conversation);
Option<nlohmann::json> delete_conversation(const std::string& conversation_id);
+ Option<bool> check_conversation_exists(const std::string& conversation_id);
+ Option<std::unordered_set<std::string>> get_conversation_ids();
Option<nlohmann::json> get_all_conversations();
static constexpr size_t MAX_TOKENS = 3000;
- Option<int> init(Store* store);
+ Option<bool> init(ReplicationState* raft_server);
void clear_expired_conversations();
void run();
void stop();
void _set_ttl_offset(size_t offset) {
TTL_OFFSET = offset;
}
+
+ Option<bool> validate_conversation_store_schema(Collection* collection);
+ Option<bool> validate_conversation_store_collection(const std::string& collection);
+ Option<bool> add_conversation_collection(const std::string& collection);
+ Option<bool> remove_conversation_collection(const std::string& collection);
+ Option<Collection*> get_conversation_collection(const std::string& conversation_id);
private:
ConversationManager() {}
- std::unordered_map<std::string, nlohmann::json> conversations;
std::mutex conversations_mutex;
-
- static constexpr char* CONVERSATION_RPEFIX = "$CNVP";
- Store* store;
-
- static const std::string get_conversation_key(const std::string& conversation_id);
-
+ ReplicationState* raft_server;
static constexpr size_t CONVERSATION_TTL = 60 * 60 * 24;
size_t TTL_OFFSET = 0;
+ size_t MAX_CONVERSATIONS_TO_DELETE_ONCE = 5;
std::atomic<bool> quit = false;
std::condition_variable cv;
+ std::unordered_map<std::string, uint32_t> conversation_collection_map;
+ std::unordered_map<std::string, std::string> conversation_mapper;
};
\ No newline at end of file
diff --git a/include/conversation_model_manager.h b/include/conversation_model_manager.h
index 5782e884..2e43ee16 100644
--- a/include/conversation_model_manager.h
+++ b/include/conversation_model_manager.h
@@ -7,6 +7,7 @@
#include <option.h>
#include "store.h"
#include "sole.hpp"
+#include "collection.h"
class ConversationModelManager
{
@@ -21,16 +22,17 @@ class ConversationModelManager
static Option<nlohmann::json> delete_model(const std::string& model_id);
static Option<nlohmann::json> update_model(const std::string& model_id, nlohmann::json model);
static Option<nlohmann::json> get_all_models();
-
-
static Option<int> init(Store* store);
+ static Option<nlohmann::json> migrate_model(nlohmann::json model);
private:
static inline std::unordered_map<std::string, nlohmann::json> models;
static inline std::shared_mutex models_mutex;
static constexpr char* MODEL_NEXT_ID = "$CVMN";
static constexpr char* MODEL_KEY_PREFIX = "$CVMP";
-
+ static inline int64_t DEFAULT_CONVERSATION_COLLECTION_SUFFIX = 0;
static inline Store* store;
static const std::string get_model_key(const std::string& model_id);
+ static Option<Collection*> get_default_conversation_collection();
+
};
\ No newline at end of file
diff --git a/include/core_api.h b/include/core_api.h
index 780f7b3a..ebdf1062 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -213,4 +213,4 @@ bool del_conversation_model(const std::shared_ptr<http_req>& req, const std::sha
bool get_conversation_models(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
-bool put_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
\ No newline at end of file
+bool put_conversation_model(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
diff --git a/src/collection.cpp b/src/collection.cpp
index 43ddd06d..6d306b26 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -77,6 +77,8 @@ Collection::~Collection() {
VQModelManager::get_instance().delete_model(vq_model->get_model_name());
}
}
+
+ ConversationManager::get_instance().remove_conversation_collection(name);
}
uint32_t Collection::get_next_seq_id() {
@@ -2791,39 +2793,24 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
return Option<nlohmann::json>(formatted_answer_op.code(), formatted_answer_op.error());
}
- if(has_conversation_history) {
- ConversationManager::get_instance().append_conversation(conversation_id, formatted_question_op.get());
- ConversationManager::get_instance().append_conversation(conversation_id, formatted_answer_op.get());
- auto get_conversation_op = ConversationManager::get_instance().get_conversation(conversation_id);
- if(!get_conversation_op.ok()) {
- return Option<nlohmann::json>(get_conversation_op.code(), get_conversation_op.error());
- }
-
- auto conversation_history = get_conversation_op.get();
- if(exclude_fields.count("conversation_history") == 0) {
- result["conversation"]["conversation_history"] = conversation_history;
- }
- result["conversation"]["conversation_id"] = conversation_id;
- } else {
- nlohmann::json conversation_history = nlohmann::json::array();
- conversation_history.push_back(formatted_question_op.get());
- conversation_history.push_back(formatted_answer_op.get());
+ nlohmann::json conversation_history = nlohmann::json::array();
+ conversation_history.push_back(formatted_question_op.get());
+ conversation_history.push_back(formatted_answer_op.get());
- auto create_conversation_op = ConversationManager::get_instance().create_conversation(conversation_history);
- if(!create_conversation_op.ok()) {
- return Option<nlohmann::json>(create_conversation_op.code(), create_conversation_op.error());
- }
+ auto add_conversation_op = ConversationManager::get_instance().add_conversation(conversation_history, conversation_model["conversation_collection"].get<std::string>(), conversation_id);
+ if(!add_conversation_op.ok()) {
+ return Option<nlohmann::json>(add_conversation_op.code(), add_conversation_op.error());
+ }
- auto get_conversation_op = ConversationManager::get_instance().get_conversation(create_conversation_op.get());
- if(!get_conversation_op.ok()) {
- return Option<nlohmann::json>(get_conversation_op.code(), get_conversation_op.error());
- }
+ auto get_conversation_op = ConversationManager::get_instance().get_conversation(add_conversation_op.get());
+ if(!get_conversation_op.ok()) {
+ return Option<nlohmann::json>(get_conversation_op.code(), get_conversation_op.error());
+ }
- if(exclude_fields.count("conversation_history") == 0) {
- result["conversation"]["conversation_history"] = get_conversation_op.get();
- }
- result["conversation"]["conversation_id"] = create_conversation_op.get();
+ if(exclude_fields.count("conversation_history") == 0) {
+ result["conversation"]["conversation_history"] = get_conversation_op.get();
}
+ result["conversation"]["conversation_id"] = add_conversation_op.get();
}
result["facet_counts"] = nlohmann::json::array();
diff --git a/src/conversation_manager.cpp b/src/conversation_manager.cpp
index bf71618b..9e5ed11e 100644
--- a/src/conversation_manager.cpp
+++ b/src/conversation_manager.cpp
@@ -1,75 +1,146 @@
#include "conversation_manager.h"
#include "logger.h"
#include <chrono>
+#include "http_client.h"
+#include "core_api.h"
-Option<std::string> ConversationManager::create_conversation(const nlohmann::json& conversation) {
+Option<std::string> ConversationManager::add_conversation(const nlohmann::json& conversation, const std::string& conversation_collection, const std::string& id) {
std::unique_lock lock(conversations_mutex);
if(!conversation.is_array()) {
return Option<std::string>(400, "Conversation is not an array");
}
- std::string conversation_id = sole::uuid4().str();
- auto conversation_key = get_conversation_key(conversation_id);
- nlohmann::json conversation_store_json;
- conversation_store_json["id"] = conversation_id;
- conversation_store_json["conversation"] = conversation;
- conversation_store_json["last_updated"] = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
- conversation_store_json["ttl"] = CONVERSATION_TTL;
- bool insert_op = store->insert(conversation_key, conversation_store_json.dump(0));
- if(!insert_op) {
- return Option<std::string>(500, "Error while inserting conversation into the store");
+ if(!id.empty()) {
+ auto conversation_exists = check_conversation_exists(id);
+ if(!conversation_exists.ok()) {
+ return Option<std::string>(conversation_exists.code(), conversation_exists.error());
+ }
}
- conversations[conversation_id] = conversation_store_json;
-
- return Option<std::string>(conversation_id);
-}
+ std::string conversation_id = id.empty() ? sole::uuid4().str() : id;
-Option<nlohmann::json> ConversationManager::get_conversation(const std::string& conversation_id) {
- std::unique_lock lock(conversations_mutex);
- auto conversation = conversations.find(conversation_id);
- if (conversation == conversations.end()) {
- return Option<nlohmann::json>(404, "Conversation not found");
+ auto collection = CollectionManager::get_instance().get_collection(conversation_collection).get();
+ if(!collection) {
+ return Option<std::string>(404, "Conversation store collection not found");
}
- return Option<nlohmann::json>(conversation->second);
-}
+ std::string body;
+
+ for(const auto& message : conversation) {
+ if(!message.is_object()) {
+ return Option<std::string>(400, "Message is not an object");
+ }
+
+ // key is role, value is message
+ const auto& message_it = message.items().begin();
+ if(message_it == message.items().end()) {
+ return Option<std::string>(400, "Message is empty");
+ }
-Option<bool> ConversationManager::append_conversation(const std::string& conversation_id, const nlohmann::json& message) {
- std::unique_lock lock(conversations_mutex);
- auto conversation_it = conversations.find(conversation_id);
- if (conversation_it == conversations.end()) {
- return Option<bool>(404, "Conversation not found");
+ if(!message_it.value().is_string()) {
+ return Option<std::string>(400, "Role and message must be strings");
+ }
+
+ nlohmann::json message_json;
+ message_json["conversation_id"] = conversation_id;
+ message_json["role"] = message_it.key();
+ message_json["message"] = message_it.value();
+ message_json["timestamp"] = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ body += message_json.dump(-1) + "\n";
}
- if(!message.is_object() && !message.is_array()) {
- return Option<bool>(400, "Message is not an object or array");
+ if(!raft_server) {
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
+
+ req->params["action"] = "emplace";
+ req->params["collection"] = conversation_collection;
+ req->body = body;
+
+ auto api_res = post_import_documents(req, resp);
+ if(!api_res) {
+ return Option<std::string>(resp->status_code, resp->body);
+ }
+
+ conversation_mapper[conversation_id] = conversation_collection;
+ return Option<std::string>(conversation_id);
}
- nlohmann::json conversation = conversation_it->second;
- if(!message.is_array()) {
- conversation["conversation"].push_back(message);
- } else {
- for(auto& m : message) {
- conversation["conversation"].push_back(m);
+ std::string leader_url = raft_server->get_leader_url();
+
+ if(!leader_url.empty()) {
+ std::string base_url = leader_url + "collections/" + conversation_collection;
+ std::string res;
+ std::string url = base_url + "/documents/import?action=emplace";
+ std::map<std::string, std::string> res_headers;
+
+ long status = HttpClient::post_response(url, body, res, res_headers, {}, 10*1000, true);
+
+ if(status != 200) {
+ LOG(ERROR) << "Error while creating conversation: " << res;
+ LOG(ERROR) << "Status: " << status;
+ return Option<std::string>(400, "Error while creating conversation");
+ } else {
+ conversation_mapper[conversation_id] = conversation_collection;
+ return Option<std::string>(conversation_id);
}
+ } else {
+ return Option<std::string>(500, "Leader URL is empty");
}
+}
- conversation["last_updated"] = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+Option<nlohmann::json> ConversationManager::get_conversation(const std::string& conversation_id) {
- auto conversation_key = get_conversation_key(conversation_id);
- bool insert_op = store->insert(conversation_key, conversation.dump(0));
- if(!insert_op) {
- return Option<bool>(500, "Error while inserting conversation into the store");
+ auto collection_op = get_conversation_collection(conversation_id);
+ if(!collection_op.ok()) {
+ return Option<nlohmann::json>(collection_op.code(), collection_op.error());
+ }
+ auto collection = collection_op.get();
+
+ nlohmann::json res;
+ size_t total = 0;
+ std::vector<sort_by> sort_by_vec = {{"timestamp", sort_field_const::asc}};
+ auto search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, sort_by_vec, {}, 250);
+ if(!search_res.ok()) {
+ return Option<nlohmann::json>(400, "Error while searching conversation store: " + search_res.error());
+ }
+ auto search_res_json = search_res.get();
+ total = search_res_json["found"].get<uint32_t>();
+ if(total == 0) {
+ return Option<nlohmann::json>(404, "Conversation not found");
}
- conversations[conversation_id] = conversation;
+ res["conversation"] = nlohmann::json::array();
+ for(auto& hit : search_res_json["hits"]) {
+ nlohmann::json message;
+ message[hit["document"]["role"]] = hit["document"]["message"];
+ res["conversation"].push_back(message);
+ }
+ res["id"] = conversation_id;
+ res["ttl"] = CONVERSATION_TTL;
+ res["last_updated"] = (search_res_json["hits"].size() > 0) ? search_res_json["hits"][search_res_json["hits"].size() - 1]["document"]["timestamp"].get<uint32_t>() : 0;
+
+ if(total > 250) {
+ while(total > 0) {
+ search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, sort_by_vec, {}, 250, search_res_json["page"].get<uint32_t>() + 1);
+ if(!search_res.ok()) {
+ return Option<nlohmann::json>(400, "Error while searching conversation store: " + search_res.error());
+ }
+ search_res_json = search_res.get();
+ for(auto& hit : search_res_json["hits"]) {
+ nlohmann::json message;
+ message[hit["document"]["role"]] = hit["document"]["message"];
+ res["conversation"].push_back(message);
+ }
+ res["last_updated"] = search_res_json["hits"][search_res_json["hits"].size() - 1]["document"]["timestamp"];
+ total -= search_res_json["hits"].size();
+ }
+ }
- return Option<bool>(true);
+ return Option<nlohmann::json>(res);
}
-
// pop front elements until the conversation is less than MAX_TOKENS
Option<nlohmann::json> ConversationManager::truncate_conversation(nlohmann::json conversation, size_t limit) {
if(!conversation.is_array()) {
@@ -92,96 +163,148 @@ Option<nlohmann::json> ConversationManager::truncate_conversation(nlohmann::json
Option<nlohmann::json> ConversationManager::delete_conversation(const std::string& conversation_id) {
std::unique_lock lock(conversations_mutex);
- auto conversation = conversations.find(conversation_id);
- if (conversation == conversations.end()) {
- return Option<nlohmann::json>(404, "Conversation not found");
+
+ auto conversation_exists = check_conversation_exists(conversation_id);
+ if(!conversation_exists.ok()) {
+ return Option<nlohmann::json>(conversation_exists.code(), conversation_exists.error());
+ }
+
+ auto conversation_collection_op = get_conversation_collection(conversation_id);
+ if(!conversation_collection_op.ok()) {
+ return Option<nlohmann::json>(conversation_collection_op.code(), conversation_collection_op.error());
}
+ auto conversation_collection = conversation_collection_op.get()->get_name();
+ if(!raft_server) {
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
- Option<nlohmann::json> conversation_res(conversation->second);
+ req->params["filter_by"] = "conversation_id:" + conversation_id;
+ req->params["collection"] = conversation_collection;
- auto conversation_key = get_conversation_key(conversation_id);
- bool delete_op = store->remove(conversation_key);
- if(!delete_op) {
- return Option<nlohmann::json>(500, "Error while deleting conversation from the store");
+ auto api_res = del_remove_documents(req, resp);
+ if(!api_res) {
+ return Option<nlohmann::json>(resp->status_code, resp->body);
+ }
+
+ nlohmann::json res_json;
+ res_json["id"] = conversation_id;
+ conversation_mapper.erase(conversation_id);
+ return Option<nlohmann::json>(res_json);
}
- conversations.erase(conversation);
- return conversation_res;
-}
+ auto leader_url = raft_server->get_leader_url();
-Option<nlohmann::json> ConversationManager::get_all_conversations() {
- std::unique_lock lock(conversations_mutex);
- nlohmann::json all_conversations = nlohmann::json::array();
- for(auto& conversation : conversations) {
- all_conversations.push_back(conversation.second);
+ if(leader_url.empty()) {
+ return Option<nlohmann::json>(500, "Leader URL is empty");
}
- return Option<nlohmann::json>(all_conversations);
-}
+ std::string base_url = leader_url + "collections/" + conversation_collection;
+ std::string res;
+ std::string url = base_url + "/documents?filter_by=conversation_id:" + conversation_id;
+ std::map<std::string, std::string> res_headers;
-Option<int> ConversationManager::init(Store* store) {
+ long status = HttpClient::delete_response(url, res, res_headers, 10*1000, true);
+
+ if(status != 200) {
+ LOG(ERROR) << "Error while deleting conversation: " << res;
+ LOG(ERROR) << "Status: " << status;
+ return Option<nlohmann::json>(400, "Error while deleting conversation");
+ } else {
+ nlohmann::json res_json;
+ res_json["conversation_id"] = conversation_id;
+ conversation_mapper.erase(conversation_id);
+ return Option<nlohmann::json>(res_json);
+ }
+}
- if(store == nullptr) {
- return Option<int>(500, "Store is null");
+Option<nlohmann::json> ConversationManager::get_all_conversations() {
+ auto conversation_ids_op = get_conversation_ids();
+ if(!conversation_ids_op.ok()) {
+ return Option<nlohmann::json>(conversation_ids_op.code(), conversation_ids_op.error());
+ }
+ auto conversation_ids = conversation_ids_op.get();
+ nlohmann::json res;
+ res = nlohmann::json::array();
+
+ for(auto& conversation_id : conversation_ids) {
+ auto conversation_op = get_conversation(conversation_id);
+ if(!conversation_op.ok()) {
+ return Option<nlohmann::json>(conversation_op.code(), conversation_op.error());
+ }
+ res.push_back(conversation_op.get());
}
- std::unique_lock lock(conversations_mutex);
- ConversationManager::store = store;
-
+ return Option<nlohmann::json>(res);
+}
- std::vector<std::string> conversation_strs;
- store->scan_fill(std::string(CONVERSATION_RPEFIX) + "_", std::string(CONVERSATION_RPEFIX) + "`", conversation_strs);
-
- size_t loaded_conversations = 0;
+Option<bool> ConversationManager::init(ReplicationState* raft_server) {
- for(auto& conversation_str : conversation_strs) {
- nlohmann::json conversation_json = nlohmann::json::parse(conversation_str);
- if(conversation_json.count("id") == 0) {
- LOG(INFO) << "key: " << conversation_str << " is missing id";
- continue;
- }
- conversations[conversation_json["id"]] = conversation_json;
- loaded_conversations++;
+ if(raft_server == nullptr) {
+ return Option<bool>(400, "Raft server is null");
}
- return Option<int>(loaded_conversations);
+ this->raft_server = raft_server;
+
+ return Option<bool>(true);
}
-const std::string ConversationManager::get_conversation_key(const std::string& conversation_id) {
- return std::string(CONVERSATION_RPEFIX) + "_" + conversation_id;
-}
+
void ConversationManager::clear_expired_conversations() {
std::unique_lock lock(conversations_mutex);
int cleared_conversations = 0;
+ auto conversation_ids_op = get_conversation_ids();
+ if(!conversation_ids_op.ok()) {
+ LOG(ERROR) << conversation_ids_op.error();
+ return;
+ }
- for(auto it = conversations.begin(); it != conversations.end();) {
- const std::string& conversation_id = it->first;
- nlohmann::json conversation_json = it->second;
- if(conversation_json.count("last_updated") == 0 || conversation_json.count("ttl") == 0) {
- bool delete_op = store->remove(get_conversation_key(conversation_id));
- if(!delete_op) {
- LOG(ERROR) << "Error while deleting conversation from the store";
- }
- it = conversations.erase(it);
- cleared_conversations++;
- continue;
- }
- int last_updated = conversation_json["last_updated"];
- int ttl = conversation_json["ttl"];
- if(last_updated + ttl < TTL_OFFSET + std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()) {
- auto delete_op = store->remove(get_conversation_key(conversation_id));
- if(!delete_op) {
- LOG(ERROR) << "Error while deleting conversation from the store";
- }
- it = conversations.erase(it);
- cleared_conversations++;
- } else {
- it++;
+ auto conversation_ids = conversation_ids_op.get();
+ std::vector<std::string> conversation_ids_to_delete;
+ std::vector<sort_by> sort_by_vec = {{"timestamp", sort_field_const::desc}};
+
+ for(auto& conversation_id : conversation_ids) {
+ auto collection_op = get_conversation_collection(conversation_id);
+ if(!collection_op.ok()) {
+ LOG(ERROR) << collection_op.error();
+ continue;
+ }
+ auto collection = collection_op.get();
+ auto search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, sort_by_vec, {}, 1);
+ if(!search_res.ok()) {
+ LOG(ERROR) << "Error while searching conversation store: " << search_res.error();
+ continue;
}
+ auto search_res_json = search_res.get();
+ if(search_res_json["hits"].size() == 0) {
+ continue;
+ }
+ auto last_updated = search_res_json["hits"][0]["document"]["timestamp"].get<uint32_t>();
+ if(std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count() - last_updated + TTL_OFFSET > CONVERSATION_TTL) {
+ conversation_ids_to_delete.push_back(conversation_id);
+ }
+
+ if(conversation_ids_to_delete.size() >= MAX_CONVERSATIONS_TO_DELETE_ONCE) {
+ break;
+ }
+ }
+
+ if(conversation_ids_to_delete.empty()) {
+ return;
+ }
+
+ for(auto& conversation_id : conversation_ids_to_delete) {
+ lock.unlock();
+ auto delete_op = delete_conversation(conversation_id);
+ lock.lock();
+ if(!delete_op.ok()) {
+ LOG(ERROR) << delete_op.error();
+ continue;
+ }
+ cleared_conversations++;
}
LOG(INFO) << "Cleared " << cleared_conversations << " expired conversations";
@@ -197,31 +320,56 @@ Option<nlohmann::json> ConversationManager::update_conversation(nlohmann::json c
return Option<nlohmann::json>(400, "Conversation is missing id");
}
- const std::string& conversation_id = conversation["id"];
+ if(!conversation["id"].is_string()) {
+ return Option<nlohmann::json>(400, "Conversation id must be a string");
+ }
- auto conversation_it = conversations.find(conversation_id);
- if (conversation_it == conversations.end()) {
- return Option<nlohmann::json>(404, "Conversation not found");
+ if(conversation.count("conversation") == 0) {
+ return Option<nlohmann::json>(400, "Conversation is missing conversation");
}
- auto conversation_key = get_conversation_key(conversation_id);
+ if(!conversation["conversation"].is_array()) {
+ return Option<nlohmann::json>(400, "Conversation history is not an array");
+ }
- auto actual_conversation = conversation_it->second;
+ for(auto& message : conversation["conversation"]) {
+ if(!message.is_object()) {
+ return Option<nlohmann::json>(400, "Message is not an object");
+ }
- if(conversation.count("ttl") == 0) {
- return Option<nlohmann::json>(400, "Only `ttl` can be updated");
+ const auto& message_it = message.items().begin();
+ if(message_it == message.items().end()) {
+ return Option<nlohmann::json>(400, "Message is empty");
+ }
+
+ if(!message_it.value().is_string()) {
+ return Option<nlohmann::json>(400, "Role and message must be strings");
+ }
+ }
+
+ const std::string& conversation_id = conversation["id"];
+
+ auto conversation_exists = check_conversation_exists(conversation_id);
+ if(!conversation_exists.ok()) {
+ return Option<nlohmann::json>(conversation_exists.code(), conversation_exists.error());
}
-
- actual_conversation["ttl"] = conversation["ttl"];
- bool insert_op = store->insert(conversation_key, actual_conversation.dump(0));
- if(!insert_op) {
- return Option<nlohmann::json>(500, "Error while inserting conversation into the store");
+ auto delete_op = delete_conversation(conversation_id);
+ if(!delete_op.ok()) {
+ return Option<nlohmann::json>(delete_op.code(), delete_op.error());
}
- conversations[conversation_id] = actual_conversation;
+ auto create_op = add_conversation(conversation["conversation"], conversation_id);
+ if(!create_op.ok()) {
+ return Option<nlohmann::json>(create_op.code(), create_op.error());
+ }
+
+ auto actual_conversation_op = get_conversation(conversation_id);
+ if(!actual_conversation_op.ok()) {
+ return Option<nlohmann::json>(actual_conversation_op.code(), actual_conversation_op.error());
+ }
- return Option<nlohmann::json>(actual_conversation);
+ return Option<nlohmann::json>(actual_conversation_op.get());
}
void ConversationManager::run() {
@@ -241,4 +389,174 @@ void ConversationManager::run() {
void ConversationManager::stop() {
quit = true;
cv.notify_all();
+}
+
+
+
+Option<bool> ConversationManager::validate_conversation_store_schema(Collection* collection) {
+ const auto& schema = collection->get_schema();
+
+ if(schema.count("conversation_id") == 0) {
+ return Option<bool>(400, "Schema is missing `conversation_id` field");
+ }
+
+ if(schema.count("role") == 0) {
+ return Option<bool>(400, "Schema is missing `role` field");
+ }
+
+ if(schema.count("message") == 0) {
+ return Option<bool>(400, "Schema is missing `message` field");
+ }
+
+ if(schema.count("timestamp") == 0) {
+ return Option<bool>(400, "Schema is missing `timestamp` field");
+ }
+
+ if(schema.at("conversation_id").type != field_types::STRING) {
+ return Option<bool>(400, "`conversation_id` field must be a string");
+ }
+
+ if(!schema.at("conversation_id").facet) {
+ return Option<bool>(400, "`conversation_id` field must be a facet");
+ }
+
+ if(schema.at("role").type != field_types::STRING) {
+ return Option<bool>(400, "`role` field must be a string");
+ }
+
+ if(schema.at("message").type != field_types::STRING) {
+ return Option<bool>(400, "`message` field must be a string");
+ }
+
+ if(schema.at("timestamp").type != field_types::INT32) {
+ return Option<bool>(400, "`timestamp` field must be an integer");
+ }
+
+ if(!schema.at("timestamp").sort) {
+ return Option<bool>(400, "`timestamp` field must be a sort field");
+ }
+
+ return Option<bool>(true);
+}
+
+
+Option<bool> ConversationManager::check_conversation_exists(const std::string& conversation_id) {
+ auto collection_op = get_conversation_collection(conversation_id);
+ if(!collection_op.ok()) {
+ return Option<bool>(collection_op.code(), collection_op.error());
+ }
+
+ auto collection = collection_op.get();
+
+ nlohmann::json res;
+ size_t total = 0;
+ auto search_res = collection->search("*", {}, "conversation_id:" + conversation_id, {}, {}, {}, 250);
+ if(!search_res.ok()) {
+ return Option<bool>(400, "Error while searching conversation store: " + search_res.error());
+ }
+ auto search_res_json = search_res.get();
+ total = search_res_json["found"].get<uint32_t>();
+ if(total == 0) {
+ return Option<bool>(404, "Conversation not found");
+ }
+
+ return Option<bool>(true);
+}
+
+Option<std::unordered_set<std::string>> ConversationManager::get_conversation_ids() {
+ std::unordered_set<std::string> conversation_ids;
+ for(const auto& conversation_id : conversation_mapper) {
+ conversation_ids.insert(conversation_id.first);
+ }
+
+ return Option<std::unordered_set<std::string>>(conversation_ids);
+}
+
+Option<bool> ConversationManager::add_conversation_collection(const std::string& collection) {
+ std::unique_lock lock(conversations_mutex);
+ if(conversation_collection_map.count(collection) > 0) {
+ conversation_collection_map[collection]++;
+ } else {
+ auto collection_ptr = CollectionManager::get_instance().get_collection(collection).get();
+ if(!collection_ptr) {
+ return Option<bool>(404, "Collection not found");
+ }
+
+ auto validate_op = validate_conversation_store_schema(collection_ptr);
+ if(!validate_op.ok()) {
+ return Option<bool>(validate_op.code(), validate_op.error());
+ }
+
+ conversation_collection_map[collection] = 1;
+ }
+
+ return Option<bool>(true);
+}
+
+Option<bool> ConversationManager::remove_conversation_collection(const std::string& collection) {
+ std::unique_lock lock(conversations_mutex);
+ if(conversation_collection_map.count(collection) == 0) {
+ return Option<bool>(404, "Collection not found");
+ }
+
+ conversation_collection_map[collection]--;
+
+ if(conversation_collection_map[collection] == 0) {
+ std::vector<std::string> conversations_to_delete;
+ for(auto& conversation : conversation_mapper) {
+ if(conversation.second == collection) {
+ conversations_to_delete.push_back(conversation.first);
+ }
+ }
+ for(auto conversation_id : conversations_to_delete) {
+ conversation_mapper.erase(conversation_id);
+ }
+ conversation_collection_map.erase(collection);
+ }
+
+ return Option<bool>(true);
+}
+
+Option<Collection*> ConversationManager::get_conversation_collection(const std::string& conversation_id) {
+
+ if(conversation_mapper.count(conversation_id) > 0) {
+ auto collection = CollectionManager::get_instance().get_collection(conversation_mapper[conversation_id]).get();
+ if(collection) {
+ return Option<Collection*>(collection);
+ }
+ }
+
+ for(auto& collection : conversation_collection_map) {
+ auto collection_ptr = CollectionManager::get_instance().get_collection(collection.first).get();
+ if(!collection_ptr) {
+ continue;
+ }
+
+ auto search_res = collection_ptr->search("*", {}, "conversation_id:" + conversation_id, {}, {}, {}, 1);
+ if(!search_res.ok()) {
+ continue;
+ }
+
+ auto search_res_json = search_res.get();
+ if(search_res_json["found"].get<uint32_t>() > 0) {
+ conversation_mapper[conversation_id] = collection.first;
+ return Option<Collection*>(collection_ptr);
+ }
+ }
+
+ return Option<Collection*>(404, "Conversation not found");
+}
+
+Option<bool> ConversationManager::validate_conversation_store_collection(const std::string& collection) {
+ auto collection_ptr = CollectionManager::get_instance().get_collection(collection).get();
+ if(!collection_ptr) {
+ return Option<bool>(404, "Collection not found");
+ }
+
+ auto validate_op = validate_conversation_store_schema(collection_ptr);
+ if(!validate_op.ok()) {
+ return Option<bool>(validate_op.code(), validate_op.error());
+ }
+
+ return Option<bool>(true);
}
\ No newline at end of file
diff --git a/src/conversation_model.cpp b/src/conversation_model.cpp
index b5a2b678..068deadf 100644
--- a/src/conversation_model.cpp
+++ b/src/conversation_model.cpp
@@ -28,6 +28,15 @@ Option<bool> ConversationModel::validate_model(const nlohmann::json& model_confi
return Option<bool>(400, "Property `max_bytes` is not provided or not a positive integer.");
}
+ if(model_config.count("conversation_collection") == 0 || !model_config["conversation_collection"].is_string()) {
+ return Option<bool>(400, "Property `conversation_collection` is not provided or not a string.");
+ }
+
+ auto validate_converson_collection_op = ConversationManager::get_instance().validate_conversation_store_collection(model_config["conversation_collection"].get<std::string>());
+ if(!validate_converson_collection_op.ok()) {
+ return Option<bool>(400, validate_converson_collection_op.error());
+ }
+
const std::string model_namespace = get_model_namespace(model_config["model_name"].get<std::string>());
if(model_namespace == "openai") {
return OpenAIConversationModel::validate_model(model_config);
diff --git a/src/conversation_model_manager.cpp b/src/conversation_model_manager.cpp
index 4cd81511..57aef3c9 100644
--- a/src/conversation_model_manager.cpp
+++ b/src/conversation_model_manager.cpp
@@ -1,5 +1,6 @@
#include "conversation_model_manager.h"
#include "conversation_model.h"
+#include "conversation_manager.h"
Option<nlohmann::json> ConversationModelManager::get_model(const std::string& model_id) {
std::shared_lock lock(models_mutex);
@@ -29,6 +30,8 @@ Option<nlohmann::json> ConversationModelManager::add_model(nlohmann::json model,
models[model_id] = model;
+
+ ConversationManager::get_instance().add_conversation_collection(model["conversation_collection"]);
return Option<nlohmann::json>(model);
}
@@ -43,7 +46,8 @@ Option<nlohmann::json> ConversationModelManager::delete_model(const std::string&
auto model_key = get_model_key(model_id);
bool delete_op = store->remove(model_key);
-
+
+ ConversationManager::get_instance().remove_conversation_collection(model["conversation_collection"]);
models.erase(it);
return Option<nlohmann::json>(model);
}
@@ -78,6 +82,11 @@ Option<nlohmann::json> ConversationModelManager::update_model(const std::string&
return Option<nlohmann::json>(500, "Error while inserting model into the store");
}
+ if(it->second["conversation_collection"] != model["conversation_collection"]) {
+ ConversationManager::get_instance().remove_conversation_collection(it->second["conversation_collection"]);
+ ConversationManager::get_instance().add_conversation_collection(model["conversation_collection"]);
+ }
+
models[model_id] = model;
return Option<nlohmann::json>(model);
@@ -95,6 +104,18 @@ Option<int> ConversationModelManager::init(Store* store) {
nlohmann::json model_json = nlohmann::json::parse(model_str);
std::string model_id = model_json["id"];
models[model_id] = model_json;
+ if(model_json.count("conversation_collection") == 0) {
+ auto delete_op = delete_model(model_id);
+ if(!delete_op.ok()) {
+ return Option<int>(delete_op.code(), delete_op.error());
+ }
+ auto migrate_op = migrate_model(model_json);
+ if(!migrate_op.ok()) {
+ return Option<int>(migrate_op.code(), migrate_op.error());
+ }
+ model_json = migrate_op.get();
+ }
+ ConversationManager::get_instance().add_conversation_collection(model_json["conversation_collection"]);
loaded_models++;
}
@@ -103,4 +124,65 @@ Option<int> ConversationModelManager::init(Store* store) {
const std::string ConversationModelManager::get_model_key(const std::string& model_id) {
return std::string(MODEL_KEY_PREFIX) + "_" + model_id;
+}
+
+
+Option<Collection*> ConversationModelManager::get_default_conversation_collection() {
+ int64_t time_epoch;
+ if(DEFAULT_CONVERSATION_COLLECTION_SUFFIX == 0) {
+ time_epoch = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ DEFAULT_CONVERSATION_COLLECTION_SUFFIX = time_epoch;
+ } else {
+ time_epoch = DEFAULT_CONVERSATION_COLLECTION_SUFFIX;
+ }
+
+ std::string collection_id = "default_conversation_history_" + std::to_string(time_epoch);
+ nlohmann::json schema_json = R"({
+ "fields": [
+ {
+ "name": "conversation_id",
+ "type": "string",
+ "facet": true
+ },
+ {
+ "name": "role",
+ "type": "string"
+ },
+ {
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "name": "timestamp",
+ "type": "int32",
+ "sort": true
+ }
+ ]
+ })"_json;
+ schema_json["name"] = collection_id;
+
+ auto get_res = CollectionManager::get_instance().get_collection(collection_id).get();
+ if(get_res) {
+ return Option<Collection*>(get_res);
+ }
+
+ auto create_res = CollectionManager::get_instance().create_collection(schema_json);
+ if(!create_res.ok()) {
+ return Option<Collection*>(create_res.code(), create_res.error());
+ }
+ return Option<Collection*>(create_res.get());
+}
+
+Option<nlohmann::json> ConversationModelManager::migrate_model(nlohmann::json model) {
+ auto model_id = model["id"];
+ auto default_collection = get_default_conversation_collection();
+ if(!default_collection.ok()) {
+ return Option<nlohmann::json>(default_collection.code(), default_collection.error());
+ }
+ model["conversation_collection"] = default_collection.get()->get_name();
+ auto add_res = add_model(model, model_id);
+ if(!add_res.ok()) {
+ return Option<nlohmann::json>(add_res.code(), add_res.error());
+ }
+ return Option<nlohmann::json>(model);
}
\ No newline at end of file
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 3a9dac74..61920ff0 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -866,43 +866,26 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_p
StringUtils::split(req->params["exclude_fields"], exclude_fields, ",");
bool exclude_conversation_history = std::find(exclude_fields.begin(), exclude_fields.end(), "conversation_history") != exclude_fields.end();
- if(conversation_history) {
- std::string conversation_id = orig_req_params["conversation_id"];
- ConversationManager::get_instance().append_conversation(conversation_id, formatted_question_op.get());
- ConversationManager::get_instance().append_conversation(conversation_id, formatted_answer_op.get());
- auto get_conversation_op = ConversationManager::get_instance().get_conversation(conversation_id);
- if(!get_conversation_op.ok()) {
- res->set_400(get_conversation_op.error());
- return false;
- }
-
- auto conversation_history = get_conversation_op.get();
- if(!exclude_conversation_history) {
- response["conversation"]["conversation_history"] = conversation_history;
- } else {
- response["conversation"]["conversation_id"] = conversation_id;
- }
- } else {
- nlohmann::json conversation_history = nlohmann::json::array();
- conversation_history.push_back(formatted_question_op.get());
- conversation_history.push_back(formatted_answer_op.get());
-
- auto create_conversation_op = ConversationManager::get_instance().create_conversation(conversation_history);
- if(!create_conversation_op.ok()) {
- res->set_400(create_conversation_op.error());
- return false;
- }
+ nlohmann::json conversation_history = nlohmann::json::array();
+ conversation_history.push_back(formatted_question_op.get());
+ conversation_history.push_back(formatted_answer_op.get());
+ std::string conversation_id = conversation_history ? orig_req_params["conversation_id"] : "";
+
+ auto add_conversation_op = ConversationManager::get_instance().add_conversation(conversation_history, conversation_model["conversation_collection"], conversation_id);
+ if(!add_conversation_op.ok()) {
+ res->set_400(add_conversation_op.error());
+ return false;
+ }
- auto get_conversation_op = ConversationManager::get_instance().get_conversation(create_conversation_op.get());
- if(!get_conversation_op.ok()) {
- res->set_400(get_conversation_op.error());
- return false;
- }
- if(!exclude_conversation_history) {
- response["conversation"]["conversation_history"] = get_conversation_op.get();
- }
- response["conversation"]["conversation_id"] = create_conversation_op.get();
+ auto get_conversation_op = ConversationManager::get_instance().get_conversation(add_conversation_op.get());
+ if(!get_conversation_op.ok()) {
+ res->set_400(get_conversation_op.error());
+ return false;
+ }
+ if(!exclude_conversation_history) {
+ response["conversation"]["conversation_history"] = get_conversation_op.get();
}
+ response["conversation"]["conversation_id"] = add_conversation_op.get();
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index c0379461..dd3fd032 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -450,13 +450,6 @@ int run_server(const Config & config, const std::string & version, void (*master
}
EmbedderManager::set_model_dir(config.get_data_dir() + "/models");
- auto conversations_init = ConversationManager::get_instance().init(&store);
-
- if(!conversations_init.ok()) {
- LOG(INFO) << "Failed to initialize conversation manager: " << conversations_init.error();
- } else {
- LOG(INFO) << "Loaded " << conversations_init.get() << "(s) conversations.";
- }
auto conversation_models_init = ConversationModelManager::init(&store);
@@ -475,6 +468,12 @@ int run_server(const Config & config, const std::string & version, void (*master
num_collections_parallel_load,
config.get_num_documents_parallel_load());
+ auto conversations_init = ConversationManager::get_instance().init(&replication_state);
+
+ if(!conversations_init.ok()) {
+ LOG(INFO) << "Failed to initialize conversation manager: " << conversations_init.error();
+ }
+
std::thread raft_thread([&replication_state, &config, &state_dir,
&app_thread_pool, &server_thread_pool, &replication_thread_pool, batch_indexer]() {
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index c4a0301b..5613aeeb 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -31,7 +31,31 @@ protected:
collectionManager.load(8, 1000);
ConversationModelManager::init(store);
- ConversationManager::get_instance().init(store);
+ nlohmann::json schema_json = R"({
+ "name": "conversation_store",
+ "fields": [
+ {
+ "name": "conversation_id",
+ "type": "string",
+ "facet": true
+ },
+ {
+ "name": "role",
+ "type": "string"
+ },
+ {
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "name": "timestamp",
+ "type": "int32",
+ "sort": true
+ }
+ ]
+ })"_json;
+
+ collectionManager.create_collection(schema_json);
}
virtual void SetUp() {
@@ -2873,7 +2897,8 @@ TEST_F(CollectionVectorTest, TestQAConversation) {
auto conversation_model_config = R"({
"model_name": "openai/gpt-3.5-turbo",
- "max_bytes: 1000
+ "max_bytes: 1000,
+ "conversation_collection": "conversation_store",
})"_json;
conversation_model_config["api_key"] = api_key;
@@ -3320,7 +3345,8 @@ TEST_F(CollectionVectorTest, InvalidMultiSearchConversation) {
auto conversation_model_config = R"({
"model_name": "openai/gpt-3.5-turbo",
- "max_bytes": 1000
+ "max_bytes": 1000,
+ "conversation_collection": "conversation_store"
})"_json;
conversation_model_config["api_key"] = api_key;
@@ -3396,6 +3422,28 @@ TEST_F(CollectionVectorTest, InvalidMultiSearchConversation) {
ASSERT_EQ(res_json["message"], "`conversation` cannot be used in POST body. Please set `conversation` as a query parameter in the request, instead of inside the POST body");
}
+TEST_F(CollectionVectorTest, TestMigratingConversationModel) {
+ auto conversation_model_config = R"({
+ "model_name": "openai/gpt-3.5-turbo",
+ "max_bytes": 1000,
+ "conversation_collection": "conversation_store"
+ })"_json;
+
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
+ return;
+ }
+
+ auto api_key = std::string(std::getenv("api_key"));
+
+ auto migrate_res = ConversationModelManager::migrate_model(conversation_model_config);
+ ASSERT_TRUE(migrate_res.ok());
+ auto migrated_model = migrate_res.get();
+ ASSERT_TRUE(migrated_model.count("conversation_collection") == 1);
+
+ auto collection = CollectionManager::get_instance().get_collection("conversation_store").get();
+ ASSERT_TRUE(collection != nullptr);
+}
TEST_F(CollectionVectorTest, TestVectorQueryQs) {
diff --git a/test/conversation_test.cpp b/test/conversation_test.cpp
index d8546945..3f6c2400 100644
--- a/test/conversation_test.cpp
+++ b/test/conversation_test.cpp
@@ -4,34 +4,62 @@
class ConversationTest : public ::testing::Test {
protected:
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Store* store;
+ std::atomic<bool> quit = false;
void SetUp() override {
std::string state_dir_path = "/tmp/typesense_test/conversation_test";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
+
store = new Store(state_dir_path);
- ConversationManager::get_instance().init(store);
+ collectionManager.init(store, 1.0, "auth_key", quit);
+ collectionManager.load(8, 1000);
+
+ nlohmann::json schema_json = R"({
+ "name": "conversation_store",
+ "fields": [
+ {
+ "name": "conversation_id",
+ "type": "string",
+ "facet": true
+ },
+ {
+ "name": "role",
+ "type": "string"
+ },
+ {
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "name": "timestamp",
+ "type": "int32",
+ "sort": true
+ }
+ ]
+ })"_json;
+
+ collectionManager.create_collection(schema_json);
+ ConversationManager::get_instance().add_conversation_collection("conversation_store");
}
void TearDown() override {
- auto conversations = ConversationManager::get_instance().get_all_conversations();
- for (auto& conversation : conversations.get()) {
- ConversationManager::get_instance().delete_conversation(conversation["id"]);
- }
+ ConversationManager::get_instance().remove_conversation_collection("conversation_store");
+ collectionManager.dispose();
delete store;
}
-
- Store* store;
};
TEST_F(ConversationTest, CreateConversation) {
nlohmann::json conversation = nlohmann::json::array();
- auto create_res = ConversationManager::get_instance().create_conversation(conversation);
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");
ASSERT_TRUE(create_res.ok());
}
TEST_F(ConversationTest, CreateConversationInvalidType) {
nlohmann::json conversation = nlohmann::json::object();
- auto create_res = ConversationManager::get_instance().create_conversation(conversation);
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");
ASSERT_FALSE(create_res.ok());
ASSERT_EQ(create_res.code(), 400);
ASSERT_EQ(create_res.error(), "Conversation is not an array");
@@ -49,14 +77,14 @@ TEST_F(ConversationTest, AppendConversation) {
nlohmann::json message = nlohmann::json::object();
message["user"] = "Hello";
conversation.push_back(message);
- auto create_res = ConversationManager::get_instance().create_conversation(conversation);
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");
ASSERT_TRUE(create_res.ok());
std::string conversation_id = create_res.get();
- auto append_res = ConversationManager::get_instance().append_conversation(conversation_id, message);
+ auto append_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store", conversation_id);
ASSERT_TRUE(append_res.ok());
- ASSERT_EQ(append_res.get(), true);
+ ASSERT_EQ(append_res.get(), conversation_id);
auto get_res = ConversationManager::get_instance().get_conversation(conversation_id);
@@ -73,32 +101,36 @@ TEST_F(ConversationTest, AppendInvalidConversation) {
nlohmann::json conversation = nlohmann::json::array();
nlohmann::json message = nlohmann::json::object();
message["user"] = "Hello";
- auto create_res = ConversationManager::get_instance().create_conversation(conversation);
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");
ASSERT_TRUE(create_res.ok());
std::string conversation_id = create_res.get();
message = "invalid";
- auto append_res = ConversationManager::get_instance().append_conversation(conversation_id, message);
+ auto append_res = ConversationManager::get_instance().add_conversation(message, "conversation_store", conversation_id);
ASSERT_FALSE(append_res.ok());
ASSERT_EQ(append_res.code(), 400);
- ASSERT_EQ(append_res.error(), "Message is not an object or array");
+ ASSERT_EQ(append_res.error(), "Conversation is not an array");
}
TEST_F(ConversationTest, DeleteConversation) {
nlohmann::json conversation = nlohmann::json::array();
- auto create_res = ConversationManager::get_instance().create_conversation(conversation);
+ nlohmann::json message = nlohmann::json::object();
+ message["user"] = "Hello";
+ conversation.push_back(message);
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");
ASSERT_TRUE(create_res.ok());
std::string conversation_id = create_res.get();
+ LOG(INFO) << conversation_id;
auto delete_res = ConversationManager::get_instance().delete_conversation(conversation_id);
+ LOG(INFO) << delete_res.error();
ASSERT_TRUE(delete_res.ok());
auto delete_res_json = delete_res.get();
ASSERT_EQ(delete_res_json["id"], conversation_id);
- ASSERT_TRUE(delete_res_json["conversation"].is_array());
auto get_res = ConversationManager::get_instance().get_conversation(conversation_id);
ASSERT_FALSE(get_res.ok());
@@ -155,7 +187,7 @@ TEST_F(ConversationTest, TestConversationExpire) {
nlohmann::json message = nlohmann::json::object();
message["user"] = "Hello";
conversation.push_back(message);
- auto create_res = ConversationManager::get_instance().create_conversation(conversation);
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store");
ASSERT_TRUE(create_res.ok());
std::string conversation_id = create_res.get();
@@ -169,7 +201,9 @@ TEST_F(ConversationTest, TestConversationExpire) {
ASSERT_EQ(get_res.get()["conversation"].size(), 1);
ConversationManager::get_instance()._set_ttl_offset(24 * 60 * 60 * 2);
+ LOG(INFO) << "Clearing expired conversations";
ConversationManager::get_instance().clear_expired_conversations();
+ LOG(INFO) << "Cleared expired conversations";
get_res = ConversationManager::get_instance().get_conversation(conversation_id);
ASSERT_FALSE(get_res.ok());
@@ -179,4 +213,143 @@ TEST_F(ConversationTest, TestConversationExpire) {
ConversationManager::get_instance()._set_ttl_offset(0);
}
+TEST_F(ConversationTest, TestRemoveConversationCollection) {
+ nlohmann::json schema_json = R"({
+ "name": "conversation_store2",
+ "fields": [
+ {
+ "name": "conversation_id",
+ "type": "string",
+ "facet": true
+ },
+ {
+ "name": "role",
+ "type": "string"
+ },
+ {
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "name": "timestamp",
+ "type": "int32",
+ "sort": true
+ }
+ ]
+ })"_json;
+ LOG(INFO) << "Creating collection";
+ collectionManager.create_collection(schema_json);
+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");
+ LOG(INFO) << "Collection created";
+ nlohmann::json conversation = nlohmann::json::array();
+ nlohmann::json message = nlohmann::json::object();
+ message["user"] = "Hello";
+ conversation.push_back(message);
+
+ LOG(INFO) << "Adding conversation";
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store2");
+ ASSERT_TRUE(create_res.ok());
+
+ LOG(INFO) << "Getting conversation";
+ auto get_res = ConversationManager::get_instance().get_conversation(create_res.get());
+ ASSERT_TRUE(get_res.ok());
+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);
+
+
+ LOG(INFO) << "Removing collection";
+ auto remove_res = ConversationManager::get_instance().remove_conversation_collection("conversation_store2");
+ ASSERT_TRUE(remove_res.ok());
+
+ LOG(INFO) << "Getting conversation";
+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());
+ ASSERT_FALSE(get_res.ok());
+ ASSERT_EQ(get_res.code(), 404);
+ ASSERT_EQ(get_res.error(), "Conversation not found");
+
+ LOG(INFO) << "Adding collection again";
+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");
+
+
+ LOG(INFO) << "Adding conversation";
+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());
+ ASSERT_TRUE(get_res.ok());
+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);
+
+ LOG(INFO) << "Dropping collection";
+ collectionManager.drop_collection("conversation_store2");
+
+ LOG(INFO) << "Getting conversation";
+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());
+ ASSERT_FALSE(get_res.ok());
+ ASSERT_EQ(get_res.code(), 404);
+ ASSERT_EQ(get_res.error(), "Conversation not found");
+
+ LOG(INFO) << "TestRemoveConversationCollection done";
+}
+
+TEST_F(ConversationTest, TestMultipleRefConversationCollection) {
+ nlohmann::json schema_json = R"({
+ "name": "conversation_store2",
+ "fields": [
+ {
+ "name": "conversation_id",
+ "type": "string",
+ "facet": true
+ },
+ {
+ "name": "role",
+ "type": "string"
+ },
+ {
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "name": "timestamp",
+ "type": "int32",
+ "sort": true
+ }
+ ]
+ })"_json;
+
+ collectionManager.create_collection(schema_json);
+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");
+ ConversationManager::get_instance().add_conversation_collection("conversation_store2");
+
+ nlohmann::json conversation = nlohmann::json::array();
+ nlohmann::json message = nlohmann::json::object();
+ message["user"] = "Hello";
+ conversation.push_back(message);
+
+ auto create_res = ConversationManager::get_instance().add_conversation(conversation, "conversation_store2");
+ ASSERT_TRUE(create_res.ok());
+
+ auto get_res = ConversationManager::get_instance().get_conversation(create_res.get());
+ ASSERT_TRUE(get_res.ok());
+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);
+
+ auto remove_res = ConversationManager::get_instance().remove_conversation_collection("conversation_store2");
+ ASSERT_TRUE(remove_res.ok());
+ get_res = ConversationManager::get_instance().get_conversation(create_res.get());
+ ASSERT_TRUE(get_res.ok());
+ ASSERT_EQ(get_res.get()["conversation"].size(), 1);
+}
+
+TEST_F(ConversationTest, TestInvalidConversationCollection) {
+ nlohmann::json schema_json = R"({
+ "name": "conversation_store2",
+ "fields": [
+ {
+ "name": "lorem",
+ "type": "string"
+ }
+ ]
+ })"_json;
+
+ collectionManager.create_collection(schema_json);
+ auto res = ConversationManager::get_instance().add_conversation_collection("conversation_store2");
+ ASSERT_FALSE(res.ok());
+ ASSERT_EQ(res.code(), 400);
+ ASSERT_EQ(res.error(), "Schema is missing `conversation_id` field");
+}
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index 87f884e2..abae043f 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -29,7 +29,32 @@ protected:
collectionManager.load(8, 1000);
ConversationModelManager::init(store);
- ConversationManager::get_instance().init(store);
+ nlohmann::json schema_json = R"({
+ "name": "conversation_store",
+ "fields": [
+ {
+ "name": "conversation_id",
+ "type": "string",
+ "facet": true
+ },
+ {
+ "name": "role",
+ "type": "string"
+ },
+ {
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "name": "timestamp",
+ "type": "int32",
+ "sort": true
+ }
+ ]
+ })"_json;
+
+ collectionManager.create_collection(schema_json);
+ ConversationManager::get_instance().add_conversation_collection("conversation_store");
}
virtual void SetUp() {
@@ -1389,7 +1414,8 @@ TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) {
TEST_F(CoreAPIUtilsTest, TestConversationModels) {
nlohmann::json model_config = R"({
"model_name": "openai/gpt-3.5-turbo",
- "max_bytes": 10000
+ "max_bytes": 10000,
+ "conversation_collection": "conversation_store"
})"_json;
EmbedderManager::set_model_dir("/tmp/typesense_test/models");
@@ -1430,6 +1456,7 @@ TEST_F(CoreAPIUtilsTest, TestConversationModels) {
TEST_F(CoreAPIUtilsTest, TestInvalidConversationModels) {
// test with no model_name
nlohmann::json model_config = R"({
+ "conversation_collection": "conversation_store"
})"_json;
if (std::getenv("api_key") == nullptr) {
@@ -1517,6 +1544,25 @@ TEST_F(CoreAPIUtilsTest, TestInvalidConversationModels) {
ASSERT_EQ(400, resp->status_code);
ASSERT_EQ("Property `max_bytes` must be a positive number.", nlohmann::json::parse(resp->body)["message"]);
+
+ model_config["max_bytes"] = 10000;
+ model_config["conversation_collection"] = 123;
+
+ // test with conversation_collection as integer
+ req->body = model_config.dump();
+ post_conversation_model(req, resp);
+
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("Property `conversation_collection` is not provided or not a string.", nlohmann::json::parse(resp->body)["message"]);
+
+ // test with conversation_collection as empty string
+ model_config["conversation_collection"] = "";
+
+ req->body = model_config.dump();
+ post_conversation_model(req, resp);
+
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("Collection not found", nlohmann::json::parse(resp->body)["message"]);
}
TEST_F(CoreAPIUtilsTest, DeleteNonExistingDoc) {