Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add support for vLLM RAG (#1563)
Ozan Armağan6e24c06e35ab
diff --git a/include/conversation_manager.h b/include/conversation_manager.hindex 6abeffe1..3cc5e74f 100644--- a/include/conversation_manager.h+++ b/include/conversation_manager.h@@ -24,9 +24,8 @@ class ConversationManager {Option<std::string> create_conversation(const nlohmann::json& conversation);Option<nlohmann::json> get_conversation(const std::string& conversation_id);Option<bool> append_conversation(const std::string& conversation_id, const nlohmann::json& message);- Option<nlohmann::json> truncate_conversation(nlohmann::json conversation);+ Option<nlohmann::json> truncate_conversation(nlohmann::json conversation, size_t limit);Option<nlohmann::json> update_conversation(nlohmann::json conversation);- static size_t get_token_count(const nlohmann::json& message);Option<nlohmann::json> delete_conversation(const std::string& conversation_id);Option<nlohmann::json> get_all_conversations();static constexpr size_t MAX_TOKENS = 3000;diff --git a/include/conversation_model.h b/include/conversation_model.hindex d98ba270..06527cc6 100644--- a/include/conversation_model.h+++ b/include/conversation_model.h@@ -22,9 +22,11 @@ class ConversationModel {static Option<std::string> get_standalone_question(const nlohmann::json& conversation_history, const std::string& question, const nlohmann::json& model_config);static Option<nlohmann::json> format_question(const std::string& message, const nlohmann::json& model_config);static Option<nlohmann::json> format_answer(const std::string& message, const nlohmann::json& model_config);- // max token count for the context search results- static Option<size_t> max_context_tokens(const nlohmann::json& model_config);- private:+ static Option<size_t> get_minimum_required_bytes(const nlohmann::json& model_config);+ protected:+ static const inline std::string CONVERSATION_HISTORY = "\n\n<Conversation history>\n";+ static const inline std::string QUESTION = "\n\n<Question>\n";+ static const inline std::string STANDALONE_QUESTION_PROMPT = "\n\n<Standalone question>\n";};class OpenAIConversationModel : public ConversationModel {@@ -34,17 +36,21 @@ class OpenAIConversationModel : public ConversationModel {static Option<std::string> get_standalone_question(const nlohmann::json& conversation_history, const std::string& question, const nlohmann::json& model_config);static Option<nlohmann::json> format_question(const std::string& message);static Option<nlohmann::json> format_answer(const std::string& message);+ // max_bytes must be greater than or equal to the minimum required bytes+ static const size_t get_minimum_required_bytes() {+ return DATA_STR.size() + QUESTION_STR.size() + ANSWER_STR.size();+ }static const inline std::string STANDALONE_QUESTION_PROMPT = R"(Rewrite the follow-up question on top of a human-assistant conversation history as a standalone question that encompasses all pertinent context.)";// prevent instantiationOpenAIConversationModel() = delete;- static size_t max_context_tokens() {- return 3000;- }private:static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";static constexpr char* OPENAI_CHAT_COMPLETION = "https://api.openai.com/v1/chat/completions";+ static const inline std::string DATA_STR = "<Data>\n";+ static const inline std::string QUESTION_STR = "\n\n<Question>\n";+ static const inline std::string ANSWER_STR = "\n\n<Answer>";};class CFConversationModel : public ConversationModel {@@ -61,10 +67,38 @@ class CFConversationModel : public ConversationModel {static const inline std::string INFO_PROMPT = "You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and do not mention provided context directly, act like already knowing the context.";// prevent instantiationCFConversationModel() = delete;- static size_t max_context_tokens() {- return 768;+ static const size_t get_minimum_required_bytes() {+ return CONTEXT_INFO.size() + SPLITTER_STR.size() + QUERY_STR.size() + ANSWER_STR.size();}private:static const inline std::vector<std::string> CF_MODEL_NAMES{"mistral/mistral-7b-instruct-v0.1"};static const std::string get_model_url(const std::string& model_name, const std::string& account_id);+ static const inline std::string CONTEXT_INFO = "Context information is below.\n";+ static const inline std::string SPLITTER_STR = "---------------------\n";+ static const inline std::string QUERY_STR = "Given the context information and not prior knowledge, answer the query. Context is JSON format, do not return data directly, answer like a human assistant.\nQuery: ";+ static const inline std::string ANSWER_STR = "\n\nAnswer:\n";};++class vLLMConversationModel : public ConversationModel {+ public:+ static Option<std::string> get_answer(const std::string& context, const std::string& prompt, const std::string& system_prompt, const nlohmann::json& model_config);+ static Option<bool> validate_model(const nlohmann::json& model_config);+ static Option<std::string> get_standalone_question(const nlohmann::json& conversation_history, const std::string& question, const nlohmann::json& model_config);+ static Option<nlohmann::json> format_question(const std::string& message);+ static Option<nlohmann::json> format_answer(const std::string& message);+ static const inline std::string STANDALONE_QUESTION_PROMPT = R"(+ Rewrite the follow-up question on top of a human-assistant conversation history as a standalone question that encompasses all pertinent context.+ )";+ // prevent instantiation+ vLLMConversationModel() = delete;+ // max_bytes must be greater than or equal to the minimum required bytes+ static const size_t get_minimum_required_bytes() {+ return DATA_STR.size() + QUESTION_STR.size() + ANSWER_STR.size();+ }+ private:+ static const std::string get_list_models_url(const std::string& url);+ static const std::string get_chat_completion_url(const std::string& url);+ static const inline std::string DATA_STR = "<Data>\n";+ static const inline std::string QUESTION_STR = "\n\n<Question>\n";+ static const inline std::string ANSWER_STR = "\n\n<Answer>";+};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 125c355e..123c93c3 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1823,15 +1823,6 @@ Option<nlohmann::json> Collection::search(std::string raw_query,auto conversation_history = conversation_history_op.get();- auto truncate_conversation_history = ConversationManager::get_instance().truncate_conversation(conversation_history_op.get()["conversation"]);-- conversation_history["conversation"] = truncate_conversation_history.get();--- if(!truncate_conversation_history.ok()) {- return Option<nlohmann::json>(400, truncate_conversation_history.error());- }-auto conversation_model_op = ConversationModelManager::get_model(conversation_model_id);auto standalone_question_op = ConversationModel::get_standalone_question(conversation_history, raw_query, conversation_model_op.get());@@ -2717,13 +2708,16 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();- auto max_docs_token = ConversationModel::max_context_tokens(conversation_model);- if(!max_docs_token.ok()) {- return Option<nlohmann::json>(max_docs_token.code(), max_docs_token.error());+ auto min_required_bytes_op = ConversationModel::get_minimum_required_bytes(conversation_model);+ if(!min_required_bytes_op.ok()) {+ return Option<nlohmann::json>(min_required_bytes_op.code(), min_required_bytes_op.error());+ }+ auto min_required_bytes = min_required_bytes_op.get();+ if(conversation_model["max_bytes"].get<size_t>() < min_required_bytes + raw_query.size()) {+ return Option<nlohmann::json>(400, "`max_bytes` of the conversation model is less than the minimum required bytes(" + std::to_string(min_required_bytes) + ").");}-// remove document with lowest score until total tokens is less than MAX_TOKENS- while(ConversationManager::get_instance().get_token_count(docs_array) > max_docs_token.get()) {+ while(docs_array.dump(0).size() > conversation_model["max_bytes"].get<size_t>() - min_required_bytes - raw_query.size()) {try {if(docs_array.empty()) {break;diff --git a/src/conversation_manager.cpp b/src/conversation_manager.cppindex 8e4f1949..bf71618b 100644--- a/src/conversation_manager.cpp+++ b/src/conversation_manager.cpp@@ -70,20 +70,15 @@ Option<bool> ConversationManager::append_conversation(const std::string& convers}-size_t ConversationManager::get_token_count(const nlohmann::json& message) {- // from OpenAI API docs:- // A helpful rule of thumb is that one token generally corresponds to ~4 characters of text for common English text.- // This translates to roughly ¾ of a word (so 100 tokens ~= 75 words).-- return message.dump(0).size() / 4;-}-// pop front elements until the conversation is less than MAX_TOKENS-Option<nlohmann::json> ConversationManager::truncate_conversation(nlohmann::json conversation) {+Option<nlohmann::json> ConversationManager::truncate_conversation(nlohmann::json conversation, size_t limit) {if(!conversation.is_array()) {return Option<nlohmann::json>(400, "Conversation history is not an array");}- while(get_token_count(conversation) > MAX_TOKENS) {+ if(limit <= 0) {+ return Option<nlohmann::json>(400, "Limit must be positive integer");+ }+ while(conversation.dump(0).size() > limit) {// pop front element from json arraytry {conversation.erase(0);diff --git a/src/conversation_model.cpp b/src/conversation_model.cppindex 659112e5..b5a2b678 100644--- a/src/conversation_model.cpp+++ b/src/conversation_model.cpp@@ -24,11 +24,17 @@ Option<bool> ConversationModel::validate_model(const nlohmann::json& model_confireturn Option<bool>(400, "Property `system_prompt` is not a string.");}+ if(model_config.count("max_bytes") == 0 || !model_config["max_bytes"].is_number_unsigned() || model_config["max_bytes"].get<size_t>() == 0) {+ return Option<bool>(400, "Property `max_bytes` is not provided or not a positive integer.");+ }+const std::string model_namespace = get_model_namespace(model_config["model_name"].get<std::string>());if(model_namespace == "openai") {return OpenAIConversationModel::validate_model(model_config);} else if(model_namespace == "cf") {return CFConversationModel::validate_model(model_config);+ } else if(model_namespace == "vllm") {+ return vLLMConversationModel::validate_model(model_config);}return Option<bool>(400, "Model namespace `" + model_namespace + "` is not supported.");@@ -47,9 +53,11 @@ Option<std::string> ConversationModel::get_answer(const std::string& context, coreturn OpenAIConversationModel::get_answer(context, prompt, system_prompt, model_config);} else if(model_namespace == "cf") {return CFConversationModel::get_answer(context, prompt, system_prompt, model_config);+ } else if(model_namespace == "vllm") {+ return vLLMConversationModel::get_answer(context, prompt, system_prompt, model_config);}- throw Option<std::string>(400, "Model namespace " + model_namespace + " is not supported.");+ return Option<std::string>(400, "Model namespace " + model_namespace + " is not supported.");}Option<std::string> ConversationModel::get_standalone_question(const nlohmann::json& conversation_history, const std::string& question, const nlohmann::json& model_config) {@@ -59,9 +67,11 @@ Option<std::string> ConversationModel::get_standalone_question(const nlohmann::jreturn OpenAIConversationModel::get_standalone_question(conversation_history, question, model_config);} else if(model_namespace == "cf") {return CFConversationModel::get_standalone_question(conversation_history, question, model_config);+ } else if(model_namespace == "vllm") {+ return vLLMConversationModel::get_standalone_question(conversation_history, question, model_config);}- throw Option<std::string>(400, "Model namespace " + model_namespace + " is not supported.");+ return Option<std::string>(400, "Model namespace " + model_namespace + " is not supported.");}Option<nlohmann::json> ConversationModel::format_question(const std::string& message, const nlohmann::json& model_config) {@@ -71,9 +81,11 @@ Option<nlohmann::json> ConversationModel::format_question(const std::string& mesreturn OpenAIConversationModel::format_question(message);} else if(model_namespace == "cf") {return CFConversationModel::format_question(message);+ } else if(model_namespace == "vllm") {+ return vLLMConversationModel::format_question(message);}- throw Option<nlohmann::json>(400, "Model namespace " + model_namespace + " is not supported.");+ return Option<nlohmann::json>(400, "Model namespace " + model_namespace + " is not supported.");}Option<nlohmann::json> ConversationModel::format_answer(const std::string& message, const nlohmann::json& model_config) {@@ -83,24 +95,27 @@ Option<nlohmann::json> ConversationModel::format_answer(const std::string& messareturn OpenAIConversationModel::format_answer(message);} else if(model_namespace == "cf") {return CFConversationModel::format_answer(message);+ } else if(model_namespace == "vllm") {+ return vLLMConversationModel::format_answer(message);}- throw Option<nlohmann::json>(400, "Model namespace " + model_namespace + " is not supported.");+ return Option<nlohmann::json>(400, "Model namespace " + model_namespace + " is not supported.");}-Option<size_t> ConversationModel::max_context_tokens(const nlohmann::json& model_config) {+Option<size_t> ConversationModel::get_minimum_required_bytes(const nlohmann::json& model_config) {const std::string model_namespace = get_model_namespace(model_config["model_name"].get<std::string>());if(model_namespace == "openai") {- return Option<size_t>(OpenAIConversationModel::max_context_tokens());+ return Option<size_t>(OpenAIConversationModel::get_minimum_required_bytes());} else if(model_namespace == "cf") {- return Option<size_t>(CFConversationModel::max_context_tokens());+ return Option<size_t>(CFConversationModel::get_minimum_required_bytes());+ } else if(model_namespace == "vllm") {+ return Option<size_t>(vLLMConversationModel::get_minimum_required_bytes());}- throw Option<size_t>(400, "Model namespace " + model_namespace + " is not supported.");+ return Option<size_t>(400, "Model namespace " + model_namespace + " is not supported.");}-Option<bool> OpenAIConversationModel::validate_model(const nlohmann::json& model_config) {if(model_config.count("api_key") == 0) {return Option<bool>(400, "API key is not provided");@@ -208,7 +223,7 @@ Option<std::string> OpenAIConversationModel::get_answer(const std::string& contenlohmann::json message = nlohmann::json::object();message["role"] = "user";- message["content"] = "<Data>\n" + context + "\n\n<Question>\n" + prompt + "\n\n<Answer>";+ message["content"] = DATA_STR + context + QUESTION_STR + prompt + ANSWER_STR;req_body["messages"].push_back(message);std::string res;@@ -234,6 +249,14 @@ Option<std::string> OpenAIConversationModel::get_answer(const std::string& contenlohmann::json json_res;try {json_res = nlohmann::json::parse(res);++ if(json_res.count("choices") == 0 || json_res["choices"].size() == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }++ if(json_res["choices"][0].count("message") == 0 || json_res["choices"][0]["message"].count("content") == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }} catch (const std::exception& e) {throw Option<std::string>(400, "Got malformed response from OpenAI API.");}@@ -243,6 +266,11 @@ Option<std::string> OpenAIConversationModel::get_answer(const std::string& conteOption<std::string> OpenAIConversationModel::get_standalone_question(const nlohmann::json& conversation_history,const std::string& question, const nlohmann::json& model_config) {+ const size_t min_required_bytes = CONVERSATION_HISTORY.size() + QUESTION.size() + STANDALONE_QUESTION_PROMPT.size() + question.size();+ if(model_config["max_bytes"].get<size_t>() < min_required_bytes) {+ return Option<std::string>(400, "Max bytes is not enough to generate standalone question.");+ }+const std::string model_name = EmbedderManager::get_model_name_without_namespace(model_config["model_name"].get<std::string>());const std::string api_key = model_config["api_key"].get<std::string>();std::unordered_map<std::string, std::string> headers;@@ -257,7 +285,16 @@ Option<std::string> OpenAIConversationModel::get_standalone_question(const nlohmstd::string standalone_question = STANDALONE_QUESTION_PROMPT;standalone_question += "\n\n<Conversation history>\n";- for(auto& message : conversation_history["conversation"]) {+ auto conversation = conversation_history["conversation"];+ auto max_conversation_length = model_config["max_bytes"].get<size_t>() - min_required_bytes;+ auto truncate_conversation_op = ConversationManager::get_instance().truncate_conversation(conversation, max_conversation_length);+ if(!truncate_conversation_op.ok()) {+ return Option<std::string>(400, truncate_conversation_op.error());+ }++ auto truncated_conversation = truncate_conversation_op.get();++ for(auto& message : truncated_conversation) {if(message.count("user") == 0 && message.count("assistant") == 0) {return Option<std::string>(400, "Conversation history is not valid");}@@ -296,6 +333,13 @@ Option<std::string> OpenAIConversationModel::get_standalone_question(const nlohmnlohmann::json json_res;try {json_res = nlohmann::json::parse(res);+ if(json_res.count("choices") == 0 || json_res["choices"].size() == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }++ if(json_res["choices"][0].count("message") == 0 || json_res["choices"][0]["message"].count("content") == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }} catch (const std::exception& e) {return Option<std::string>(400, "Got malformed response from OpenAI API.");}@@ -413,18 +457,10 @@ Option<std::string> CFConversationModel::get_answer(const std::string& context,nlohmann::json message = nlohmann::json::object();message["role"] = "user";- message["content"] = R"(- Context information is below.- ---------------------- )" + context + R"(- ---------------------- Given the context information and not prior knowledge, answer the query. Context is JSON format, do not return data directly, answer like a human assistant.- Query: )" + prompt + R"(-- Answer:- )";+ message["content"] = CONTEXT_INFO + SPLITTER_STR + context + QUERY_STR + prompt + ANSWER_STR;req_body["messages"].push_back(message);+std::string res;auto url = get_model_url(model_name, account_id);auto res_code = RemoteEmbedder::call_remote_api("POST_STREAM", url, req_body.dump(), res, res_headers, headers);@@ -458,6 +494,11 @@ Option<std::string> CFConversationModel::get_answer(const std::string& context,Option<std::string> CFConversationModel::get_standalone_question(const nlohmann::json& conversation_history,const std::string& question, const nlohmann::json& model_config) {+ const size_t min_required_bytes = CONVERSATION_HISTORY.size() + QUESTION.size() + STANDALONE_QUESTION_PROMPT.size() + question.size();+ if(model_config["max_bytes"].get<size_t>() < min_required_bytes) {+ return Option<std::string>(400, "Max bytes is not enough to generate standalone question.");+ }+const std::string model_name = EmbedderManager::get_model_name_without_namespace(model_config["model_name"].get<std::string>());const std::string api_key = model_config["api_key"].get<std::string>();const std::string account_id = model_config["account_id"].get<std::string>();@@ -472,9 +513,17 @@ Option<std::string> CFConversationModel::get_standalone_question(const nlohmann:std::string res;std::string standalone_question = STANDALONE_QUESTION_PROMPT;+ auto conversation = conversation_history["conversation"];+ auto max_conversation_length = model_config["max_bytes"].get<size_t>() - min_required_bytes;+ auto truncate_conversation_op = ConversationManager::get_instance().truncate_conversation(conversation, max_conversation_length);- standalone_question += "\n\n<Conversation history>\n";- for(auto& message : conversation_history["conversation"]) {+ if(!truncate_conversation_op.ok()) {+ return Option<std::string>(400, "Conversation history is not valid");+ }++ auto truncated_conversation = truncate_conversation_op.get();++ for(auto& message : truncated_conversation) {if(message.count("user") == 0 && message.count("assistant") == 0) {return Option<std::string>(400, "Conversation history is not valid");}@@ -578,4 +627,252 @@ Option<std::string> CFConversationModel::parse_stream_response(const std::stringLOG(ERROR) << "Response: " << res;return Option<std::string>(400, "Got malformed response from Cloudflare API.");}-}\ No newline at end of file+}++Option<bool> vLLMConversationModel::validate_model(const nlohmann::json& model_config) {+ if(model_config.count("vllm_url") == 0) {+ return Option<bool>(400, "vLLM URL is not provided");+ }++ if(!model_config["vllm_url"].is_string()) {+ return Option<bool>(400, "vLLM URL is not a string");+ }++ std::unordered_map<std::string, std::string> headers;+ std::map<std::string, std::string> res_headers;+ std::string res;+ auto res_code = RemoteEmbedder::call_remote_api("GET", get_list_models_url(model_config["vllm_url"]), "", res, res_headers, headers);++ if(res_code == 408) {+ return Option<bool>(408, "vLLM API timeout.");+ }++ if (res_code != 200) {+ nlohmann::json json_res;+ try {+ json_res = nlohmann::json::parse(res);+ } catch (const std::exception& e) {+ return Option<bool>(400, "vLLM API error: " + res);+ }+ if(json_res.count("message") == 0) {+ return Option<bool>(400, "vLLM API error: " + res);+ }+ return Option<bool>(400, "vLLM API error: " + nlohmann::json::parse(res)["message"].get<std::string>());+ }++ nlohmann::json models_json;+ try {+ models_json = nlohmann::json::parse(res);+ } catch (const std::exception& e) {+ return Option<bool>(400, "Got malformed response from vLLM API.");+ }+ bool found = false;+ // extract model name by removing "vLLM/" prefix+ auto model_name_without_namespace = EmbedderManager::get_model_name_without_namespace(model_config["model_name"].get<std::string>());+ for (auto& model : models_json["data"]) {+ if (model["id"] == model_name_without_namespace) {+ found = true;+ break;+ }+ }++ if(!found) {+ return Option<bool>(400, "Property `model_name` is not a valid vLLM model.");+ }++ nlohmann::json req_body;+ headers["Content-Type"] = "application/json";+ req_body["model"] = model_name_without_namespace;+ req_body["messages"] = R"([+ {+ "role":"user",+ "content":"hello"+ }+ ])"_json;+ std::string chat_res;++ res_code = RemoteEmbedder::call_remote_api("POST", get_chat_completion_url(model_config["vllm_url"]), req_body.dump(-1), chat_res, res_headers, headers);++ if(res_code == 408) {+ return Option<bool>(408, "vLLM API timeout.");+ }++ if (res_code != 200) {+ nlohmann::json json_res;+ try {+ json_res = nlohmann::json::parse(res);+ } catch (const std::exception& e) {+ return Option<bool>(400, "vLLM API error: " + res);+ }+ if(json_res.count("message") == 0) {+ return Option<bool>(400, "vLLM API error: " + res);+ }+ return Option<bool>(400, "vLLM API error: " + nlohmann::json::parse(res)["message"].get<std::string>());+ }++ return Option<bool>(true);+}++Option<std::string> vLLMConversationModel::get_answer(const std::string& context, const std::string& prompt,+ const std::string& system_prompt, const nlohmann::json& model_config) {+ const std::string model_name = EmbedderManager::get_model_name_without_namespace(model_config["model_name"].get<std::string>());+ const std::string vllm_url = model_config["vllm_url"].get<std::string>();++ std::unordered_map<std::string, std::string> headers;+ std::map<std::string, std::string> res_headers;+ headers["Content-Type"] = "application/json";+ nlohmann::json req_body;+ req_body["model"] = model_name;+ req_body["messages"] = nlohmann::json::array();++ if(!system_prompt.empty()) {+ nlohmann::json system_message = nlohmann::json::object();+ system_message["role"] = "system";+ system_message["content"] = system_prompt;+ req_body["messages"].push_back(system_message);+ }++ nlohmann::json message = nlohmann::json::object();+ message["role"] = "user";+ message["content"] = DATA_STR + context + QUESTION_STR + prompt + ANSWER_STR;+ req_body["messages"].push_back(message);++ std::string res;+ auto res_code = RemoteEmbedder::call_remote_api("POST", get_chat_completion_url(vllm_url), req_body.dump(), res, res_headers, headers);++ if(res_code == 408) {+ throw Option<std::string>(400, "vLLM API timeout.");+ }++ if (res_code != 200) {+ nlohmann::json json_res;+ try {+ json_res = nlohmann::json::parse(res);+ } catch (const std::exception& e) {+ return Option<std::string>(400, "vLLM API error: " + res);+ }+ if(json_res.count("message") == 0) {+ return Option<std::string>(400, "vLLM API error: " + res);+ }+ return Option<std::string>(400, "vLLM API error: " + nlohmann::json::parse(res)["message"].get<std::string>());+ }++ nlohmann::json json_res;+ try {+ json_res = nlohmann::json::parse(res);++ if(json_res.count("choices") == 0 || json_res["choices"].size() == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }++ if(json_res["choices"][0].count("message") == 0 || json_res["choices"][0]["message"].count("content") == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }+ } catch (const std::exception& e) {+ throw Option<std::string>(400, "Got malformed response from vLLM API.");+ }++ return Option<std::string>(json_res["choices"][0]["message"]["content"].get<std::string>());+}++Option<std::string> vLLMConversationModel::get_standalone_question(const nlohmann::json& conversation_history,+ const std::string& question, const nlohmann::json& model_config) {+ const size_t min_required_bytes = CONVERSATION_HISTORY.size() + QUESTION.size() + STANDALONE_QUESTION_PROMPT.size() + question.size();+ if(model_config["max_bytes"].get<size_t>() < min_required_bytes) {+ return Option<std::string>(400, "Max bytes is not enough to generate standalone question.");+ }++ const std::string model_name = EmbedderManager::get_model_name_without_namespace(model_config["model_name"].get<std::string>());+ const std::string vllm_url = model_config["vllm_url"].get<std::string>();+ std::unordered_map<std::string, std::string> headers;+ std::map<std::string, std::string> res_headers;+ headers["Content-Type"] = "application/json";+ nlohmann::json req_body;+ req_body["model"] = model_name;+ req_body["messages"] = nlohmann::json::array();+ std::string res;++ std::string standalone_question = STANDALONE_QUESTION_PROMPT;+ auto conversation = conversation_history["conversation"];+ auto max_conversation_length = model_config["max_bytes"].get<size_t>() - min_required_bytes;+ auto truncate_conversation_op = ConversationManager::get_instance().truncate_conversation(conversation, max_conversation_length);++ if(!truncate_conversation_op.ok()) {+ return Option<std::string>(400, "Conversation history is not valid");+ }++ auto truncated_conversation = truncate_conversation_op.get();++ for(auto& message : truncated_conversation) {+ if(message.count("user") == 0 && message.count("assistant") == 0) {+ return Option<std::string>(400, "Conversation history is not valid");+ }++ standalone_question += message.dump(0) + "\n";+ }++ standalone_question += "\n\n<Question>\n" + question;+ standalone_question += "\n\n<Standalone question>\n";++ nlohmann::json message = nlohmann::json::object();+ message["role"] = "user";+ message["content"] = standalone_question;++ req_body["messages"].push_back(message);++ auto res_code = RemoteEmbedder::call_remote_api("POST", get_chat_completion_url(vllm_url), req_body.dump(), res, res_headers, headers);++ if(res_code == 408) {+ return Option<std::string>(400, "vLLM API timeout.");+ }++ if (res_code != 200) {+ nlohmann::json json_res;+ try {+ json_res = nlohmann::json::parse(res);+ } catch (const std::exception& e) {+ return Option<std::string>(400, "vLLM API error: " + res);+ }+ if(json_res.count("message") == 0) {+ return Option<std::string>(400, "vLLM API error: " + res);+ }+ return Option<std::string>(400, "vLLM API error: " + nlohmann::json::parse(res)["message"].get<std::string>());+ }++ nlohmann::json json_res;+ try {+ json_res = nlohmann::json::parse(res);++ if(json_res.count("choices") == 0 || json_res["choices"].size() == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }++ if(json_res["choices"][0].count("message") == 0 || json_res["choices"][0]["message"].count("content") == 0) {+ return Option<std::string>(400, "Got malformed response from OpenAI API.");+ }+ } catch (const std::exception& e) {+ return Option<std::string>(400, "Got malformed response from vLLM API.");+ }++ return Option<std::string>(json_res["choices"][0]["message"]["content"].get<std::string>());+}++Option<nlohmann::json> vLLMConversationModel::format_question(const std::string& message) {+ nlohmann::json json = nlohmann::json::object();+ json["user"] = message;+ return Option<nlohmann::json>(json);+}++Option<nlohmann::json> vLLMConversationModel::format_answer(const std::string& message) {+ nlohmann::json json = nlohmann::json::object();+ json["assistant"] = message;+ return Option<nlohmann::json>(json);+}++const std::string vLLMConversationModel::get_list_models_url(const std::string& vllm_url) {+ return vllm_url.back() == '/' ? vllm_url + "v1/models" : vllm_url + "/v1/models";+}++const std::string vLLMConversationModel::get_chat_completion_url(const std::string& vllm_url) {+ return vllm_url.back() == '/' ? vllm_url + "v1/chat/completions" : vllm_url + "/v1/chat/completions";+}diff --git a/src/core_api.cpp b/src/core_api.cppindex c1c78b8b..60926709 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -734,16 +734,20 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_pconst std::string& conversation_model_id = orig_req_params["conversation_model_id"];auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();- // We have to pop a document from the search result with max size- // Until we do not exceed MAX_TOKENS limit- auto max_docs_token = ConversationModel::max_context_tokens(conversation_model);- if(!max_docs_token.ok()) {- res->set_400(max_docs_token.error());+ auto min_required_bytes_op = ConversationModel::get_minimum_required_bytes(conversation_model);+ if(!min_required_bytes_op.ok()) {+ res->set_400(min_required_bytes_op.error());+ return false;+ }+ auto min_required_bytes = min_required_bytes_op.get();+ auto prompt = req->params["q"];+ if(conversation_model["max_bytes"].get<size_t>() < min_required_bytes + prompt.size()) {+ res->set_400("`max_bytes` of the conversation model is less than the minimum required bytes(" + std::to_string(min_required_bytes) + ").");return false;}// remove document with lowest score until total tokens is less than MAX_TOKENS- while(ConversationManager::get_instance().get_token_count(result_docs_arr) > max_docs_token.get()) {+ while(result_docs_arr.dump(0).size() > conversation_model["max_bytes"].get<size_t>() - min_required_bytes - prompt.size()) {// sort the result_docs_arr by size descendingstd::sort(result_docs_arr.begin(), result_docs_arr.end(), [](const auto& a, const auto& b) {return a.size() > b.size();@@ -755,7 +759,6 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_p}}-// Make result_docs_arr 1Dnlohmann::json result_docs = nlohmann::json::array();for(const auto& result_doc : result_docs_arr) {@@ -764,8 +767,6 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_p}}- auto prompt = req->params["q"];-auto answer_op = ConversationModel::get_answer(result_docs.dump(0), prompt, conversation_model);if(!answer_op.ok()) {@@ -783,7 +784,6 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_preturn false;}-auto formatted_answer_op = ConversationModel::format_answer(answer_op.get(), conversation_model);if(!formatted_answer_op.ok()) {res->set_400(formatted_answer_op.error());diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 5258e61c..a1e3e0f0 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -2872,7 +2872,8 @@ TEST_F(CollectionVectorTest, TestQAConversation) {auto api_key = std::string(std::getenv("api_key"));auto conversation_model_config = R"({- "model_name": "openai/gpt-3.5-turbo"+ "model_name": "openai/gpt-3.5-turbo",+ "max_bytes: 1000})"_json;conversation_model_config["api_key"] = api_key;@@ -3318,7 +3319,8 @@ TEST_F(CollectionVectorTest, InvalidMultiSearchConversation) {auto api_key = std::string(std::getenv("api_key"));auto conversation_model_config = R"({- "model_name": "openai/gpt-3.5-turbo"+ "model_name": "openai/gpt-3.5-turbo",+ "max_bytes": 1000})"_json;conversation_model_config["api_key"] = api_key;diff --git a/test/conversation_test.cpp b/test/conversation_test.cppindex 375e5a3c..d8546945 100644--- a/test/conversation_test.cpp+++ b/test/conversation_test.cpp@@ -121,26 +121,34 @@ TEST_F(ConversationTest, TruncateConversation) {conversation.push_back(message);}- auto truncated = ConversationManager::get_instance().truncate_conversation(conversation);+ auto truncated = ConversationManager::get_instance().truncate_conversation(conversation, 100);ASSERT_TRUE(truncated.ok());ASSERT_TRUE(truncated.get().size() < conversation.size());+ ASSERT_TRUE(truncated.get().dump(0).size() < 100);}TEST_F(ConversationTest, TruncateConversationEmpty) {nlohmann::json conversation = nlohmann::json::array();- auto truncated = ConversationManager::get_instance().truncate_conversation(conversation);+ auto truncated = ConversationManager::get_instance().truncate_conversation(conversation, 100);ASSERT_TRUE(truncated.ok());ASSERT_TRUE(truncated.get().size() == 0);}TEST_F(ConversationTest, TruncateConversationInvalidType) {nlohmann::json conversation = nlohmann::json::object();- auto truncated = ConversationManager::get_instance().truncate_conversation(conversation);+ auto truncated = ConversationManager::get_instance().truncate_conversation(conversation, 100);ASSERT_FALSE(truncated.ok());ASSERT_EQ(truncated.code(), 400);ASSERT_EQ(truncated.error(), "Conversation history is not an array");}+TEST_F(ConversationTest, TruncateConversationInvalidLimit) {+ nlohmann::json conversation = nlohmann::json::array();+ auto truncated = ConversationManager::get_instance().truncate_conversation(conversation, 0);+ ASSERT_FALSE(truncated.ok());+ ASSERT_EQ(truncated.code(), 400);+ ASSERT_EQ(truncated.error(), "Limit must be positive integer");+}TEST_F(ConversationTest, TestConversationExpire) {nlohmann::json conversation = nlohmann::json::array();diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex ba07c356..446f16b0 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -1388,7 +1388,8 @@ TEST_F(CoreAPIUtilsTest, SampleGzipIndexTest) {TEST_F(CoreAPIUtilsTest, TestConversationModels) {nlohmann::json model_config = R"({- "model_name": "openai/gpt-3.5-turbo"+ "model_name": "openai/gpt-3.5-turbo",+ "max_bytes": 10000})"_json;EmbedderManager::set_model_dir("/tmp/typesense_test/models");@@ -1490,6 +1491,32 @@ TEST_F(CoreAPIUtilsTest, TestInvalidConversationModels) {ASSERT_EQ(400, resp->status_code);ASSERT_EQ("Property `model_name` is not provided or not a string.", nlohmann::json::parse(resp->body)["message"]);++ model_config["model_name"] = "openai/gpt-3.5-turbo";++ // test without max_bytes+ post_conversation_model(req, resp);++ ASSERT_EQ(400, resp->status_code);+ ASSERT_EQ("Property `max_bytes` is not provided or not a number.", nlohmann::json::parse(resp->body)["message"]);++ // test with max_bytes as string+ model_config["max_bytes"] = "10000";++ req->body = model_config.dump();+ post_conversation_model(req, resp);++ ASSERT_EQ(400, resp->status_code);+ ASSERT_EQ("Property `max_bytes` is not provided or not a number.", nlohmann::json::parse(resp->body)["message"]);++ // test with max_bytes as negative number+ model_config["max_bytes"] = -10000;++ req->body = model_config.dump();+ post_conversation_model(req, resp);++ ASSERT_EQ(400, resp->status_code);+ ASSERT_EQ("Property `max_bytes` must be a positive number.", nlohmann::json::parse(resp->body)["message"]);}TEST_F(CoreAPIUtilsTest, DeleteNonExistingDoc) {