Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix context length for RAG models (#1544)
Ozan Armağanded3a5ec08a5
diff --git a/include/conversation_model.h b/include/conversation_model.hindex d8ff13fe..61ce16ff 100644--- a/include/conversation_model.h+++ b/include/conversation_model.h@@ -22,6 +22,8 @@ class ConversationModel {static Option<std::string> get_standalone_question(const nlohmann::json& conversation_history, const std::string& question, const nlohmann::json& model_config);static Option<nlohmann::json> format_question(const std::string& message, const nlohmann::json& model_config);static Option<nlohmann::json> format_answer(const std::string& message, const nlohmann::json& model_config);+ // max token count for the context search results+ static Option<size_t> max_context_tokens(const nlohmann::json& model_config);private:};@@ -37,6 +39,9 @@ class OpenAIConversationModel : public ConversationModel {)";// prevent instantiationOpenAIConversationModel() = delete;+ static size_t max_context_tokens() {+ return 3000;+ }private:static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";static constexpr char* OPENAI_CHAT_COMPLETION = "https://api.openai.com/v1/chat/completions";@@ -50,11 +55,14 @@ class CFConversationModel : public ConversationModel {static Option<nlohmann::json> format_question(const std::string& message);static Option<nlohmann::json> format_answer(const std::string& message);static const inline std::string STANDALONE_QUESTION_PROMPT = R"(- [INST]Rewrite the follow-up question on top of a human-assistant conversation history as a standalone question that encompasses all pertinent context.[/INST]+ Rewrite the follow-up question on top of a human-assistant conversation history as a standalone question that encompasses all pertinent context.Use 1024 characters maximum.)";- static const inline std::string INFO_PROMPT = "[INST] You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise. [/INST]";+ static const inline std::string INFO_PROMPT = "You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and do not mention provided context directly, act like already knowing the context.";// prevent instantiationCFConversationModel() = delete;+ static size_t max_context_tokens() {+ return 768;+ }private:static const inline std::vector<std::string> CF_MODEL_NAMES{"mistral/mistral-7b-instruct-v0.1"};static const std::string get_model_url(const std::string& model_name, const std::string& account_id);diff --git a/src/collection.cpp b/src/collection.cppindex 71912b9f..ce1cfab4 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2712,17 +2712,24 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}}+ auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();+ auto max_docs_token = ConversationModel::max_context_tokens(conversation_model);+ if(!max_docs_token.ok()) {+ return Option<nlohmann::json>(max_docs_token.code(), max_docs_token.error());+ }+// remove document with lowest score until total tokens is less than MAX_TOKENS- while(ConversationManager::get_instance().get_token_count(docs_array) > ConversationManager::get_instance().MAX_TOKENS) {+ while(ConversationManager::get_instance().get_token_count(docs_array) > max_docs_token.get()) {try {+ if(docs_array.empty()) {+ break;+ }docs_array.erase(docs_array.size() - 1);} catch(...) {return Option<nlohmann::json>(400, "Failed to remove document from search results.");}}- auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();-bool has_conversation_history = !conversation_id.empty();auto qa_op = ConversationModel::get_answer(docs_array.dump(0), raw_query, conversation_model);if(!qa_op.ok()) {diff --git a/src/conversation_model.cpp b/src/conversation_model.cppindex afdcf660..b482a462 100644--- a/src/conversation_model.cpp+++ b/src/conversation_model.cpp@@ -86,6 +86,18 @@ Option<nlohmann::json> ConversationModel::format_answer(const std::string& messathrow Option<nlohmann::json>(400, "Model namespace " + model_namespace + " is not supported.");}+Option<size_t> ConversationModel::max_context_tokens(const nlohmann::json& model_config) {+ const std::string model_namespace = get_model_namespace(model_config["model_name"].get<std::string>());++ if(model_namespace == "openai") {+ return Option<size_t>(OpenAIConversationModel::max_context_tokens());+ } else if(model_namespace == "cf") {+ return Option<size_t>(CFConversationModel::max_context_tokens());+ }++ throw Option<size_t>(400, "Model namespace " + model_namespace + " is not supported.");+}+Option<bool> OpenAIConversationModel::validate_model(const nlohmann::json& model_config) {if(model_config.count("api_key") == 0) {@@ -398,11 +410,17 @@ Option<std::string> CFConversationModel::get_answer(const std::string& context,}nlohmann::json message = nlohmann::json::object();- message["role"] = "system";- message["content"] = INFO_PROMPT;- req_body["messages"].push_back(message);message["role"] = "user";- message["content"] = "[INST]Context:\n" + context + "\n\nQuestion:\n" + prompt + "\n\nAnswer:[/INST]";+ message["content"] = R"(+ Context information is below.+ ---------------------+ )" + context + R"(+ ---------------------+ Given the context information and not prior knowledge, answer the query. Context is JSON format, do not return data directly, answer like a human assistant.+ Query: )" + prompt + R"(++ Answer:+ )";req_body["messages"].push_back(message);std::string res;@@ -417,7 +435,10 @@ Option<std::string> CFConversationModel::get_answer(const std::string& context,nlohmann::json json_res;try {json_res = nlohmann::json::parse(res);- json_res = nlohmann::json::parse(json_res["response"].get<std::string>());+ if(json_res.count("response") == 0 || json_res["response"].size() == 0) {+ return Option<std::string>(400, "Cloudflare API error: " + res);+ }+ json_res = nlohmann::json::parse(json_res["response"][0].get<std::string>());} catch (const std::exception& e) {throw Option<std::string>(400, "Cloudflare API error: " + res);}diff --git a/src/core_api.cpp b/src/core_api.cppindex 3d7a893b..c1c78b8b 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -687,6 +687,10 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_pnlohmann::json result_docs_arr = nlohmann::json::array();int res_index = 0;for(const auto& result : response["results"]) {+ if(result.count("code") != 0) {+ continue;+ }+nlohmann::json result_docs = nlohmann::json::array();std::vector<std::string> vector_fields;@@ -728,9 +732,18 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_presult_docs_arr.push_back(result_docs);}+ const std::string& conversation_model_id = orig_req_params["conversation_model_id"];+ auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();// We have to pop a document from the search result with max size// Until we do not exceed MAX_TOKENS limit- while(ConversationManager::get_instance().get_token_count(result_docs_arr) > ConversationManager::get_instance().MAX_TOKENS) {+ auto max_docs_token = ConversationModel::max_context_tokens(conversation_model);+ if(!max_docs_token.ok()) {+ res->set_400(max_docs_token.error());+ return false;+ }++ // remove document with lowest score until total tokens is less than MAX_TOKENS+ while(ConversationManager::get_instance().get_token_count(result_docs_arr) > max_docs_token.get()) {// sort the result_docs_arr by size descendingstd::sort(result_docs_arr.begin(), result_docs_arr.end(), [](const auto& a, const auto& b) {return a.size() > b.size();@@ -751,9 +764,6 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_p}}- const std::string& conversation_model_id = orig_req_params["conversation_model_id"];- auto conversation_model = ConversationModelManager::get_model(conversation_model_id).get();-auto prompt = req->params["q"];auto answer_op = ConversationModel::get_answer(result_docs.dump(0), prompt, conversation_model);