Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Added OpenAI integration for embeddings
ozanarmagan096543dcd6f0
diff --git a/include/field.h b/include/field.hindex d4361db9..5fbb386e 100644--- a/include/field.h+++ b/include/field.h@@ -52,6 +52,8 @@ namespace fields {static const std::string reference = "reference";static const std::string embed_from = "embed_from";static const std::string model_name = "model_name";+ static const std::string openai_api_key = "openai_api_key";+ static const std::string model_parameters = "model_parameters";}enum vector_distance_type_t {@@ -78,7 +80,7 @@ struct field {size_t num_dim;std::vector<std::string> embed_from;- std::string model_name;+ nlohmann::json model_parameters;vector_distance_type_t vec_dist;static constexpr int VAL_UNKNOWN = 2;@@ -89,9 +91,10 @@ struct field {field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,- int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &embed_from = {}, const std::string& model_name = "") :+ int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &embed_from = {},+ const nlohmann::json& model_parameters = nlohmann::json()) :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),- nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed_from(embed_from), model_name(model_name) {+ nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed_from(embed_from), model_parameters(model_parameters) {set_computed_defaults(sort, infix);}@@ -321,8 +324,12 @@ struct field {}if(!field.embed_from.empty()) {field_val[fields::embed_from] = field.embed_from;- if(!field.model_name.empty()) {- field_val[fields::model_name] = field.model_name;+ if(!field.model_parameters.empty()) {+ field_val[fields::model_parameters] = field.model_parameters;+ // hide openai api key+ if(field_val[fields::model_parameters].count("openai_api_key") != 0) {+ field_val[fields::model_parameters]["openai_api_key"] = "<hidden>";+ }}}fields_json.push_back(field_val);diff --git a/include/http_client.h b/include/http_client.hindex 9d07f391..4fa9b685 100644--- a/include/http_client.h+++ b/include/http_client.h@@ -32,7 +32,7 @@ private:static size_t curl_req_send_callback(char* buffer, size_t size, size_t nitems, void *userdata);- static long perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers);+ static long perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers, struct curl_slist *chunk = nullptr);public:static HttpClient & get_instance() {@@ -48,13 +48,13 @@ public:static long download_file(const std::string& url, const std::string& file_path);static long get_response(const std::string& url, std::string& response,- std::map<std::string, std::string>& res_headers, long timeout_ms=4000);+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers = {}, long timeout_ms=4000);static long delete_response(const std::string& url, std::string& response,std::map<std::string, std::string>& res_headers, long timeout_ms=120000);static long post_response(const std::string & url, const std::string & body, std::string & response,- std::map<std::string, std::string>& res_headers, long timeout_ms=4000);+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers = {}, long timeout_ms=4000);static long post_response_async(const std::string &url, const std::shared_ptr<http_req> request,const std::shared_ptr<http_res> response,diff --git a/include/index.h b/include/index.hindex 332e9fa2..a80dff23 100644--- a/include/index.h+++ b/include/index.h@@ -538,7 +538,10 @@ private:static Option<bool> embed_fields(nlohmann::json& document,const tsl::htrie_map<char, field>& embedding_fields,- const tsl::htrie_map<char, field> & search_schema);+ const tsl::htrie_map<char, field> & search_schema);+ static Option<bool> batch_embed_fields(std::vector<nlohmann::json*>& documents,+ const tsl::htrie_map<char, field>& embedding_fields,+ const tsl::htrie_map<char, field> & search_schema);public:// for limiting number of results on multiple candidates / query rewritesdiff --git a/include/text_embedder.h b/include/text_embedder.hindex c1cb33fd..869644f8 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -3,6 +3,7 @@#include <core/session/onnxruntime_cxx_api.h>#include <tokenizer/bert_tokenizer.hpp>#include <vector>+#include "option.h"struct encoded_input_t {std::vector<int64_t> input_ids;@@ -14,15 +15,26 @@ struct encoded_input_t {class TextEmbedder {public:TextEmbedder(const std::string& model_path);+ TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key);~TextEmbedder();- std::vector<float> Embed(const std::string& text);+ Option<std::vector<float>> Embed(const std::string& text);+ Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);++ bool is_openai() {+ return !openai_api_key.empty();+ }static bool is_model_valid(const std::string& model_path, unsigned int& num_dims);+ static Option<bool> is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims);private:- Ort::Session* session_;+ std::unique_ptr<Ort::Session> session_;Ort::Env env_;encoded_input_t Encode(const std::string& text);- BertTokenizer* tokenizer_;+ std::unique_ptr<BertTokenizer> tokenizer_;static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);std::string output_tensor_name;+ std::string openai_api_key;+ std::string openai_model_path;+ static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";+ static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";};diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex 192044c8..f19bc8d9 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -1,5 +1,5 @@#pragma once-+#include <memory>#include <filesystem>#include <unordered_map>#include <openssl/md5.h>@@ -8,6 +8,7 @@#include "http_client.h"#include "text_embedder.h"+// singleton classclass TextEmbedderManager {public:@@ -21,24 +22,33 @@ public:TextEmbedderManager(const TextEmbedderManager&) = delete;TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;- TextEmbedder* get_text_embedder(const std::string& model_path) {- if (text_embedders.find(model_path) == text_embedders.end()) {- text_embedders[model_path] = new TextEmbedder(model_path);+ TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {+ if(model_parameters.count("model_name") == 0) {+ if(text_embedders[DEFAULT_MODEL_NAME] == nullptr) {+ text_embedders[DEFAULT_MODEL_NAME] = std::make_shared<TextEmbedder>(DEFAULT_MODEL_NAME);+ }+ return text_embedders[DEFAULT_MODEL_NAME].get();+ } else {+ const std::string& model_name = model_parameters.at("model_name");+ if(text_embedders[model_name] == nullptr) {+ if(model_parameters.count("openai_api_key") == 0) {+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);+ } else {+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("openai_api_key"));+ }+ }+ return text_embedders[model_name].get();}- return text_embedders[model_path];+}void delete_text_embedder(const std::string& model_path) {if (text_embedders.find(model_path) != text_embedders.end()) {- delete text_embedders[model_path];text_embedders.erase(model_path);}}void delete_all_text_embedders() {- for (auto& text_embedder : text_embedders) {- delete text_embedder.second;- }text_embedders.clear();}@@ -120,7 +130,7 @@ public:}private:TextEmbedderManager() = default;- std::unordered_map<std::string, TextEmbedder*> text_embedders;+ std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;};diff --git a/src/collection.cpp b/src/collection.cppindex d6aa3403..525cc753 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -244,8 +244,12 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::embed_from] = coll_field.embed_from;}- if(coll_field.model_name.size() > 0) {- field_json[fields::model_name] = coll_field.model_name;+ if(coll_field.model_parameters.size() > 0) {+ field_json[fields::model_parameters] = coll_field.model_parameters;+ // Hide OpenAI API key from the response.+ if(field_json[fields::model_parameters].count(fields::openai_api_key) != 0) {+ field_json[fields::model_parameters][fields::openai_api_key] = "<hidden>";+ }}if(coll_field.num_dim > 0) {@@ -1184,11 +1188,15 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();- auto embedder = embedder_manager.get_text_embedder(search_field.model_name.size() > 0 ? search_field.model_name : TextEmbedderManager::DEFAULT_MODEL_NAME);+ auto embedder = embedder_manager.get_text_embedder(search_field.model_parameters);std::string embed_query = "query: " + raw_query;+ auto embedding_op = embedder->Embed(embed_query);+ if(!embedding_op.ok()) {+ return Option<nlohmann::json>(400, embedding_op.error());+ }- std::vector<float> embedding = embedder->Embed(embed_query);+ std::vector<float> embedding = embedding_op.get();vector_query._reset();vector_query.values = embedding;vector_query.field_name = field_name;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 28bd0f5a..765da8dc 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -62,10 +62,9 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::embed_from] = std::vector<std::string>();}- if(field_obj.count(fields::model_name) == 0) {- field_obj[fields::model_name] = "";+ if(field_obj.count(fields::model_parameters) == 0) {+ field_obj[fields::model_parameters] = nlohmann::json::object();}-vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;if(field_obj.count(fields::vec_dist) != 0) {@@ -79,7 +78,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed_from],- field_obj[fields::model_name]);+ field_obj[fields::model_parameters]);// value of `sort` depends on field typeif(field_obj.count(fields::sort) == 0) {diff --git a/src/field.cpp b/src/field.cppindex 6c572b99..ec9e8f7c 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -672,26 +672,37 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}- if(field_json.count(fields::model_name) > 0 && field_json.count(fields::embed_from) == 0) {- return Option<bool>(400, "Property `" + fields::model_name + "` can only be used with `" + fields::embed_from + "`.");+ if(field_json.count(fields::model_parameters) > 0 && field_json.count(fields::embed_from) == 0) {+ return Option<bool>(400, "Property `" + fields::model_parameters + "` can only be used with `" + fields::embed_from + "`.");}if(field_json.count(fields::embed_from) != 0) {// If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)field_json[fields::num_dim] = static_cast<unsigned int>(384);- if(field_json.count(fields::model_name) != 0) {- unsigned int num_dim = 0;- if(!field_json[fields::model_name].is_string()) {- return Option<bool>(400, "Property `" + fields::model_name + "` must be a string.");- }- if(field_json[fields::model_name].get<std::string>().empty()) {- return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");- }-- if(TextEmbedder::is_model_valid(field_json[fields::model_name].get<std::string>(), num_dim)) {- field_json[fields::num_dim] = num_dim;- } else {- return Option<bool>(400, "Property `" + fields::model_name + "` must be a valid model path.");+ if(field_json.count(fields::model_parameters) != 0) {+ auto model_parameters = field_json[fields::model_parameters];+ if(model_parameters.count(fields::model_name) != 0) {+ unsigned int num_dim = 0;+ if(!model_parameters[fields::model_name].is_string()) {+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a string.");+ }+ if(model_parameters[fields::model_name].get<std::string>().empty()) {+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");+ }+ if(model_parameters.count("openai_api_key") != 0) {+ auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::openai_api_key].get<std::string>(), num_dim);+ if(res.ok()) {+ field_json[fields::num_dim] = num_dim;+ } else {+ return Option<bool>(res.code(), res.error());+ }+ } else {+ if(TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), num_dim)) {+ field_json[fields::num_dim] = num_dim;+ } else {+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a valid model path.");+ }+ }}}} else {@@ -699,10 +710,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}- if(field_json.count(fields::model_name) == 0) {- field_json[fields::model_name] = "";- }-auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);@@ -784,8 +791,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::optional], field_json[fields::index], field_json[fields::locale],field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,- field_json[fields::reference], field_json[fields::embed_from].get<std::vector<std::string>>(),- field_json[fields::model_name])+ field_json[fields::reference], field_json[fields::embed_from].get<std::vector<std::string>>(),+ field_json[fields::model_parameters]));if (!field_json[fields::reference].get<std::string>().empty()) {diff --git a/src/http_client.cpp b/src/http_client.cppindex a06b8aea..652f0971 100644--- a/src/http_client.cpp+++ b/src/http_client.cpp@@ -16,16 +16,24 @@ struct client_state_t: public req_state_t {};long HttpClient::post_response(const std::string &url, const std::string &body, std::string &response,- std::map<std::string, std::string>& res_headers, long timeout_ms) {+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers, long timeout_ms) {CURL *curl = init_curl(url, response);if(curl == nullptr) {return 500;}curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());- return perform_curl(curl, res_headers);++ struct curl_slist *chunk = nullptr;+ for(const auto& header: headers) {+ std::string header_str = header.first + ": " + header.second;+ chunk = curl_slist_append(chunk, header_str.c_str());+ }++ return perform_curl(curl, res_headers, chunk);}+long HttpClient::post_response_async(const std::string &url, const std::shared_ptr<http_req> request,const std::shared_ptr<http_res> response, HttpServer* server) {deferred_req_res_t* req_res = new deferred_req_res_t(request, response, server, false);@@ -82,13 +90,18 @@ long HttpClient::delete_response(const std::string &url, std::string &response,}long HttpClient::get_response(const std::string &url, std::string &response,- std::map<std::string, std::string>& res_headers, long timeout_ms) {+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers, long timeout_ms) {CURL *curl = init_curl(url, response);if(curl == nullptr) {return 500;}+ struct curl_slist *chunk = nullptr;+ for(const auto& header: headers) {+ std::string header_str = header.first + ": " + header.second;+ chunk = curl_slist_append(chunk, header_str.c_str());+ }- return perform_curl(curl, res_headers);+ return perform_curl(curl, res_headers, chunk);}void HttpClient::init(const std::string &api_key) {@@ -115,8 +128,7 @@ void HttpClient::init(const std::string &api_key) {}}-long HttpClient::perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers) {- struct curl_slist *chunk = nullptr;+long HttpClient::perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers, struct curl_slist *chunk) {std::string api_key_header = std::string("x-typesense-api-key: ") + HttpClient::api_key;chunk = curl_slist_append(chunk, api_key_header.c_str());curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);diff --git a/src/index.cpp b/src/index.cppindex 6e99d0d0..52b7d647 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -418,6 +418,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteconst bool do_validation) {// runs in a partitioned thread+ std::vector<nlohmann::json*> docs_to_embed;for(size_t i = 0; i < batch_size; i++) {index_record& index_rec = iter_batch[batch_start_index + i];@@ -453,9 +454,11 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteget_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,index_rec.del_doc);scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);- embed_fields(index_rec.new_doc, embedding_fields, search_schema);+ docs_to_embed.push_back(&index_rec.new_doc);+ //embed_fields(index_rec.new_doc, embedding_fields, search_schema);} else {- embed_fields(index_rec.doc, embedding_fields, search_schema);+ docs_to_embed.push_back(&index_rec.doc);+ //embed_fields(index_rec.doc, embedding_fields, search_schema);}compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);@@ -485,6 +488,14 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteindex_rec.index_failure(400, e.what());}}++ auto embed_op = batch_embed_fields(docs_to_embed, embedding_fields, search_schema);+ if(!embed_op.ok()) {+ for(size_t i = 0; i < batch_size; i++) {+ index_record& index_rec = iter_batch[batch_start_index + i];+ index_rec.index_failure(embed_op.code(), embed_op.error());+ }+ }}size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,@@ -6349,11 +6360,50 @@ Option<bool> Index::embed_fields(nlohmann::json& document,}}TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();- auto embedder = embedder_manager.get_text_embedder(field.model_name.size() > 0 ? field.model_name : TextEmbedderManager::DEFAULT_MODEL_NAME);- std::vector<float> embedding = embedder->Embed(text_to_embed);- document[field.name] = embedding;+ auto embedder = embedder_manager.get_text_embedder(field.model_parameters);+ auto embedding_op = embedder->Embed(text_to_embed);++ if(!embedding_op.ok()) {+ return Option<bool>(400, embedding_op.error());+ }++ document[field.name] = embedding_op.get();}+ return Option<bool>(true);+}+Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,+ const tsl::htrie_map<char, field>& embedding_fields,+ const tsl::htrie_map<char, field> & search_schema) {+ for(const auto& field : embedding_fields) {+ std::vector<std::string> text_to_embed;+ for(auto& document : documents) {+ std::string text = "passage: ";+ for(const auto& field_name : field.embed_from) {+ auto field_it = search_schema.find(field_name);+ if(field_it.value().type == field_types::STRING) {+ text += (*document)[field_name].get<std::string>() + " ";+ } else if(field_it.value().type == field_types::STRING_ARRAY) {+ for(const auto& val : (*document)[field_name]) {+ text += val.get<std::string>() + " ";+ }+ }+ }+ text_to_embed.push_back(text);+ }+ TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();+ auto embedder = embedder_manager.get_text_embedder(field.model_parameters);+ auto embedding_op = embedder->batch_embed(text_to_embed);++ if(!embedding_op.ok()) {+ return Option<bool>(400, embedding_op.error());+ }++ auto embeddings = embedding_op.get();+ for(size_t i = 0; i < documents.size(); i++) {+ (*documents[i])[field.name] = embeddings[i];+ }+ }return Option<bool>(true);}diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex 05ffa59e..de2f4cc8 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -11,14 +11,13 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {Ort::SessionOptions session_options;std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);LOG(INFO) << "Loading model from: " << abs_path;- session_ = new Ort::Session(env_, abs_path.c_str(), session_options);+ session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path());std::stringstream ss;ss << stream.rdbuf();auto vocab_ = ss.str();- tokenizer_ = new BertTokenizer(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),+ tokenizer_ = std::make_unique<BertTokenizer>(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));-auto output_tensor_count = session_->GetOutputCount();for (size_t i = 0; i < output_tensor_count; i++) {auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();@@ -30,6 +29,11 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {}}+TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key) : openai_api_key(openai_api_key), openai_model_path(openai_model_path) {++}++encoded_input_t TextEmbedder::Encode(const std::string& text) {auto encoded = tokenizer_->Encode(tokenizer_->Tokenize(ustring(text)));@@ -59,46 +63,97 @@ std::vector<float> TextEmbedder::mean_pooling(const std::vector<std::vector<floareturn pooled_output;}-std::vector<float> TextEmbedder::Embed(const std::string& text) {- auto encoded_input = Encode(text);- // create input tensor object from data values- Ort::AllocatorWithDefaultOptions allocator;- Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);- std::vector<Ort::Value> input_tensors;- std::vector<std::vector<int64_t>> input_shapes;- std::vector<const char*> input_node_names = {"input_ids", "attention_mask", "token_type_ids"};- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.attention_mask.data(), encoded_input.attention_mask.size(), input_shapes[1].data(), input_shapes[1].size()));- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));-- //LOG(INFO) << "Running model";- // create output tensor object- std::vector<const char*> output_node_names = {output_tensor_name.c_str()};- std::vector<int64_t> output_node_dims {1, static_cast<int64_t>(encoded_input.input_ids.size()), 384}; // batch_size x seq_length x hidden_size- auto output_tensor = session_->Run(Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(), input_tensors.size(), output_node_names.data(), output_node_names.size());- std::vector<std::vector<float>> output;- float* data = output_tensor[0].GetTensorMutableData<float>();- // print output tensor shape- auto shape = output_tensor[0].GetTensorTypeAndShapeInfo().GetShape();-- for (int i = 0; i < shape[1]; i++) {- std::vector<float> temp;- for (int j = 0; j < shape[2]; j++) {- temp.push_back(data[i * shape[2] + j]);+Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {+ if(is_openai()) {+ HttpClient& client = HttpClient::get_instance();+ std::unordered_map<std::string, std::string> headers;+ std::map<std::string, std::string> res_headers;+ headers["Authorization"] = "Bearer " + openai_api_key;+ headers["Content-Type"] = "application/json";+ std::string res;+ nlohmann::json req_body;+ req_body["input"] = text;+ // remove "openai/" prefix+ req_body["model"] = openai_model_path.substr(7);+ auto res_code = client.post_response(TextEmbedder::OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);+ if (res_code != 200) {+ LOG(ERROR) << "OpenAI API error: " << res;+ return Option<std::vector<float>>(400, "OpenAI API error: " + res);}- output.push_back(temp);+ return Option<std::vector<float>>(nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>());+ } else {+ auto encoded_input = Encode(text);+ // create input tensor object from data values+ Ort::AllocatorWithDefaultOptions allocator;+ Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);+ std::vector<Ort::Value> input_tensors;+ std::vector<std::vector<int64_t>> input_shapes;+ std::vector<const char*> input_node_names = {"input_ids", "attention_mask", "token_type_ids"};+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.attention_mask.data(), encoded_input.attention_mask.size(), input_shapes[1].data(), input_shapes[1].size()));+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));++ //LOG(INFO) << "Running model";+ // create output tensor object+ std::vector<const char*> output_node_names = {output_tensor_name.c_str()};+ std::vector<int64_t> output_node_dims {1, static_cast<int64_t>(encoded_input.input_ids.size()), 384}; // batch_size x seq_length x hidden_size+ auto output_tensor = session_->Run(Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(), input_tensors.size(), output_node_names.data(), output_node_names.size());+ std::vector<std::vector<float>> output;+ float* data = output_tensor[0].GetTensorMutableData<float>();+ // print output tensor shape+ auto shape = output_tensor[0].GetTensorTypeAndShapeInfo().GetShape();++ for (int i = 0; i < shape[1]; i++) {+ std::vector<float> temp;+ for (int j = 0; j < shape[2]; j++) {+ temp.push_back(data[i * shape[2] + j]);+ }+ output.push_back(temp);+ }+ auto pooled_output = mean_pooling(output);++ return Option<std::vector<float>>(pooled_output);}- auto pooled_output = mean_pooling(output);+}- return pooled_output;+Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vector<std::string>& inputs) {+ std::vector<std::vector<float>> outputs;+ if(!is_openai()) {+ // for now only openai is supported for batch embedding+ for(const auto& input : inputs) {+ outputs.push_back(Embed(input).get());+ }+ } else {+ nlohmann::json req_body;+ req_body["input"] = inputs;+ // remove "openai/" prefix+ req_body["model"] = openai_model_path.substr(7);+ std::unordered_map<std::string, std::string> headers;+ headers["Authorization"] = "Bearer " + openai_api_key;+ headers["Content-Type"] = "application/json";+ std::map<std::string, std::string> res_headers;+ std::string res;+ HttpClient& client = HttpClient::get_instance();++ auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);++ if(res_code != 200) {+ LOG(ERROR) << "OpenAI API error: " << res;+ return Option<std::vector<std::vector<float>>>(400, res);+ }++ nlohmann::json res_json = nlohmann::json::parse(res);+ for(auto& data : res_json["data"]) {+ outputs.push_back(data["embedding"].get<std::vector<float>>());+ }+ }+ return Option<std::vector<std::vector<float>>>(outputs);}TextEmbedder::~TextEmbedder() {- delete tokenizer_;- delete session_;}@@ -155,3 +210,58 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& nreturn true;}+++Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims) {+ if (openai_model_path.empty() || openai_api_key.empty() || openai_model_path.length() < 7) {+ return Option<bool>(400, "Invalid OpenAI model path or API key");+ }++ HttpClient& client = HttpClient::get_instance();+ std::unordered_map<std::string, std::string> headers;+ std::map<std::string, std::string> res_headers;+ headers["Authorization"] = "Bearer " + openai_api_key;+ std::string res;+ auto res_code = client.get_response(TextEmbedder::OPENAI_LIST_MODELS, res, res_headers, headers);+ if (res_code != 200) {+ LOG(ERROR) << "OpenAI API error: " << res;+ return Option<bool>(400, "OpenAI API error: " + res);+ }++ auto models_json = nlohmann::json::parse(res);+ bool found = false;+ // extract model name by removing "openai/" prefix+ auto model_name = openai_model_path.substr(7);+ for (auto& model : models_json["data"]) {+ if (model["id"] == model_name) {+ found = true;+ break;+ }+ }++ if (!found) {+ return Option<bool>(400, "OpenAI model not found");+ }+++ // This part is hard coded for now. Because OpenAI API does not provide a way to get the output dimensions of the model.+ if(model_name.find("-ada-") != std::string::npos) {+ if(model_name.substr(model_name.length() - 3) == "002") {+ num_dims = 1536;+ } else {+ num_dims = 1024;+ }+ }+ else if(model_name.find("-davinci-") != std::string::npos) {+ num_dims = 12288;+ } else if(model_name.find("-curie-") != std::string::npos) {+ num_dims = 4096;+ } else if(model_name.find("-babbage-") != std::string::npos) {+ num_dims = 2048;+ } else {+ num_dims = 768;+ }+++ return Option<bool>(true);+}\ No newline at end of filediff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 3acc794c..14e6e27b 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1646,12 +1646,12 @@ TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {ASSERT_EQ("Property `embed_from` must be an array.", field_op.error());}-TEST_F(CollectionAllFieldsTest, ModelPathWithoutEmbedFrom) {+TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {TextEmbedderManager::model_dir = "/tmp/models";nlohmann::json field_json;field_json["name"] = "embedding";field_json["type"] = "float[]";- field_json["model_name"] = "model";+ field_json["model_parameters"] = {{"model_name", "bert-base-uncased"}};std::vector<field> fields;std::string fallback_field_type;@@ -1660,7 +1660,7 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutEmbedFrom) {auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);ASSERT_FALSE(field_op.ok());- ASSERT_EQ("Property `model_name` can only be used with `embed_from`.", field_op.error());+ ASSERT_EQ("Property `model_parameters` can only be used with `embed_from`.", field_op.error());}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 43572401..218e199a 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4,6 +4,7 @@#include <fstream>#include <algorithm>#include <filesystem>+#include <cstdlib>#include <collection_manager.h>#include "collection.h"#include "text_embedder_manager.h"@@ -4993,3 +4994,39 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {// check if the embedding field is updatedASSERT_NE(embedding_field, updated_embedding_field);}+++TEST_F(CollectionTest, DISABLED_CreateOpenAIEmbeddingField) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "openai/text-embedding-ada-002"}}+ ]+ })"_json;++ if (std::getenv("OPENAI_API_KEY") == nullptr) {+ LOG(INFO) << "Skipping test as OPENAI_API_KEY is not set.";+ return;+ }++ auto openai_api_key = std::string(std::getenv("OPENAI_API_KEY"));+ schema["fields"][1]["model_parameters"]["openai_api_key"] = openai_api_key;+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");+ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ auto summary = op.get()->get_summary_json();+ ASSERT_EQ("openai/text-embedding-ada-002", summary["fields"][1]["model_parameters"]["model_name"]);+ ASSERT_EQ(1536, summary["fields"][1]["num_dim"]);+ // make sure openai_api_key is <hidden>+ ASSERT_EQ("<hidden>", summary["fields"][1]["model_parameters"]["openai_api_key"]);++ nlohmann::json doc;+ doc["name"] = "butter";++ auto add_op = op.get()->add(doc.dump());+ ASSERT_TRUE(add_op.ok());+ ASSERT_EQ(1536, add_op.get()["embedding"].size());+}++