Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Added OpenAI integration for embeddings

ozanarmagan096543dcd6f0
diff --git a/include/field.h b/include/field.h
index d4361db9..5fbb386e 100644
--- a/include/field.h
+++ b/include/field.h
@@ -52,6 +52,8 @@ namespace fields {
static const std::string reference = "reference";
static const std::string embed_from = "embed_from";
static const std::string model_name = "model_name";
+ static const std::string openai_api_key = "openai_api_key";
+ static const std::string model_parameters = "model_parameters";
}
enum vector_distance_type_t {
@@ -78,7 +80,7 @@ struct field {
size_t num_dim;
std::vector<std::string> embed_from;
- std::string model_name;
+ nlohmann::json model_parameters;
vector_distance_type_t vec_dist;
static constexpr int VAL_UNKNOWN = 2;
@@ -89,9 +91,10 @@ struct field {
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,
- int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &embed_from = {}, const std::string& model_name = "") :
+ int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &embed_from = {},
+ const nlohmann::json& model_parameters = nlohmann::json()) :
name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),
- nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed_from(embed_from), model_name(model_name) {
+ nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), embed_from(embed_from), model_parameters(model_parameters) {
set_computed_defaults(sort, infix);
}
@@ -321,8 +324,12 @@ struct field {
}
if(!field.embed_from.empty()) {
field_val[fields::embed_from] = field.embed_from;
- if(!field.model_name.empty()) {
- field_val[fields::model_name] = field.model_name;
+ if(!field.model_parameters.empty()) {
+ field_val[fields::model_parameters] = field.model_parameters;
+ // hide openai api key
+ if(field_val[fields::model_parameters].count("openai_api_key") != 0) {
+ field_val[fields::model_parameters]["openai_api_key"] = "<hidden>";
+ }
}
}
fields_json.push_back(field_val);
diff --git a/include/http_client.h b/include/http_client.h
index 9d07f391..4fa9b685 100644
--- a/include/http_client.h
+++ b/include/http_client.h
@@ -32,7 +32,7 @@ private:
static size_t curl_req_send_callback(char* buffer, size_t size, size_t nitems, void *userdata);
- static long perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers);
+ static long perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers, struct curl_slist *chunk = nullptr);
public:
static HttpClient & get_instance() {
@@ -48,13 +48,13 @@ public:
static long download_file(const std::string& url, const std::string& file_path);
static long get_response(const std::string& url, std::string& response,
- std::map<std::string, std::string>& res_headers, long timeout_ms=4000);
+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers = {}, long timeout_ms=4000);
static long delete_response(const std::string& url, std::string& response,
std::map<std::string, std::string>& res_headers, long timeout_ms=120000);
static long post_response(const std::string & url, const std::string & body, std::string & response,
- std::map<std::string, std::string>& res_headers, long timeout_ms=4000);
+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers = {}, long timeout_ms=4000);
static long post_response_async(const std::string &url, const std::shared_ptr<http_req> request,
const std::shared_ptr<http_res> response,
diff --git a/include/index.h b/include/index.h
index 332e9fa2..a80dff23 100644
--- a/include/index.h
+++ b/include/index.h
@@ -538,7 +538,10 @@ private:
static Option<bool> embed_fields(nlohmann::json& document,
const tsl::htrie_map<char, field>& embedding_fields,
- const tsl::htrie_map<char, field> & search_schema);
+ const tsl::htrie_map<char, field> & search_schema);
+ static Option<bool> batch_embed_fields(std::vector<nlohmann::json*>& documents,
+ const tsl::htrie_map<char, field>& embedding_fields,
+ const tsl::htrie_map<char, field> & search_schema);
public:
// for limiting number of results on multiple candidates / query rewrites
diff --git a/include/text_embedder.h b/include/text_embedder.h
index c1cb33fd..869644f8 100644
--- a/include/text_embedder.h
+++ b/include/text_embedder.h
@@ -3,6 +3,7 @@
#include <core/session/onnxruntime_cxx_api.h>
#include <tokenizer/bert_tokenizer.hpp>
#include <vector>
+#include "option.h"
struct encoded_input_t {
std::vector<int64_t> input_ids;
@@ -14,15 +15,26 @@ struct encoded_input_t {
class TextEmbedder {
public:
TextEmbedder(const std::string& model_path);
+ TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key);
~TextEmbedder();
- std::vector<float> Embed(const std::string& text);
+ Option<std::vector<float>> Embed(const std::string& text);
+ Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);
+
+ bool is_openai() {
+ return !openai_api_key.empty();
+ }
static bool is_model_valid(const std::string& model_path, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims);
private:
- Ort::Session* session_;
+ std::unique_ptr<Ort::Session> session_;
Ort::Env env_;
encoded_input_t Encode(const std::string& text);
- BertTokenizer* tokenizer_;
+ std::unique_ptr<BertTokenizer> tokenizer_;
static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);
std::string output_tensor_name;
+ std::string openai_api_key;
+ std::string openai_model_path;
+ static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";
+ static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";
};
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index 192044c8..f19bc8d9 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -1,5 +1,5 @@
#pragma once
-
+#include <memory>
#include <filesystem>
#include <unordered_map>
#include <openssl/md5.h>
@@ -8,6 +8,7 @@
#include "http_client.h"
#include "text_embedder.h"
+
// singleton class
class TextEmbedderManager {
public:
@@ -21,24 +22,33 @@ public:
TextEmbedderManager(const TextEmbedderManager&) = delete;
TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;
- TextEmbedder* get_text_embedder(const std::string& model_path) {
- if (text_embedders.find(model_path) == text_embedders.end()) {
- text_embedders[model_path] = new TextEmbedder(model_path);
+ TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {
+ if(model_parameters.count("model_name") == 0) {
+ if(text_embedders[DEFAULT_MODEL_NAME] == nullptr) {
+ text_embedders[DEFAULT_MODEL_NAME] = std::make_shared<TextEmbedder>(DEFAULT_MODEL_NAME);
+ }
+ return text_embedders[DEFAULT_MODEL_NAME].get();
+ } else {
+ const std::string& model_name = model_parameters.at("model_name");
+ if(text_embedders[model_name] == nullptr) {
+ if(model_parameters.count("openai_api_key") == 0) {
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);
+ } else {
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("openai_api_key"));
+ }
+ }
+ return text_embedders[model_name].get();
}
- return text_embedders[model_path];
+
}
void delete_text_embedder(const std::string& model_path) {
if (text_embedders.find(model_path) != text_embedders.end()) {
- delete text_embedders[model_path];
text_embedders.erase(model_path);
}
}
void delete_all_text_embedders() {
- for (auto& text_embedder : text_embedders) {
- delete text_embedder.second;
- }
text_embedders.clear();
}
@@ -120,7 +130,7 @@ public:
}
private:
TextEmbedderManager() = default;
- std::unordered_map<std::string, TextEmbedder*> text_embedders;
+ std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index d6aa3403..525cc753 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -244,8 +244,12 @@ nlohmann::json Collection::get_summary_json() const {
field_json[fields::embed_from] = coll_field.embed_from;
}
- if(coll_field.model_name.size() > 0) {
- field_json[fields::model_name] = coll_field.model_name;
+ if(coll_field.model_parameters.size() > 0) {
+ field_json[fields::model_parameters] = coll_field.model_parameters;
+ // Hide OpenAI API key from the response.
+ if(field_json[fields::model_parameters].count(fields::openai_api_key) != 0) {
+ field_json[fields::model_parameters][fields::openai_api_key] = "<hidden>";
+ }
}
if(coll_field.num_dim > 0) {
@@ -1184,11 +1188,15 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
- auto embedder = embedder_manager.get_text_embedder(search_field.model_name.size() > 0 ? search_field.model_name : TextEmbedderManager::DEFAULT_MODEL_NAME);
+ auto embedder = embedder_manager.get_text_embedder(search_field.model_parameters);
std::string embed_query = "query: " + raw_query;
+ auto embedding_op = embedder->Embed(embed_query);
+ if(!embedding_op.ok()) {
+ return Option<nlohmann::json>(400, embedding_op.error());
+ }
- std::vector<float> embedding = embedder->Embed(embed_query);
+ std::vector<float> embedding = embedding_op.get();
vector_query._reset();
vector_query.values = embedding;
vector_query.field_name = field_name;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 28bd0f5a..765da8dc 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -62,10 +62,9 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field_obj[fields::embed_from] = std::vector<std::string>();
}
- if(field_obj.count(fields::model_name) == 0) {
- field_obj[fields::model_name] = "";
+ if(field_obj.count(fields::model_parameters) == 0) {
+ field_obj[fields::model_parameters] = nlohmann::json::object();
}
-
vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;
if(field_obj.count(fields::vec_dist) != 0) {
@@ -79,7 +78,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],
-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],
field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::embed_from],
- field_obj[fields::model_name]);
+ field_obj[fields::model_parameters]);
// value of `sort` depends on field type
if(field_obj.count(fields::sort) == 0) {
diff --git a/src/field.cpp b/src/field.cpp
index 6c572b99..ec9e8f7c 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -672,26 +672,37 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
}
- if(field_json.count(fields::model_name) > 0 && field_json.count(fields::embed_from) == 0) {
- return Option<bool>(400, "Property `" + fields::model_name + "` can only be used with `" + fields::embed_from + "`.");
+ if(field_json.count(fields::model_parameters) > 0 && field_json.count(fields::embed_from) == 0) {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "` can only be used with `" + fields::embed_from + "`.");
}
if(field_json.count(fields::embed_from) != 0) {
// If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)
field_json[fields::num_dim] = static_cast<unsigned int>(384);
- if(field_json.count(fields::model_name) != 0) {
- unsigned int num_dim = 0;
- if(!field_json[fields::model_name].is_string()) {
- return Option<bool>(400, "Property `" + fields::model_name + "` must be a string.");
- }
- if(field_json[fields::model_name].get<std::string>().empty()) {
- return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");
- }
-
- if(TextEmbedder::is_model_valid(field_json[fields::model_name].get<std::string>(), num_dim)) {
- field_json[fields::num_dim] = num_dim;
- } else {
- return Option<bool>(400, "Property `" + fields::model_name + "` must be a valid model path.");
+ if(field_json.count(fields::model_parameters) != 0) {
+ auto model_parameters = field_json[fields::model_parameters];
+ if(model_parameters.count(fields::model_name) != 0) {
+ unsigned int num_dim = 0;
+ if(!model_parameters[fields::model_name].is_string()) {
+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a string.");
+ }
+ if(model_parameters[fields::model_name].get<std::string>().empty()) {
+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");
+ }
+ if(model_parameters.count("openai_api_key") != 0) {
+ auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::openai_api_key].get<std::string>(), num_dim);
+ if(res.ok()) {
+ field_json[fields::num_dim] = num_dim;
+ } else {
+ return Option<bool>(res.code(), res.error());
+ }
+ } else {
+ if(TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), num_dim)) {
+ field_json[fields::num_dim] = num_dim;
+ } else {
+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a valid model path.");
+ }
+ }
}
}
} else {
@@ -699,10 +710,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
- if(field_json.count(fields::model_name) == 0) {
- field_json[fields::model_name] = "";
- }
-
auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);
@@ -784,8 +791,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],
field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,
- field_json[fields::reference], field_json[fields::embed_from].get<std::vector<std::string>>(),
- field_json[fields::model_name])
+ field_json[fields::reference], field_json[fields::embed_from].get<std::vector<std::string>>(),
+ field_json[fields::model_parameters])
);
if (!field_json[fields::reference].get<std::string>().empty()) {
diff --git a/src/http_client.cpp b/src/http_client.cpp
index a06b8aea..652f0971 100644
--- a/src/http_client.cpp
+++ b/src/http_client.cpp
@@ -16,16 +16,24 @@ struct client_state_t: public req_state_t {
};
long HttpClient::post_response(const std::string &url, const std::string &body, std::string &response,
- std::map<std::string, std::string>& res_headers, long timeout_ms) {
+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers, long timeout_ms) {
CURL *curl = init_curl(url, response);
if(curl == nullptr) {
return 500;
}
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
- return perform_curl(curl, res_headers);
+
+ struct curl_slist *chunk = nullptr;
+ for(const auto& header: headers) {
+ std::string header_str = header.first + ": " + header.second;
+ chunk = curl_slist_append(chunk, header_str.c_str());
+ }
+
+ return perform_curl(curl, res_headers, chunk);
}
+
long HttpClient::post_response_async(const std::string &url, const std::shared_ptr<http_req> request,
const std::shared_ptr<http_res> response, HttpServer* server) {
deferred_req_res_t* req_res = new deferred_req_res_t(request, response, server, false);
@@ -82,13 +90,18 @@ long HttpClient::delete_response(const std::string &url, std::string &response,
}
long HttpClient::get_response(const std::string &url, std::string &response,
- std::map<std::string, std::string>& res_headers, long timeout_ms) {
+ std::map<std::string, std::string>& res_headers, const std::unordered_map<std::string, std::string>& headers, long timeout_ms) {
CURL *curl = init_curl(url, response);
if(curl == nullptr) {
return 500;
}
+ struct curl_slist *chunk = nullptr;
+ for(const auto& header: headers) {
+ std::string header_str = header.first + ": " + header.second;
+ chunk = curl_slist_append(chunk, header_str.c_str());
+ }
- return perform_curl(curl, res_headers);
+ return perform_curl(curl, res_headers, chunk);
}
void HttpClient::init(const std::string &api_key) {
@@ -115,8 +128,7 @@ void HttpClient::init(const std::string &api_key) {
}
}
-long HttpClient::perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers) {
- struct curl_slist *chunk = nullptr;
+long HttpClient::perform_curl(CURL *curl, std::map<std::string, std::string>& res_headers, struct curl_slist *chunk) {
std::string api_key_header = std::string("x-typesense-api-key: ") + HttpClient::api_key;
chunk = curl_slist_append(chunk, api_key_header.c_str());
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
diff --git a/src/index.cpp b/src/index.cpp
index 6e99d0d0..52b7d647 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -418,6 +418,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
const bool do_validation) {
// runs in a partitioned thread
+ std::vector<nlohmann::json*> docs_to_embed;
for(size_t i = 0; i < batch_size; i++) {
index_record& index_rec = iter_batch[batch_start_index + i];
@@ -453,9 +454,11 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,
index_rec.del_doc);
scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);
- embed_fields(index_rec.new_doc, embedding_fields, search_schema);
+ docs_to_embed.push_back(&index_rec.new_doc);
+ //embed_fields(index_rec.new_doc, embedding_fields, search_schema);
} else {
- embed_fields(index_rec.doc, embedding_fields, search_schema);
+ docs_to_embed.push_back(&index_rec.doc);
+ //embed_fields(index_rec.doc, embedding_fields, search_schema);
}
compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);
@@ -485,6 +488,14 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
index_rec.index_failure(400, e.what());
}
}
+
+ auto embed_op = batch_embed_fields(docs_to_embed, embedding_fields, search_schema);
+ if(!embed_op.ok()) {
+ for(size_t i = 0; i < batch_size; i++) {
+ index_record& index_rec = iter_batch[batch_start_index + i];
+ index_rec.index_failure(embed_op.code(), embed_op.error());
+ }
+ }
}
size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,
@@ -6349,11 +6360,50 @@ Option<bool> Index::embed_fields(nlohmann::json& document,
}
}
TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
- auto embedder = embedder_manager.get_text_embedder(field.model_name.size() > 0 ? field.model_name : TextEmbedderManager::DEFAULT_MODEL_NAME);
- std::vector<float> embedding = embedder->Embed(text_to_embed);
- document[field.name] = embedding;
+ auto embedder = embedder_manager.get_text_embedder(field.model_parameters);
+ auto embedding_op = embedder->Embed(text_to_embed);
+
+ if(!embedding_op.ok()) {
+ return Option<bool>(400, embedding_op.error());
+ }
+
+ document[field.name] = embedding_op.get();
}
+ return Option<bool>(true);
+}
+Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
+ const tsl::htrie_map<char, field>& embedding_fields,
+ const tsl::htrie_map<char, field> & search_schema) {
+ for(const auto& field : embedding_fields) {
+ std::vector<std::string> text_to_embed;
+ for(auto& document : documents) {
+ std::string text = "passage: ";
+ for(const auto& field_name : field.embed_from) {
+ auto field_it = search_schema.find(field_name);
+ if(field_it.value().type == field_types::STRING) {
+ text += (*document)[field_name].get<std::string>() + " ";
+ } else if(field_it.value().type == field_types::STRING_ARRAY) {
+ for(const auto& val : (*document)[field_name]) {
+ text += val.get<std::string>() + " ";
+ }
+ }
+ }
+ text_to_embed.push_back(text);
+ }
+ TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
+ auto embedder = embedder_manager.get_text_embedder(field.model_parameters);
+ auto embedding_op = embedder->batch_embed(text_to_embed);
+
+ if(!embedding_op.ok()) {
+ return Option<bool>(400, embedding_op.error());
+ }
+
+ auto embeddings = embedding_op.get();
+ for(size_t i = 0; i < documents.size(); i++) {
+ (*documents[i])[field.name] = embeddings[i];
+ }
+ }
return Option<bool>(true);
}
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index 05ffa59e..de2f4cc8 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -11,14 +11,13 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {
Ort::SessionOptions session_options;
std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);
LOG(INFO) << "Loading model from: " << abs_path;
- session_ = new Ort::Session(env_, abs_path.c_str(), session_options);
+ session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);
std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path());
std::stringstream ss;
ss << stream.rdbuf();
auto vocab_ = ss.str();
- tokenizer_ = new BertTokenizer(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),
+ tokenizer_ = std::make_unique<BertTokenizer>(vocab_, true, true, ustring("[UNK]"), ustring("[SEP]"), ustring("[PAD]"),
ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));
-
auto output_tensor_count = session_->GetOutputCount();
for (size_t i = 0; i < output_tensor_count; i++) {
auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
@@ -30,6 +29,11 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {
}
}
+TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key) : openai_api_key(openai_api_key), openai_model_path(openai_model_path) {
+
+}
+
+
encoded_input_t TextEmbedder::Encode(const std::string& text) {
auto encoded = tokenizer_->Encode(tokenizer_->Tokenize(ustring(text)));
@@ -59,46 +63,97 @@ std::vector<float> TextEmbedder::mean_pooling(const std::vector<std::vector<floa
return pooled_output;
}
-std::vector<float> TextEmbedder::Embed(const std::string& text) {
- auto encoded_input = Encode(text);
- // create input tensor object from data values
- Ort::AllocatorWithDefaultOptions allocator;
- Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
- std::vector<Ort::Value> input_tensors;
- std::vector<std::vector<int64_t>> input_shapes;
- std::vector<const char*> input_node_names = {"input_ids", "attention_mask", "token_type_ids"};
- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});
- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});
- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});
- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));
- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.attention_mask.data(), encoded_input.attention_mask.size(), input_shapes[1].data(), input_shapes[1].size()));
- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));
-
- //LOG(INFO) << "Running model";
- // create output tensor object
- std::vector<const char*> output_node_names = {output_tensor_name.c_str()};
- std::vector<int64_t> output_node_dims {1, static_cast<int64_t>(encoded_input.input_ids.size()), 384}; // batch_size x seq_length x hidden_size
- auto output_tensor = session_->Run(Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(), input_tensors.size(), output_node_names.data(), output_node_names.size());
- std::vector<std::vector<float>> output;
- float* data = output_tensor[0].GetTensorMutableData<float>();
- // print output tensor shape
- auto shape = output_tensor[0].GetTensorTypeAndShapeInfo().GetShape();
-
- for (int i = 0; i < shape[1]; i++) {
- std::vector<float> temp;
- for (int j = 0; j < shape[2]; j++) {
- temp.push_back(data[i * shape[2] + j]);
+Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {
+ if(is_openai()) {
+ HttpClient& client = HttpClient::get_instance();
+ std::unordered_map<std::string, std::string> headers;
+ std::map<std::string, std::string> res_headers;
+ headers["Authorization"] = "Bearer " + openai_api_key;
+ headers["Content-Type"] = "application/json";
+ std::string res;
+ nlohmann::json req_body;
+ req_body["input"] = text;
+ // remove "openai/" prefix
+ req_body["model"] = openai_model_path.substr(7);
+ auto res_code = client.post_response(TextEmbedder::OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
+ if (res_code != 200) {
+ LOG(ERROR) << "OpenAI API error: " << res;
+ return Option<std::vector<float>>(400, "OpenAI API error: " + res);
}
- output.push_back(temp);
+ return Option<std::vector<float>>(nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>());
+ } else {
+ auto encoded_input = Encode(text);
+ // create input tensor object from data values
+ Ort::AllocatorWithDefaultOptions allocator;
+ Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
+ std::vector<Ort::Value> input_tensors;
+ std::vector<std::vector<int64_t>> input_shapes;
+ std::vector<const char*> input_node_names = {"input_ids", "attention_mask", "token_type_ids"};
+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});
+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});
+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});
+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));
+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.attention_mask.data(), encoded_input.attention_mask.size(), input_shapes[1].data(), input_shapes[1].size()));
+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));
+
+ //LOG(INFO) << "Running model";
+ // create output tensor object
+ std::vector<const char*> output_node_names = {output_tensor_name.c_str()};
+ std::vector<int64_t> output_node_dims {1, static_cast<int64_t>(encoded_input.input_ids.size()), 384}; // batch_size x seq_length x hidden_size
+ auto output_tensor = session_->Run(Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(), input_tensors.size(), output_node_names.data(), output_node_names.size());
+ std::vector<std::vector<float>> output;
+ float* data = output_tensor[0].GetTensorMutableData<float>();
+ // print output tensor shape
+ auto shape = output_tensor[0].GetTensorTypeAndShapeInfo().GetShape();
+
+ for (int i = 0; i < shape[1]; i++) {
+ std::vector<float> temp;
+ for (int j = 0; j < shape[2]; j++) {
+ temp.push_back(data[i * shape[2] + j]);
+ }
+ output.push_back(temp);
+ }
+ auto pooled_output = mean_pooling(output);
+
+ return Option<std::vector<float>>(pooled_output);
}
- auto pooled_output = mean_pooling(output);
+}
- return pooled_output;
+Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vector<std::string>& inputs) {
+ std::vector<std::vector<float>> outputs;
+ if(!is_openai()) {
+ // for now only openai is supported for batch embedding
+ for(const auto& input : inputs) {
+ outputs.push_back(Embed(input).get());
+ }
+ } else {
+ nlohmann::json req_body;
+ req_body["input"] = inputs;
+ // remove "openai/" prefix
+ req_body["model"] = openai_model_path.substr(7);
+ std::unordered_map<std::string, std::string> headers;
+ headers["Authorization"] = "Bearer " + openai_api_key;
+ headers["Content-Type"] = "application/json";
+ std::map<std::string, std::string> res_headers;
+ std::string res;
+ HttpClient& client = HttpClient::get_instance();
+
+ auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
+
+ if(res_code != 200) {
+ LOG(ERROR) << "OpenAI API error: " << res;
+ return Option<std::vector<std::vector<float>>>(400, res);
+ }
+
+ nlohmann::json res_json = nlohmann::json::parse(res);
+ for(auto& data : res_json["data"]) {
+ outputs.push_back(data["embedding"].get<std::vector<float>>());
+ }
+ }
+ return Option<std::vector<std::vector<float>>>(outputs);
}
TextEmbedder::~TextEmbedder() {
- delete tokenizer_;
- delete session_;
}
@@ -155,3 +210,58 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& n
return true;
}
+
+
+Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims) {
+ if (openai_model_path.empty() || openai_api_key.empty() || openai_model_path.length() < 7) {
+ return Option<bool>(400, "Invalid OpenAI model path or API key");
+ }
+
+ HttpClient& client = HttpClient::get_instance();
+ std::unordered_map<std::string, std::string> headers;
+ std::map<std::string, std::string> res_headers;
+ headers["Authorization"] = "Bearer " + openai_api_key;
+ std::string res;
+ auto res_code = client.get_response(TextEmbedder::OPENAI_LIST_MODELS, res, res_headers, headers);
+ if (res_code != 200) {
+ LOG(ERROR) << "OpenAI API error: " << res;
+ return Option<bool>(400, "OpenAI API error: " + res);
+ }
+
+ auto models_json = nlohmann::json::parse(res);
+ bool found = false;
+ // extract model name by removing "openai/" prefix
+ auto model_name = openai_model_path.substr(7);
+ for (auto& model : models_json["data"]) {
+ if (model["id"] == model_name) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ return Option<bool>(400, "OpenAI model not found");
+ }
+
+
+ // This part is hard coded for now. Because OpenAI API does not provide a way to get the output dimensions of the model.
+ if(model_name.find("-ada-") != std::string::npos) {
+ if(model_name.substr(model_name.length() - 3) == "002") {
+ num_dims = 1536;
+ } else {
+ num_dims = 1024;
+ }
+ }
+ else if(model_name.find("-davinci-") != std::string::npos) {
+ num_dims = 12288;
+ } else if(model_name.find("-curie-") != std::string::npos) {
+ num_dims = 4096;
+ } else if(model_name.find("-babbage-") != std::string::npos) {
+ num_dims = 2048;
+ } else {
+ num_dims = 768;
+ }
+
+
+ return Option<bool>(true);
+}
\ No newline at end of file
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 3acc794c..14e6e27b 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1646,12 +1646,12 @@ TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {
ASSERT_EQ("Property `embed_from` must be an array.", field_op.error());
}
-TEST_F(CollectionAllFieldsTest, ModelPathWithoutEmbedFrom) {
+TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {
TextEmbedderManager::model_dir = "/tmp/models";
nlohmann::json field_json;
field_json["name"] = "embedding";
field_json["type"] = "float[]";
- field_json["model_name"] = "model";
+ field_json["model_parameters"] = {{"model_name", "bert-base-uncased"}};
std::vector<field> fields;
std::string fallback_field_type;
@@ -1660,7 +1660,7 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutEmbedFrom) {
auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);
ASSERT_FALSE(field_op.ok());
- ASSERT_EQ("Property `model_name` can only be used with `embed_from`.", field_op.error());
+ ASSERT_EQ("Property `model_parameters` can only be used with `embed_from`.", field_op.error());
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 43572401..218e199a 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4,6 +4,7 @@
#include <fstream>
#include <algorithm>
#include <filesystem>
+#include <cstdlib>
#include <collection_manager.h>
#include "collection.h"
#include "text_embedder_manager.h"
@@ -4993,3 +4994,39 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {
// check if the embedding field is updated
ASSERT_NE(embedding_field, updated_embedding_field);
}
+
+
+TEST_F(CollectionTest, DISABLED_CreateOpenAIEmbeddingField) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "openai/text-embedding-ada-002"}}
+ ]
+ })"_json;
+
+ if (std::getenv("OPENAI_API_KEY") == nullptr) {
+ LOG(INFO) << "Skipping test as OPENAI_API_KEY is not set.";
+ return;
+ }
+
+ auto openai_api_key = std::string(std::getenv("OPENAI_API_KEY"));
+ schema["fields"][1]["model_parameters"]["openai_api_key"] = openai_api_key;
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ auto summary = op.get()->get_summary_json();
+ ASSERT_EQ("openai/text-embedding-ada-002", summary["fields"][1]["model_parameters"]["model_name"]);
+ ASSERT_EQ(1536, summary["fields"][1]["num_dim"]);
+ // make sure openai_api_key is <hidden>
+ ASSERT_EQ("<hidden>", summary["fields"][1]["model_parameters"]["openai_api_key"]);
+
+ nlohmann::json doc;
+ doc["name"] = "butter";
+
+ auto add_op = op.get()->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+ ASSERT_EQ(1536, add_op.get()["embedding"].size());
+}
+
+