Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix for bug when creating collections with using embedding with OpenAI API

ozanarmaganc49ca4f7e436
diff --git a/src/index.cpp b/src/index.cpp
index f6ae888f..f3d74a76 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -454,10 +454,18 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
get_doc_changes(index_rec.operation, search_schema, index_rec.doc, index_rec.old_doc,
index_rec.new_doc, index_rec.del_doc);
scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);
+
for(auto& field: index_rec.doc.items()) {
- if(embedding_fields.find(field.key()) != embedding_fields.end()) {
- docs_to_embed.push_back(&index_rec.doc);
- break;
+ for(auto& embedding_field : embedding_fields) {
+ if(!embedding_field.embed[fields::from].is_null()) {
+ auto embed_from_vector = embedding_field.embed[fields::from].get<std::vector<std::string>>();
+ for(auto& embed_from: embed_from_vector) {
+ if(embed_from == field.key()) {
+ docs_to_embed.push_back(&index_rec.new_doc);
+ break;
+ }
+ }
+ }
}
}
} else {
diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cpp
index d1a05317..604e1071 100644
--- a/src/text_embedder_remote.cpp
+++ b/src/text_embedder_remote.cpp
@@ -64,17 +64,21 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,
req_body["input"] = "typesense";
// remove "openai/" prefix
req_body["model"] = model_name_without_namespace;
- res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
+
+ std::string embedding_res;
+ headers["Content-Type"] = "application/json";
+ res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);
+
if (res_code != 200) {
- nlohmann::json json_res = nlohmann::json::parse(res);
+ nlohmann::json json_res = nlohmann::json::parse(embedding_res);
if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
- return Option<bool>(400, "OpenAI API error: " + res);
+ return Option<bool>(400, "OpenAI API error: " + embedding_res);
}
return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
}
- auto embedding = nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>();
+ auto embedding = nlohmann::json::parse(embedding_res)["data"][0]["embedding"].get<std::vector<float>>();
num_dims = embedding.size();
return Option<bool>(true);
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 69a17e0e..e73953ec 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -5000,6 +5000,27 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {
ASSERT_NE(embedding_field, updated_embedding_field);
}
+TEST_F(CollectionTest, CreateCollectionWithOpenAI) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "openai/text-embedding-ada-002"}}}
+ ]
+ })"_json;
+
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
+ return;
+ }
+
+ auto api_key = std::string(std::getenv("api_key"));
+ schema["fields"][1]["embed"]["model_config"]["api_key"] = api_key;
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+}
+
TEST_F(CollectionTest, CreateOpenAIEmbeddingField) {
nlohmann::json schema = R"({