Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix for bug when creating collections with using embedding with OpenAI API
ozanarmaganc49ca4f7e436
diff --git a/src/index.cpp b/src/index.cppindex f6ae888f..f3d74a76 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -454,10 +454,18 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteget_doc_changes(index_rec.operation, search_schema, index_rec.doc, index_rec.old_doc,index_rec.new_doc, index_rec.del_doc);scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);+for(auto& field: index_rec.doc.items()) {- if(embedding_fields.find(field.key()) != embedding_fields.end()) {- docs_to_embed.push_back(&index_rec.doc);- break;+ for(auto& embedding_field : embedding_fields) {+ if(!embedding_field.embed[fields::from].is_null()) {+ auto embed_from_vector = embedding_field.embed[fields::from].get<std::vector<std::string>>();+ for(auto& embed_from: embed_from_vector) {+ if(embed_from == field.key()) {+ docs_to_embed.push_back(&index_rec.new_doc);+ break;+ }+ }+ }}}} else {diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex d1a05317..604e1071 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -64,17 +64,21 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,req_body["input"] = "typesense";// remove "openai/" prefixreq_body["model"] = model_name_without_namespace;- res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);++ std::string embedding_res;+ headers["Content-Type"] = "application/json";+ res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);+if (res_code != 200) {- nlohmann::json json_res = nlohmann::json::parse(res);+ nlohmann::json json_res = nlohmann::json::parse(embedding_res);if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {- return Option<bool>(400, "OpenAI API error: " + res);+ return Option<bool>(400, "OpenAI API error: " + embedding_res);}return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());}- auto embedding = nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>();+ auto embedding = nlohmann::json::parse(embedding_res)["data"][0]["embedding"].get<std::vector<float>>();num_dims = embedding.size();return Option<bool>(true);}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 69a17e0e..e73953ec 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -5000,6 +5000,27 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {ASSERT_NE(embedding_field, updated_embedding_field);}+TEST_F(CollectionTest, CreateCollectionWithOpenAI) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "openai/text-embedding-ada-002"}}}+ ]+ })"_json;++ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";+ return;+ }++ auto api_key = std::string(std::getenv("api_key"));+ schema["fields"][1]["embed"]["model_config"]["api_key"] = api_key;+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");+ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+}+TEST_F(CollectionTest, CreateOpenAIEmbeddingField) {nlohmann::json schema = R"({