Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review changes
ozanarmagancc54fb817616
diff --git a/include/collection.h b/include/collection.hindex 2444b9cd..2b658bb6 100644--- a/include/collection.h+++ b/include/collection.h@@ -213,7 +213,7 @@ private:Option<bool> persist_collection_meta();- Option<bool> batch_alter_data(std::vector<field>& alter_fields,+ Option<bool> batch_alter_data(const std::vector<field>& alter_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type);@@ -566,6 +566,8 @@ public:std::vector<std::string>& reordered_search_fields) const;Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);++ static void process_embedding_field_delete(const std::string& model_name);};template<class T>diff --git a/include/collection_manager.h b/include/collection_manager.hindex 03ef6b43..42f102a5 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -201,4 +201,5 @@ public:Option<bool> upsert_preset(const std::string & preset_name, const nlohmann::json& preset_config);Option<bool> delete_preset(const std::string & preset_name);+};\ No newline at end of filediff --git a/include/field.h b/include/field.hindex 5557d54b..6d262eba 100644--- a/include/field.h+++ b/include/field.h@@ -422,12 +422,12 @@ struct field {static Option<bool> json_fields_to_fields(bool enable_nested_fields,nlohmann::json& fields_json,std::string& fallback_field_type,- std::vector<field>& the_fields, const std::string& collection_name = "");+ std::vector<field>& the_fields);static Option<bool> validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema,nlohmann::json& field_json,const nlohmann::json& fields_json,- field& the_field, const std::string& collection_name);+ field& the_field);static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,diff --git a/src/collection.cpp b/src/collection.cppindex ecbd0d01..38133b3f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3765,7 +3765,7 @@ Option<bool> Collection::persist_collection_meta() {return Option<bool>(true);}-Option<bool> Collection::batch_alter_data(std::vector<field>& alter_fields,+Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type) {// Update schema with additions (deletions can only be made later)@@ -3800,7 +3800,8 @@ Option<bool> Collection::batch_alter_data(std::vector<field>& alter_fields,auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();auto model_name = f.embed[fields::model_config][fields::model_name].get<std::string>();if(text_embedders.count(model_name) == 0) {- TextEmbedderManager::get_instance().validate_and_init_model(f.embed[fields::model_config], f.num_dim);+ size_t dummy_num_dim = 0;+ TextEmbedderManager::get_instance().validate_and_init_model(f.embed[fields::model_config], dummy_num_dim);}embedding_fields.emplace(f.name, f);}@@ -4298,7 +4299,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,}if(!f.embed.empty()) {- auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f, name);+ auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f);if(!validate_res.ok()) {return validate_res;@@ -5032,23 +5033,23 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t k}void Collection::remove_embedding_field(const std::string& field_name) {- field del_field;-- if(embedding_fields.find(field_name) != embedding_fields.end()) {- del_field = embedding_fields[field_name];- } else {+ if(embedding_fields.find(field_name) == embedding_fields.end()) {return;}+ auto del_field = embedding_fields[field_name];+embedding_fields.erase(field_name);auto model_name = del_field.embed[fields::model_config]["model_name"].get<std::string>();+ process_embedding_field_delete(model_name);+}+void Collection::process_embedding_field_delete(const std::string& model_name) {auto collections = CollectionManager::get_instance().get_collections();-bool found = false;- for(auto& collection: collections) {+ for(const auto& collection: collections) {auto embedding_fields_other = collection->embedding_fields;for(auto& embedding_field: embedding_fields_other) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex e3229d19..fc2992d7 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -528,38 +528,20 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& cols_lock.unlock();- auto embedding_fields = collection->get_embedding_fields();-std::unique_lock u_lock(mutex);collections.erase(actual_coll_name);collection_id_names.erase(collection->get_collection_id());- for(auto& embedding_field : embedding_fields) {- bool found = false;- auto model_name = embedding_field.embed[fields::model_config]["model_name"].get<std::string>();- for(auto& collection: collections) {- auto embedding_fields_other = collection.second->get_embedding_fields();+ u_lock.unlock();- for(auto& embedding_field: embedding_fields_other) {- if(embedding_field.embed.count(fields::model_config) != 0) {- auto model_config = embedding_field.embed[fields::model_config];- if(model_config["model_name"].get<std::string>() == model_name) {- found = true;- break;- }- }- }- }+ const auto& embedding_fields = collection->get_embedding_fields();- if(!found) {- LOG(INFO) << "Deleting text embedder: " << model_name;- TextEmbedderManager::get_instance().delete_text_embedder(embedding_field.embed[fields::model_config]["model_name"].get<std::string>());- }+ for(const auto& embedding_field : embedding_fields) {+ auto model_name = embedding_field.embed[fields::model_config]["model_name"].get<std::string>();+ Collection::process_embedding_field_delete(model_name);}- u_lock.unlock();-// don't hold any collection manager locks here, since this can take some timedelete collection;@@ -1287,7 +1269,7 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsostd::string fallback_field_type;std::vector<field> fields;auto parse_op = field::json_fields_to_fields(req_json[ENABLE_NESTED_FIELDS].get<bool>(),- req_json["fields"], fallback_field_type, fields, req_json["name"]);+ req_json["fields"], fallback_field_type, fields);if(!parse_op.ok()) {return Option<Collection*>(parse_op.code(), parse_op.error());diff --git a/src/field.cpp b/src/field.cppindex c3eea682..d053e578 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -1081,7 +1081,7 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {}Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,- std::vector<field>& the_fields, const std::string& collection_name) {+ std::vector<field>& the_fields) {size_t num_auto_detect_fields = 0;const tsl::htrie_map<char, field> dummy_search_schema;@@ -1094,7 +1094,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::j}if(!the_fields.empty() && !the_fields.back().embed.empty()) {- auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back(), collection_name);+ auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back());if(!validate_res.ok()) {return validate_res;}@@ -1110,7 +1110,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::jOption<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema, nlohmann::json& field_json,const nlohmann::json& fields_json,- field& the_field, const std::string& collection_name) {+ field& the_field) {const std::string err_msg = "Property `" + fields::embed + "." + fields::from +"` can only refer to string or string array fields.";diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex d076c029..9d329374 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -2386,8 +2386,6 @@ TEST_F(CollectionVectorTest, TestUnloadingModelsOnDrop) {auto alter_op = coll->alter(alter_schema);ASSERT_TRUE(alter_op.ok());- LOG(INFO) << "created second collection";-text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();ASSERT_EQ(1, text_embedders.size());