Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Dropped field should be removed properly from embed_from.
Kishore Nallan5120148b5330
diff --git a/include/collection.h b/include/collection.hindex f95b7c9f..e9ca2b2a 100644--- a/include/collection.h+++ b/include/collection.h@@ -162,7 +162,7 @@ private:void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);- void process_remove_field_for_embedding_fields(const field& the_field, std::vector<field>& garbage_fields);+ void process_remove_field_for_embedding_fields(const field& del_field, std::vector<field>& garbage_embed_fields);void curate_results(string& actual_query, const string& filter_query, bool enable_overrides, bool already_segmented,const std::map<size_t, std::vector<std::string>>& pinned_hits,diff --git a/src/collection.cpp b/src/collection.cppindex 8883f4ab..62e0e0a2 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3942,7 +3942,6 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {}}-// hide credentials in the alter payload returnfor(auto& field_json : alter_payload["fields"]) {if(field_json[fields::embed].count(fields::model_config) != 0) {@@ -3955,8 +3954,6 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {}}--return Option<bool>(true);}@@ -4904,27 +4901,44 @@ Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_ha}// Removes the dropped field from embed_from of all embedding fields.-void Collection::process_remove_field_for_embedding_fields(const field& the_field, std::vector<field>& garbage_fields) {+void Collection::process_remove_field_for_embedding_fields(const field& del_field,+ std::vector<field>& garbage_embed_fields) {for(auto& field : fields) {if(field.embed.count(fields::from) == 0) {continue;}- auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();- embed_from.erase(std::remove_if(embed_from.begin(), embed_from.end(), [&the_field](std::string field_name) {- return the_field.name == field_name;- }));- field.embed[fields::from] = std::move(embed_from);- embedding_fields[field.name] = field;- // mark this embedding field as "garbage" if it has no more embed_from fields- if(embed_from.empty()) {- embedding_fields.erase(field.name);- garbage_fields.push_back(field);+ bool found_field = false;+ nlohmann::json& embed_from_names = field.embed[fields::from];+ for(auto it = embed_from_names.begin(); it != embed_from_names.end();) {+ LOG(INFO) << it.value();+ if(it.value() == del_field.name) {+ it = embed_from_names.erase(it);+ found_field = true;+ } else {+ it++;+ }}-+ if(found_field) {+ // mark this embedding field as "garbage" if it has no more embed_from fields+ if(embed_from_names.empty()) {+ garbage_embed_fields.push_back(field);+ } else {+ // the dropped field was present in `embed_from`, so we have to update the field objects+ field.embed[fields::from] = embed_from_names;+ embedding_fields[field.name].embed[fields::from] = embed_from_names;+ }+ }}+ for(auto& garbage_field: garbage_embed_fields) {+ embedding_fields.erase(garbage_field.name);+ search_schema.erase(garbage_field.name);+ fields.erase(std::remove_if(fields.begin(), fields.end(), [&garbage_field](const auto &f) {+ return f.name == garbage_field.name;+ }), fields.end());+ }}void Collection::hide_credential(nlohmann::json& json, const std::string& credential_name) {diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex 7bc6f32c..0d8364fd 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1580,9 +1580,13 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {nlohmann::json schema = R"({"name": "objects","fields": [- {"name": "names", "type": "string[]"},- {"name": "category", "type":"string"},- {"name": "embedding", "type":"float[]", "embed":{"from": ["names","category"], "model_config": {"model_name": "ts/e5-small"}}}+ {"name": "title", "type": "string"},+ {"name": "names", "type": "string[]"},+ {"name": "category", "type":"string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["names","category"],+ "model_config": {"model_name": "ts/e5-small"}}},+ {"name": "embedding2", "type":"float[]", "embed":{"from": ["names"],+ "model_config": {"model_name": "ts/e5-small"}}}]})"_json;@@ -1594,20 +1598,28 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {LOG(INFO) << "Created collection";+ auto embedding_fields = coll->get_embedding_fields();+ ASSERT_EQ(2, embedding_fields.size());+ ASSERT_EQ(2, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());+ ASSERT_EQ(1, embedding_fields["embedding2"].embed[fields::from].get<std::vector<std::string>>().size());++ auto coll_schema = coll->get_schema();+ ASSERT_EQ(5, coll_schema.size());++ auto the_fields = coll->get_fields();+ ASSERT_EQ(5, the_fields.size());+auto schema_changes = R"({"fields": [{"name": "names", "drop": true}]})"_json;-- auto embedding_fields = coll->get_embedding_fields();- ASSERT_EQ(2, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());-auto alter_op = coll->alter(schema_changes);ASSERT_TRUE(alter_op.ok());embedding_fields = coll->get_embedding_fields();+ ASSERT_EQ(1, embedding_fields.size());ASSERT_EQ(1, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());ASSERT_EQ("category", embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>()[0]);@@ -1623,6 +1635,16 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {embedding_fields = coll->get_embedding_fields();ASSERT_EQ(0, embedding_fields.size());ASSERT_EQ(0, coll->_get_index()->_get_vector_index().size());++ // only title remains++ coll_schema = coll->get_schema();+ ASSERT_EQ(1, coll_schema.size());+ ASSERT_EQ("title", coll_schema["title"].name);++ the_fields = coll->get_fields();+ ASSERT_EQ(1, the_fields.size());+ ASSERT_EQ("title", the_fields[0].name);}TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {