Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Dropped field should be removed properly from embed_from.

Kishore Nallan5120148b5330
diff --git a/include/collection.h b/include/collection.h
index f95b7c9f..e9ca2b2a 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -162,7 +162,7 @@ private:
void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
- void process_remove_field_for_embedding_fields(const field& the_field, std::vector<field>& garbage_fields);
+ void process_remove_field_for_embedding_fields(const field& del_field, std::vector<field>& garbage_embed_fields);
void curate_results(string& actual_query, const string& filter_query, bool enable_overrides, bool already_segmented,
const std::map<size_t, std::vector<std::string>>& pinned_hits,
diff --git a/src/collection.cpp b/src/collection.cpp
index 8883f4ab..62e0e0a2 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3942,7 +3942,6 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
}
}
-
// hide credentials in the alter payload return
for(auto& field_json : alter_payload["fields"]) {
if(field_json[fields::embed].count(fields::model_config) != 0) {
@@ -3955,8 +3954,6 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
}
}
-
-
return Option<bool>(true);
}
@@ -4904,27 +4901,44 @@ Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_ha
}
// Removes the dropped field from embed_from of all embedding fields.
-void Collection::process_remove_field_for_embedding_fields(const field& the_field, std::vector<field>& garbage_fields) {
+void Collection::process_remove_field_for_embedding_fields(const field& del_field,
+ std::vector<field>& garbage_embed_fields) {
for(auto& field : fields) {
if(field.embed.count(fields::from) == 0) {
continue;
}
- auto embed_from = field.embed[fields::from].get<std::vector<std::string>>();
- embed_from.erase(std::remove_if(embed_from.begin(), embed_from.end(), [&the_field](std::string field_name) {
- return the_field.name == field_name;
- }));
- field.embed[fields::from] = std::move(embed_from);
- embedding_fields[field.name] = field;
- // mark this embedding field as "garbage" if it has no more embed_from fields
- if(embed_from.empty()) {
- embedding_fields.erase(field.name);
- garbage_fields.push_back(field);
+ bool found_field = false;
+ nlohmann::json& embed_from_names = field.embed[fields::from];
+ for(auto it = embed_from_names.begin(); it != embed_from_names.end();) {
+ LOG(INFO) << it.value();
+ if(it.value() == del_field.name) {
+ it = embed_from_names.erase(it);
+ found_field = true;
+ } else {
+ it++;
+ }
}
-
+ if(found_field) {
+ // mark this embedding field as "garbage" if it has no more embed_from fields
+ if(embed_from_names.empty()) {
+ garbage_embed_fields.push_back(field);
+ } else {
+ // the dropped field was present in `embed_from`, so we have to update the field objects
+ field.embed[fields::from] = embed_from_names;
+ embedding_fields[field.name].embed[fields::from] = embed_from_names;
+ }
+ }
}
+ for(auto& garbage_field: garbage_embed_fields) {
+ embedding_fields.erase(garbage_field.name);
+ search_schema.erase(garbage_field.name);
+ fields.erase(std::remove_if(fields.begin(), fields.end(), [&garbage_field](const auto &f) {
+ return f.name == garbage_field.name;
+ }), fields.end());
+ }
}
void Collection::hide_credential(nlohmann::json& json, const std::string& credential_name) {
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index 7bc6f32c..0d8364fd 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1580,9 +1580,13 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {
nlohmann::json schema = R"({
"name": "objects",
"fields": [
- {"name": "names", "type": "string[]"},
- {"name": "category", "type":"string"},
- {"name": "embedding", "type":"float[]", "embed":{"from": ["names","category"], "model_config": {"model_name": "ts/e5-small"}}}
+ {"name": "title", "type": "string"},
+ {"name": "names", "type": "string[]"},
+ {"name": "category", "type":"string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["names","category"],
+ "model_config": {"model_name": "ts/e5-small"}}},
+ {"name": "embedding2", "type":"float[]", "embed":{"from": ["names"],
+ "model_config": {"model_name": "ts/e5-small"}}}
]
})"_json;
@@ -1594,20 +1598,28 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {
LOG(INFO) << "Created collection";
+ auto embedding_fields = coll->get_embedding_fields();
+ ASSERT_EQ(2, embedding_fields.size());
+ ASSERT_EQ(2, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());
+ ASSERT_EQ(1, embedding_fields["embedding2"].embed[fields::from].get<std::vector<std::string>>().size());
+
+ auto coll_schema = coll->get_schema();
+ ASSERT_EQ(5, coll_schema.size());
+
+ auto the_fields = coll->get_fields();
+ ASSERT_EQ(5, the_fields.size());
+
auto schema_changes = R"({
"fields": [
{"name": "names", "drop": true}
]
})"_json;
-
- auto embedding_fields = coll->get_embedding_fields();
- ASSERT_EQ(2, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());
-
auto alter_op = coll->alter(schema_changes);
ASSERT_TRUE(alter_op.ok());
embedding_fields = coll->get_embedding_fields();
+ ASSERT_EQ(1, embedding_fields.size());
ASSERT_EQ(1, embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>().size());
ASSERT_EQ("category", embedding_fields["embedding"].embed[fields::from].get<std::vector<std::string>>()[0]);
@@ -1623,6 +1635,16 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {
embedding_fields = coll->get_embedding_fields();
ASSERT_EQ(0, embedding_fields.size());
ASSERT_EQ(0, coll->_get_index()->_get_vector_index().size());
+
+ // only title remains
+
+ coll_schema = coll->get_schema();
+ ASSERT_EQ(1, coll_schema.size());
+ ASSERT_EQ("title", coll_schema["title"].name);
+
+ the_fields = coll->get_fields();
+ ASSERT_EQ(1, the_fields.size());
+ ASSERT_EQ("title", the_fields[0].name);
}
TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {