Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Review changes

ozanarmagancc54fb817616
diff --git a/include/collection.h b/include/collection.h
index 2444b9cd..2b658bb6 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -213,7 +213,7 @@ private:
Option<bool> persist_collection_meta();
- Option<bool> batch_alter_data(std::vector<field>& alter_fields,
+ Option<bool> batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
const std::string& this_fallback_field_type);
@@ -566,6 +566,8 @@ public:
std::vector<std::string>& reordered_search_fields) const;
Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);
+
+ static void process_embedding_field_delete(const std::string& model_name);
};
template<class T>
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 03ef6b43..42f102a5 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -201,4 +201,5 @@ public:
Option<bool> upsert_preset(const std::string & preset_name, const nlohmann::json& preset_config);
Option<bool> delete_preset(const std::string & preset_name);
+
};
\ No newline at end of file
diff --git a/include/field.h b/include/field.h
index 5557d54b..6d262eba 100644
--- a/include/field.h
+++ b/include/field.h
@@ -422,12 +422,12 @@ struct field {
static Option<bool> json_fields_to_fields(bool enable_nested_fields,
nlohmann::json& fields_json,
std::string& fallback_field_type,
- std::vector<field>& the_fields, const std::string& collection_name = "");
+ std::vector<field>& the_fields);
static Option<bool> validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema,
nlohmann::json& field_json,
const nlohmann::json& fields_json,
- field& the_field, const std::string& collection_name);
+ field& the_field);
static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,
diff --git a/src/collection.cpp b/src/collection.cpp
index ecbd0d01..38133b3f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3765,7 +3765,7 @@ Option<bool> Collection::persist_collection_meta() {
return Option<bool>(true);
}
-Option<bool> Collection::batch_alter_data(std::vector<field>& alter_fields,
+Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
const std::string& this_fallback_field_type) {
// Update schema with additions (deletions can only be made later)
@@ -3800,7 +3800,8 @@ Option<bool> Collection::batch_alter_data(std::vector<field>& alter_fields,
auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
auto model_name = f.embed[fields::model_config][fields::model_name].get<std::string>();
if(text_embedders.count(model_name) == 0) {
- TextEmbedderManager::get_instance().validate_and_init_model(f.embed[fields::model_config], f.num_dim);
+ size_t dummy_num_dim = 0;
+ TextEmbedderManager::get_instance().validate_and_init_model(f.embed[fields::model_config], dummy_num_dim);
}
embedding_fields.emplace(f.name, f);
}
@@ -4298,7 +4299,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
}
if(!f.embed.empty()) {
- auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f, name);
+ auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f);
if(!validate_res.ok()) {
return validate_res;
@@ -5032,23 +5033,23 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t k
}
void Collection::remove_embedding_field(const std::string& field_name) {
- field del_field;
-
- if(embedding_fields.find(field_name) != embedding_fields.end()) {
- del_field = embedding_fields[field_name];
- } else {
+ if(embedding_fields.find(field_name) == embedding_fields.end()) {
return;
}
+ auto del_field = embedding_fields[field_name];
+
embedding_fields.erase(field_name);
auto model_name = del_field.embed[fields::model_config]["model_name"].get<std::string>();
+ process_embedding_field_delete(model_name);
+}
+void Collection::process_embedding_field_delete(const std::string& model_name) {
auto collections = CollectionManager::get_instance().get_collections();
-
bool found = false;
- for(auto& collection: collections) {
+ for(const auto& collection: collections) {
auto embedding_fields_other = collection->embedding_fields;
for(auto& embedding_field: embedding_fields_other) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index e3229d19..fc2992d7 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -528,38 +528,20 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col
s_lock.unlock();
- auto embedding_fields = collection->get_embedding_fields();
-
std::unique_lock u_lock(mutex);
collections.erase(actual_coll_name);
collection_id_names.erase(collection->get_collection_id());
- for(auto& embedding_field : embedding_fields) {
- bool found = false;
- auto model_name = embedding_field.embed[fields::model_config]["model_name"].get<std::string>();
- for(auto& collection: collections) {
- auto embedding_fields_other = collection.second->get_embedding_fields();
+ u_lock.unlock();
- for(auto& embedding_field: embedding_fields_other) {
- if(embedding_field.embed.count(fields::model_config) != 0) {
- auto model_config = embedding_field.embed[fields::model_config];
- if(model_config["model_name"].get<std::string>() == model_name) {
- found = true;
- break;
- }
- }
- }
- }
+ const auto& embedding_fields = collection->get_embedding_fields();
- if(!found) {
- LOG(INFO) << "Deleting text embedder: " << model_name;
- TextEmbedderManager::get_instance().delete_text_embedder(embedding_field.embed[fields::model_config]["model_name"].get<std::string>());
- }
+ for(const auto& embedding_field : embedding_fields) {
+ auto model_name = embedding_field.embed[fields::model_config]["model_name"].get<std::string>();
+ Collection::process_embedding_field_delete(model_name);
}
- u_lock.unlock();
-
// don't hold any collection manager locks here, since this can take some time
delete collection;
@@ -1287,7 +1269,7 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
std::string fallback_field_type;
std::vector<field> fields;
auto parse_op = field::json_fields_to_fields(req_json[ENABLE_NESTED_FIELDS].get<bool>(),
- req_json["fields"], fallback_field_type, fields, req_json["name"]);
+ req_json["fields"], fallback_field_type, fields);
if(!parse_op.ok()) {
return Option<Collection*>(parse_op.code(), parse_op.error());
diff --git a/src/field.cpp b/src/field.cpp
index c3eea682..d053e578 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -1081,7 +1081,7 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {
}
Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,
- std::vector<field>& the_fields, const std::string& collection_name) {
+ std::vector<field>& the_fields) {
size_t num_auto_detect_fields = 0;
const tsl::htrie_map<char, field> dummy_search_schema;
@@ -1094,7 +1094,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::j
}
if(!the_fields.empty() && !the_fields.back().embed.empty()) {
- auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back(), collection_name);
+ auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back());
if(!validate_res.ok()) {
return validate_res;
}
@@ -1110,7 +1110,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::j
Option<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema, nlohmann::json& field_json,
const nlohmann::json& fields_json,
- field& the_field, const std::string& collection_name) {
+ field& the_field) {
const std::string err_msg = "Property `" + fields::embed + "." + fields::from +
"` can only refer to string or string array fields.";
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index d076c029..9d329374 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -2386,8 +2386,6 @@ TEST_F(CollectionVectorTest, TestUnloadingModelsOnDrop) {
auto alter_op = coll->alter(alter_schema);
ASSERT_TRUE(alter_op.ok());
- LOG(INFO) << "created second collection";
-
text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
ASSERT_EQ(1, text_embedders.size());