Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Unload models from RAM when it is unused

ozanarmagan61469b4d8447
diff --git a/include/field.h b/include/field.h
index 6d262eba..5557d54b 100644
--- a/include/field.h
+++ b/include/field.h
@@ -422,12 +422,12 @@ struct field {
static Option<bool> json_fields_to_fields(bool enable_nested_fields,
nlohmann::json& fields_json,
std::string& fallback_field_type,
- std::vector<field>& the_fields);
+ std::vector<field>& the_fields, const std::string& collection_name = "");
static Option<bool> validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema,
nlohmann::json& field_json,
const nlohmann::json& fields_json,
- field& the_field);
+ field& the_field, const std::string& collection_name);
static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index 2681e0b9..c841ced5 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -36,7 +36,10 @@ public:
TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;
Option<TextEmbedder*> get_text_embedder(const nlohmann::json& model_config);
-
+ void add_text_embedder_to_collection(const std::string& collection_name, const std::string& model_name);
+ void remove_text_embedder_from_collection(const std::string& collection_name, const std::string& model_name);
+ void remove_collection(const std::string& collection_name);
+
void delete_text_embedder(const std::string& model_path);
void delete_all_text_embedders();
@@ -83,6 +86,8 @@ private:
std::unordered_map<std::string, text_embedding_model> public_models;
std::mutex text_embedders_mutex;
+ std::unordered_map<std::string, std::unordered_set<std::string>> text_embedder_to_collections, collection_to_text_embedders;
+
static Option<std::string> get_namespace(const std::string& model_name);
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 0ddd0bcf..482e71a7 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3798,6 +3798,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
if(f.embed.count(fields::from) != 0) {
found_embedding_field = true;
embedding_fields.emplace(f.name, f);
+ TextEmbedderManager::get_instance().add_text_embedder_to_collection(name, f.embed[fields::model_config]["model_name"]);
}
fields.push_back(f);
@@ -3909,6 +3910,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
if(del_field.embed.count(fields::from) != 0) {
embedding_fields.erase(del_field.name);
+ TextEmbedderManager::get_instance().remove_text_embedder_from_collection(name, del_field.embed[fields::model_config]["model_name"]);
}
if(del_field.name == ".*") {
@@ -4224,6 +4226,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
if(found_field && field_it.value().embed.count(fields::from) != 0) {
updated_embedding_fields.erase(field_it.key());
+ TextEmbedderManager::get_instance().remove_text_embedder_from_collection(name, field_it.value().embed[fields::model_config]["model_name"]);
}
if(found_field) {
@@ -4233,6 +4236,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
if(field_it.value().embed.count(fields::from) != 0) {
updated_embedding_fields.erase(field_it.key());
+ TextEmbedderManager::get_instance().remove_text_embedder_from_collection(name, field_it.value().embed[fields::model_config]["model_name"]);
}
// should also remove children if the field being dropped is an object
@@ -4247,6 +4251,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
if(prefix_kv.value().embed.count(fields::from) != 0) {
updated_embedding_fields.erase(prefix_kv.key());
+ TextEmbedderManager::get_instance().remove_text_embedder_from_collection(name, prefix_kv.value().embed[fields::model_config]["model_name"]);
}
}
}
@@ -4293,7 +4298,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
}
if(!f.embed.empty()) {
- auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f);
+ auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f, name);
if(!validate_res.ok()) {
return validate_res;
@@ -4988,6 +4993,7 @@ void Collection::process_remove_field_for_embedding_fields(const field& del_fiel
fields.erase(std::remove_if(fields.begin(), fields.end(), [&garbage_field](const auto &f) {
return f.name == garbage_field.name;
}), fields.end());
+ TextEmbedderManager::get_instance().remove_text_embedder_from_collection(name, garbage_field.embed[fields::model_config]["model_name"].get<std::string>());
}
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 657a906a..1d9e0e1c 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -88,6 +88,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
}
field_obj[fields::num_dim] = num_dim;
+ TextEmbedderManager::get_instance().add_text_embedder_to_collection(this_collection_name, model_config["model_name"].get<std::string>());
LOG(INFO) << "Model init done.";
}
@@ -531,6 +532,9 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col
std::unique_lock u_lock(mutex);
collections.erase(actual_coll_name);
collection_id_names.erase(collection->get_collection_id());
+
+ TextEmbedderManager::get_instance().remove_collection(actual_coll_name);
+
u_lock.unlock();
// don't hold any collection manager locks here, since this can take some time
@@ -1260,7 +1264,7 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
std::string fallback_field_type;
std::vector<field> fields;
auto parse_op = field::json_fields_to_fields(req_json[ENABLE_NESTED_FIELDS].get<bool>(),
- req_json["fields"], fallback_field_type, fields);
+ req_json["fields"], fallback_field_type, fields, req_json["name"]);
if(!parse_op.ok()) {
return Option<Collection*>(parse_op.code(), parse_op.error());
diff --git a/src/field.cpp b/src/field.cpp
index d053e578..b72bad33 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -1081,7 +1081,7 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {
}
Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,
- std::vector<field>& the_fields) {
+ std::vector<field>& the_fields, const std::string& collection_name) {
size_t num_auto_detect_fields = 0;
const tsl::htrie_map<char, field> dummy_search_schema;
@@ -1094,7 +1094,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::j
}
if(!the_fields.empty() && !the_fields.back().embed.empty()) {
- auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back());
+ auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back(), collection_name);
if(!validate_res.ok()) {
return validate_res;
}
@@ -1110,7 +1110,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::j
Option<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema, nlohmann::json& field_json,
const nlohmann::json& fields_json,
- field& the_field) {
+ field& the_field, const std::string& collection_name) {
const std::string err_msg = "Property `" + fields::embed + "." + fields::from +
"` can only refer to string or string array fields.";
@@ -1140,6 +1140,7 @@ Option<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, fie
if(!res.ok()) {
return Option<bool>(res.code(), res.error());
}
+ TextEmbedderManager::get_instance().add_text_embedder_to_collection(collection_name, model_config[fields::model_name].get<std::string>());
LOG(INFO) << "Model init done.";
field_json[fields::num_dim] = num_dim;
diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cpp
index 89400a79..e47178c3 100644
--- a/src/text_embedder_manager.cpp
+++ b/src/text_embedder_manager.cpp
@@ -394,3 +394,76 @@ bool TextEmbedderManager::is_remote_model(const std::string& model_name) {
auto model_namespace = get_namespace(model_name);
return model_namespace.ok() && (model_namespace.get() == "openai" || model_namespace.get() == "google" || model_namespace.get() == "gcp");
}
+
+void TextEmbedderManager::add_text_embedder_to_collection(const std::string& collection_name, const std::string& model_name) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ auto text_embedder_it = text_embedders.find(model_name);
+ if(text_embedder_it == text_embedders.end()) {
+ LOG(ERROR) << "Text embedder not found: " << model_name;
+ return;
+ }
+
+ if(text_embedder_to_collections.find(model_name) == text_embedder_to_collections.end()) {
+ text_embedder_to_collections.emplace(model_name, std::unordered_set<std::string>{collection_name});
+ } else {
+ text_embedder_to_collections[model_name].emplace(collection_name);
+ }
+
+ if(collection_to_text_embedders.find(collection_name) == collection_to_text_embedders.end()) {
+ collection_to_text_embedders.emplace(collection_name, std::unordered_set<std::string>{model_name});
+ } else {
+ collection_to_text_embedders[collection_name].emplace(model_name);
+ }
+
+ LOG(INFO) << "Added text embedder: " << model_name << " to collection: " << collection_name;
+}
+
+
+void TextEmbedderManager::remove_text_embedder_from_collection(const std::string& collection_name, const std::string& model_name) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ auto text_embedder_it = text_embedders.find(model_name);
+ if(text_embedder_it == text_embedders.end()) {
+ LOG(ERROR) << "Text embedder not found: " << model_name;
+ return;
+ }
+
+ if(text_embedder_to_collections.find(model_name) == text_embedder_to_collections.end()) {
+ LOG(ERROR) << "Text embedder not found in any collection: " << model_name;
+ return;
+ }
+
+ if(collection_to_text_embedders.find(collection_name) == collection_to_text_embedders.end()) {
+ LOG(ERROR) << "Collection not found: " << collection_name;
+ return;
+ }
+
+ text_embedder_to_collections[model_name].erase(collection_name);
+ collection_to_text_embedders[collection_name].erase(model_name);
+
+ if(text_embedder_to_collections[model_name].empty()) {
+ text_embedder_to_collections.erase(model_name);
+ text_embedders.erase(model_name);
+ }
+
+ LOG(INFO) << "Removed text embedder: " << model_name << " from collection: " << collection_name;
+}
+
+
+void TextEmbedderManager::remove_collection(const std::string& collection_name) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ if(collection_to_text_embedders.find(collection_name) == collection_to_text_embedders.end()) {
+ LOG(ERROR) << "Collection not found: " << collection_name;
+ return;
+ }
+
+ for(const auto& model_name : collection_to_text_embedders[collection_name]) {
+ text_embedder_to_collections[model_name].erase(collection_name);
+ if(text_embedder_to_collections[model_name].empty()) {
+ text_embedder_to_collections.erase(model_name);
+ text_embedders.erase(model_name);
+ }
+ }
+
+ collection_to_text_embedders.erase(collection_name);
+ LOG(INFO) << "Removed all text embedders for collection: " << collection_name;
+}
\ No newline at end of file
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 4eb84853..e99390da 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -31,6 +31,7 @@ protected:
virtual void TearDown() {
collectionManager.dispose();
+ TextEmbedderManager::get_instance().delete_all_text_embedders();
delete store;
}
};
@@ -2229,4 +2230,80 @@ TEST_F(CollectionVectorTest, QueryByNotAutoEmbeddingVectorField) {
ASSERT_FALSE(search_res.ok());
ASSERT_EQ("Vector field `embedding` is not an auto-embedding field, do not use `query_by` with it, use `vector_query` instead.", search_res.error());
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionVectorTest, TestUnloadingModelsOnCollectionDelete) {
+ nlohmann::json actual_schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto schema = actual_schema;
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ auto delete_op = collectionManager.drop_collection("test", true);
+
+ ASSERT_TRUE(delete_op.ok());
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+
+ // create another collection
+ schema = actual_schema;
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ coll = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ // create second collection
+ schema = actual_schema;
+ schema["name"] = "test2";
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll2 = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ delete_op = collectionManager.drop_collection("test", true);
+ ASSERT_TRUE(delete_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ delete_op = collectionManager.drop_collection("test2", true);
+ ASSERT_TRUE(delete_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+}
\ No newline at end of file