Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallan191d62a0cbb8
diff --cc include/collection.h
index 920b4597,35cefee1..75bf51ae
--- a/include/collection.h
+++ b/include/collection.h
@@@ -363,10 -354,10 +365,12 @@@ public
tsl::htrie_map<char, field> get_embedding_fields();
+ tsl::htrie_map<char, field> get_embedding_fields_unsafe();
+
std::string get_default_sorting_field();
+ Option<bool> add_reference_helper_fields(nlohmann::json& document);
+
Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,
const index_operation_t& operation,
const DIRTY_VALUES dirty_values,
diff --cc include/collection_manager.h
index 828421fc,b92f29e5..c70ce6ad
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@@ -205,10 -202,5 +205,12 @@@ public
Option<bool> delete_preset(const std::string & preset_name);
+ static void _get_reference_collection_names(const std::string& filter_query,
+ std::set<std::string>& reference_collection_names);
+
+ void add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair);
+
+ std::map<std::string, std::set<reference_pair>> _get_referenced_in_backlog() const;
++
+ void process_embedding_field_delete(const std::string& model_name);
-};
+};
diff --cc src/collection.cpp
index f699edad,eb9dee36..89f07fe2
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -5617,52 -5036,17 +5652,67 @@@ Option<bool> Collection::truncate_after
return Option<bool>(true);
}
+void Collection::reference_populate_sort_mapping(int *sort_order, std::vector<size_t> &geopoint_indices,
+ std::vector<sort_by> &sort_fields_std,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t> *, 3> &field_values)
+ const {
+ std::shared_lock lock(mutex);
+ index->populate_sort_mapping_with_lock(sort_order, geopoint_indices, sort_fields_std, field_values);
+}
+
+int64_t Collection::reference_string_sort_score(const string &field_name, const uint32_t& seq_id) const {
+ std::shared_lock lock(mutex);
+ return index->reference_string_sort_score(field_name, seq_id);
+}
+
+bool Collection::is_referenced_in(const std::string& collection_name) const {
+ std::shared_lock lock(mutex);
+ return referenced_in.count(collection_name) > 0;
+}
+
+void Collection::add_referenced_in(const reference_pair& pair) {
+ return add_referenced_in(pair.collection, pair.field);
+}
+
+void Collection::add_referenced_ins(const std::set<reference_pair>& pairs) {
+ std::shared_lock lock(mutex);
+ for (const auto &pair: pairs) {
+ referenced_in.emplace(pair.collection, pair.field);
+ }
+}
+
+void Collection::add_referenced_in(const std::string& collection_name, const std::string& field_name) {
+ std::shared_lock lock(mutex);
+ referenced_in.emplace(collection_name, field_name);
+}
+
+Option<std::string> Collection::get_reference_field(const std::string& collection_name) const {
+ std::shared_lock lock(mutex);
+
+ if (referenced_in.count(collection_name) == 0) {
+ return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"
+ + collection_name + "`.");
+ }
+
+ return Option<std::string>(referenced_in.at(collection_name));
+}
+
+Option<uint32_t> Collection::get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const {
+ std::shared_lock lock(mutex);
+ return index->get_sort_indexed_field_value(field_name, seq_id);
+}
++
+ void Collection::remove_embedding_field(const std::string& field_name) {
+ if(embedding_fields.find(field_name) == embedding_fields.end()) {
+ return;
+ }
+
+ const auto& del_field = embedding_fields[field_name];
+ const auto& model_name = del_field.embed[fields::model_config]["model_name"].get<std::string>();
+ embedding_fields.erase(field_name);
+ CollectionManager::get_instance().process_embedding_field_delete(model_name);
+ }
+
+ tsl::htrie_map<char, field> Collection::get_embedding_fields_unsafe() {
+ return embedding_fields;
-}
++}
diff --cc src/collection_manager.cpp
index 3d5c306d,5ad7d814..f9b3720a
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -1441,16 -1121,11 +1449,15 @@@ Option<bool> CollectionManager::do_sear
facet_sample_percent,
facet_sample_threshold,
offset,
+ HASH,
remote_embedding_timeout_ms,
remote_embedding_num_tries,
+ stopwords_set,
+ facet_return_parent,
+ ref_include_fields_vec,
+ drop_tokens_mode,
prioritize_num_matching_fields,
- group_missing_values
- );
- group_missing_values,
- drop_tokens_mode);
++ group_missing_values);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
@@@ -1898,12 -1573,28 +1905,38 @@@ Option<Collection*> CollectionManager::
return Option<Collection*>(new_coll);
}
+void CollectionManager::add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair) {
+ std::shared_lock lock(mutex);
+ referenced_in_backlog[collection_name].insert(pair);
+}
+
+std::map<std::string, std::set<reference_pair>> CollectionManager::_get_referenced_in_backlog() const {
+ std::shared_lock lock(mutex);
+ return referenced_in_backlog;
- }
++}
++
+ void CollectionManager::process_embedding_field_delete(const std::string& model_name) {
+ std::shared_lock lock(mutex);
+ bool found = false;
+
+ for(const auto& collection: collections) {
+ // will be deadlock if we try to acquire lock on collection here
+ // caller of this function should have already acquired lock on collection
+ const auto& embedding_fields = collection.second->get_embedding_fields_unsafe();
+
+ for(const auto& embedding_field: embedding_fields) {
+ if(embedding_field.embed.count(fields::model_config) != 0) {
+ const auto& model_config = embedding_field.embed[fields::model_config];
+ if(model_config["model_name"].get<std::string>() == model_name) {
+ found = true;
+ break;
+ }
+ }
+ }
+ }
+
+ if(!found) {
+ LOG(INFO) << "Deleting text embedder: " << model_name;
+ TextEmbedderManager::get_instance().delete_text_embedder(model_name);
+ }
-}
++}
diff --cc test/collection_vector_search_test.cpp
index 34f5a405,55049641..c6bd963d
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -2234,3 -2231,270 +2249,269 @@@ TEST_F(CollectionVectorTest, QueryByNot
ASSERT_EQ("Vector field `embedding` is not an auto-embedding field, do not use `query_by` with it, use `vector_query` instead.", search_res.error());
}
+
+ TEST_F(CollectionVectorTest, TestUnloadingModelsOnCollectionDelete) {
+ nlohmann::json actual_schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto schema = actual_schema;
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ auto delete_op = collectionManager.drop_collection("test", true);
+
+ ASSERT_TRUE(delete_op.ok());
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+
+ // create another collection
+ schema = actual_schema;
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ coll = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ // create second collection
+ schema = actual_schema;
+ schema["name"] = "test2";
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll2 = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ delete_op = collectionManager.drop_collection("test", true);
+ ASSERT_TRUE(delete_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ delete_op = collectionManager.drop_collection("test2", true);
+ ASSERT_TRUE(delete_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+ }
+
+ TEST_F(CollectionVectorTest, TestUnloadingModelsOnDrop) {
+ nlohmann::json actual_schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto schema = actual_schema;
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ nlohmann::json drop_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec",
+ "drop": true
+ }
+ ]
+ })"_json;
+
+ auto drop_op = coll->alter(drop_schema);
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+
+ // create another collection
+ schema = actual_schema;
+ schema["name"] = "test2";
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll2 = collection_create_op.get();
+
+ nlohmann::json alter_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ auto alter_op = coll->alter(alter_schema);
+ ASSERT_TRUE(alter_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ drop_op = coll2->alter(drop_schema);
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ drop_op = coll->alter(drop_schema);
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+ }
+
-
+ TEST_F(CollectionVectorTest, TestUnloadModelsCollectionHaveTwoEmbeddingField) {
+ nlohmann::json actual_schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ },
+ {
+ "name": "title_vec2",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto schema = actual_schema;
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ nlohmann::json drop_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec",
+ "drop": true
+ }
+ ]
+ })"_json;
+
+ auto alter_op = coll->alter(drop_schema);
+ ASSERT_TRUE(alter_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ drop_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec2",
+ "drop": true
+ }
+ ]
+ })"_json;
+
+ alter_op = coll->alter(drop_schema);
+ ASSERT_TRUE(alter_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+
+ // create another collection
+ schema = actual_schema;
+ schema["name"] = "test2";
+
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll2 = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ // drop collection
+ auto drop_op = collectionManager.drop_collection("test2", true);
+
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
-}
++}