Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-facets
Kishore Nallan191d62a0cbb8
diff --cc include/collection.hindex 920b4597,35cefee1..75bf51ae--- a/include/collection.h+++ b/include/collection.h@@@ -363,10 -354,10 +365,12 @@@ publictsl::htrie_map<char, field> get_embedding_fields();+ tsl::htrie_map<char, field> get_embedding_fields_unsafe();+std::string get_default_sorting_field();+ Option<bool> add_reference_helper_fields(nlohmann::json& document);+Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,const index_operation_t& operation,const DIRTY_VALUES dirty_values,diff --cc include/collection_manager.hindex 828421fc,b92f29e5..c70ce6ad--- a/include/collection_manager.h+++ b/include/collection_manager.h@@@ -205,10 -202,5 +205,12 @@@ publicOption<bool> delete_preset(const std::string & preset_name);+ static void _get_reference_collection_names(const std::string& filter_query,+ std::set<std::string>& reference_collection_names);++ void add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair);++ std::map<std::string, std::set<reference_pair>> _get_referenced_in_backlog() const;+++ void process_embedding_field_delete(const std::string& model_name);-};+};diff --cc src/collection.cppindex f699edad,eb9dee36..89f07fe2--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -5617,52 -5036,17 +5652,67 @@@ Option<bool> Collection::truncate_afterreturn Option<bool>(true);}+void Collection::reference_populate_sort_mapping(int *sort_order, std::vector<size_t> &geopoint_indices,+ std::vector<sort_by> &sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t> *, 3> &field_values)+ const {+ std::shared_lock lock(mutex);+ index->populate_sort_mapping_with_lock(sort_order, geopoint_indices, sort_fields_std, field_values);+}++int64_t Collection::reference_string_sort_score(const string &field_name, const uint32_t& seq_id) const {+ std::shared_lock lock(mutex);+ return index->reference_string_sort_score(field_name, seq_id);+}++bool Collection::is_referenced_in(const std::string& collection_name) const {+ std::shared_lock lock(mutex);+ return referenced_in.count(collection_name) > 0;+}++void Collection::add_referenced_in(const reference_pair& pair) {+ return add_referenced_in(pair.collection, pair.field);+}++void Collection::add_referenced_ins(const std::set<reference_pair>& pairs) {+ std::shared_lock lock(mutex);+ for (const auto &pair: pairs) {+ referenced_in.emplace(pair.collection, pair.field);+ }+}++void Collection::add_referenced_in(const std::string& collection_name, const std::string& field_name) {+ std::shared_lock lock(mutex);+ referenced_in.emplace(collection_name, field_name);+}++Option<std::string> Collection::get_reference_field(const std::string& collection_name) const {+ std::shared_lock lock(mutex);++ if (referenced_in.count(collection_name) == 0) {+ return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"+ + collection_name + "`.");+ }++ return Option<std::string>(referenced_in.at(collection_name));+}++Option<uint32_t> Collection::get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const {+ std::shared_lock lock(mutex);+ return index->get_sort_indexed_field_value(field_name, seq_id);+}+++ void Collection::remove_embedding_field(const std::string& field_name) {+ if(embedding_fields.find(field_name) == embedding_fields.end()) {+ return;+ }++ const auto& del_field = embedding_fields[field_name];+ const auto& model_name = del_field.embed[fields::model_config]["model_name"].get<std::string>();+ embedding_fields.erase(field_name);+ CollectionManager::get_instance().process_embedding_field_delete(model_name);+ }++ tsl::htrie_map<char, field> Collection::get_embedding_fields_unsafe() {+ return embedding_fields;-}++}diff --cc src/collection_manager.cppindex 3d5c306d,5ad7d814..f9b3720a--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -1441,16 -1121,11 +1449,15 @@@ Option<bool> CollectionManager::do_searfacet_sample_percent,facet_sample_threshold,offset,+ HASH,remote_embedding_timeout_ms,remote_embedding_num_tries,+ stopwords_set,+ facet_return_parent,+ ref_include_fields_vec,+ drop_tokens_mode,prioritize_num_matching_fields,- group_missing_values- );- group_missing_values,- drop_tokens_mode);++ group_missing_values);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();@@@ -1898,12 -1573,28 +1905,38 @@@ Option<Collection*> CollectionManager::return Option<Collection*>(new_coll);}+void CollectionManager::add_referenced_in_backlog(const std::string& collection_name, reference_pair&& pair) {+ std::shared_lock lock(mutex);+ referenced_in_backlog[collection_name].insert(pair);+}++std::map<std::string, std::set<reference_pair>> CollectionManager::_get_referenced_in_backlog() const {+ std::shared_lock lock(mutex);+ return referenced_in_backlog;- }++}+++ void CollectionManager::process_embedding_field_delete(const std::string& model_name) {+ std::shared_lock lock(mutex);+ bool found = false;++ for(const auto& collection: collections) {+ // will be deadlock if we try to acquire lock on collection here+ // caller of this function should have already acquired lock on collection+ const auto& embedding_fields = collection.second->get_embedding_fields_unsafe();++ for(const auto& embedding_field: embedding_fields) {+ if(embedding_field.embed.count(fields::model_config) != 0) {+ const auto& model_config = embedding_field.embed[fields::model_config];+ if(model_config["model_name"].get<std::string>() == model_name) {+ found = true;+ break;+ }+ }+ }+ }++ if(!found) {+ LOG(INFO) << "Deleting text embedder: " << model_name;+ TextEmbedderManager::get_instance().delete_text_embedder(model_name);+ }-}++}diff --cc test/collection_vector_search_test.cppindex 34f5a405,55049641..c6bd963d--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@@ -2234,3 -2231,270 +2249,269 @@@ TEST_F(CollectionVectorTest, QueryByNotASSERT_EQ("Vector field `embedding` is not an auto-embedding field, do not use `query_by` with it, use `vector_query` instead.", search_res.error());}++ TEST_F(CollectionVectorTest, TestUnloadingModelsOnCollectionDelete) {+ nlohmann::json actual_schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto schema = actual_schema;+ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(1, text_embedders.size());++ auto delete_op = collectionManager.drop_collection("test", true);++ ASSERT_TRUE(delete_op.ok());+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());++ // create another collection+ schema = actual_schema;+ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ coll = collection_create_op.get();++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ // create second collection+ schema = actual_schema;+ schema["name"] = "test2";+ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll2 = collection_create_op.get();++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(1, text_embedders.size());++ delete_op = collectionManager.drop_collection("test", true);+ ASSERT_TRUE(delete_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ delete_op = collectionManager.drop_collection("test2", true);+ ASSERT_TRUE(delete_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());+ }++ TEST_F(CollectionVectorTest, TestUnloadingModelsOnDrop) {+ nlohmann::json actual_schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto schema = actual_schema;+ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(1, text_embedders.size());++ nlohmann::json drop_schema = R"({+ "fields": [+ {+ "name": "title_vec",+ "drop": true+ }+ ]+ })"_json;++ auto drop_op = coll->alter(drop_schema);+ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());++ // create another collection+ schema = actual_schema;+ schema["name"] = "test2";+ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll2 = collection_create_op.get();++ nlohmann::json alter_schema = R"({+ "fields": [+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ auto alter_op = coll->alter(alter_schema);+ ASSERT_TRUE(alter_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ drop_op = coll2->alter(drop_schema);+ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ drop_op = coll->alter(drop_schema);+ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());+ }+-+ TEST_F(CollectionVectorTest, TestUnloadModelsCollectionHaveTwoEmbeddingField) {+ nlohmann::json actual_schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ },+ {+ "name": "title_vec2",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto schema = actual_schema;+ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ nlohmann::json drop_schema = R"({+ "fields": [+ {+ "name": "title_vec",+ "drop": true+ }+ ]+ })"_json;++ auto alter_op = coll->alter(drop_schema);+ ASSERT_TRUE(alter_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ drop_schema = R"({+ "fields": [+ {+ "name": "title_vec2",+ "drop": true+ }+ ]+ })"_json;++ alter_op = coll->alter(drop_schema);+ ASSERT_TRUE(alter_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());++ // create another collection+ schema = actual_schema;+ schema["name"] = "test2";++ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll2 = collection_create_op.get();++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ // drop collection+ auto drop_op = collectionManager.drop_collection("test2", true);++ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());-}++}