Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.26-facets' into v0.26-facets
Ozan Armağanfa11f4cd7efc
diff --cc include/collection.hindex f9fe4642,2ad4742c..b6163e07--- a/include/collection.h+++ b/include/collection.h@@@ -285,8 -295,14 +295,13 @@@ privatetsl::htrie_set<char>& include_fields_full,tsl::htrie_set<char>& exclude_fields_full) const;- Option<uint32_t> get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const;+ Option<std::string> get_referenced_in_field(const std::string& collection_name) const;+ Option<bool> get_related_ids(const std::string& ref_collection_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const;+- static void hide_credential(nlohmann::json& json, const std::string& credential_name);++ void remove_embedding_field(const std::string& field_name);public:@@@ -495,11 -523,10 +523,14 @@@const size_t remote_embedding_num_tries = 2,const std::string& stopwords_set="",const std::vector<std::string>& facet_return_parent = {},+ const bool converstaion = false,+ const int conversation_model_id = -1,+ const std::string& system_prompt = "",+ int conversation_id = -1,- const std::vector<ref_include_fields>& ref_include_fields_vec = {}) const;+ const std::vector<ref_include_fields>& ref_include_fields_vec = {},+ const std::string& drop_tokens_mode = "right_to_left",+ const bool prioritize_num_matching_fields = true,+ const bool group_missing_values = true) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;@@@ -605,12 -630,13 +634,15 @@@void add_referenced_in(const std::string& collection_name, const std::string& field_name);- Option<std::string> get_reference_field(const std::string& collection_name) const;+ Option<std::string> get_referenced_in_field_with_lock(const std::string& collection_name) const;++ Option<bool> get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,+ std::vector<uint32_t>& result) const;- Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;+ Option<uint32_t> get_sort_index_value_with_lock(const std::string& field_name, const uint32_t& seq_id) const;+ static void hide_credential(nlohmann::json& json, const std::string& credential_name);+friend class filter_result_iterator_t;};diff --cc include/text_embedder_remote.hindex 28b80848,1c39489a..8b9e682e--- a/include/text_embedder_remote.h+++ b/include/text_embedder_remote.h@@@ -25,12 -25,13 +25,13 @@@ struct embedding_res_tclass RemoteEmbedder {protected:static Option<bool> validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties);- static long call_remote_api(const std::string& method, const std::string& url, const std::string& req_body, std::string& res_body, std::map<std::string, std::string>& res_headers, std::unordered_map<std::string, std::string>& req_headers);static inline ReplicationState* raft_server = nullptr;public:+ static long call_remote_api(const std::string& method, const std::string& url, const std::string& req_body, std::string& res_body, std::map<std::string, std::string>& res_headers, std::unordered_map<std::string, std::string>& req_headers);virtual nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) = 0;virtual embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) = 0;- virtual std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) = 0;+ virtual std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200,+ const size_t remote_embedding_timeout_ms = 60000, const size_t remote_embedding_num_tries = 2) = 0;static const std::string get_model_key(const nlohmann::json& model_config);static void init(ReplicationState* rs) {raft_server = rs;diff --cc src/collection.cppindex 38b53a01,24b3d331..3e3e40e5--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -19,9 -20,7 +20,10 @@@#include "vector_query_ops.h"#include "text_embedder_manager.h"#include "stopwords_manager.h"+#include "conversation_model.h"+#include "conversation_manager.h"+#include "conversation_model_manager.h"+ #include "field.h"const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";@@@ -1391,13 -1406,12 +1409,15 @@@ Option<nlohmann::json> Collection::searconst size_t remote_embedding_timeout_ms,const size_t remote_embedding_num_tries,const std::string& stopwords_set,-- const std::vector<std::string>& facet_return_parent,++ const std::vector<std::string>& facet_return_parent,=+ const bool conversation,+ const int conversation_model_id,+ const std::string& system_prompt,+ int conversation_id,- const std::vector<ref_include_fields>& ref_include_fields_vec) const {-+ const std::vector<ref_include_fields>& ref_include_fields_vec,+ const std::string& drop_tokens_mode,+ const bool prioritize_num_matching_fields,+ const bool group_missing_values) const {-std::shared_lock lock(mutex);// setup thread local vars@@@ -1624,7 -1624,9 +1673,8 @@@return Option<nlohmann::json>(400, error);}- for(const std::string & field_name: processed_search_fields) {-+ for(const auto& processed_search_field: processed_search_fields) {+ const auto& field_name = processed_search_field.name;field search_field = search_schema.at(field_name);if(!search_field.index) {std::string error = "Field `" + field_name + "` is marked as a non-indexed field in the schema.";diff --cc src/collection_manager.cppindex 535fdc77,bf3baa70..db6bea8d--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -977,11 -1115,9 +1116,14 @@@ Option<bool> CollectionManager::do_searconst char *FACET_SAMPLE_PERCENT = "facet_sample_percent";const char *FACET_SAMPLE_THRESHOLD = "facet_sample_threshold";+ const char *CONVERSATION = "conversation";+ const char *CONVERSATION_ID = "conversation_id";+ const char *SYSTEM_PROMPT = "system_prompt";+ const char *CONVERSATION_MODEL_ID = "conversation_model_id";++ const char *DROP_TOKENS_MODE = "drop_tokens_mode";+ const char *PRIORITIZE_NUM_MATCHING_FIELDS = "prioritize_num_matching_fields";+// enrich params with values from embedded paramsfor(auto& item: embedded_params.items()) {if(item.key() == "expires_at") {@@@ -1102,11 -1239,9 +1245,15 @@@size_t facet_sample_percent = 100;size_t facet_sample_threshold = 0;+ bool conversation = false;+ size_t conversation_id = std::numeric_limits<size_t>::max();+ std::string system_prompt;+ size_t conversation_model_id = std::numeric_limits<size_t>::max();++ std::string drop_tokens_mode_str = "right_to_left";+ bool prioritize_num_matching_fields = true;+++std::unordered_map<std::string, size_t*> unsigned_int_values = {{MIN_LEN_1TYPO, &min_len_1typo},{MIN_LEN_2TYPO, &min_len_2typo},@@@ -1145,7 -1278,7 +1292,8 @@@{HIGHLIGHT_END_TAG, &highlight_end_tag},{PINNED_HITS, &pinned_hits_str},{HIDDEN_HITS, &hidden_hits_str},+ {SYSTEM_PROMPT, &system_prompt},+ {DROP_TOKENS_MODE, &drop_tokens_mode_str},};std::unordered_map<std::string, bool*> bool_values = {@@@ -1155,7 -1288,8 +1303,9 @@@{EXHAUSTIVE_SEARCH, &exhaustive_search},{ENABLE_OVERRIDES, &enable_overrides},{ENABLE_HIGHLIGHT_V1, &enable_highlight_v1},+ {CONVERSATION, &conversation},+ {PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},+ {GROUP_MISSING_VALUES, &group_missing_values},};std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {@@@ -1357,12 -1490,10 +1507,15 @@@remote_embedding_num_tries,stopwords_set,facet_return_parent,+ conversation,+ (conversation_model_id == std::numeric_limits<size_t>::max()) ? -1 : static_cast<int>(conversation_model_id),+ system_prompt,+ (conversation_id == std::numeric_limits<size_t>::max()) ? -1 : static_cast<int>(conversation_id),- ref_include_fields_vec);-+ ref_include_fields_vec,+ drop_tokens_mode_str,+ prioritize_num_matching_fields,+ group_missing_values);++uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --cc src/typesense_server_utils.cppindex 4979f881,97cfbd00..8319355f--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@@ -475,17 -459,10 +477,22 @@@ int run_server(const Config & config, cAnalyticsManager::get_instance().run(&replication_state);});+ std::thread conersation_garbage_collector_thread([]() {+ LOG(INFO) << "Conversation garbage collector thread started.";+ int last_clear_time = 0;+ while(!brpc::IsAskedToQuit()) {+ if(last_clear_time + 60 < std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()) {+ last_clear_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();+ ConversationManager::clear_expired_conversations();+ }+ std::this_thread::sleep_for(std::chrono::seconds(5));+ }+ });+++ HouseKeeper::get_instance().init(config.get_housekeeping_interval());+ std::thread housekeeping_thread([]() {+ HouseKeeper::get_instance().run();+ });RemoteEmbedder::init(&replication_state);diff --cc test/collection_vector_search_test.cppindex 19590486,5181c76b..00caa694--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@@ -1862,99 -2055,787 +2059,883 @@@ TEST_F(CollectionVectorTest, TestMultilASSERT_TRUE(semantic_results.ok());}-+ TEST_F(CollectionVectorTest, TestTwoEmbeddingFieldsSamePrefix) {+ nlohmann::json schema = R"({+ "name": "docs",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ },+ {+ "name": "embedding_en",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "title": "john doe"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto semantic_results = coll1->search("john", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>());++ ASSERT_TRUE(semantic_results.ok());+ }++ TEST_F(CollectionVectorTest, TestOneEmbeddingOneKeywordFieldsHaveSamePrefix) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "title": "john doe"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto keyword_results = coll1->search("john", {"title"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>());++ ASSERT_TRUE(keyword_results.ok());+ }++ TEST_F(CollectionVectorTest, HybridSearchOnlyKeyworMatchDoNotHaveVectorDistance) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "title": "john doe"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ // hybrid search with empty vector (to pass distance threshold param)+ std::string vec_query = "embedding:([], distance_threshold: 0.05)";++ auto hybrid_results = coll1->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, vec_query);++ ASSERT_TRUE(hybrid_results.ok());++ ASSERT_EQ(1, hybrid_results.get()["hits"].size());+ ASSERT_EQ(0, hybrid_results.get()["hits"][0].count("vector_distance"));+ }++ TEST_F(CollectionVectorTest, QueryByNotAutoEmbeddingVectorField) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "num_dim": 384+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto search_res = coll->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([0.96826, 0.94, 0.39557, 0.306488])");++ ASSERT_FALSE(search_res.ok());++ ASSERT_EQ("Vector field `embedding` is not an auto-embedding field, do not use `query_by` with it, use `vector_query` instead.", search_res.error());+ }++ TEST_F(CollectionVectorTest, TestUnloadingModelsOnCollectionDelete) {+ nlohmann::json actual_schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto schema = actual_schema;+ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();-+++ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(1, text_embedders.size());++ auto delete_op = collectionManager.drop_collection("test", true);++ ASSERT_TRUE(delete_op.ok());+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());++ // create another collection+ schema = actual_schema;+ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ coll = collection_create_op.get();++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ // create second collection+ schema = actual_schema;+ schema["name"] = "test2";+ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll2 = collection_create_op.get();++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(1, text_embedders.size());++ delete_op = collectionManager.drop_collection("test", true);+ ASSERT_TRUE(delete_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ delete_op = collectionManager.drop_collection("test2", true);+ ASSERT_TRUE(delete_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());+ }++ TEST_F(CollectionVectorTest, TestUnloadingModelsOnDrop) {+ nlohmann::json actual_schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto schema = actual_schema;+ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(1, text_embedders.size());++ nlohmann::json drop_schema = R"({+ "fields": [+ {+ "name": "title_vec",+ "drop": true+ }+ ]+ })"_json;++ auto drop_op = coll->alter(drop_schema);+ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());++ // create another collection+ schema = actual_schema;+ schema["name"] = "test2";+ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll2 = collection_create_op.get();++ nlohmann::json alter_schema = R"({+ "fields": [+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ auto alter_op = coll->alter(alter_schema);+ ASSERT_TRUE(alter_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ drop_op = coll2->alter(drop_schema);+ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ drop_op = coll->alter(drop_schema);+ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());+ }++ TEST_F(CollectionVectorTest, TestUnloadModelsCollectionHaveTwoEmbeddingField) {+ nlohmann::json actual_schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ },+ {+ "name": "title_vec2",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto schema = actual_schema;+ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ nlohmann::json drop_schema = R"({+ "fields": [+ {+ "name": "title_vec",+ "drop": true+ }+ ]+ })"_json;++ auto alter_op = coll->alter(drop_schema);+ ASSERT_TRUE(alter_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ drop_schema = R"({+ "fields": [+ {+ "name": "title_vec2",+ "drop": true+ }+ ]+ })"_json;++ alter_op = coll->alter(drop_schema);+ ASSERT_TRUE(alter_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());++ // create another collection+ schema = actual_schema;+ schema["name"] = "test2";++ collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll2 = collection_create_op.get();++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(1, text_embedders.size());++ // drop collection+ auto drop_op = collectionManager.drop_collection("test2", true);++ ASSERT_TRUE(drop_op.ok());++ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();+ ASSERT_EQ(0, text_embedders.size());+ }++ TEST_F(CollectionVectorTest, TestHybridSearchAlphaParam) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "name",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "name"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "soccer"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "basketball"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "volleyball"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());+++ // do hybrid search+ auto hybrid_results = coll->search("sports", {"name", "embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ ASSERT_EQ(3, hybrid_results["hits"].size());++ // check scores+ ASSERT_FLOAT_EQ(0.3, hybrid_results["hits"][0]["hybrid_search_info"]["rank_fusion_score"].get<float>());+ ASSERT_FLOAT_EQ(0.15, hybrid_results["hits"][1]["hybrid_search_info"]["rank_fusion_score"].get<float>());+ ASSERT_FLOAT_EQ(0.10, hybrid_results["hits"][2]["hybrid_search_info"]["rank_fusion_score"].get<float>());++ // do hybrid search with alpha = 0.5+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], alpha:0.5)").get();+ ASSERT_EQ(3, hybrid_results["hits"].size());++ // check scores+ ASSERT_FLOAT_EQ(0.5, hybrid_results["hits"][0]["hybrid_search_info"]["rank_fusion_score"].get<float>());+ ASSERT_FLOAT_EQ(0.25, hybrid_results["hits"][1]["hybrid_search_info"]["rank_fusion_score"].get<float>());+ ASSERT_FLOAT_EQ(0.16666667, hybrid_results["hits"][2]["hybrid_search_info"]["rank_fusion_score"].get<float>());+ }++ TEST_F(CollectionVectorTest, TestHybridSearchInvalidAlpha) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "name",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "name"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();+++ // do hybrid search with alpha = 1.5+ auto hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], alpha:1.5)");++ ASSERT_FALSE(hybrid_results.ok());+ ASSERT_EQ("Malformed vector query string: "+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());++ // do hybrid search with alpha = -0.5+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], alpha:-0.5)");++ ASSERT_FALSE(hybrid_results.ok());+ ASSERT_EQ("Malformed vector query string: "+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());++ // do hybrid search with alpha as string+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], alpha:\"0.5\")");++ ASSERT_FALSE(hybrid_results.ok());+ ASSERT_EQ("Malformed vector query string: "+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());++ }++ TEST_F(CollectionVectorTest, TestSearchNonIndexedEmbeddingField) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "name",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "index": false,+ "optional": true,+ "embed": {+ "from": [+ "name"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "soccer"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto search_res = coll->search("soccer", {"name", "embedding"}, "", {}, {}, {0});+ ASSERT_FALSE(search_res.ok());++ ASSERT_EQ("Field `embedding` is marked as a non-indexed field in the schema.", search_res.error());+ }++ TEST_F(CollectionVectorTest, TestSearchNonIndexedVectorField) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "vec",+ "type": "float[]",+ "index": false,+ "optional": true,+ "num_dim": 2+ }+ ]+ })"_json;++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "vec": [0.1, 0.2]+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto search_result = coll->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "vec:([0.96826, 0.94])");++ ASSERT_FALSE(search_result.ok());+ ASSERT_EQ("Field `vec` is marked as a non-indexed field in the schema.", search_result.error());+ }++ TEST_F(CollectionVectorTest, TestSemanticSearchAfterUpdate) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "name",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "name"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "soccer",+ "id": "0"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "basketball",+ "id": "1"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "typesense",+ "id": "2"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "potato",+ "id": "3"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], id:0, k:1)");++ ASSERT_TRUE(result.ok());+ ASSERT_EQ(1, result.get()["hits"].size());+ ASSERT_EQ("basketball", result.get()["hits"][0]["document"]["name"]);++ auto update_op = coll->add(R"({+ "name": "onion",+ "id": "0"+ })"_json.dump(), index_operation_t::UPDATE, "0");++ ASSERT_TRUE(update_op.ok());++ result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], id:0, k:1)");++ ASSERT_TRUE(result.ok());+ ASSERT_EQ(1, result.get()["hits"].size());+ ASSERT_EQ("potato", result.get()["hits"][0]["document"]["name"]);+ }+++TEST_F(CollectionVectorTest, TestQAConversation) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";+ return;+ }++ auto api_key = std::string(std::getenv("api_key"));++ auto conversation_model_config = R"({+ "model_name": "openai/gpt-3.5-turbo"+ })"_json;++ conversation_model_config["api_key"] = api_key;++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto model_add_op = ConversationModelManager::add_model(conversation_model_config);++ ASSERT_TRUE(model_add_op.ok());++ auto add_op = coll->add(R"({+ "product_name": "moisturizer",+ "category": "beauty"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "shampoo",+ "category": "beauty"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "shirt",+ "category": "clothing"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "product_name": "pants",+ "category": "clothing"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto results_op = coll->search("how many products are there for clothing category?", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>(), {},+ 10, "", 30, 4, "", 1, "", "", {}, 3, "<mark>", "</mark>", {}, 4294967295UL, true, false,+ true, "", false, 6000000UL, 4, 7, fallback, 4, {off}, 32767UL, 32767UL, 2, 2, false, "",+ true, 0, max_score, 100, 0, 0, HASH, 30000, 2, "", {}, true, 0);++ ASSERT_TRUE(results_op.ok());++ auto results = results_op.get();++ ASSERT_EQ(4, results["hits"].size());+ ASSERT_TRUE(results.contains("conversation"));+ ASSERT_TRUE(results["conversation"].is_object());+ ASSERT_EQ("how many products are there for clothing category?", results["conversation"]["query"]);+ ASSERT_EQ(0, results["conversation"]["conversation_id"]);+++ // test getting conversation history+ auto history_op = ConversationManager::get_conversation(0);++ ASSERT_TRUE(history_op.ok());++ auto history = history_op.get();++ ASSERT_TRUE(history.is_object());+ ASSERT_TRUE(history.contains("conversation"));+ ASSERT_TRUE(history["conversation"].is_array ());++ ASSERT_EQ("how many products are there for clothing category?", history["conversation"][0]["user"]);- }++}