Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.26-facets' into v0.26-facets

Ozan Armağanfa11f4cd7efc
diff --cc include/collection.h
index f9fe4642,2ad4742c..b6163e07
--- a/include/collection.h
+++ b/include/collection.h
@@@ -285,8 -295,14 +295,13 @@@ private
tsl::htrie_set<char>& include_fields_full,
tsl::htrie_set<char>& exclude_fields_full) const;
- Option<uint32_t> get_reference_doc_id(const std::string& ref_collection_name, const uint32_t& seq_id) const;
+ Option<std::string> get_referenced_in_field(const std::string& collection_name) const;
+ Option<bool> get_related_ids(const std::string& ref_collection_name, const uint32_t& seq_id,
+ std::vector<uint32_t>& result) const;
+
- static void hide_credential(nlohmann::json& json, const std::string& credential_name);
+
+ void remove_embedding_field(const std::string& field_name);
public:
@@@ -495,11 -523,10 +523,14 @@@
const size_t remote_embedding_num_tries = 2,
const std::string& stopwords_set="",
const std::vector<std::string>& facet_return_parent = {},
+ const bool converstaion = false,
+ const int conversation_model_id = -1,
+ const std::string& system_prompt = "",
+ int conversation_id = -1,
- const std::vector<ref_include_fields>& ref_include_fields_vec = {}) const;
+ const std::vector<ref_include_fields>& ref_include_fields_vec = {},
+ const std::string& drop_tokens_mode = "right_to_left",
+ const bool prioritize_num_matching_fields = true,
+ const bool group_missing_values = true) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
@@@ -605,12 -630,13 +634,15 @@@
void add_referenced_in(const std::string& collection_name, const std::string& field_name);
- Option<std::string> get_reference_field(const std::string& collection_name) const;
+ Option<std::string> get_referenced_in_field_with_lock(const std::string& collection_name) const;
+
+ Option<bool> get_related_ids_with_lock(const std::string& field_name, const uint32_t& seq_id,
+ std::vector<uint32_t>& result) const;
- Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;
+ Option<uint32_t> get_sort_index_value_with_lock(const std::string& field_name, const uint32_t& seq_id) const;
+ static void hide_credential(nlohmann::json& json, const std::string& credential_name);
+
friend class filter_result_iterator_t;
};
diff --cc include/text_embedder_remote.h
index 28b80848,1c39489a..8b9e682e
--- a/include/text_embedder_remote.h
+++ b/include/text_embedder_remote.h
@@@ -25,12 -25,13 +25,13 @@@ struct embedding_res_t
class RemoteEmbedder {
protected:
static Option<bool> validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties);
- static long call_remote_api(const std::string& method, const std::string& url, const std::string& req_body, std::string& res_body, std::map<std::string, std::string>& res_headers, std::unordered_map<std::string, std::string>& req_headers);
static inline ReplicationState* raft_server = nullptr;
public:
+ static long call_remote_api(const std::string& method, const std::string& url, const std::string& req_body, std::string& res_body, std::map<std::string, std::string>& res_headers, std::unordered_map<std::string, std::string>& req_headers);
virtual nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) = 0;
virtual embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) = 0;
- virtual std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) = 0;
+ virtual std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200,
+ const size_t remote_embedding_timeout_ms = 60000, const size_t remote_embedding_num_tries = 2) = 0;
static const std::string get_model_key(const nlohmann::json& model_config);
static void init(ReplicationState* rs) {
raft_server = rs;
diff --cc src/collection.cpp
index 38b53a01,24b3d331..3e3e40e5
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -19,9 -20,7 +20,10 @@@
#include "vector_query_ops.h"
#include "text_embedder_manager.h"
#include "stopwords_manager.h"
+#include "conversation_model.h"
+#include "conversation_manager.h"
+#include "conversation_model_manager.h"
+ #include "field.h"
const std::string override_t::MATCH_EXACT = "exact";
const std::string override_t::MATCH_CONTAINS = "contains";
@@@ -1391,13 -1406,12 +1409,15 @@@ Option<nlohmann::json> Collection::sear
const size_t remote_embedding_timeout_ms,
const size_t remote_embedding_num_tries,
const std::string& stopwords_set,
-- const std::vector<std::string>& facet_return_parent,
++ const std::vector<std::string>& facet_return_parent,=
+ const bool conversation,
+ const int conversation_model_id,
+ const std::string& system_prompt,
+ int conversation_id,
- const std::vector<ref_include_fields>& ref_include_fields_vec) const {
-
+ const std::vector<ref_include_fields>& ref_include_fields_vec,
+ const std::string& drop_tokens_mode,
+ const bool prioritize_num_matching_fields,
+ const bool group_missing_values) const {
-
std::shared_lock lock(mutex);
// setup thread local vars
@@@ -1624,7 -1624,9 +1673,8 @@@
return Option<nlohmann::json>(400, error);
}
- for(const std::string & field_name: processed_search_fields) {
-
+ for(const auto& processed_search_field: processed_search_fields) {
+ const auto& field_name = processed_search_field.name;
field search_field = search_schema.at(field_name);
if(!search_field.index) {
std::string error = "Field `" + field_name + "` is marked as a non-indexed field in the schema.";
diff --cc src/collection_manager.cpp
index 535fdc77,bf3baa70..db6bea8d
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -977,11 -1115,9 +1116,14 @@@ Option<bool> CollectionManager::do_sear
const char *FACET_SAMPLE_PERCENT = "facet_sample_percent";
const char *FACET_SAMPLE_THRESHOLD = "facet_sample_threshold";
+ const char *CONVERSATION = "conversation";
+ const char *CONVERSATION_ID = "conversation_id";
+ const char *SYSTEM_PROMPT = "system_prompt";
+ const char *CONVERSATION_MODEL_ID = "conversation_model_id";
+
+ const char *DROP_TOKENS_MODE = "drop_tokens_mode";
+ const char *PRIORITIZE_NUM_MATCHING_FIELDS = "prioritize_num_matching_fields";
+
// enrich params with values from embedded params
for(auto& item: embedded_params.items()) {
if(item.key() == "expires_at") {
@@@ -1102,11 -1239,9 +1245,15 @@@
size_t facet_sample_percent = 100;
size_t facet_sample_threshold = 0;
+ bool conversation = false;
+ size_t conversation_id = std::numeric_limits<size_t>::max();
+ std::string system_prompt;
+ size_t conversation_model_id = std::numeric_limits<size_t>::max();
+
+ std::string drop_tokens_mode_str = "right_to_left";
+ bool prioritize_num_matching_fields = true;
+
++
std::unordered_map<std::string, size_t*> unsigned_int_values = {
{MIN_LEN_1TYPO, &min_len_1typo},
{MIN_LEN_2TYPO, &min_len_2typo},
@@@ -1145,7 -1278,7 +1292,8 @@@
{HIGHLIGHT_END_TAG, &highlight_end_tag},
{PINNED_HITS, &pinned_hits_str},
{HIDDEN_HITS, &hidden_hits_str},
+ {SYSTEM_PROMPT, &system_prompt},
+ {DROP_TOKENS_MODE, &drop_tokens_mode_str},
};
std::unordered_map<std::string, bool*> bool_values = {
@@@ -1155,7 -1288,8 +1303,9 @@@
{EXHAUSTIVE_SEARCH, &exhaustive_search},
{ENABLE_OVERRIDES, &enable_overrides},
{ENABLE_HIGHLIGHT_V1, &enable_highlight_v1},
+ {CONVERSATION, &conversation},
+ {PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},
+ {GROUP_MISSING_VALUES, &group_missing_values},
};
std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {
@@@ -1357,12 -1490,10 +1507,15 @@@
remote_embedding_num_tries,
stopwords_set,
facet_return_parent,
+ conversation,
+ (conversation_model_id == std::numeric_limits<size_t>::max()) ? -1 : static_cast<int>(conversation_model_id),
+ system_prompt,
+ (conversation_id == std::numeric_limits<size_t>::max()) ? -1 : static_cast<int>(conversation_id),
- ref_include_fields_vec);
-
+ ref_include_fields_vec,
+ drop_tokens_mode_str,
+ prioritize_num_matching_fields,
+ group_missing_values);
++
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --cc src/typesense_server_utils.cpp
index 4979f881,97cfbd00..8319355f
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@@ -475,17 -459,10 +477,22 @@@ int run_server(const Config & config, c
AnalyticsManager::get_instance().run(&replication_state);
});
+ std::thread conersation_garbage_collector_thread([]() {
+ LOG(INFO) << "Conversation garbage collector thread started.";
+ int last_clear_time = 0;
+ while(!brpc::IsAskedToQuit()) {
+ if(last_clear_time + 60 < std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()) {
+ last_clear_time = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ ConversationManager::clear_expired_conversations();
+ }
+ std::this_thread::sleep_for(std::chrono::seconds(5));
+ }
+ });
++
+ HouseKeeper::get_instance().init(config.get_housekeeping_interval());
+ std::thread housekeeping_thread([]() {
+ HouseKeeper::get_instance().run();
+ });
RemoteEmbedder::init(&replication_state);
diff --cc test/collection_vector_search_test.cpp
index 19590486,5181c76b..00caa694
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -1862,99 -2055,787 +2059,883 @@@ TEST_F(CollectionVectorTest, TestMultil
ASSERT_TRUE(semantic_results.ok());
}
-
+ TEST_F(CollectionVectorTest, TestTwoEmbeddingFieldsSamePrefix) {
+ nlohmann::json schema = R"({
+ "name": "docs",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ },
+ {
+ "name": "embedding_en",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "title": "john doe"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto semantic_results = coll1->search("john", {"embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>());
+
+ ASSERT_TRUE(semantic_results.ok());
+ }
+
+ TEST_F(CollectionVectorTest, TestOneEmbeddingOneKeywordFieldsHaveSamePrefix) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "title": "john doe"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto keyword_results = coll1->search("john", {"title"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>());
+
+ ASSERT_TRUE(keyword_results.ok());
+ }
+
+ TEST_F(CollectionVectorTest, HybridSearchOnlyKeyworMatchDoNotHaveVectorDistance) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "title": "john doe"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ // hybrid search with empty vector (to pass distance threshold param)
+ std::string vec_query = "embedding:([], distance_threshold: 0.05)";
+
+ auto hybrid_results = coll1->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, vec_query);
+
+ ASSERT_TRUE(hybrid_results.ok());
+
+ ASSERT_EQ(1, hybrid_results.get()["hits"].size());
+ ASSERT_EQ(0, hybrid_results.get()["hits"][0].count("vector_distance"));
+ }
+
+ TEST_F(CollectionVectorTest, QueryByNotAutoEmbeddingVectorField) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "num_dim": 384
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto search_res = coll->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([0.96826, 0.94, 0.39557, 0.306488])");
+
+ ASSERT_FALSE(search_res.ok());
+
+ ASSERT_EQ("Vector field `embedding` is not an auto-embedding field, do not use `query_by` with it, use `vector_query` instead.", search_res.error());
+ }
+
+ TEST_F(CollectionVectorTest, TestUnloadingModelsOnCollectionDelete) {
+ nlohmann::json actual_schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto schema = actual_schema;
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
-
++
+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ auto delete_op = collectionManager.drop_collection("test", true);
+
+ ASSERT_TRUE(delete_op.ok());
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+
+ // create another collection
+ schema = actual_schema;
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ coll = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ // create second collection
+ schema = actual_schema;
+ schema["name"] = "test2";
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll2 = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ delete_op = collectionManager.drop_collection("test", true);
+ ASSERT_TRUE(delete_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ delete_op = collectionManager.drop_collection("test2", true);
+ ASSERT_TRUE(delete_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+ }
+
+ TEST_F(CollectionVectorTest, TestUnloadingModelsOnDrop) {
+ nlohmann::json actual_schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto schema = actual_schema;
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+
+ ASSERT_EQ(1, text_embedders.size());
+
+ nlohmann::json drop_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec",
+ "drop": true
+ }
+ ]
+ })"_json;
+
+ auto drop_op = coll->alter(drop_schema);
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+
+ // create another collection
+ schema = actual_schema;
+ schema["name"] = "test2";
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll2 = collection_create_op.get();
+
+ nlohmann::json alter_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ auto alter_op = coll->alter(alter_schema);
+ ASSERT_TRUE(alter_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ drop_op = coll2->alter(drop_schema);
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ drop_op = coll->alter(drop_schema);
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+ }
+
+ TEST_F(CollectionVectorTest, TestUnloadModelsCollectionHaveTwoEmbeddingField) {
+ nlohmann::json actual_schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ },
+ {
+ "name": "title_vec2",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto schema = actual_schema;
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+ auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ nlohmann::json drop_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec",
+ "drop": true
+ }
+ ]
+ })"_json;
+
+ auto alter_op = coll->alter(drop_schema);
+ ASSERT_TRUE(alter_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ drop_schema = R"({
+ "fields": [
+ {
+ "name": "title_vec2",
+ "drop": true
+ }
+ ]
+ })"_json;
+
+ alter_op = coll->alter(drop_schema);
+ ASSERT_TRUE(alter_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+
+ // create another collection
+ schema = actual_schema;
+ schema["name"] = "test2";
+
+ collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll2 = collection_create_op.get();
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(1, text_embedders.size());
+
+ // drop collection
+ auto drop_op = collectionManager.drop_collection("test2", true);
+
+ ASSERT_TRUE(drop_op.ok());
+
+ text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
+ ASSERT_EQ(0, text_embedders.size());
+ }
+
+ TEST_F(CollectionVectorTest, TestHybridSearchAlphaParam) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "name": "soccer"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "basketball"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "volleyball"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+
+ // do hybrid search
+ auto hybrid_results = coll->search("sports", {"name", "embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ ASSERT_EQ(3, hybrid_results["hits"].size());
+
+ // check scores
+ ASSERT_FLOAT_EQ(0.3, hybrid_results["hits"][0]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.15, hybrid_results["hits"][1]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.10, hybrid_results["hits"][2]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+
+ // do hybrid search with alpha = 0.5
+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:0.5)").get();
+ ASSERT_EQ(3, hybrid_results["hits"].size());
+
+ // check scores
+ ASSERT_FLOAT_EQ(0.5, hybrid_results["hits"][0]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.25, hybrid_results["hits"][1]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.16666667, hybrid_results["hits"][2]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ }
+
+ TEST_F(CollectionVectorTest, TestHybridSearchInvalidAlpha) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+
+ // do hybrid search with alpha = 1.5
+ auto hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:1.5)");
+
+ ASSERT_FALSE(hybrid_results.ok());
+ ASSERT_EQ("Malformed vector query string: "
+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());
+
+ // do hybrid search with alpha = -0.5
+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:-0.5)");
+
+ ASSERT_FALSE(hybrid_results.ok());
+ ASSERT_EQ("Malformed vector query string: "
+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());
+
+ // do hybrid search with alpha as string
+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:\"0.5\")");
+
+ ASSERT_FALSE(hybrid_results.ok());
+ ASSERT_EQ("Malformed vector query string: "
+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());
+
+ }
+
+ TEST_F(CollectionVectorTest, TestSearchNonIndexedEmbeddingField) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "index": false,
+ "optional": true,
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "name": "soccer"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_res = coll->search("soccer", {"name", "embedding"}, "", {}, {}, {0});
+ ASSERT_FALSE(search_res.ok());
+
+ ASSERT_EQ("Field `embedding` is marked as a non-indexed field in the schema.", search_res.error());
+ }
+
+ TEST_F(CollectionVectorTest, TestSearchNonIndexedVectorField) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "vec",
+ "type": "float[]",
+ "index": false,
+ "optional": true,
+ "num_dim": 2
+ }
+ ]
+ })"_json;
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "vec": [0.1, 0.2]
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_result = coll->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "vec:([0.96826, 0.94])");
+
+ ASSERT_FALSE(search_result.ok());
+ ASSERT_EQ("Field `vec` is marked as a non-indexed field in the schema.", search_result.error());
+ }
+
+ TEST_F(CollectionVectorTest, TestSemanticSearchAfterUpdate) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "name": "soccer",
+ "id": "0"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "basketball",
+ "id": "1"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "typesense",
+ "id": "2"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "potato",
+ "id": "3"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], id:0, k:1)");
+
+ ASSERT_TRUE(result.ok());
+ ASSERT_EQ(1, result.get()["hits"].size());
+ ASSERT_EQ("basketball", result.get()["hits"][0]["document"]["name"]);
+
+ auto update_op = coll->add(R"({
+ "name": "onion",
+ "id": "0"
+ })"_json.dump(), index_operation_t::UPDATE, "0");
+
+ ASSERT_TRUE(update_op.ok());
+
+ result = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], id:0, k:1)");
+
+ ASSERT_TRUE(result.ok());
+ ASSERT_EQ(1, result.get()["hits"].size());
+ ASSERT_EQ("potato", result.get()["hits"][0]["document"]["name"]);
+ }
++
+TEST_F(CollectionVectorTest, TestQAConversation) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
+ return;
+ }
+
+ auto api_key = std::string(std::getenv("api_key"));
+
+ auto conversation_model_config = R"({
+ "model_name": "openai/gpt-3.5-turbo"
+ })"_json;
+
+ conversation_model_config["api_key"] = api_key;
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto model_add_op = ConversationModelManager::add_model(conversation_model_config);
+
+ ASSERT_TRUE(model_add_op.ok());
+
+ auto add_op = coll->add(R"({
+ "product_name": "moisturizer",
+ "category": "beauty"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "shampoo",
+ "category": "beauty"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "shirt",
+ "category": "clothing"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "product_name": "pants",
+ "category": "clothing"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto results_op = coll->search("how many products are there for clothing category?", {"embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>(), {},
+ 10, "", 30, 4, "", 1, "", "", {}, 3, "<mark>", "</mark>", {}, 4294967295UL, true, false,
+ true, "", false, 6000000UL, 4, 7, fallback, 4, {off}, 32767UL, 32767UL, 2, 2, false, "",
+ true, 0, max_score, 100, 0, 0, HASH, 30000, 2, "", {}, true, 0);
+
+ ASSERT_TRUE(results_op.ok());
+
+ auto results = results_op.get();
+
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_TRUE(results.contains("conversation"));
+ ASSERT_TRUE(results["conversation"].is_object());
+ ASSERT_EQ("how many products are there for clothing category?", results["conversation"]["query"]);
+ ASSERT_EQ(0, results["conversation"]["conversation_id"]);
+
+
+ // test getting conversation history
+ auto history_op = ConversationManager::get_conversation(0);
+
+ ASSERT_TRUE(history_op.ok());
+
+ auto history = history_op.get();
+
+ ASSERT_TRUE(history.is_object());
+ ASSERT_TRUE(history.contains("conversation"));
+ ASSERT_TRUE(history["conversation"].is_array ());
+
+ ASSERT_EQ("how many products are there for clothing category?", history["conversation"][0]["user"]);
- }
++}