Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallan91209a17dc14
diff --cc include/collection.h
index 75bf51ae,1a4780f2..aab3ce31
--- a/include/collection.h
+++ b/include/collection.h
@@@ -496,15 -472,11 +498,15 @@@ public
const size_t facet_sample_percent = 100,
const size_t facet_sample_threshold = 0,
const size_t page_offset = 0,
+ facet_index_type_t facet_index_type = HASH,
const size_t remote_embedding_timeout_ms = 30000,
const size_t remote_embedding_num_tries = 2,
+ const std::string& stopwords_set="",
+ const std::vector<std::string>& facet_return_parent = {},
+ const std::vector<ref_include_fields>& ref_include_fields_vec = {},
- const drop_tokens_mode_t drop_tokens_mode = right_to_left,
++ const std::string& drop_tokens_mode = "right_to_left",
const bool prioritize_num_matching_fields = true,
- const bool group_missing_values = true,
- const std::string& drop_tokens_mode = "right_to_left") const;
+ const bool group_missing_values = true) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --cc include/index.h
index 79c49c8b,ad38e00e..2dbc88d6
--- a/include/index.h
+++ b/include/index.h
@@@ -672,8 -698,8 +684,10 @@@ public
const size_t max_extra_suffix, const size_t facet_query_num_typos,
const bool filter_curated_hits, enable_t split_join_tokens,
const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold,
- const std::string& collection_name, facet_index_type_t facet_index_type = DETECT,
- const drop_tokens_mode_t drop_tokens_mode = right_to_left) const;
+ const std::string& collection_name,
- const drop_tokens_param_t drop_tokens_mode) const;
++ const drop_tokens_param_t drop_tokens_mode,
++ facet_index_type_t facet_index_type = DETECT
++ ) const;
void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
const bool is_update);
diff --cc src/collection.cpp
index 89f07fe2,27275a4b..bdb56b5c
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -1413,15 -1113,11 +1413,15 @@@ Option<nlohmann::json> Collection::sear
const size_t facet_sample_percent,
const size_t facet_sample_threshold,
const size_t page_offset,
+ facet_index_type_t facet_index_type,
const size_t remote_embedding_timeout_ms,
const size_t remote_embedding_num_tries,
+ const std::string& stopwords_set,
+ const std::vector<std::string>& facet_return_parent,
+ const std::vector<ref_include_fields>& ref_include_fields_vec,
- const drop_tokens_mode_t drop_tokens_mode,
++ const std::string& drop_tokens_mode,
const bool prioritize_num_matching_fields,
- const bool group_missing_values,
- const std::string& drop_tokens_mode) const {
+ const bool group_missing_values) const {
std::shared_lock lock(mutex);
diff --cc src/collection_manager.cpp
index 107bc1a7,9019eabb..6129c63c
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -1415,13 -1072,7 +1415,7 @@@ Option<bool> CollectionManager::do_sear
Index::NUM_CANDIDATES_DEFAULT_MIN);
}
- auto drop_tokens_mode_op = magic_enum::enum_cast<drop_tokens_mode_t>(drop_tokens_mode_str);
- drop_tokens_mode_t drop_tokens_mode;
- if(drop_tokens_mode_op.has_value()) {
- drop_tokens_mode = drop_tokens_mode_op.value();
- }
-
- Option<nlohmann::json> result_op = collection->search(raw_query, search_fields, simple_filter_query, facet_fields,
+ Option<nlohmann::json> result_op = collection->search(raw_query, search_fields, filter_query, facet_fields,
sort_fields, num_typos,
per_page,
page,
@@@ -1464,15 -1115,11 +1458,15 @@@
facet_sample_percent,
facet_sample_threshold,
offset,
+ HASH,
remote_embedding_timeout_ms,
remote_embedding_num_tries,
+ stopwords_set,
+ facet_return_parent,
+ ref_include_fields_vec,
- drop_tokens_mode,
++ drop_tokens_mode_str,
prioritize_num_matching_fields,
- group_missing_values,
- drop_tokens_mode_str);
+ group_missing_values);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --cc src/index.cpp
index b4055da7,71f3e829..4f1877fc
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -1858,8 -2306,7 +1858,9 @@@ Option<bool> Index::run_search(search_a
search_params->facet_sample_percent,
search_params->facet_sample_threshold,
collection_name,
- facet_index_type,
-- search_params->drop_tokens_mode);
++ search_params->drop_tokens_mode,
++ facet_index_type
++ );
}
void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
@@@ -2310,19 -2759,22 +2311,20 @@@ Option<bool> Index::search(std::vector<
const vector_query_t& vector_query,
size_t facet_sample_percent, size_t facet_sample_threshold,
const std::string& collection_name,
- facet_index_type_t facet_index_type,
- const drop_tokens_mode_t drop_tokens_mode) const {
- const drop_tokens_param_t drop_tokens_mode) const {
++ const drop_tokens_param_t drop_tokens_mode,
++ facet_index_type_t facet_index_type
++ ) const {
std::shared_lock lock(mutex);
- uint32_t filter_ids_length = 0;
- auto rearrange_op = rearrange_filter_tree(filter_tree_root, filter_ids_length, collection_name);
- if (!rearrange_op.ok()) {
- return rearrange_op;
- }
+ auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root);
+ std::unique_ptr<filter_result_iterator_t> filter_iterator_guard(filter_result_iterator);
- filter_result_t filter_result;
- auto filter_op = recursive_filter(filter_tree_root, filter_result, collection_name);
- if (!filter_op.ok()) {
- return filter_op;
+ auto filter_init_op = filter_result_iterator->init_status();
+ if (!filter_init_op.ok()) {
+ return filter_init_op;
}
- if (filter_tree_root != nullptr && filter_result.count == 0) {
+ if (filter_tree_root != nullptr && !filter_result_iterator->is_valid) {
return Option(true);
}
@@@ -2843,12 -3200,11 +2857,12 @@@
if(has_text_match) {
// For hybrid search, we need to give weight to text match and vector search
- constexpr float TEXT_MATCH_WEIGHT = 0.7;
- constexpr float VECTOR_SEARCH_WEIGHT = 1.0 - TEXT_MATCH_WEIGHT;
+ const float VECTOR_SEARCH_WEIGHT = vector_query.alpha;
+ const float TEXT_MATCH_WEIGHT = 1.0 - VECTOR_SEARCH_WEIGHT;
- VectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);
+ VectorFilterFunctor filterFunctor(filter_result_iterator);
auto& field_vector_index = vector_index.at(vector_query.field_name);
+
std::vector<std::pair<float, size_t>> dist_labels;
// use k as 100 by default for ensuring results stability in pagination
size_t default_k = 100;
diff --cc test/collection_grouping_test.cpp
index b98fd02f,8a4a3330..8a69c55f
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@@ -645,7 -643,7 +645,7 @@@ TEST_F(CollectionGroupingTest, ControlM
{}, {}, {"brand"}, 2,
"<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, false).get();
- 100, 0, 0, 30000, 2, true, false).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", true, false).get();
ASSERT_EQ(3, res["grouped_hits"].size());
ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());
@@@ -668,7 -666,7 +668,7 @@@
{}, {}, {"brand"}, 2,
"<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, true).get();
- 100, 0, 0, 30000, 2, true, true).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", true, true).get();
ASSERT_EQ(2, res["grouped_hits"].size());
@@@ -885,164 -883,4 +885,164 @@@ TEST_F(CollectionGroupingTest, GroupSor
ASSERT_EQ(res.ok(), false);
ASSERT_EQ(res.error(), "group_by parameters should not be empty when using sort_by group_found");
+}
+
+TEST_F(CollectionGroupingTest, SkipToReverseGroupBy) {
+ std::vector<field> fields = {
+ field("brand", field_types::STRING, true, true),
+ };
+
+ Collection* coll2 = collectionManager.get_collection("coll2").get();
+ if(coll2 == nullptr) {
+ coll2 = collectionManager.create_collection("coll2", 1, fields).get();
+ }
+
+ nlohmann::json doc;
+
+ doc["id"] = "0";
+ doc["brand"] = nullptr;
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ auto res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, false).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", true, false).get();
+
+ ASSERT_EQ(1, res["grouped_hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][0]["group_key"].size());
+ ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
+ ASSERT_EQ("0", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());
+
+ doc["id"] = "1";
+ doc["brand"] = "adidas";
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "2";
+ doc["brand"] = "puma";
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "3";
+ doc["brand"] = nullptr;
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "4";
+ doc["brand"] = "nike";
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+
+ res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, false).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", true, false).get();
+
+ ASSERT_EQ(5, res["grouped_hits"].size());
+
+ ASSERT_EQ("nike", res["grouped_hits"][0]["group_key"][0].get<std::string>());
+ ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
+ ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
+ ASSERT_EQ("3", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ("puma", res["grouped_hits"][2]["group_key"][0].get<std::string>());
+ ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());
+
+ ASSERT_EQ("adidas", res["grouped_hits"][3]["group_key"][0].get<std::string>());
+ ASSERT_EQ(1, res["grouped_hits"][3]["hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][4]["group_key"].size());
+ ASSERT_EQ(1, res["grouped_hits"][4]["hits"].size());
+ ASSERT_EQ("0", res["grouped_hits"][4]["hits"][0]["document"]["id"].get<std::string>());
+
+ res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, true).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", true, true).get();
+
+ ASSERT_EQ(4, res["grouped_hits"].size());
+
+ ASSERT_EQ("nike", res["grouped_hits"][0]["group_key"][0].get<std::string>());
+ ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
+ ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());
+ ASSERT_EQ("3", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ("puma", res["grouped_hits"][2]["group_key"][0].get<std::string>());
+ ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());
+}
+
+TEST_F(CollectionGroupingTest, GroupByMultipleFacetFields) {
+ auto res = coll_group->search("*", {}, "", {"brand", "colors"}, {}, {0}, 50, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"size"}, 2).get();
+
+ ASSERT_EQ(12, res["found_docs"].get<size_t>());
+ ASSERT_EQ(3, res["found"].get<size_t>());
+ ASSERT_EQ(3, res["grouped_hits"].size());
+ ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
+
+ ASSERT_EQ(2, res["grouped_hits"][0]["found"].get<int32_t>());
+ ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());
+ ASSERT_EQ(11, res["grouped_hits"][0]["hits"][0]["document"]["size"].get<size_t>());
+ ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][0]["hits"][1]["document"]["rating"].get<float>());
+ ASSERT_STREQ("1", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_EQ(7, res["grouped_hits"][1]["found"].get<int32_t>());
+ ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());
+ ASSERT_STREQ("4", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][1]["hits"][1]["document"]["rating"].get<float>());
+ ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_EQ(3, res["grouped_hits"][2]["found"].get<int32_t>());
+ ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][2]["hits"][0]["document"]["rating"].get<float>());
+ ASSERT_STREQ("2", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][2]["hits"][1]["document"]["rating"].get<float>());
+ ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
+
+ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+
+ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+
+ ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+
+ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
+ ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
+
+
+ ASSERT_STREQ("colors", res["facet_counts"][1]["field_name"].get<std::string>().c_str());
+
+ ASSERT_EQ(3, (int) res["facet_counts"][1]["counts"][0]["count"]);
+ ASSERT_STREQ("blue", res["facet_counts"][1]["counts"][0]["value"].get<std::string>().c_str());
+
+ ASSERT_EQ(3, (int) res["facet_counts"][1]["counts"][1]["count"]);
+ ASSERT_STREQ("white", res["facet_counts"][1]["counts"][1]["value"].get<std::string>().c_str());
+
+ ASSERT_EQ(1, (int) res["facet_counts"][1]["counts"][2]["count"]);
+ ASSERT_STREQ("red", res["facet_counts"][1]["counts"][2]["value"].get<std::string>().c_str());
}
diff --cc test/collection_specific_more_test.cpp
index c89d761a,86d0de41..706a8827
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@@ -1883,7 -1873,7 +1883,7 @@@ TEST_F(CollectionSpecificMoreTest, Disa
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true);
- 100, 0, 0, 30000, 2, true);
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", true);
auto res = coll1->search("beta", {"name", "brand"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 5,
@@@ -1891,7 -1881,7 +1891,7 @@@
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, false).get();
- 100, 0, 0, 30000, 2, false).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", false).get();
size_t score1 = std::stoul(res["hits"][0]["text_match_info"]["score"].get<std::string>());
size_t score2 = std::stoul(res["hits"][1]["text_match_info"]["score"].get<std::string>());
@@@ -1902,7 -1892,7 +1902,7 @@@
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true).get();
- 100, 0, 0, 30000, 2, true).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, "right_to_left", true).get();
ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", res["hits"][1]["document"]["id"].get<std::string>());
@@@ -2413,7 -2403,7 +2413,7 @@@ TEST_F(CollectionSpecificMoreTest, Drop
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
- 0, HASH, 30000, 2, "", {}, {}, left_to_right).get();
- 0, 30000, 2, true, true, "left_to_right").get();
++ 0, HASH, 30000, 2, "", {}, {}, "left_to_right").get();
ASSERT_EQ(1, res["hits"].size());
ASSERT_EQ("1", res["hits"][0]["document"]["id"].get<std::string>());
@@@ -2423,10 -2413,48 +2423,48 @@@
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
- 0, HASH, 30000, 2, "", {}, {}, right_to_left).get();
- 0, 30000, 2, true, true, "right_to_left").get();
++ 0, HASH, 30000, 2, "", {}, {}, "right_to_left").get();
ASSERT_EQ(1, res["hits"].size());
ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
+
+ // search on both sides
+ res = coll1->search("alpha gamma", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {false}, drop_tokens_threshold,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
- 0, 30000, 2, true, true, "both_sides:3").get();
++ 0, HASH, 30000, 2, "", {}, {}, "both_sides:3").get();
+ ASSERT_EQ(2, res["hits"].size());
+
+ // but must follow token limit
+ res = coll1->search("alpha gamma", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {false}, drop_tokens_threshold,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
- 0, 30000, 2, true, true, "both_sides:1").get();
++ 0, HASH, 30000, 2, "", {}, {}, "both_sides:1").get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
+
+ // validation checks
+ auto res_op = coll1->search("alpha gamma", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {false}, drop_tokens_threshold,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
- 0, 30000, 2, true, true, "all_sides");
++ 0, HASH, 30000, 2, "", {}, {}, "all_sides");
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Invalid format for drop tokens mode.", res_op.error());
+
+ res_op = coll1->search("alpha gamma", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {false}, drop_tokens_threshold,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,
+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,
- 0, 30000, 2, true, true, "both_sides:x");
++ 0, HASH, 30000, 2, "", {}, {}, "both_sides:x");
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Invalid format for drop tokens mode.", res_op.error());
}
TEST_F(CollectionSpecificMoreTest, DoNotHighlightFieldsForSpecialCharacterQuery) {
diff --cc test/collection_vector_search_test.cpp
index c6bd963d,b22686d1..bfdb2f78
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -2515,3 -2498,154 +2515,154 @@@ TEST_F(CollectionVectorTest, TestUnload
text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
ASSERT_EQ(0, text_embedders.size());
}
+
+ TEST_F(CollectionVectorTest, TestHybridSearchAlphaParam) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "name": "soccer"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "basketball"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "volleyball"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+
+ // do hybrid search
+ auto hybrid_results = coll->search("sports", {"name", "embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ ASSERT_EQ(3, hybrid_results["hits"].size());
+
+ // check scores
+ ASSERT_FLOAT_EQ(0.3, hybrid_results["hits"][0]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.15, hybrid_results["hits"][1]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.10, hybrid_results["hits"][2]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+
+ // do hybrid search with alpha = 0.5
+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:0.5)").get();
+ ASSERT_EQ(3, hybrid_results["hits"].size());
+
+ // check scores
+ ASSERT_FLOAT_EQ(0.5, hybrid_results["hits"][0]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.25, hybrid_results["hits"][1]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ ASSERT_FLOAT_EQ(0.16666667, hybrid_results["hits"][2]["hybrid_search_info"]["rank_fusion_score"].get<float>());
+ }
+
+ TEST_F(CollectionVectorTest, TestHybridSearchInvalidAlpha) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+
+ // do hybrid search with alpha = 1.5
+ auto hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:1.5)");
+
+ ASSERT_FALSE(hybrid_results.ok());
+ ASSERT_EQ("Malformed vector query string: "
+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());
+
+ // do hybrid search with alpha = -0.5
+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:-0.5)");
+
+ ASSERT_FALSE(hybrid_results.ok());
+ ASSERT_EQ("Malformed vector query string: "
+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());
+
+ // do hybrid search with alpha as string
+ hybrid_results = coll->search("sports", {"name", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([], alpha:\"0.5\")");
+
+ ASSERT_FALSE(hybrid_results.ok());
+ ASSERT_EQ("Malformed vector query string: "
+ "`alpha` parameter must be a float between 0.0-1.0.", hybrid_results.error());
+
-}
++}