Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Revert "Do grouping in two pass (#1677)"
Kishore Nallanb9d70433b4e7
diff --git a/include/hll/dense_array.h b/include/hll/dense_array.hdeleted file mode 100644index ac07bbe6..00000000--- a/include/hll/dense_array.h+++ /dev/null@@ -1,123 +0,0 @@-// Source : https://github.com/iwiwi/hyperloglog-hip-#ifndef HYPERLOGLOG_HIP_DENSE_ARRAY_H_-#define HYPERLOGLOG_HIP_DENSE_ARRAY_H_--#include <algorithm>-#include <climits>-#include <cstdint>-#include <memory>-#include <type_traits>--namespace hyperloglog_hip {-template<size_t NumRegisterBits, typename Value = uint8_t>-class dense_array {- public:- typedef Value value_type;-- dense_array(size_t num_registers)- : data_(new value_type[data_length(num_registers)]()) {}-- value_type get(size_t pos) {- const size_t b = pos * num_register_bits();- const size_t i1 = b / num_value_bits();- const size_t o1 = b - i1 * num_value_bits();- const size_t n1 = num_value_bits() - o1;- value_type v = data_[i1] >> o1;-- if (n1 > num_register_bits()) {- v &= (value_type(1) << num_register_bits()) - 1;- }- else if (n1 < num_register_bits()) {- const size_t i2 = i1 + 1;- const size_t n2 = num_register_bits() - n1;- v |= (data_[i2] & ((value_type(1) << n2) - 1)) << n1;- }- return v;- }-- void set(size_t pos, value_type val) {- const size_t b = pos * num_register_bits();-- const size_t i1 = b / num_value_bits();- const size_t o1 = b - i1 * num_value_bits();- const size_t n1 = std::min(num_value_bits() - o1, num_register_bits());- data_[i1] &= value_type(-1) ^ (((value_type(1) << n1) - 1) << o1);- data_[i1] |= val << o1;-- if (n1 < num_register_bits()) {- const size_t i2 = i1 + 1;- const size_t n2 = num_register_bits() - n1;- data_[i2] &= value_type(-1) ^ ((value_type(1) << n2) - 1);- data_[i2] |= val >> n1;- }- }-- private:- std::unique_ptr<value_type[]> data_;-- static constexpr size_t num_register_bits() {- return NumRegisterBits;- }-- static constexpr size_t num_value_bits() {- return sizeof(Value) * CHAR_BIT;- }-- static constexpr size_t data_length(size_t num_registers) {- return (num_registers * num_register_bits() + num_value_bits() - 1) / num_value_bits();- }-- static_assert(std::is_unsigned<value_type>::value,- "Value should be an unsigned integral type.");-- static_assert(sizeof(value_type) * CHAR_BIT >= NumRegisterBits,- "Value should have at least NumRegisterBits bits.");-};--template<typename Value>-class dense_array_primitive {- public:- typedef Value value_type;-- dense_array_primitive(size_t size) : data_(new value_type[size]()) {}- virtual ~dense_array_primitive() {}-- value_type get(size_t pos) const {- return data_[pos];- }-- void set(size_t pos, value_type val) {- data_[pos] = val;- }-- private:- std::unique_ptr<value_type[]> data_;-};--template<>-class dense_array<8, uint8_t> : public dense_array_primitive<uint8_t> {- public:- dense_array(size_t size) : dense_array_primitive(size) {}-};--template<>-class dense_array<16, uint16_t> : public dense_array_primitive<uint16_t> {- public:- dense_array(size_t size) : dense_array_primitive(size) {}-};--template<>-class dense_array<32, uint32_t> : public dense_array_primitive<uint32_t> {- public:- dense_array(size_t size) : dense_array_primitive(size) {}-};--template<>-class dense_array<64, uint64_t> : public dense_array_primitive<uint64_t> {- public:- dense_array(size_t size) : dense_array_primitive(size) {}-};-} // namespace hyperloglog_hip--#endif // HYEPRLOGLOG_HIP_DENSE_ARRAY_H_-diff --git a/include/hll/distinct_counter.h b/include/hll/distinct_counter.hdeleted file mode 100644index 0a3deb62..00000000--- a/include/hll/distinct_counter.h+++ /dev/null@@ -1,64 +0,0 @@-// Source : https://github.com/iwiwi/hyperloglog-hip-#ifndef HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_-#define HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_--#include <algorithm>-#include <cstdint>-#include <cmath>-#include "dense_array.h"--namespace hyperloglog_hip {-template<typename Key, typename Hash = std::hash<Key>, int NumRegisterBits = 5>-class distinct_counter {- public:- typedef Key key_type;- typedef Hash hash_type;-- distinct_counter(size_t num_bucket_bits = 12)- : num_bucket_bits_(num_bucket_bits), M_(1 << num_bucket_bits),- c_(0), s_(1 << num_bucket_bits) {}-- void insert(const key_type &v) {- static constexpr uint64_t num_register_bits = NumRegisterBits;- static constexpr uint64_t register_limit = (uint64_t(1) << num_register_bits) - 1;-- const uint64_t h = hash_(v) * magic1() + magic2();- const uint64_t h0 = h & ((uint64_t(1) << num_bucket_bits_) - 1);- const uint64_t h1 = h >> num_bucket_bits_;-- const uint64_t b_old = M_.get(h0);- const uint64_t b_new = h1 == 0 ? register_limit :- std::min(register_limit, uint64_t(1 + __builtin_ctzl(h1)));-- if (b_new > b_old) {- M_.set(h0, b_new);- c_ += 1.0 / (s_ / (uint64_t(1) << num_bucket_bits_));- s_ -= 1.0 / (uint64_t(1) << b_old);- if (b_new < register_limit) {- s_ += 1.0 / (uint64_t(1) << b_new);- }- }- }-- size_t count() const {- return round(c_);- }-- private:- const size_t num_bucket_bits_;- dense_array<NumRegisterBits> M_;- double c_, s_;- hash_type hash_;-- static constexpr uint64_t magic1() {- return 9223372036854775837ULL;- }-- static constexpr uint64_t magic2() {- return 1234567890123456789ULL;- }-};-} // namespace hyperloglog_hip--#endif // HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_-diff --git a/include/topster.h b/include/topster.hindex f168eefa..a471564c 100644--- a/include/topster.h+++ b/include/topster.h@@ -7,9 +7,6 @@#include <unordered_map>#include <field.h>#include "filter_result_iterator.h"-#include "hll/distinct_counter.h"--#define HYPERLOGLOG_THRESHOLD 2048struct KV {int8_t match_score_index{};@@ -139,17 +136,10 @@ struct Topster {spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;size_t distinct;- //hyperloglog lib counter for counting total unique group values more than 512- hyperloglog_hip::distinct_counter<uint64_t> hyperloglog_counter;- std::set<uint64_t> groups_found; //to keep count of total unique group less than 512- uint32_t groups_found_count = 0; //to keep track of groups found in current pass- bool is_first_pass_completed = false;-explicit Topster(size_t capacity): Topster(capacity, 0) {}- explicit Topster(size_t capacity, size_t distinct, bool is_first_pass_completed = false): MAX_SIZE(capacity),- size(0), distinct(distinct), is_first_pass_completed(is_first_pass_completed) {+ explicit Topster(size_t capacity, size_t distinct): MAX_SIZE(capacity), size(0), distinct(distinct) {// we allocate data first to get a memory block whose indices are then assigned to `kvs`// we use separate **kvs for easier pointer swapsdata = new KV[capacity];@@ -194,32 +184,24 @@ struct Topster {LOG(INFO) << "kv key: " << mkv.first << " => " << mkv.second->scores[mkv.second->match_score_index];}*/- /* returns either 0 or 1- * 1 -> distinct_id was added to group_kv_map in second pass, which will aggregate found counts to groups_processed- * 0 -> distinct_id was added to kv_map in first pass- * -1 -> distinct_id was not added- */-+ int ret = 1;+bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller(kv, kvs[0]);size_t heap_op_index = 0;if(!distinct && less_than_min_heap) {// for non-distinct, if incoming value is smaller than min-heap ignore- return -1;+ return 0;}bool SIFT_DOWN = true;- if(distinct && is_first_pass_completed) {- if(kv_map.count(kv->distinct_key) == 0) {- return -1;- }-- const auto& doc_seq_id_exists =+ if(distinct) {+ const auto& doc_seq_id_exists =(group_doc_seq_ids.find(kv->key) != group_doc_seq_ids.end());if(doc_seq_id_exists) {- return -1;+ ret = 2;}group_doc_seq_ids.emplace(kv->key);@@ -231,16 +213,14 @@ struct Topster {Topster* g_topster = new Topster(distinct, 0);g_topster->add(kv);group_kv_map.insert({kv->distinct_key, g_topster});- groups_found_count++;}- return 1;+ return ret;} else { // not distinct//LOG(INFO) << "Searching for key: " << kv->key;- auto key = distinct ? kv->distinct_key : kv->key;- const auto& found_it = kv_map.find(key);+ const auto& found_it = kv_map.find(kv->key);bool is_duplicate_key = (found_it != kv_map.end());/*@@ -257,15 +237,14 @@ struct Topster {bool smaller_than_existing = is_smaller(kv, existing_kv);if(smaller_than_existing) {- return -1;+ return 0;}SIFT_DOWN = true;// replace existing kv and sift downheap_op_index = existing_kv->array_index;- auto heap_op_index_key = distinct ? kvs[heap_op_index]->distinct_key : kvs[heap_op_index]->key;- kv_map.erase(heap_op_index_key);+ kv_map.erase(kvs[heap_op_index]->key);} else { // not duplicateif(size < MAX_SIZE) {@@ -278,24 +257,12 @@ struct Topster {// we have to replace min heap element since array is fullSIFT_DOWN = true;heap_op_index = 0;- auto heap_op_index_key = distinct ? kvs[heap_op_index]->distinct_key : kvs[heap_op_index]->key;- kv_map.erase(heap_op_index_key);+ kv_map.erase(kvs[heap_op_index]->key);}}// kv will be copied into the pointer at heap_op_index- kv_map.emplace(key, kvs[heap_op_index]);-- if(distinct) {- hyperloglog_counter.insert(kv->distinct_key);-- if(groups_found.size() < HYPERLOGLOG_THRESHOLD) {- groups_found.insert(kv->distinct_key);- groups_found_count = groups_found.size();- } else {- groups_found_count = hyperloglog_counter.count();- }- }+ kv_map.emplace(kv->key, kvs[heap_op_index]);}// we have to replace the existing element in the heap and sift down@@ -333,7 +300,7 @@ struct Topster {}}- return 0;+ return ret;}static bool is_greater(const struct KV* i, const struct KV* j) {@@ -373,18 +340,4 @@ struct Topster {KV* getKV(uint32_t index) {return kvs[index];}-- const size_t get_total_unique_groups() const {- auto groups_count = groups_found.size();- return groups_count < 512 ? groups_count : hyperloglog_counter.count();- }-- void set_first_pass_complete() {- is_first_pass_completed = true;- groups_found_count = 0;- }-- const size_t get_current_groups_count() const {- return groups_found_count;- }};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex f9f0fe4a..00290da5 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2441,7 +2441,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,// for grouping we have to aggregate group set sizes to a count valueif(group_limit) {- total = search_params->topster->get_total_unique_groups() + override_result_kvs.size();+ total = search_params->groups_processed.size() + override_result_kvs.size();} else {total = search_params->all_result_ids_len;}@@ -3630,7 +3630,7 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<Kif(group_count_index >= 0) {const auto& itr = groups_processed.find(kv_head->distinct_key);if(itr != groups_processed.end()) {- kv_head->scores[group_count_index] = (int)itr->second * group_sort_order;+ kv_head->scores[group_count_index] = itr->second * group_sort_order;}}gtopster.add(kv_head);diff --git a/src/index.cpp b/src/index.cppindex 872f2fd4..6a236711 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1565,7 +1565,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {- if(index_topster->distinct && index_topster->is_first_pass_completed) {+ if(index_topster->distinct) {for(auto &group_topster_entry: index_topster->group_kv_map) {Topster* group_topster = group_topster_entry.second;for(const auto& map_kv: group_topster->kv_map) {@@ -2215,58 +2215,6 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& coconst std::vector<facet_index_type_t>& facet_index_types, bool enable_typos_for_numerical_tokens,bool enable_synonyms, bool synonym_prefix, uint32_t synonym_num_typos,bool enable_typos_for_alpha_numerical_tokens) {- if(search_params->group_limit != 0) {- search(search_params->field_query_tokens,- search_params->search_fields,- search_params->match_type,- search_params->filter_tree_root, search_params->facets, search_params->facet_query,- search_params->max_facet_values,- search_params->included_ids, search_params->excluded_ids,- search_params->sort_fields_std, search_params->num_typos,- search_params->topster, search_params->curated_topster,- search_params->per_page, search_params->offset, search_params->token_order,- search_params->prefixes, search_params->drop_tokens_threshold,- search_params->all_result_ids_len, search_params->groups_processed,- search_params->searched_queries,- search_params->qtoken_set,- search_params->raw_result_kvs, search_params->override_result_kvs,- search_params->typo_tokens_threshold,- search_params->group_limit,- search_params->group_by_fields,- search_params->group_missing_values,- search_params->default_sorting_field,- search_params->prioritize_exact_match,- search_params->prioritize_token_position,- search_params->prioritize_num_matching_fields,- search_params->exhaustive_search,- search_params->concurrency,- search_params->search_cutoff_ms,- search_params->min_len_1typo,- search_params->min_len_2typo,- search_params->max_candidates,- search_params->infixes,- search_params->max_extra_prefix,- search_params->max_extra_suffix,- search_params->facet_query_num_typos,- search_params->filter_curated_hits,- search_params->split_join_tokens,- search_params->vector_query,- search_params->facet_sample_percent,- search_params->facet_sample_threshold,- collection_name,- search_params->drop_tokens_mode,- facet_index_types,- enable_typos_for_numerical_tokens,- enable_synonyms,- synonym_prefix,- synonym_num_typos,- search_params->enable_lazy_filter,- enable_typos_for_alpha_numerical_tokens- );- search_params->topster->set_first_pass_complete();- search_params->curated_topster->set_first_pass_complete();-- }auto res = search(search_params->field_query_tokens,search_params->search_fields,search_params->match_type,@@ -2314,6 +2262,7 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->enable_lazy_filter,enable_typos_for_alpha_numerical_tokens);+return res;}@@ -2947,7 +2896,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consKV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);int ret = topster->add(&kv);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}@@ -3110,7 +3059,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conskv.vector_distance = vec_dist_score;int ret = topster->add(&kv);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}@@ -3581,7 +3530,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consauto ret = topster->add(&kv);vec_search_ids.push_back(seq_id);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}}@@ -4263,8 +4212,7 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fresume_typo_loop:- auto current_groups_count = topster ? topster->get_current_groups_count() : 0;- auto results_count = group_limit != 0 ? current_groups_count : all_result_ids_len;+ auto results_count = group_limit != 0 ? groups_processed.size() : all_result_ids_len;if(!exhaustive_search && results_count >= typo_tokens_threshold) {// if typo threshold is breached, we are donereturn Option<bool>(true);@@ -4695,7 +4643,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token}int ret = topster->add(&kv);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}result_ids.push_back(seq_id);@@ -5496,7 +5444,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = actual_topster->add(&kv);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}@@ -5671,7 +5619,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vKV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = actual_topster->add(&kv);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}@@ -6033,8 +5981,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootsearched_queries.push_back({});- topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct, topster->is_first_pass_completed);- topsters[thread_id]->kv_map = topster->kv_map;+ topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);auto& compute_sort_score_status = compute_sort_score_statuses[thread_id] = nullptr;thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,@@ -6096,7 +6043,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootKV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = topsters[thread_id]->add(&kv);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {tgroups_processed[thread_id][distinct_id]++;}if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {@@ -6611,7 +6558,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;KV kv(query_index, seq_id, distinct_id, match_score_index, scores);int ret = topster->add(&kv);- if(group_limit != 0 && ret > 0) {+ if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 0ce80dbf..5f37eba3 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -461,7 +461,7 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {"", 10,{}, {}, {"colors"}, 2).get();- ASSERT_EQ(10, res["found_docs"].get<size_t>());+ ASSERT_EQ(9, res["found_docs"].get<size_t>());ASSERT_EQ(4, res["found"].get<size_t>());ASSERT_EQ(4, res["grouped_hits"].size());@@ -755,54 +755,147 @@ TEST_F(CollectionGroupingTest, SortingOnGroupCount) {ASSERT_EQ(7, res2["grouped_hits"][2]["found"].get<int32_t>());}-TEST_F(CollectionGroupingTest, SortingByGroupCount) {+TEST_F(CollectionGroupingTest, SortingMoreThanMaxTopsterSize) {++ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("brand", field_types::STRING, true, true),+ field("size", field_types::INT32, true, false),+ field("colors", field_types::STRING, true, false),+ field("rating", field_types::FLOAT, true, false)+ };++ Collection* coll3 = collectionManager.get_collection("coll3").get();+ if(coll3 == nullptr) {+ coll3 = collectionManager.create_collection("coll3", 4, fields, "rating").get();+ }++ for(auto i = 0; i < 150; i++) {+ auto group_id = i;+ for(auto j = 0; j < 4; j++) {+ nlohmann::json doc;+ doc["title"] = "Omega Casual Poplin Shirt";+ doc["brand"] = "Omega";+ doc["size"] = group_id;+ doc["colors"] = "blue";+ doc["rating"] = 4.5;++ ASSERT_TRUE(coll3->add(doc.dump()).ok());+ }+ }++ for(auto i = 150; i < 250; i++) {+ auto group_id = i;+ for(auto j = 0; j < 3; j++) {+ nlohmann::json doc;+ doc["title"] = "Beta Casual Poplin Shirt";+ doc["brand"] = "Beta";+ doc["size"] = group_id;+ doc["colors"] = "white";+ doc["rating"] = 4.3;++ ASSERT_TRUE(coll3->add(doc.dump()).ok());+ }+ }++ for(auto i = 250; i < 300; i++) {+ auto group_id = i;+ for(auto j = 0; j < 2; j++) {+ nlohmann::json doc;+ doc["title"] = "Zeta Casual Poplin Shirt";+ doc["brand"] = "Zeta";+ doc["size"] = group_id;+ doc["colors"] = "red";+ doc["rating"] = 4.6;++ ASSERT_TRUE(coll3->add(doc.dump()).ok());+ }+ }+//first search in desc orderstd::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};- auto res = coll_group->search("shirt", {"title"}, "", {"brand"}, sort_fields, {0}, 10, 1, FREQUENCY,+ auto res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 2, FREQUENCY,{false}, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10,- {}, {}, {"brand"}, 10).get();- ASSERT_EQ(12, res["found_docs"].get<size_t>());- ASSERT_EQ(5, res["found"].get<size_t>());- ASSERT_EQ(5, res["grouped_hits"].size());+ {}, {}, {"size"}, 2).get();- ASSERT_EQ(4, res["grouped_hits"][0]["found"].get<int32_t>());+ ASSERT_EQ(1000, res["found_docs"].get<size_t>());+ ASSERT_EQ(300, res["found"].get<size_t>());+ ASSERT_EQ(100, res["grouped_hits"].size());- ASSERT_EQ(3, res["grouped_hits"][1]["found"].get<int32_t>());+ ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());- ASSERT_EQ(2, res["grouped_hits"][2]["found"].get<int32_t>());+ ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());- ASSERT_EQ(2, res["grouped_hits"][3]["found"].get<int32_t>());+ ASSERT_EQ(3, res["grouped_hits"][50]["found"].get<int32_t>());++ ASSERT_EQ(3, res["grouped_hits"][99]["found"].get<int32_t>());+++ res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 3, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"size"}, 2).get();++ ASSERT_EQ(1000, res["found_docs"].get<size_t>());+ ASSERT_EQ(300, res["found"].get<size_t>());+ ASSERT_EQ(100, res["grouped_hits"].size());++ ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());++ ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());++ ASSERT_EQ(2, res["grouped_hits"][50]["found"].get<int32_t>());++ ASSERT_EQ(2, res["grouped_hits"][99]["found"].get<int32_t>());- ASSERT_EQ(1, res["grouped_hits"][4]["found"].get<int32_t>());//search in asc orderstd::vector<sort_by> sort_fields2 = {sort_by("_group_found", "ASC")};- auto res2 = coll_group->search("shirt", {"title"}, "", {"brand"}, sort_fields2, {0}, 10, 1, FREQUENCY,+ auto res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 1, FREQUENCY,{false}, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10,- {}, {}, {"brand"}, 10).get();+ {}, {}, {"size"}, 2).get();- ASSERT_EQ(12, res2["found_docs"].get<size_t>());- ASSERT_EQ(5, res2["found"].get<size_t>());- ASSERT_EQ(5, res2["grouped_hits"].size());+ ASSERT_EQ(1000, res2["found_docs"].get<size_t>());+ ASSERT_EQ(300, res2["found"].get<size_t>());+ ASSERT_EQ(100, res2["grouped_hits"].size());- ASSERT_EQ(1, res2["grouped_hits"][0]["found"].get<int32_t>());+ ASSERT_EQ(2, res2["grouped_hits"][0]["found"].get<int32_t>());ASSERT_EQ(2, res2["grouped_hits"][1]["found"].get<int32_t>());- ASSERT_EQ(2, res2["grouped_hits"][2]["found"].get<int32_t>());+ ASSERT_EQ(3, res2["grouped_hits"][50]["found"].get<int32_t>());- ASSERT_EQ(3, res2["grouped_hits"][3]["found"].get<int32_t>());+ ASSERT_EQ(3, res2["grouped_hits"][99]["found"].get<int32_t>());- ASSERT_EQ(4, res2["grouped_hits"][4]["found"].get<int32_t>());+ res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 2, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"size"}, 2).get();++ ASSERT_EQ(1000, res2["found_docs"].get<size_t>());+ ASSERT_EQ(300, res2["found"].get<size_t>());+ ASSERT_EQ(100, res2["grouped_hits"].size());++ ASSERT_EQ(3, res2["grouped_hits"][0]["found"].get<int32_t>());++ ASSERT_EQ(3, res2["grouped_hits"][1]["found"].get<int32_t>());++ ASSERT_EQ(4, res2["grouped_hits"][50]["found"].get<int32_t>());++ ASSERT_EQ(4, res2["grouped_hits"][99]["found"].get<int32_t>());}TEST_F(CollectionGroupingTest, GroupSortingWithoutGroupingFields) {@@ -1039,7 +1132,7 @@ TEST_F(CollectionGroupingTest, GroupByMultipleFacetFieldsWithPinning) {"", 10,{"3:1,4:2"}, {}, {"size"}, 2).get();- ASSERT_EQ(7, res["found_docs"].get<size_t>());+ ASSERT_EQ(5, res["found_docs"].get<size_t>());ASSERT_EQ(4, res["found"].get<size_t>());ASSERT_EQ(4, res["grouped_hits"].size());diff --git a/test/topster_test.cpp b/test/topster_test.cppindex 30311b83..fc34ad51 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -179,7 +179,7 @@ TEST(TopsterTest, MaxFloatValues) {}TEST(TopsterTest, DistinctIntValues) {- Topster dist_topster(5, 2);+ Topster dist_topster(5, 2);struct {uint16_t query_index;@@ -214,20 +214,9 @@ TEST(TopsterTest, DistinctIntValues) {dist_topster.add(&kv);}- dist_topster.set_first_pass_complete();- for(int i = 0; i < 14; i++) {- int64_t scores[3];- scores[0] = int64_t(data[i].match_score);- scores[1] = data[i].primary_attr;- scores[2] = data[i].secondary_attr;-- KV kv(data[i].query_index, i+100, data[i].distinct_key, 0, scores);- dist_topster.add(&kv);- }-dist_topster.sort();- std::vector<uint64_t> distinct_ids = {10, 5, 8, 4, 1};+ std::vector<uint64_t> distinct_ids = {4, 1, 8, 5, 9};for(uint32_t i = 0; i < dist_topster.size; i++) {EXPECT_EQ(distinct_ids[i], dist_topster.getDistinctKeyAt(i));@@ -235,15 +224,15 @@ TEST(TopsterTest, DistinctIntValues) {if(distinct_ids[i] == 1) {EXPECT_EQ(12, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);- EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);- EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);+ EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);+ EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);}if(distinct_ids[i] == 5) {- EXPECT_EQ(10, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);+ EXPECT_EQ(9, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);- EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);- EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);+ EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);+ EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);}}}\ No newline at end of file