Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Do grouping in two pass (#1677)
Krunal Gandhidccf6eb18648
diff --git a/include/hll/dense_array.h b/include/hll/dense_array.hnew file mode 100644index 00000000..ac07bbe6--- /dev/null+++ b/include/hll/dense_array.h@@ -0,0 +1,123 @@+// Source : https://github.com/iwiwi/hyperloglog-hip+#ifndef HYPERLOGLOG_HIP_DENSE_ARRAY_H_+#define HYPERLOGLOG_HIP_DENSE_ARRAY_H_++#include <algorithm>+#include <climits>+#include <cstdint>+#include <memory>+#include <type_traits>++namespace hyperloglog_hip {+template<size_t NumRegisterBits, typename Value = uint8_t>+class dense_array {+ public:+ typedef Value value_type;++ dense_array(size_t num_registers)+ : data_(new value_type[data_length(num_registers)]()) {}++ value_type get(size_t pos) {+ const size_t b = pos * num_register_bits();+ const size_t i1 = b / num_value_bits();+ const size_t o1 = b - i1 * num_value_bits();+ const size_t n1 = num_value_bits() - o1;+ value_type v = data_[i1] >> o1;++ if (n1 > num_register_bits()) {+ v &= (value_type(1) << num_register_bits()) - 1;+ }+ else if (n1 < num_register_bits()) {+ const size_t i2 = i1 + 1;+ const size_t n2 = num_register_bits() - n1;+ v |= (data_[i2] & ((value_type(1) << n2) - 1)) << n1;+ }+ return v;+ }++ void set(size_t pos, value_type val) {+ const size_t b = pos * num_register_bits();++ const size_t i1 = b / num_value_bits();+ const size_t o1 = b - i1 * num_value_bits();+ const size_t n1 = std::min(num_value_bits() - o1, num_register_bits());+ data_[i1] &= value_type(-1) ^ (((value_type(1) << n1) - 1) << o1);+ data_[i1] |= val << o1;++ if (n1 < num_register_bits()) {+ const size_t i2 = i1 + 1;+ const size_t n2 = num_register_bits() - n1;+ data_[i2] &= value_type(-1) ^ ((value_type(1) << n2) - 1);+ data_[i2] |= val >> n1;+ }+ }++ private:+ std::unique_ptr<value_type[]> data_;++ static constexpr size_t num_register_bits() {+ return NumRegisterBits;+ }++ static constexpr size_t num_value_bits() {+ return sizeof(Value) * CHAR_BIT;+ }++ static constexpr size_t data_length(size_t num_registers) {+ return (num_registers * num_register_bits() + num_value_bits() - 1) / num_value_bits();+ }++ static_assert(std::is_unsigned<value_type>::value,+ "Value should be an unsigned integral type.");++ static_assert(sizeof(value_type) * CHAR_BIT >= NumRegisterBits,+ "Value should have at least NumRegisterBits bits.");+};++template<typename Value>+class dense_array_primitive {+ public:+ typedef Value value_type;++ dense_array_primitive(size_t size) : data_(new value_type[size]()) {}+ virtual ~dense_array_primitive() {}++ value_type get(size_t pos) const {+ return data_[pos];+ }++ void set(size_t pos, value_type val) {+ data_[pos] = val;+ }++ private:+ std::unique_ptr<value_type[]> data_;+};++template<>+class dense_array<8, uint8_t> : public dense_array_primitive<uint8_t> {+ public:+ dense_array(size_t size) : dense_array_primitive(size) {}+};++template<>+class dense_array<16, uint16_t> : public dense_array_primitive<uint16_t> {+ public:+ dense_array(size_t size) : dense_array_primitive(size) {}+};++template<>+class dense_array<32, uint32_t> : public dense_array_primitive<uint32_t> {+ public:+ dense_array(size_t size) : dense_array_primitive(size) {}+};++template<>+class dense_array<64, uint64_t> : public dense_array_primitive<uint64_t> {+ public:+ dense_array(size_t size) : dense_array_primitive(size) {}+};+} // namespace hyperloglog_hip++#endif // HYEPRLOGLOG_HIP_DENSE_ARRAY_H_+diff --git a/include/hll/distinct_counter.h b/include/hll/distinct_counter.hnew file mode 100644index 00000000..0a3deb62--- /dev/null+++ b/include/hll/distinct_counter.h@@ -0,0 +1,64 @@+// Source : https://github.com/iwiwi/hyperloglog-hip+#ifndef HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_+#define HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_++#include <algorithm>+#include <cstdint>+#include <cmath>+#include "dense_array.h"++namespace hyperloglog_hip {+template<typename Key, typename Hash = std::hash<Key>, int NumRegisterBits = 5>+class distinct_counter {+ public:+ typedef Key key_type;+ typedef Hash hash_type;++ distinct_counter(size_t num_bucket_bits = 12)+ : num_bucket_bits_(num_bucket_bits), M_(1 << num_bucket_bits),+ c_(0), s_(1 << num_bucket_bits) {}++ void insert(const key_type &v) {+ static constexpr uint64_t num_register_bits = NumRegisterBits;+ static constexpr uint64_t register_limit = (uint64_t(1) << num_register_bits) - 1;++ const uint64_t h = hash_(v) * magic1() + magic2();+ const uint64_t h0 = h & ((uint64_t(1) << num_bucket_bits_) - 1);+ const uint64_t h1 = h >> num_bucket_bits_;++ const uint64_t b_old = M_.get(h0);+ const uint64_t b_new = h1 == 0 ? register_limit :+ std::min(register_limit, uint64_t(1 + __builtin_ctzl(h1)));++ if (b_new > b_old) {+ M_.set(h0, b_new);+ c_ += 1.0 / (s_ / (uint64_t(1) << num_bucket_bits_));+ s_ -= 1.0 / (uint64_t(1) << b_old);+ if (b_new < register_limit) {+ s_ += 1.0 / (uint64_t(1) << b_new);+ }+ }+ }++ size_t count() const {+ return round(c_);+ }++ private:+ const size_t num_bucket_bits_;+ dense_array<NumRegisterBits> M_;+ double c_, s_;+ hash_type hash_;++ static constexpr uint64_t magic1() {+ return 9223372036854775837ULL;+ }++ static constexpr uint64_t magic2() {+ return 1234567890123456789ULL;+ }+};+} // namespace hyperloglog_hip++#endif // HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_+diff --git a/include/topster.h b/include/topster.hindex a471564c..f168eefa 100644--- a/include/topster.h+++ b/include/topster.h@@ -7,6 +7,9 @@#include <unordered_map>#include <field.h>#include "filter_result_iterator.h"+#include "hll/distinct_counter.h"++#define HYPERLOGLOG_THRESHOLD 2048struct KV {int8_t match_score_index{};@@ -136,10 +139,17 @@ struct Topster {spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;size_t distinct;+ //hyperloglog lib counter for counting total unique group values more than 512+ hyperloglog_hip::distinct_counter<uint64_t> hyperloglog_counter;+ std::set<uint64_t> groups_found; //to keep count of total unique group less than 512+ uint32_t groups_found_count = 0; //to keep track of groups found in current pass+ bool is_first_pass_completed = false;+explicit Topster(size_t capacity): Topster(capacity, 0) {}- explicit Topster(size_t capacity, size_t distinct): MAX_SIZE(capacity), size(0), distinct(distinct) {+ explicit Topster(size_t capacity, size_t distinct, bool is_first_pass_completed = false): MAX_SIZE(capacity),+ size(0), distinct(distinct), is_first_pass_completed(is_first_pass_completed) {// we allocate data first to get a memory block whose indices are then assigned to `kvs`// we use separate **kvs for easier pointer swapsdata = new KV[capacity];@@ -184,24 +194,32 @@ struct Topster {LOG(INFO) << "kv key: " << mkv.first << " => " << mkv.second->scores[mkv.second->match_score_index];}*/- int ret = 1;-+ /* returns either 0 or 1+ * 1 -> distinct_id was added to group_kv_map in second pass, which will aggregate found counts to groups_processed+ * 0 -> distinct_id was added to kv_map in first pass+ * -1 -> distinct_id was not added+ */+bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller(kv, kvs[0]);size_t heap_op_index = 0;if(!distinct && less_than_min_heap) {// for non-distinct, if incoming value is smaller than min-heap ignore- return 0;+ return -1;}bool SIFT_DOWN = true;- if(distinct) {- const auto& doc_seq_id_exists =+ if(distinct && is_first_pass_completed) {+ if(kv_map.count(kv->distinct_key) == 0) {+ return -1;+ }++ const auto& doc_seq_id_exists =(group_doc_seq_ids.find(kv->key) != group_doc_seq_ids.end());if(doc_seq_id_exists) {- ret = 2;+ return -1;}group_doc_seq_ids.emplace(kv->key);@@ -213,14 +231,16 @@ struct Topster {Topster* g_topster = new Topster(distinct, 0);g_topster->add(kv);group_kv_map.insert({kv->distinct_key, g_topster});+ groups_found_count++;}- return ret;+ return 1;} else { // not distinct//LOG(INFO) << "Searching for key: " << kv->key;+ auto key = distinct ? kv->distinct_key : kv->key;- const auto& found_it = kv_map.find(kv->key);+ const auto& found_it = kv_map.find(key);bool is_duplicate_key = (found_it != kv_map.end());/*@@ -237,14 +257,15 @@ struct Topster {bool smaller_than_existing = is_smaller(kv, existing_kv);if(smaller_than_existing) {- return 0;+ return -1;}SIFT_DOWN = true;// replace existing kv and sift downheap_op_index = existing_kv->array_index;- kv_map.erase(kvs[heap_op_index]->key);+ auto heap_op_index_key = distinct ? kvs[heap_op_index]->distinct_key : kvs[heap_op_index]->key;+ kv_map.erase(heap_op_index_key);} else { // not duplicateif(size < MAX_SIZE) {@@ -257,12 +278,24 @@ struct Topster {// we have to replace min heap element since array is fullSIFT_DOWN = true;heap_op_index = 0;- kv_map.erase(kvs[heap_op_index]->key);+ auto heap_op_index_key = distinct ? kvs[heap_op_index]->distinct_key : kvs[heap_op_index]->key;+ kv_map.erase(heap_op_index_key);}}// kv will be copied into the pointer at heap_op_index- kv_map.emplace(kv->key, kvs[heap_op_index]);+ kv_map.emplace(key, kvs[heap_op_index]);++ if(distinct) {+ hyperloglog_counter.insert(kv->distinct_key);++ if(groups_found.size() < HYPERLOGLOG_THRESHOLD) {+ groups_found.insert(kv->distinct_key);+ groups_found_count = groups_found.size();+ } else {+ groups_found_count = hyperloglog_counter.count();+ }+ }}// we have to replace the existing element in the heap and sift down@@ -300,7 +333,7 @@ struct Topster {}}- return ret;+ return 0;}static bool is_greater(const struct KV* i, const struct KV* j) {@@ -340,4 +373,18 @@ struct Topster {KV* getKV(uint32_t index) {return kvs[index];}++ const size_t get_total_unique_groups() const {+ auto groups_count = groups_found.size();+ return groups_count < 512 ? groups_count : hyperloglog_counter.count();+ }++ void set_first_pass_complete() {+ is_first_pass_completed = true;+ groups_found_count = 0;+ }++ const size_t get_current_groups_count() const {+ return groups_found_count;+ }};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex b9cded16..6ef051ba 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2388,7 +2388,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,// for grouping we have to aggregate group set sizes to a count valueif(group_limit) {- total = search_params->groups_processed.size() + override_result_kvs.size();+ total = search_params->topster->get_total_unique_groups() + override_result_kvs.size();} else {total = search_params->all_result_ids_len;}@@ -3592,7 +3592,7 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<Kif(group_count_index >= 0) {const auto& itr = groups_processed.find(kv_head->distinct_key);if(itr != groups_processed.end()) {- kv_head->scores[group_count_index] = itr->second * group_sort_order;+ kv_head->scores[group_count_index] = (int)itr->second * group_sort_order;}}gtopster.add(kv_head);diff --git a/src/index.cpp b/src/index.cppindex d5436131..22f273dc 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1571,7 +1571,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {- if(index_topster->distinct) {+ if(index_topster->distinct && index_topster->is_first_pass_completed) {for(auto &group_topster_entry: index_topster->group_kv_map) {Topster* group_topster = group_topster_entry.second;for(const auto& map_kv: group_topster->kv_map) {@@ -2221,53 +2221,106 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cofacet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,bool enable_synonyms, bool synonym_prefix, uint32_t synonym_num_typos,bool enable_typos_for_alpha_numerical_tokens) {- return search(search_params->field_query_tokens,- search_params->search_fields,- search_params->match_type,- search_params->filter_tree_root, search_params->facets, search_params->facet_query,- search_params->max_facet_values,- search_params->included_ids, search_params->excluded_ids,- search_params->sort_fields_std, search_params->num_typos,- search_params->topster, search_params->curated_topster,- search_params->per_page, search_params->offset, search_params->token_order,- search_params->prefixes, search_params->drop_tokens_threshold,- search_params->all_result_ids_len, search_params->groups_processed,- search_params->searched_queries,- search_params->qtoken_set,- search_params->raw_result_kvs, search_params->override_result_kvs,- search_params->typo_tokens_threshold,- search_params->group_limit,- search_params->group_by_fields,- search_params->group_missing_values,- search_params->default_sorting_field,- search_params->prioritize_exact_match,- search_params->prioritize_token_position,- search_params->prioritize_num_matching_fields,- search_params->exhaustive_search,- search_params->concurrency,- search_params->search_cutoff_ms,- search_params->min_len_1typo,- search_params->min_len_2typo,- search_params->max_candidates,- search_params->infixes,- search_params->max_extra_prefix,- search_params->max_extra_suffix,- search_params->facet_query_num_typos,- search_params->filter_curated_hits,- search_params->split_join_tokens,- search_params->vector_query,- search_params->facet_sample_percent,- search_params->facet_sample_threshold,- collection_name,- search_params->drop_tokens_mode,- facet_index_type,- enable_typos_for_numerical_tokens,- enable_synonyms,- synonym_prefix,- synonym_num_typos,- search_params->enable_lazy_filter,- enable_typos_for_alpha_numerical_tokens- );+ if(search_params->group_limit != 0) {+ search(search_params->field_query_tokens,+ search_params->search_fields,+ search_params->match_type,+ search_params->filter_tree_root, search_params->facets, search_params->facet_query,+ search_params->max_facet_values,+ search_params->included_ids, search_params->excluded_ids,+ search_params->sort_fields_std, search_params->num_typos,+ search_params->topster, search_params->curated_topster,+ search_params->per_page, search_params->offset, search_params->token_order,+ search_params->prefixes, search_params->drop_tokens_threshold,+ search_params->all_result_ids_len, search_params->groups_processed,+ search_params->searched_queries,+ search_params->qtoken_set,+ search_params->raw_result_kvs, search_params->override_result_kvs,+ search_params->typo_tokens_threshold,+ search_params->group_limit,+ search_params->group_by_fields,+ search_params->group_missing_values,+ search_params->default_sorting_field,+ search_params->prioritize_exact_match,+ search_params->prioritize_token_position,+ search_params->prioritize_num_matching_fields,+ search_params->exhaustive_search,+ search_params->concurrency,+ search_params->search_cutoff_ms,+ search_params->min_len_1typo,+ search_params->min_len_2typo,+ search_params->max_candidates,+ search_params->infixes,+ search_params->max_extra_prefix,+ search_params->max_extra_suffix,+ search_params->facet_query_num_typos,+ search_params->filter_curated_hits,+ search_params->split_join_tokens,+ search_params->vector_query,+ search_params->facet_sample_percent,+ search_params->facet_sample_threshold,+ collection_name,+ search_params->drop_tokens_mode,+ facet_index_type,+ enable_typos_for_numerical_tokens,+ enable_synonyms,+ synonym_prefix,+ synonym_num_typos,+ search_params->enable_lazy_filter,+ enable_typos_for_alpha_numerical_tokens+ );+ search_params->topster->set_first_pass_complete();+ search_params->curated_topster->set_first_pass_complete();++ }+ auto res = search(search_params->field_query_tokens,+ search_params->search_fields,+ search_params->match_type,+ search_params->filter_tree_root, search_params->facets, search_params->facet_query,+ search_params->max_facet_values,+ search_params->included_ids, search_params->excluded_ids,+ search_params->sort_fields_std, search_params->num_typos,+ search_params->topster, search_params->curated_topster,+ search_params->per_page, search_params->offset, search_params->token_order,+ search_params->prefixes, search_params->drop_tokens_threshold,+ search_params->all_result_ids_len, search_params->groups_processed,+ search_params->searched_queries,+ search_params->qtoken_set,+ search_params->raw_result_kvs, search_params->override_result_kvs,+ search_params->typo_tokens_threshold,+ search_params->group_limit,+ search_params->group_by_fields,+ search_params->group_missing_values,+ search_params->default_sorting_field,+ search_params->prioritize_exact_match,+ search_params->prioritize_token_position,+ search_params->prioritize_num_matching_fields,+ search_params->exhaustive_search,+ search_params->concurrency,+ search_params->search_cutoff_ms,+ search_params->min_len_1typo,+ search_params->min_len_2typo,+ search_params->max_candidates,+ search_params->infixes,+ search_params->max_extra_prefix,+ search_params->max_extra_suffix,+ search_params->facet_query_num_typos,+ search_params->filter_curated_hits,+ search_params->split_join_tokens,+ search_params->vector_query,+ search_params->facet_sample_percent,+ search_params->facet_sample_threshold,+ collection_name,+ search_params->drop_tokens_mode,+ facet_index_type,+ enable_typos_for_numerical_tokens,+ enable_synonyms,+ synonym_prefix,+ synonym_num_typos,+ search_params->enable_lazy_filter,+ enable_typos_for_alpha_numerical_tokens+ );+ return res;}void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,@@ -2897,7 +2950,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consKV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);int ret = topster->add(&kv);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {groups_processed[distinct_id]++;}@@ -3065,7 +3118,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conskv.vector_distance = vec_dist_score;int ret = topster->add(&kv);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {groups_processed[distinct_id]++;}@@ -3536,7 +3589,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consauto ret = topster->add(&kv);vec_search_ids.push_back(seq_id);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {groups_processed[distinct_id]++;}}@@ -4221,7 +4274,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fresume_typo_loop:- auto results_count = group_limit != 0 ? groups_processed.size() : all_result_ids_len;+ auto current_groups_count = topster ? topster->get_current_groups_count() : 0;+ auto results_count = group_limit != 0 ? current_groups_count : all_result_ids_len;if(!exhaustive_search && results_count >= typo_tokens_threshold) {// if typo threshold is breached, we are donereturn Option<bool>(true);@@ -4652,7 +4706,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token}int ret = topster->add(&kv);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {groups_processed[distinct_id]++;}result_ids.push_back(seq_id);@@ -5453,7 +5507,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = actual_topster->add(&kv);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {groups_processed[distinct_id]++;}@@ -5628,7 +5682,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vKV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = actual_topster->add(&kv);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {groups_processed[distinct_id]++;}@@ -5979,7 +6033,8 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootsearched_queries.push_back({});- topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);+ topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct, topster->is_first_pass_completed);+ topsters[thread_id]->kv_map = topster->kv_map;auto& compute_sort_score_status = compute_sort_score_statuses[thread_id] = nullptr;thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,@@ -6041,7 +6096,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootKV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = topsters[thread_id]->add(&kv);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {tgroups_processed[thread_id][distinct_id]++;}if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {@@ -6556,7 +6611,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;KV kv(query_index, seq_id, distinct_id, match_score_index, scores);int ret = topster->add(&kv);- if(group_limit != 0 && ret < 2) {+ if(group_limit != 0 && ret > 0) {groups_processed[distinct_id]++;}diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 2a88e36b..8d3a14a0 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -461,7 +461,7 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {"", 10,{}, {}, {"colors"}, 2).get();- ASSERT_EQ(9, res["found_docs"].get<size_t>());+ ASSERT_EQ(10, res["found_docs"].get<size_t>());ASSERT_EQ(4, res["found"].get<size_t>());ASSERT_EQ(4, res["grouped_hits"].size());@@ -755,147 +755,54 @@ TEST_F(CollectionGroupingTest, SortingOnGroupCount) {ASSERT_EQ(7, res2["grouped_hits"][2]["found"].get<int32_t>());}-TEST_F(CollectionGroupingTest, SortingMoreThanMaxTopsterSize) {-- std::vector<field> fields = {- field("title", field_types::STRING, false),- field("brand", field_types::STRING, true, true),- field("size", field_types::INT32, true, false),- field("colors", field_types::STRING, true, false),- field("rating", field_types::FLOAT, true, false)- };-- Collection* coll3 = collectionManager.get_collection("coll3").get();- if(coll3 == nullptr) {- coll3 = collectionManager.create_collection("coll3", 4, fields, "rating").get();- }-- for(auto i = 0; i < 150; i++) {- auto group_id = i;- for(auto j = 0; j < 4; j++) {- nlohmann::json doc;- doc["title"] = "Omega Casual Poplin Shirt";- doc["brand"] = "Omega";- doc["size"] = group_id;- doc["colors"] = "blue";- doc["rating"] = 4.5;-- ASSERT_TRUE(coll3->add(doc.dump()).ok());- }- }-- for(auto i = 150; i < 250; i++) {- auto group_id = i;- for(auto j = 0; j < 3; j++) {- nlohmann::json doc;- doc["title"] = "Beta Casual Poplin Shirt";- doc["brand"] = "Beta";- doc["size"] = group_id;- doc["colors"] = "white";- doc["rating"] = 4.3;-- ASSERT_TRUE(coll3->add(doc.dump()).ok());- }- }-- for(auto i = 250; i < 300; i++) {- auto group_id = i;- for(auto j = 0; j < 2; j++) {- nlohmann::json doc;- doc["title"] = "Zeta Casual Poplin Shirt";- doc["brand"] = "Zeta";- doc["size"] = group_id;- doc["colors"] = "red";- doc["rating"] = 4.6;-- ASSERT_TRUE(coll3->add(doc.dump()).ok());- }- }-+TEST_F(CollectionGroupingTest, SortingByGroupCount) {//first search in desc orderstd::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};- auto res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 2, FREQUENCY,+ auto res = coll_group->search("shirt", {"title"}, "", {"brand"}, sort_fields, {0}, 10, 1, FREQUENCY,{false}, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10,- {}, {}, {"size"}, 2).get();-- ASSERT_EQ(1000, res["found_docs"].get<size_t>());- ASSERT_EQ(300, res["found"].get<size_t>());- ASSERT_EQ(100, res["grouped_hits"].size());-- ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());-- ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());-- ASSERT_EQ(3, res["grouped_hits"][50]["found"].get<int32_t>());-- ASSERT_EQ(3, res["grouped_hits"][99]["found"].get<int32_t>());--- res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 3, FREQUENCY,- {false}, Index::DROP_TOKENS_THRESHOLD,- spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 5,- "", 10,- {}, {}, {"size"}, 2).get();-- ASSERT_EQ(1000, res["found_docs"].get<size_t>());- ASSERT_EQ(300, res["found"].get<size_t>());- ASSERT_EQ(100, res["grouped_hits"].size());+ {}, {}, {"brand"}, 10).get();+ ASSERT_EQ(12, res["found_docs"].get<size_t>());+ ASSERT_EQ(5, res["found"].get<size_t>());+ ASSERT_EQ(5, res["grouped_hits"].size());- ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());+ ASSERT_EQ(4, res["grouped_hits"][0]["found"].get<int32_t>());- ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());+ ASSERT_EQ(3, res["grouped_hits"][1]["found"].get<int32_t>());- ASSERT_EQ(2, res["grouped_hits"][50]["found"].get<int32_t>());+ ASSERT_EQ(2, res["grouped_hits"][2]["found"].get<int32_t>());- ASSERT_EQ(2, res["grouped_hits"][99]["found"].get<int32_t>());+ ASSERT_EQ(2, res["grouped_hits"][3]["found"].get<int32_t>());+ ASSERT_EQ(1, res["grouped_hits"][4]["found"].get<int32_t>());//search in asc orderstd::vector<sort_by> sort_fields2 = {sort_by("_group_found", "ASC")};- auto res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 1, FREQUENCY,+ auto res2 = coll_group->search("shirt", {"title"}, "", {"brand"}, sort_fields2, {0}, 10, 1, FREQUENCY,{false}, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10,- {}, {}, {"size"}, 2).get();+ {}, {}, {"brand"}, 10).get();- ASSERT_EQ(1000, res2["found_docs"].get<size_t>());- ASSERT_EQ(300, res2["found"].get<size_t>());- ASSERT_EQ(100, res2["grouped_hits"].size());+ ASSERT_EQ(12, res2["found_docs"].get<size_t>());+ ASSERT_EQ(5, res2["found"].get<size_t>());+ ASSERT_EQ(5, res2["grouped_hits"].size());- ASSERT_EQ(2, res2["grouped_hits"][0]["found"].get<int32_t>());+ ASSERT_EQ(1, res2["grouped_hits"][0]["found"].get<int32_t>());ASSERT_EQ(2, res2["grouped_hits"][1]["found"].get<int32_t>());- ASSERT_EQ(3, res2["grouped_hits"][50]["found"].get<int32_t>());-- ASSERT_EQ(3, res2["grouped_hits"][99]["found"].get<int32_t>());+ ASSERT_EQ(2, res2["grouped_hits"][2]["found"].get<int32_t>());- res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 2, FREQUENCY,- {false}, Index::DROP_TOKENS_THRESHOLD,- spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 5,- "", 10,- {}, {}, {"size"}, 2).get();+ ASSERT_EQ(3, res2["grouped_hits"][3]["found"].get<int32_t>());- ASSERT_EQ(1000, res2["found_docs"].get<size_t>());- ASSERT_EQ(300, res2["found"].get<size_t>());- ASSERT_EQ(100, res2["grouped_hits"].size());-- ASSERT_EQ(3, res2["grouped_hits"][0]["found"].get<int32_t>());-- ASSERT_EQ(3, res2["grouped_hits"][1]["found"].get<int32_t>());-- ASSERT_EQ(4, res2["grouped_hits"][50]["found"].get<int32_t>());-- ASSERT_EQ(4, res2["grouped_hits"][99]["found"].get<int32_t>());+ ASSERT_EQ(4, res2["grouped_hits"][4]["found"].get<int32_t>());}TEST_F(CollectionGroupingTest, GroupSortingWithoutGroupingFields) {@@ -1132,7 +1039,7 @@ TEST_F(CollectionGroupingTest, GroupByMultipleFacetFieldsWithPinning) {"", 10,{"3:1,4:2"}, {}, {"size"}, 2).get();- ASSERT_EQ(5, res["found_docs"].get<size_t>());+ ASSERT_EQ(7, res["found_docs"].get<size_t>());ASSERT_EQ(4, res["found"].get<size_t>());ASSERT_EQ(4, res["grouped_hits"].size());diff --git a/test/topster_test.cpp b/test/topster_test.cppindex fc34ad51..30311b83 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -179,7 +179,7 @@ TEST(TopsterTest, MaxFloatValues) {}TEST(TopsterTest, DistinctIntValues) {- Topster dist_topster(5, 2);+ Topster dist_topster(5, 2);struct {uint16_t query_index;@@ -214,9 +214,20 @@ TEST(TopsterTest, DistinctIntValues) {dist_topster.add(&kv);}+ dist_topster.set_first_pass_complete();+ for(int i = 0; i < 14; i++) {+ int64_t scores[3];+ scores[0] = int64_t(data[i].match_score);+ scores[1] = data[i].primary_attr;+ scores[2] = data[i].secondary_attr;++ KV kv(data[i].query_index, i+100, data[i].distinct_key, 0, scores);+ dist_topster.add(&kv);+ }+dist_topster.sort();- std::vector<uint64_t> distinct_ids = {4, 1, 8, 5, 9};+ std::vector<uint64_t> distinct_ids = {10, 5, 8, 4, 1};for(uint32_t i = 0; i < dist_topster.size; i++) {EXPECT_EQ(distinct_ids[i], dist_topster.getDistinctKeyAt(i));@@ -224,15 +235,15 @@ TEST(TopsterTest, DistinctIntValues) {if(distinct_ids[i] == 1) {EXPECT_EQ(12, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);- EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);- EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);+ EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);+ EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);}if(distinct_ids[i] == 5) {- EXPECT_EQ(9, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);+ EXPECT_EQ(10, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);- EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);- EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);+ EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);+ EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);}}}\ No newline at end of file