Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Do grouping in two pass (#1677)

Krunal Gandhidccf6eb18648
diff --git a/include/hll/dense_array.h b/include/hll/dense_array.h
new file mode 100644
index 00000000..ac07bbe6
--- /dev/null
+++ b/include/hll/dense_array.h
@@ -0,0 +1,123 @@
+// Source : https://github.com/iwiwi/hyperloglog-hip
+#ifndef HYPERLOGLOG_HIP_DENSE_ARRAY_H_
+#define HYPERLOGLOG_HIP_DENSE_ARRAY_H_
+
+#include <algorithm>
+#include <climits>
+#include <cstdint>
+#include <memory>
+#include <type_traits>
+
+namespace hyperloglog_hip {
+template<size_t NumRegisterBits, typename Value = uint8_t>
+class dense_array {
+ public:
+ typedef Value value_type;
+
+ dense_array(size_t num_registers)
+ : data_(new value_type[data_length(num_registers)]()) {}
+
+ value_type get(size_t pos) {
+ const size_t b = pos * num_register_bits();
+ const size_t i1 = b / num_value_bits();
+ const size_t o1 = b - i1 * num_value_bits();
+ const size_t n1 = num_value_bits() - o1;
+ value_type v = data_[i1] >> o1;
+
+ if (n1 > num_register_bits()) {
+ v &= (value_type(1) << num_register_bits()) - 1;
+ }
+ else if (n1 < num_register_bits()) {
+ const size_t i2 = i1 + 1;
+ const size_t n2 = num_register_bits() - n1;
+ v |= (data_[i2] & ((value_type(1) << n2) - 1)) << n1;
+ }
+ return v;
+ }
+
+ void set(size_t pos, value_type val) {
+ const size_t b = pos * num_register_bits();
+
+ const size_t i1 = b / num_value_bits();
+ const size_t o1 = b - i1 * num_value_bits();
+ const size_t n1 = std::min(num_value_bits() - o1, num_register_bits());
+ data_[i1] &= value_type(-1) ^ (((value_type(1) << n1) - 1) << o1);
+ data_[i1] |= val << o1;
+
+ if (n1 < num_register_bits()) {
+ const size_t i2 = i1 + 1;
+ const size_t n2 = num_register_bits() - n1;
+ data_[i2] &= value_type(-1) ^ ((value_type(1) << n2) - 1);
+ data_[i2] |= val >> n1;
+ }
+ }
+
+ private:
+ std::unique_ptr<value_type[]> data_;
+
+ static constexpr size_t num_register_bits() {
+ return NumRegisterBits;
+ }
+
+ static constexpr size_t num_value_bits() {
+ return sizeof(Value) * CHAR_BIT;
+ }
+
+ static constexpr size_t data_length(size_t num_registers) {
+ return (num_registers * num_register_bits() + num_value_bits() - 1) / num_value_bits();
+ }
+
+ static_assert(std::is_unsigned<value_type>::value,
+ "Value should be an unsigned integral type.");
+
+ static_assert(sizeof(value_type) * CHAR_BIT >= NumRegisterBits,
+ "Value should have at least NumRegisterBits bits.");
+};
+
+template<typename Value>
+class dense_array_primitive {
+ public:
+ typedef Value value_type;
+
+ dense_array_primitive(size_t size) : data_(new value_type[size]()) {}
+ virtual ~dense_array_primitive() {}
+
+ value_type get(size_t pos) const {
+ return data_[pos];
+ }
+
+ void set(size_t pos, value_type val) {
+ data_[pos] = val;
+ }
+
+ private:
+ std::unique_ptr<value_type[]> data_;
+};
+
+template<>
+class dense_array<8, uint8_t> : public dense_array_primitive<uint8_t> {
+ public:
+ dense_array(size_t size) : dense_array_primitive(size) {}
+};
+
+template<>
+class dense_array<16, uint16_t> : public dense_array_primitive<uint16_t> {
+ public:
+ dense_array(size_t size) : dense_array_primitive(size) {}
+};
+
+template<>
+class dense_array<32, uint32_t> : public dense_array_primitive<uint32_t> {
+ public:
+ dense_array(size_t size) : dense_array_primitive(size) {}
+};
+
+template<>
+class dense_array<64, uint64_t> : public dense_array_primitive<uint64_t> {
+ public:
+ dense_array(size_t size) : dense_array_primitive(size) {}
+};
+} // namespace hyperloglog_hip
+
+#endif // HYEPRLOGLOG_HIP_DENSE_ARRAY_H_
+
diff --git a/include/hll/distinct_counter.h b/include/hll/distinct_counter.h
new file mode 100644
index 00000000..0a3deb62
--- /dev/null
+++ b/include/hll/distinct_counter.h
@@ -0,0 +1,64 @@
+// Source : https://github.com/iwiwi/hyperloglog-hip
+#ifndef HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_
+#define HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_
+
+#include <algorithm>
+#include <cstdint>
+#include <cmath>
+#include "dense_array.h"
+
+namespace hyperloglog_hip {
+template<typename Key, typename Hash = std::hash<Key>, int NumRegisterBits = 5>
+class distinct_counter {
+ public:
+ typedef Key key_type;
+ typedef Hash hash_type;
+
+ distinct_counter(size_t num_bucket_bits = 12)
+ : num_bucket_bits_(num_bucket_bits), M_(1 << num_bucket_bits),
+ c_(0), s_(1 << num_bucket_bits) {}
+
+ void insert(const key_type &v) {
+ static constexpr uint64_t num_register_bits = NumRegisterBits;
+ static constexpr uint64_t register_limit = (uint64_t(1) << num_register_bits) - 1;
+
+ const uint64_t h = hash_(v) * magic1() + magic2();
+ const uint64_t h0 = h & ((uint64_t(1) << num_bucket_bits_) - 1);
+ const uint64_t h1 = h >> num_bucket_bits_;
+
+ const uint64_t b_old = M_.get(h0);
+ const uint64_t b_new = h1 == 0 ? register_limit :
+ std::min(register_limit, uint64_t(1 + __builtin_ctzl(h1)));
+
+ if (b_new > b_old) {
+ M_.set(h0, b_new);
+ c_ += 1.0 / (s_ / (uint64_t(1) << num_bucket_bits_));
+ s_ -= 1.0 / (uint64_t(1) << b_old);
+ if (b_new < register_limit) {
+ s_ += 1.0 / (uint64_t(1) << b_new);
+ }
+ }
+ }
+
+ size_t count() const {
+ return round(c_);
+ }
+
+ private:
+ const size_t num_bucket_bits_;
+ dense_array<NumRegisterBits> M_;
+ double c_, s_;
+ hash_type hash_;
+
+ static constexpr uint64_t magic1() {
+ return 9223372036854775837ULL;
+ }
+
+ static constexpr uint64_t magic2() {
+ return 1234567890123456789ULL;
+ }
+};
+} // namespace hyperloglog_hip
+
+#endif // HYPERLOGLOG_HIP_DISTINCT_COUNTER_H_
+
diff --git a/include/topster.h b/include/topster.h
index a471564c..f168eefa 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -7,6 +7,9 @@
#include <unordered_map>
#include <field.h>
#include "filter_result_iterator.h"
+#include "hll/distinct_counter.h"
+
+#define HYPERLOGLOG_THRESHOLD 2048
struct KV {
int8_t match_score_index{};
@@ -136,10 +139,17 @@ struct Topster {
spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;
size_t distinct;
+ //hyperloglog lib counter for counting total unique group values more than 512
+ hyperloglog_hip::distinct_counter<uint64_t> hyperloglog_counter;
+ std::set<uint64_t> groups_found; //to keep count of total unique group less than 512
+ uint32_t groups_found_count = 0; //to keep track of groups found in current pass
+ bool is_first_pass_completed = false;
+
explicit Topster(size_t capacity): Topster(capacity, 0) {
}
- explicit Topster(size_t capacity, size_t distinct): MAX_SIZE(capacity), size(0), distinct(distinct) {
+ explicit Topster(size_t capacity, size_t distinct, bool is_first_pass_completed = false): MAX_SIZE(capacity),
+ size(0), distinct(distinct), is_first_pass_completed(is_first_pass_completed) {
// we allocate data first to get a memory block whose indices are then assigned to `kvs`
// we use separate **kvs for easier pointer swaps
data = new KV[capacity];
@@ -184,24 +194,32 @@ struct Topster {
LOG(INFO) << "kv key: " << mkv.first << " => " << mkv.second->scores[mkv.second->match_score_index];
}*/
- int ret = 1;
-
+ /* returns either 0 or 1
+ * 1 -> distinct_id was added to group_kv_map in second pass, which will aggregate found counts to groups_processed
+ * 0 -> distinct_id was added to kv_map in first pass
+ * -1 -> distinct_id was not added
+ */
+
bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller(kv, kvs[0]);
size_t heap_op_index = 0;
if(!distinct && less_than_min_heap) {
// for non-distinct, if incoming value is smaller than min-heap ignore
- return 0;
+ return -1;
}
bool SIFT_DOWN = true;
- if(distinct) {
- const auto& doc_seq_id_exists =
+ if(distinct && is_first_pass_completed) {
+ if(kv_map.count(kv->distinct_key) == 0) {
+ return -1;
+ }
+
+ const auto& doc_seq_id_exists =
(group_doc_seq_ids.find(kv->key) != group_doc_seq_ids.end());
if(doc_seq_id_exists) {
- ret = 2;
+ return -1;
}
group_doc_seq_ids.emplace(kv->key);
@@ -213,14 +231,16 @@ struct Topster {
Topster* g_topster = new Topster(distinct, 0);
g_topster->add(kv);
group_kv_map.insert({kv->distinct_key, g_topster});
+ groups_found_count++;
}
- return ret;
+ return 1;
} else { // not distinct
//LOG(INFO) << "Searching for key: " << kv->key;
+ auto key = distinct ? kv->distinct_key : kv->key;
- const auto& found_it = kv_map.find(kv->key);
+ const auto& found_it = kv_map.find(key);
bool is_duplicate_key = (found_it != kv_map.end());
/*
@@ -237,14 +257,15 @@ struct Topster {
bool smaller_than_existing = is_smaller(kv, existing_kv);
if(smaller_than_existing) {
- return 0;
+ return -1;
}
SIFT_DOWN = true;
// replace existing kv and sift down
heap_op_index = existing_kv->array_index;
- kv_map.erase(kvs[heap_op_index]->key);
+ auto heap_op_index_key = distinct ? kvs[heap_op_index]->distinct_key : kvs[heap_op_index]->key;
+ kv_map.erase(heap_op_index_key);
} else { // not duplicate
if(size < MAX_SIZE) {
@@ -257,12 +278,24 @@ struct Topster {
// we have to replace min heap element since array is full
SIFT_DOWN = true;
heap_op_index = 0;
- kv_map.erase(kvs[heap_op_index]->key);
+ auto heap_op_index_key = distinct ? kvs[heap_op_index]->distinct_key : kvs[heap_op_index]->key;
+ kv_map.erase(heap_op_index_key);
}
}
// kv will be copied into the pointer at heap_op_index
- kv_map.emplace(kv->key, kvs[heap_op_index]);
+ kv_map.emplace(key, kvs[heap_op_index]);
+
+ if(distinct) {
+ hyperloglog_counter.insert(kv->distinct_key);
+
+ if(groups_found.size() < HYPERLOGLOG_THRESHOLD) {
+ groups_found.insert(kv->distinct_key);
+ groups_found_count = groups_found.size();
+ } else {
+ groups_found_count = hyperloglog_counter.count();
+ }
+ }
}
// we have to replace the existing element in the heap and sift down
@@ -300,7 +333,7 @@ struct Topster {
}
}
- return ret;
+ return 0;
}
static bool is_greater(const struct KV* i, const struct KV* j) {
@@ -340,4 +373,18 @@ struct Topster {
KV* getKV(uint32_t index) {
return kvs[index];
}
+
+ const size_t get_total_unique_groups() const {
+ auto groups_count = groups_found.size();
+ return groups_count < 512 ? groups_count : hyperloglog_counter.count();
+ }
+
+ void set_first_pass_complete() {
+ is_first_pass_completed = true;
+ groups_found_count = 0;
+ }
+
+ const size_t get_current_groups_count() const {
+ return groups_found_count;
+ }
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index b9cded16..6ef051ba 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2388,7 +2388,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
// for grouping we have to aggregate group set sizes to a count value
if(group_limit) {
- total = search_params->groups_processed.size() + override_result_kvs.size();
+ total = search_params->topster->get_total_unique_groups() + override_result_kvs.size();
} else {
total = search_params->all_result_ids_len;
}
@@ -3592,7 +3592,7 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K
if(group_count_index >= 0) {
const auto& itr = groups_processed.find(kv_head->distinct_key);
if(itr != groups_processed.end()) {
- kv_head->scores[group_count_index] = itr->second * group_sort_order;
+ kv_head->scores[group_count_index] = (int)itr->second * group_sort_order;
}
}
gtopster.add(kv_head);
diff --git a/src/index.cpp b/src/index.cpp
index d5436131..22f273dc 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1571,7 +1571,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
}
void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {
- if(index_topster->distinct) {
+ if(index_topster->distinct && index_topster->is_first_pass_completed) {
for(auto &group_topster_entry: index_topster->group_kv_map) {
Topster* group_topster = group_topster_entry.second;
for(const auto& map_kv: group_topster->kv_map) {
@@ -2221,53 +2221,106 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& co
facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,
bool enable_synonyms, bool synonym_prefix, uint32_t synonym_num_typos,
bool enable_typos_for_alpha_numerical_tokens) {
- return search(search_params->field_query_tokens,
- search_params->search_fields,
- search_params->match_type,
- search_params->filter_tree_root, search_params->facets, search_params->facet_query,
- search_params->max_facet_values,
- search_params->included_ids, search_params->excluded_ids,
- search_params->sort_fields_std, search_params->num_typos,
- search_params->topster, search_params->curated_topster,
- search_params->per_page, search_params->offset, search_params->token_order,
- search_params->prefixes, search_params->drop_tokens_threshold,
- search_params->all_result_ids_len, search_params->groups_processed,
- search_params->searched_queries,
- search_params->qtoken_set,
- search_params->raw_result_kvs, search_params->override_result_kvs,
- search_params->typo_tokens_threshold,
- search_params->group_limit,
- search_params->group_by_fields,
- search_params->group_missing_values,
- search_params->default_sorting_field,
- search_params->prioritize_exact_match,
- search_params->prioritize_token_position,
- search_params->prioritize_num_matching_fields,
- search_params->exhaustive_search,
- search_params->concurrency,
- search_params->search_cutoff_ms,
- search_params->min_len_1typo,
- search_params->min_len_2typo,
- search_params->max_candidates,
- search_params->infixes,
- search_params->max_extra_prefix,
- search_params->max_extra_suffix,
- search_params->facet_query_num_typos,
- search_params->filter_curated_hits,
- search_params->split_join_tokens,
- search_params->vector_query,
- search_params->facet_sample_percent,
- search_params->facet_sample_threshold,
- collection_name,
- search_params->drop_tokens_mode,
- facet_index_type,
- enable_typos_for_numerical_tokens,
- enable_synonyms,
- synonym_prefix,
- synonym_num_typos,
- search_params->enable_lazy_filter,
- enable_typos_for_alpha_numerical_tokens
- );
+ if(search_params->group_limit != 0) {
+ search(search_params->field_query_tokens,
+ search_params->search_fields,
+ search_params->match_type,
+ search_params->filter_tree_root, search_params->facets, search_params->facet_query,
+ search_params->max_facet_values,
+ search_params->included_ids, search_params->excluded_ids,
+ search_params->sort_fields_std, search_params->num_typos,
+ search_params->topster, search_params->curated_topster,
+ search_params->per_page, search_params->offset, search_params->token_order,
+ search_params->prefixes, search_params->drop_tokens_threshold,
+ search_params->all_result_ids_len, search_params->groups_processed,
+ search_params->searched_queries,
+ search_params->qtoken_set,
+ search_params->raw_result_kvs, search_params->override_result_kvs,
+ search_params->typo_tokens_threshold,
+ search_params->group_limit,
+ search_params->group_by_fields,
+ search_params->group_missing_values,
+ search_params->default_sorting_field,
+ search_params->prioritize_exact_match,
+ search_params->prioritize_token_position,
+ search_params->prioritize_num_matching_fields,
+ search_params->exhaustive_search,
+ search_params->concurrency,
+ search_params->search_cutoff_ms,
+ search_params->min_len_1typo,
+ search_params->min_len_2typo,
+ search_params->max_candidates,
+ search_params->infixes,
+ search_params->max_extra_prefix,
+ search_params->max_extra_suffix,
+ search_params->facet_query_num_typos,
+ search_params->filter_curated_hits,
+ search_params->split_join_tokens,
+ search_params->vector_query,
+ search_params->facet_sample_percent,
+ search_params->facet_sample_threshold,
+ collection_name,
+ search_params->drop_tokens_mode,
+ facet_index_type,
+ enable_typos_for_numerical_tokens,
+ enable_synonyms,
+ synonym_prefix,
+ synonym_num_typos,
+ search_params->enable_lazy_filter,
+ enable_typos_for_alpha_numerical_tokens
+ );
+ search_params->topster->set_first_pass_complete();
+ search_params->curated_topster->set_first_pass_complete();
+
+ }
+ auto res = search(search_params->field_query_tokens,
+ search_params->search_fields,
+ search_params->match_type,
+ search_params->filter_tree_root, search_params->facets, search_params->facet_query,
+ search_params->max_facet_values,
+ search_params->included_ids, search_params->excluded_ids,
+ search_params->sort_fields_std, search_params->num_typos,
+ search_params->topster, search_params->curated_topster,
+ search_params->per_page, search_params->offset, search_params->token_order,
+ search_params->prefixes, search_params->drop_tokens_threshold,
+ search_params->all_result_ids_len, search_params->groups_processed,
+ search_params->searched_queries,
+ search_params->qtoken_set,
+ search_params->raw_result_kvs, search_params->override_result_kvs,
+ search_params->typo_tokens_threshold,
+ search_params->group_limit,
+ search_params->group_by_fields,
+ search_params->group_missing_values,
+ search_params->default_sorting_field,
+ search_params->prioritize_exact_match,
+ search_params->prioritize_token_position,
+ search_params->prioritize_num_matching_fields,
+ search_params->exhaustive_search,
+ search_params->concurrency,
+ search_params->search_cutoff_ms,
+ search_params->min_len_1typo,
+ search_params->min_len_2typo,
+ search_params->max_candidates,
+ search_params->infixes,
+ search_params->max_extra_prefix,
+ search_params->max_extra_suffix,
+ search_params->facet_query_num_typos,
+ search_params->filter_curated_hits,
+ search_params->split_join_tokens,
+ search_params->vector_query,
+ search_params->facet_sample_percent,
+ search_params->facet_sample_threshold,
+ collection_name,
+ search_params->drop_tokens_mode,
+ facet_index_type,
+ enable_typos_for_numerical_tokens,
+ enable_synonyms,
+ synonym_prefix,
+ synonym_num_typos,
+ search_params->enable_lazy_filter,
+ enable_typos_for_alpha_numerical_tokens
+ );
+ return res;
}
void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
@@ -2897,7 +2950,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
int ret = topster->add(&kv);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
groups_processed[distinct_id]++;
}
@@ -3065,7 +3118,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
kv.vector_distance = vec_dist_score;
int ret = topster->add(&kv);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
groups_processed[distinct_id]++;
}
@@ -3536,7 +3589,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto ret = topster->add(&kv);
vec_search_ids.push_back(seq_id);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
groups_processed[distinct_id]++;
}
}
@@ -4221,7 +4274,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
resume_typo_loop:
- auto results_count = group_limit != 0 ? groups_processed.size() : all_result_ids_len;
+ auto current_groups_count = topster ? topster->get_current_groups_count() : 0;
+ auto results_count = group_limit != 0 ? current_groups_count : all_result_ids_len;
if(!exhaustive_search && results_count >= typo_tokens_threshold) {
// if typo threshold is breached, we are done
return Option<bool>(true);
@@ -4652,7 +4706,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
}
int ret = topster->add(&kv);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
groups_processed[distinct_id]++;
}
result_ids.push_back(seq_id);
@@ -5453,7 +5507,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
int ret = actual_topster->add(&kv);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
groups_processed[distinct_id]++;
}
@@ -5628,7 +5682,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::v
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
int ret = actual_topster->add(&kv);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
groups_processed[distinct_id]++;
}
@@ -5979,7 +6033,8 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
searched_queries.push_back({});
- topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);
+ topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct, topster->is_first_pass_completed);
+ topsters[thread_id]->kv_map = topster->kv_map;
auto& compute_sort_score_status = compute_sort_score_statuses[thread_id] = nullptr;
thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
@@ -6041,7 +6096,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
int ret = topsters[thread_id]->add(&kv);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
tgroups_processed[thread_id][distinct_id]++;
}
if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {
@@ -6556,7 +6611,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;
KV kv(query_index, seq_id, distinct_id, match_score_index, scores);
int ret = topster->add(&kv);
- if(group_limit != 0 && ret < 2) {
+ if(group_limit != 0 && ret > 0) {
groups_processed[distinct_id]++;
}
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 2a88e36b..8d3a14a0 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -461,7 +461,7 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
"", 10,
{}, {}, {"colors"}, 2).get();
- ASSERT_EQ(9, res["found_docs"].get<size_t>());
+ ASSERT_EQ(10, res["found_docs"].get<size_t>());
ASSERT_EQ(4, res["found"].get<size_t>());
ASSERT_EQ(4, res["grouped_hits"].size());
@@ -755,147 +755,54 @@ TEST_F(CollectionGroupingTest, SortingOnGroupCount) {
ASSERT_EQ(7, res2["grouped_hits"][2]["found"].get<int32_t>());
}
-TEST_F(CollectionGroupingTest, SortingMoreThanMaxTopsterSize) {
-
- std::vector<field> fields = {
- field("title", field_types::STRING, false),
- field("brand", field_types::STRING, true, true),
- field("size", field_types::INT32, true, false),
- field("colors", field_types::STRING, true, false),
- field("rating", field_types::FLOAT, true, false)
- };
-
- Collection* coll3 = collectionManager.get_collection("coll3").get();
- if(coll3 == nullptr) {
- coll3 = collectionManager.create_collection("coll3", 4, fields, "rating").get();
- }
-
- for(auto i = 0; i < 150; i++) {
- auto group_id = i;
- for(auto j = 0; j < 4; j++) {
- nlohmann::json doc;
- doc["title"] = "Omega Casual Poplin Shirt";
- doc["brand"] = "Omega";
- doc["size"] = group_id;
- doc["colors"] = "blue";
- doc["rating"] = 4.5;
-
- ASSERT_TRUE(coll3->add(doc.dump()).ok());
- }
- }
-
- for(auto i = 150; i < 250; i++) {
- auto group_id = i;
- for(auto j = 0; j < 3; j++) {
- nlohmann::json doc;
- doc["title"] = "Beta Casual Poplin Shirt";
- doc["brand"] = "Beta";
- doc["size"] = group_id;
- doc["colors"] = "white";
- doc["rating"] = 4.3;
-
- ASSERT_TRUE(coll3->add(doc.dump()).ok());
- }
- }
-
- for(auto i = 250; i < 300; i++) {
- auto group_id = i;
- for(auto j = 0; j < 2; j++) {
- nlohmann::json doc;
- doc["title"] = "Zeta Casual Poplin Shirt";
- doc["brand"] = "Zeta";
- doc["size"] = group_id;
- doc["colors"] = "red";
- doc["rating"] = 4.6;
-
- ASSERT_TRUE(coll3->add(doc.dump()).ok());
- }
- }
-
+TEST_F(CollectionGroupingTest, SortingByGroupCount) {
//first search in desc order
std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};
- auto res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 2, FREQUENCY,
+ auto res = coll_group->search("shirt", {"title"}, "", {"brand"}, sort_fields, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
- {}, {}, {"size"}, 2).get();
-
- ASSERT_EQ(1000, res["found_docs"].get<size_t>());
- ASSERT_EQ(300, res["found"].get<size_t>());
- ASSERT_EQ(100, res["grouped_hits"].size());
-
- ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());
-
- ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());
-
- ASSERT_EQ(3, res["grouped_hits"][50]["found"].get<int32_t>());
-
- ASSERT_EQ(3, res["grouped_hits"][99]["found"].get<int32_t>());
-
-
- res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 3, FREQUENCY,
- {false}, Index::DROP_TOKENS_THRESHOLD,
- spp::sparse_hash_set<std::string>(),
- spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
- "", 10,
- {}, {}, {"size"}, 2).get();
-
- ASSERT_EQ(1000, res["found_docs"].get<size_t>());
- ASSERT_EQ(300, res["found"].get<size_t>());
- ASSERT_EQ(100, res["grouped_hits"].size());
+ {}, {}, {"brand"}, 10).get();
+ ASSERT_EQ(12, res["found_docs"].get<size_t>());
+ ASSERT_EQ(5, res["found"].get<size_t>());
+ ASSERT_EQ(5, res["grouped_hits"].size());
- ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());
+ ASSERT_EQ(4, res["grouped_hits"][0]["found"].get<int32_t>());
- ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());
+ ASSERT_EQ(3, res["grouped_hits"][1]["found"].get<int32_t>());
- ASSERT_EQ(2, res["grouped_hits"][50]["found"].get<int32_t>());
+ ASSERT_EQ(2, res["grouped_hits"][2]["found"].get<int32_t>());
- ASSERT_EQ(2, res["grouped_hits"][99]["found"].get<int32_t>());
+ ASSERT_EQ(2, res["grouped_hits"][3]["found"].get<int32_t>());
+ ASSERT_EQ(1, res["grouped_hits"][4]["found"].get<int32_t>());
//search in asc order
std::vector<sort_by> sort_fields2 = {sort_by("_group_found", "ASC")};
- auto res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 1, FREQUENCY,
+ auto res2 = coll_group->search("shirt", {"title"}, "", {"brand"}, sort_fields2, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
- {}, {}, {"size"}, 2).get();
+ {}, {}, {"brand"}, 10).get();
- ASSERT_EQ(1000, res2["found_docs"].get<size_t>());
- ASSERT_EQ(300, res2["found"].get<size_t>());
- ASSERT_EQ(100, res2["grouped_hits"].size());
+ ASSERT_EQ(12, res2["found_docs"].get<size_t>());
+ ASSERT_EQ(5, res2["found"].get<size_t>());
+ ASSERT_EQ(5, res2["grouped_hits"].size());
- ASSERT_EQ(2, res2["grouped_hits"][0]["found"].get<int32_t>());
+ ASSERT_EQ(1, res2["grouped_hits"][0]["found"].get<int32_t>());
ASSERT_EQ(2, res2["grouped_hits"][1]["found"].get<int32_t>());
- ASSERT_EQ(3, res2["grouped_hits"][50]["found"].get<int32_t>());
-
- ASSERT_EQ(3, res2["grouped_hits"][99]["found"].get<int32_t>());
+ ASSERT_EQ(2, res2["grouped_hits"][2]["found"].get<int32_t>());
- res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 2, FREQUENCY,
- {false}, Index::DROP_TOKENS_THRESHOLD,
- spp::sparse_hash_set<std::string>(),
- spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
- "", 10,
- {}, {}, {"size"}, 2).get();
+ ASSERT_EQ(3, res2["grouped_hits"][3]["found"].get<int32_t>());
- ASSERT_EQ(1000, res2["found_docs"].get<size_t>());
- ASSERT_EQ(300, res2["found"].get<size_t>());
- ASSERT_EQ(100, res2["grouped_hits"].size());
-
- ASSERT_EQ(3, res2["grouped_hits"][0]["found"].get<int32_t>());
-
- ASSERT_EQ(3, res2["grouped_hits"][1]["found"].get<int32_t>());
-
- ASSERT_EQ(4, res2["grouped_hits"][50]["found"].get<int32_t>());
-
- ASSERT_EQ(4, res2["grouped_hits"][99]["found"].get<int32_t>());
+ ASSERT_EQ(4, res2["grouped_hits"][4]["found"].get<int32_t>());
}
TEST_F(CollectionGroupingTest, GroupSortingWithoutGroupingFields) {
@@ -1132,7 +1039,7 @@ TEST_F(CollectionGroupingTest, GroupByMultipleFacetFieldsWithPinning) {
"", 10,
{"3:1,4:2"}, {}, {"size"}, 2).get();
- ASSERT_EQ(5, res["found_docs"].get<size_t>());
+ ASSERT_EQ(7, res["found_docs"].get<size_t>());
ASSERT_EQ(4, res["found"].get<size_t>());
ASSERT_EQ(4, res["grouped_hits"].size());
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index fc34ad51..30311b83 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -179,7 +179,7 @@ TEST(TopsterTest, MaxFloatValues) {
}
TEST(TopsterTest, DistinctIntValues) {
- Topster dist_topster(5, 2);
+ Topster dist_topster(5, 2);
struct {
uint16_t query_index;
@@ -214,9 +214,20 @@ TEST(TopsterTest, DistinctIntValues) {
dist_topster.add(&kv);
}
+ dist_topster.set_first_pass_complete();
+ for(int i = 0; i < 14; i++) {
+ int64_t scores[3];
+ scores[0] = int64_t(data[i].match_score);
+ scores[1] = data[i].primary_attr;
+ scores[2] = data[i].secondary_attr;
+
+ KV kv(data[i].query_index, i+100, data[i].distinct_key, 0, scores);
+ dist_topster.add(&kv);
+ }
+
dist_topster.sort();
- std::vector<uint64_t> distinct_ids = {4, 1, 8, 5, 9};
+ std::vector<uint64_t> distinct_ids = {10, 5, 8, 4, 1};
for(uint32_t i = 0; i < dist_topster.size; i++) {
EXPECT_EQ(distinct_ids[i], dist_topster.getDistinctKeyAt(i));
@@ -224,15 +235,15 @@ TEST(TopsterTest, DistinctIntValues) {
if(distinct_ids[i] == 1) {
EXPECT_EQ(12, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);
EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);
- EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);
- EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);
+ EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);
+ EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);
}
if(distinct_ids[i] == 5) {
- EXPECT_EQ(9, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);
+ EXPECT_EQ(10, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);
EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);
- EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);
- EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);
+ EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);
+ EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);
}
}
}
\ No newline at end of file