Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor grouping with sorting.

Kishore Nallana21628207ea2
diff --git a/include/topster.h b/include/topster.h
index f86a7b4b..4accd374 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -105,10 +105,9 @@ struct Topster {
KV *data;
KV** kvs;
- // For distinct, stores the min heap kv of each group_kv_map topster value
std::unordered_map<uint64_t, KV*> kv_map;
- std::unordered_map<uint64_t, Topster*> group_kv_map;
+ spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;
size_t distinct;
explicit Topster(size_t capacity): Topster(capacity, 0) {
@@ -172,64 +171,17 @@ struct Topster {
bool SIFT_DOWN = true;
if(distinct) {
- const auto& found_it = group_kv_map.find(kv->distinct_key);
- bool is_duplicate_key = (found_it != group_kv_map.end());
-
- if(!is_duplicate_key && less_than_min_heap) {
- // for distinct, if a non duplicate kv is < than min heap we ignore
- return false;
- }
-
- if(is_duplicate_key) {
- // if min heap (group_topster.kvs[0]) changes, we have to update kvs and sift
- Topster* group_topster = found_it->second;
- KV old_min_heap_kv = *kv_map[kv->distinct_key];
- bool added = group_topster->add(kv);
-
- if(!added) {
- return false;
- }
-
- // if new kv score is greater than previous min heap score we sift down, otherwise sift up
- SIFT_DOWN = is_greater(kv, &old_min_heap_kv);
-
- // new kv is different from old_min_heap_kv so we have to sift heap
- heap_op_index = old_min_heap_kv.array_index;
-
- // erase current min heap key from kv_map
- kv_map.erase(old_min_heap_kv.distinct_key);
-
+ // Grouping cannot be a streaming operation, so aggregate the KVs associated with every group.
+ auto kvs_it = group_kv_map.find(kv->distinct_key);
+ if(kvs_it != group_kv_map.end()) {
+ kvs_it->second->add(kv);
} else {
- // kv is guaranteed to be > current min heap: kvs[0]
- // create fresh topster for this distinct group key since it does not exist
- Topster* group_topster = new Topster(distinct, 0);
- group_topster->add(kv);
-
- // add new group key to map
- group_kv_map.emplace(kv->distinct_key, group_topster);
-
- // find heap operation index for updating kvs
-
- if(size < MAX_SIZE) {
- // there is enough space in heap we just copy to end
- SIFT_DOWN = false;
- heap_op_index = size;
- size++;
- } else {
- SIFT_DOWN = true;
-
- // max size is reached so we are forced to replace current min heap element (kvs[0])
- heap_op_index = 0;
-
- // remove current min heap group key from maps
- delete group_kv_map[kvs[heap_op_index]->distinct_key];
- group_kv_map.erase(kvs[heap_op_index]->distinct_key);
- kv_map.erase(kvs[heap_op_index]->distinct_key);
- }
+ Topster* g_topster = new Topster(distinct, 0);
+ g_topster->add(kv);
+ group_kv_map.insert({kv->distinct_key, g_topster});
}
-
- // kv will be copied into the pointer at heap_op_index
- kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);
+
+ return true;
} else { // not distinct
//LOG(INFO) << "Searching for key: " << kv->key;
@@ -334,9 +286,8 @@ struct Topster {
// topster must be sorted before iterated upon to remove dead array entries
void sort() {
- std::stable_sort(kvs, kvs + size, is_greater);
- for(auto &group_topster: group_kv_map) {
- group_topster.second->sort();
+ if(!distinct) {
+ std::stable_sort(kvs, kvs + size, is_greater);
}
}
diff --git a/src/collection.cpp b/src/collection.cpp
index f875bb68..f690e5f3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1384,12 +1384,27 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) {
if(topster->distinct) {
- for(auto &group_topster_entry: topster->group_kv_map) {
- Topster* group_topster = group_topster_entry.second;
- const std::vector<KV*> group_kvs(group_topster->kvs, group_topster->kvs+group_topster->size);
- result_kvs.emplace_back(group_kvs);
+ // we have to pick top-K groups
+ Topster gtopster(topster->MAX_SIZE);
+
+ for(auto& group_topster: topster->group_kv_map) {
+ group_topster.second->sort();
+ if(group_topster.second->size != 0) {
+ KV* kv_head = group_topster.second->getKV(0);
+ gtopster.add(kv_head);
+ }
}
+ gtopster.sort();
+
+ for(size_t i = 0; i < gtopster.size; i++) {
+ KV* kv = gtopster.getKV(i);
+ const std::vector<KV*> group_kvs(
+ topster->group_kv_map[kv->distinct_key]->kvs,
+ topster->group_kv_map[kv->distinct_key]->kvs+topster->group_kv_map[kv->distinct_key]->size
+ );
+ result_kvs.emplace_back(group_kvs);
+ }
} else {
for(uint32_t t = 0; t < topster->size; t++) {
KV* kv = topster->getKV(t);
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 09111c88..c5321b9e 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -154,8 +154,8 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {
ASSERT_EQ(10, res["found"].get<size_t>());
ASSERT_EQ(10, res["grouped_hits"].size());
- ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
+ ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
ASSERT_STREQ("Beta", res["grouped_hits"][0]["group_key"][1].get<std::string>().c_str());
// optional field should have no value in the group key component
@@ -428,4 +428,108 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionGroupingTest, GroupOrderIndependence) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("group", field_types::STRING, true),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc;
+
+ for(size_t i = 0; i < 256; i++) {
+ int64_t points = 100 + i;
+ doc["id"] = std::to_string(i);
+ doc["group"] = std::to_string(i);
+ doc["points"] = points;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ // doc id "255" will have points of 255
+ // try to insert doc id "256" with group "256" but having lesser points than all records
+
+ doc["id"] = "256";
+ doc["group"] = "256";
+ doc["points"] = 50;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // insert doc id "257" of same group "256" with greatest point
+
+ doc["id"] = "257";
+ doc["group"] = "256";
+ doc["points"] = 500;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // when we search by grouping records, sorting descending on points, both records of group "256" should show up
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ auto res = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"group"}, 10).get();
+
+ ASSERT_EQ(1, res["grouped_hits"][0]["group_key"].size());
+ ASSERT_STREQ("256", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
+}
+
+TEST_F(CollectionGroupingTest, UseHighestValueInGroupForOrdering) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("group", field_types::STRING, true),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc;
+
+ for(size_t i = 0; i < 250; i++) {
+ int64_t points = 100 + i;
+ doc["id"] = std::to_string(i);
+ doc["group"] = std::to_string(i);
+ doc["points"] = points;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ // points: 100 -> 349
+
+ // group with highest point is "249" with 349 points
+ // insert another document for that group with 50 points
+ doc["id"] = "250";
+ doc["group"] = "249";
+ doc["points"] = 50;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // now insert another new group whose points is greater than 50
+ doc["id"] = "251";
+ doc["group"] = "1000";
+ doc["points"] = 60;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ auto res = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"group"}, 10).get();
+
+ ASSERT_EQ(1, res["grouped_hits"][0]["group_key"].size());
+ ASSERT_STREQ("249", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
+}