Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor grouping with sorting.
Kishore Nallana21628207ea2
diff --git a/include/topster.h b/include/topster.hindex f86a7b4b..4accd374 100644--- a/include/topster.h+++ b/include/topster.h@@ -105,10 +105,9 @@ struct Topster {KV *data;KV** kvs;- // For distinct, stores the min heap kv of each group_kv_map topster valuestd::unordered_map<uint64_t, KV*> kv_map;- std::unordered_map<uint64_t, Topster*> group_kv_map;+ spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;size_t distinct;explicit Topster(size_t capacity): Topster(capacity, 0) {@@ -172,64 +171,17 @@ struct Topster {bool SIFT_DOWN = true;if(distinct) {- const auto& found_it = group_kv_map.find(kv->distinct_key);- bool is_duplicate_key = (found_it != group_kv_map.end());-- if(!is_duplicate_key && less_than_min_heap) {- // for distinct, if a non duplicate kv is < than min heap we ignore- return false;- }-- if(is_duplicate_key) {- // if min heap (group_topster.kvs[0]) changes, we have to update kvs and sift- Topster* group_topster = found_it->second;- KV old_min_heap_kv = *kv_map[kv->distinct_key];- bool added = group_topster->add(kv);-- if(!added) {- return false;- }-- // if new kv score is greater than previous min heap score we sift down, otherwise sift up- SIFT_DOWN = is_greater(kv, &old_min_heap_kv);-- // new kv is different from old_min_heap_kv so we have to sift heap- heap_op_index = old_min_heap_kv.array_index;-- // erase current min heap key from kv_map- kv_map.erase(old_min_heap_kv.distinct_key);-+ // Grouping cannot be a streaming operation, so aggregate the KVs associated with every group.+ auto kvs_it = group_kv_map.find(kv->distinct_key);+ if(kvs_it != group_kv_map.end()) {+ kvs_it->second->add(kv);} else {- // kv is guaranteed to be > current min heap: kvs[0]- // create fresh topster for this distinct group key since it does not exist- Topster* group_topster = new Topster(distinct, 0);- group_topster->add(kv);-- // add new group key to map- group_kv_map.emplace(kv->distinct_key, group_topster);-- // find heap operation index for updating kvs-- if(size < MAX_SIZE) {- // there is enough space in heap we just copy to end- SIFT_DOWN = false;- heap_op_index = size;- size++;- } else {- SIFT_DOWN = true;-- // max size is reached so we are forced to replace current min heap element (kvs[0])- heap_op_index = 0;-- // remove current min heap group key from maps- delete group_kv_map[kvs[heap_op_index]->distinct_key];- group_kv_map.erase(kvs[heap_op_index]->distinct_key);- kv_map.erase(kvs[heap_op_index]->distinct_key);- }+ Topster* g_topster = new Topster(distinct, 0);+ g_topster->add(kv);+ group_kv_map.insert({kv->distinct_key, g_topster});}-- // kv will be copied into the pointer at heap_op_index- kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);++ return true;} else { // not distinct//LOG(INFO) << "Searching for key: " << kv->key;@@ -334,9 +286,8 @@ struct Topster {// topster must be sorted before iterated upon to remove dead array entriesvoid sort() {- std::stable_sort(kvs, kvs + size, is_greater);- for(auto &group_topster: group_kv_map) {- group_topster.second->sort();+ if(!distinct) {+ std::stable_sort(kvs, kvs + size, is_greater);}}diff --git a/src/collection.cpp b/src/collection.cppindex f875bb68..f690e5f3 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1384,12 +1384,27 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::svoid Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) {if(topster->distinct) {- for(auto &group_topster_entry: topster->group_kv_map) {- Topster* group_topster = group_topster_entry.second;- const std::vector<KV*> group_kvs(group_topster->kvs, group_topster->kvs+group_topster->size);- result_kvs.emplace_back(group_kvs);+ // we have to pick top-K groups+ Topster gtopster(topster->MAX_SIZE);++ for(auto& group_topster: topster->group_kv_map) {+ group_topster.second->sort();+ if(group_topster.second->size != 0) {+ KV* kv_head = group_topster.second->getKV(0);+ gtopster.add(kv_head);+ }}+ gtopster.sort();++ for(size_t i = 0; i < gtopster.size; i++) {+ KV* kv = gtopster.getKV(i);+ const std::vector<KV*> group_kvs(+ topster->group_kv_map[kv->distinct_key]->kvs,+ topster->group_kv_map[kv->distinct_key]->kvs+topster->group_kv_map[kv->distinct_key]->size+ );+ result_kvs.emplace_back(group_kvs);+ }} else {for(uint32_t t = 0; t < topster->size; t++) {KV* kv = topster->getKV(t);diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 09111c88..c5321b9e 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -154,8 +154,8 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {ASSERT_EQ(10, res["found"].get<size_t>());ASSERT_EQ(10, res["grouped_hits"].size());- ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());+ ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());ASSERT_STREQ("Beta", res["grouped_hits"][0]["group_key"][1].get<std::string>().c_str());// optional field should have no value in the group key component@@ -428,4 +428,108 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());-}\ No newline at end of file+}++TEST_F(CollectionGroupingTest, GroupOrderIndependence) {+ Collection *coll1;++ std::vector<field> fields = {field("group", field_types::STRING, true),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc;++ for(size_t i = 0; i < 256; i++) {+ int64_t points = 100 + i;+ doc["id"] = std::to_string(i);+ doc["group"] = std::to_string(i);+ doc["points"] = points;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ // doc id "255" will have points of 255+ // try to insert doc id "256" with group "256" but having lesser points than all records++ doc["id"] = "256";+ doc["group"] = "256";+ doc["points"] = 50;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // insert doc id "257" of same group "256" with greatest point++ doc["id"] = "257";+ doc["group"] = "256";+ doc["points"] = 500;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // when we search by grouping records, sorting descending on points, both records of group "256" should show up++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ auto res = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"group"}, 10).get();++ ASSERT_EQ(1, res["grouped_hits"][0]["group_key"].size());+ ASSERT_STREQ("256", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());+}++TEST_F(CollectionGroupingTest, UseHighestValueInGroupForOrdering) {+ Collection *coll1;++ std::vector<field> fields = {field("group", field_types::STRING, true),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc;++ for(size_t i = 0; i < 250; i++) {+ int64_t points = 100 + i;+ doc["id"] = std::to_string(i);+ doc["group"] = std::to_string(i);+ doc["points"] = points;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ // points: 100 -> 349++ // group with highest point is "249" with 349 points+ // insert another document for that group with 50 points+ doc["id"] = "250";+ doc["group"] = "249";+ doc["points"] = 50;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // now insert another new group whose points is greater than 50+ doc["id"] = "251";+ doc["group"] = "1000";+ doc["points"] = 60;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ auto res = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"group"}, 10).get();++ ASSERT_EQ(1, res["grouped_hits"][0]["group_key"].size());+ ASSERT_STREQ("249", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());+}