Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
fixing edge case of repeated field values while group hit count
krunal13131b3e3a3455f2
diff --git a/include/topster.h b/include/topster.hindex 191be35e..4dcb184d 100644--- a/include/topster.h+++ b/include/topster.h@@ -99,6 +99,8 @@ struct Topster {std::unordered_map<uint64_t, KV*> kv_map;+ spp::sparse_hash_set<uint64_t> group_doc_seq_ids;+spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;size_t distinct;@@ -144,18 +146,25 @@ struct Topster {(*b)->array_index = a_index;}- bool add(KV* kv) {+ int add(KV* kv) {/*LOG(INFO) << "kv_map size: " << kv_map.size() << " -- kvs[0]: " << kvs[0]->scores[kvs[0]->match_score_index];for(auto& mkv: kv_map) {LOG(INFO) << "kv key: " << mkv.first << " => " << mkv.second->scores[mkv.second->match_score_index];}*/+ int ret = 1;+ const auto& doc_seq_id_exists = (group_doc_seq_ids.find(kv->key) != group_doc_seq_ids.end());+ if(doc_seq_id_exists) {+ ret = 2;+ }+ group_doc_seq_ids.emplace(kv->key);+bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller(kv, kvs[0]);size_t heap_op_index = 0;if(!distinct && less_than_min_heap) {// for non-distinct, if incoming value is smaller than min-heap ignore- return false;+ return 0;}bool SIFT_DOWN = true;@@ -171,7 +180,7 @@ struct Topster {group_kv_map.insert({kv->distinct_key, g_topster});}- return true;+ return ret;} else { // not distinct//LOG(INFO) << "Searching for key: " << kv->key;@@ -193,7 +202,7 @@ struct Topster {bool smaller_than_existing = is_smaller(kv, existing_kv);if(smaller_than_existing) {- return false;+ return 0;}SIFT_DOWN = true;@@ -256,7 +265,7 @@ struct Topster {}}- return true;+ return ret;}static bool is_greater(const struct KV* i, const struct KV* j) {diff --git a/src/index.cpp b/src/index.cppindex ddf0960c..ad299ddc 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2578,7 +2578,6 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vuint64_t distinct_id = seq_id;if (group_limit != 0) {distinct_id = get_distinct_id(group_by_fields, seq_id);- groups_processed[distinct_id]++;}int64_t scores[3] = {0};@@ -2587,7 +2586,11 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vresult_ids.push_back(seq_id);KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- topster->add(&kv);+ int ret = topster->add(&kv);++ if((group_limit != 0) && (ret < 2)) {+ groups_processed[distinct_id]++;+ }if (result_ids.size() == page * per_page) {break;@@ -2670,7 +2673,6 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vuint64_t distinct_id = seq_id;if (group_limit != 0) {distinct_id = get_distinct_id(group_by_fields, seq_id);- groups_processed[distinct_id]++;}auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :@@ -2683,7 +2685,11 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v//LOG(INFO) << "SEQ_ID: " << seq_id << ", score: " << dist_label.first;KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- topster->add(&kv);+ int ret = topster->add(&kv);++ if((group_limit != 0) && (ret < 2)) {+ groups_processed[distinct_id]++;+ }nearest_ids.push_back(seq_id);}@@ -3644,7 +3650,6 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,uint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = get_distinct_id(group_by_fields, seq_id);- groups_processed[distinct_id]++;}int64_t scores[3] = {0};@@ -3698,7 +3703,11 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,if(match_score_index != -1) {kv.scores[match_score_index] = aggregated_score;}- topster->add(&kv);++ int ret = topster->add(&kv);+ if((group_limit != 0) && (ret < 2)) {+ groups_processed[distinct_id]++;+ }result_ids.push_back(seq_id);});@@ -4163,11 +4172,13 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<seuint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = get_distinct_id(group_by_fields, seq_id);- groups_processed[distinct_id]++;}KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- actual_topster->add(&kv);+ int ret = actual_topster->add(&kv);+ if((group_limit != 0) && (ret < 2)) {+ groups_processed[distinct_id]++;+ }if(((i + 1) % (1 << 12)) == 0) {BREAK_CIRCUIT_BREAKER@@ -4498,11 +4509,14 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,uint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = get_distinct_id(group_by_fields, seq_id);- tgroups_processed[thread_id][distinct_id]++;}KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- topsters[thread_id]->add(&kv);+ int ret = topsters[thread_id]->add(&kv);++ if((group_limit != 0) && (ret < 2)) {+ tgroups_processed[thread_id][distinct_id]++;+ }if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {// check only once every 2^15 docs to reduce overhead@@ -5087,12 +5101,14 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16if(group_limit != 0) {distinct_id = get_distinct_id(group_by_fields, seq_id);- groups_processed[distinct_id]++;}//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;KV kv(query_index, seq_id, distinct_id, match_score_index, scores);- topster->add(&kv);+ int ret = topster->add(&kv);+ if((group_limit != 0) && (ret < 2)) {+ groups_processed[distinct_id]++;+ }//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 86f196d5..d2c2a7fd 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -316,19 +316,19 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {ASSERT_EQ(3, results["found"].get<size_t>());ASSERT_EQ(3, results["grouped_hits"].size());- ASSERT_EQ(21, results["grouped_hits"][0]["found"].get<int32_t>());+ ASSERT_EQ(3, results["grouped_hits"][0]["found"].get<int32_t>());ASSERT_STREQ("pop", results["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("4", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());- ASSERT_EQ(6, results["grouped_hits"][1]["found"].get<int32_t>());+ ASSERT_EQ(2, results["grouped_hits"][1]["found"].get<int32_t>());ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());ASSERT_EQ(2, results["grouped_hits"][1]["hits"].size());ASSERT_STREQ("5", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("0", results["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());- ASSERT_EQ(3, results["grouped_hits"][2]["found"].get<int32_t>());+ ASSERT_EQ(2, results["grouped_hits"][2]["found"].get<int32_t>());ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());@@ -567,3 +567,35 @@ TEST_F(CollectionGroupingTest, UseHighestValueInGroupForOrdering) {ASSERT_STREQ("249", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());}+++TEST_F(CollectionGroupingTest, RepeatedFieldNameGroupHitCount) {+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("brand", field_types::STRING, true, true),+ field("colors", field_types::STRING, true, false),+ };++ Collection* coll2 = collectionManager.get_collection("coll2").get();+ if(coll2 == nullptr) {+ coll2 = collectionManager.create_collection("coll2", 1, fields).get();+ }++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "foobar";+ doc["brand"] = "Omega";+ doc["colors"] = "foo";++ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ auto res = coll2->search("f", {"title", "colors"}, "", {}, {}, {0}, 10, 1, FREQUENCY,+ {true}, 10,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"brand"}, 2).get();++ ASSERT_EQ(1, res["grouped_hits"].size());+ ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());+}\ No newline at end of file