Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

fixing edge case of repeated field values while group hit count

krunal13131b3e3a3455f2
diff --git a/include/topster.h b/include/topster.h
index 191be35e..4dcb184d 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -99,6 +99,8 @@ struct Topster {
std::unordered_map<uint64_t, KV*> kv_map;
+ spp::sparse_hash_set<uint64_t> group_doc_seq_ids;
+
spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;
size_t distinct;
@@ -144,18 +146,25 @@ struct Topster {
(*b)->array_index = a_index;
}
- bool add(KV* kv) {
+ int add(KV* kv) {
/*LOG(INFO) << "kv_map size: " << kv_map.size() << " -- kvs[0]: " << kvs[0]->scores[kvs[0]->match_score_index];
for(auto& mkv: kv_map) {
LOG(INFO) << "kv key: " << mkv.first << " => " << mkv.second->scores[mkv.second->match_score_index];
}*/
+ int ret = 1;
+ const auto& doc_seq_id_exists = (group_doc_seq_ids.find(kv->key) != group_doc_seq_ids.end());
+ if(doc_seq_id_exists) {
+ ret = 2;
+ }
+ group_doc_seq_ids.emplace(kv->key);
+
bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller(kv, kvs[0]);
size_t heap_op_index = 0;
if(!distinct && less_than_min_heap) {
// for non-distinct, if incoming value is smaller than min-heap ignore
- return false;
+ return 0;
}
bool SIFT_DOWN = true;
@@ -171,7 +180,7 @@ struct Topster {
group_kv_map.insert({kv->distinct_key, g_topster});
}
- return true;
+ return ret;
} else { // not distinct
//LOG(INFO) << "Searching for key: " << kv->key;
@@ -193,7 +202,7 @@ struct Topster {
bool smaller_than_existing = is_smaller(kv, existing_kv);
if(smaller_than_existing) {
- return false;
+ return 0;
}
SIFT_DOWN = true;
@@ -256,7 +265,7 @@ struct Topster {
}
}
- return true;
+ return ret;
}
static bool is_greater(const struct KV* i, const struct KV* j) {
diff --git a/src/index.cpp b/src/index.cpp
index ddf0960c..ad299ddc 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2578,7 +2578,6 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
uint64_t distinct_id = seq_id;
if (group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
- groups_processed[distinct_id]++;
}
int64_t scores[3] = {0};
@@ -2587,7 +2586,11 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
result_ids.push_back(seq_id);
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- topster->add(&kv);
+ int ret = topster->add(&kv);
+
+ if((group_limit != 0) && (ret < 2)) {
+ groups_processed[distinct_id]++;
+ }
if (result_ids.size() == page * per_page) {
break;
@@ -2670,7 +2673,6 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
uint64_t distinct_id = seq_id;
if (group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
- groups_processed[distinct_id]++;
}
auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :
@@ -2683,7 +2685,11 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
//LOG(INFO) << "SEQ_ID: " << seq_id << ", score: " << dist_label.first;
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- topster->add(&kv);
+ int ret = topster->add(&kv);
+
+ if((group_limit != 0) && (ret < 2)) {
+ groups_processed[distinct_id]++;
+ }
nearest_ids.push_back(seq_id);
}
@@ -3644,7 +3650,6 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
- groups_processed[distinct_id]++;
}
int64_t scores[3] = {0};
@@ -3698,7 +3703,11 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
if(match_score_index != -1) {
kv.scores[match_score_index] = aggregated_score;
}
- topster->add(&kv);
+
+ int ret = topster->add(&kv);
+ if((group_limit != 0) && (ret < 2)) {
+ groups_processed[distinct_id]++;
+ }
result_ids.push_back(seq_id);
});
@@ -4163,11 +4172,13 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
- groups_processed[distinct_id]++;
}
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- actual_topster->add(&kv);
+ int ret = actual_topster->add(&kv);
+ if((group_limit != 0) && (ret < 2)) {
+ groups_processed[distinct_id]++;
+ }
if(((i + 1) % (1 << 12)) == 0) {
BREAK_CIRCUIT_BREAKER
@@ -4498,11 +4509,14 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
- tgroups_processed[thread_id][distinct_id]++;
}
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- topsters[thread_id]->add(&kv);
+ int ret = topsters[thread_id]->add(&kv);
+
+ if((group_limit != 0) && (ret < 2)) {
+ tgroups_processed[thread_id][distinct_id]++;
+ }
if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {
// check only once every 2^15 docs to reduce overhead
@@ -5087,12 +5101,14 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
if(group_limit != 0) {
distinct_id = get_distinct_id(group_by_fields, seq_id);
- groups_processed[distinct_id]++;
}
//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;
KV kv(query_index, seq_id, distinct_id, match_score_index, scores);
- topster->add(&kv);
+ int ret = topster->add(&kv);
+ if((group_limit != 0) && (ret < 2)) {
+ groups_processed[distinct_id]++;
+ }
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 86f196d5..d2c2a7fd 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -316,19 +316,19 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {
ASSERT_EQ(3, results["found"].get<size_t>());
ASSERT_EQ(3, results["grouped_hits"].size());
- ASSERT_EQ(21, results["grouped_hits"][0]["found"].get<int32_t>());
+ ASSERT_EQ(3, results["grouped_hits"][0]["found"].get<int32_t>());
ASSERT_STREQ("pop", results["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());
ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("4", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());
- ASSERT_EQ(6, results["grouped_hits"][1]["found"].get<int32_t>());
+ ASSERT_EQ(2, results["grouped_hits"][1]["found"].get<int32_t>());
ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());
ASSERT_EQ(2, results["grouped_hits"][1]["hits"].size());
ASSERT_STREQ("5", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());
- ASSERT_EQ(3, results["grouped_hits"][2]["found"].get<int32_t>());
+ ASSERT_EQ(2, results["grouped_hits"][2]["found"].get<int32_t>());
ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());
ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());
ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
@@ -567,3 +567,35 @@ TEST_F(CollectionGroupingTest, UseHighestValueInGroupForOrdering) {
ASSERT_STREQ("249", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
}
+
+
+TEST_F(CollectionGroupingTest, RepeatedFieldNameGroupHitCount) {
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("brand", field_types::STRING, true, true),
+ field("colors", field_types::STRING, true, false),
+ };
+
+ Collection* coll2 = collectionManager.get_collection("coll2").get();
+ if(coll2 == nullptr) {
+ coll2 = collectionManager.create_collection("coll2", 1, fields).get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "foobar";
+ doc["brand"] = "Omega";
+ doc["colors"] = "foo";
+
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ auto res = coll2->search("f", {"title", "colors"}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2).get();
+
+ ASSERT_EQ(1, res["grouped_hits"].size());
+ ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());
+}
\ No newline at end of file