Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Expose group by params.
kishorencb7dcb7367bc6
diff --git a/TODO.md b/TODO.mdindex 69298247..96f2a7e3 100644--- a/TODO.md+++ b/TODO.md@@ -97,6 +97,7 @@- ~~Have a LOG(ERROR) level~~- ~~Handle SIGTERM which is sent when process is killed~~- ~~Use snappy compression for storage~~+- Test for overriding result on second page- atleast 1 token match for proceeding with drop tokens- support wildcard query with filters- API for optimizing on disk storagediff --git a/include/collection.h b/include/collection.hindex 4807d091..48aa4938 100644--- a/include/collection.h+++ b/include/collection.h@@ -235,7 +235,9 @@ public:const std::string & highlight_full_fields = "",size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,const std::map<std::string, size_t>& pinned_hits={},- const std::vector<std::string>& hidden_hits={});+ const std::vector<std::string>& hidden_hits={},+ const std::vector<std::string>& group_by_fields={},+ const size_t group_limit = 0);Option<nlohmann::json> get(const std::string & id);diff --git a/include/index.h b/include/index.hindex 994aa6e5..b142c9a3 100644--- a/include/index.h+++ b/include/index.h@@ -38,11 +38,13 @@ struct search_args {bool prefix;size_t drop_tokens_threshold;size_t typo_tokens_threshold;+ std::vector<std::string> group_by_fields;+ size_t group_limit;size_t all_result_ids_len;std::vector<std::vector<art_leaf*>> searched_queries;Topster* topster;Topster* curated_topster;- std::vector<KV*> raw_result_kvs;+ std::vector<std::vector<KV*>> raw_result_kvs;std::vector<KV*> override_result_kvs;Option<uint32_t> outcome;@@ -54,16 +56,18 @@ struct search_args {std::vector<facet> facets, std::vector<uint32_t> included_ids, std::vector<uint32_t> excluded_ids,std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,- size_t drop_tokens_threshold, size_t typo_tokens_threshold):+ size_t drop_tokens_threshold, size_t typo_tokens_threshold,+ const std::vector<std::string>& group_by_fields, size_t group_limit):query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),max_facet_values(max_facet_values), per_page(per_page),page(page), token_order(token_order), prefix(prefix),drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),+ group_by_fields(group_by_fields), group_limit(group_limit),all_result_ids_len(0), outcome(0) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory- topster = new Topster(topster_size);+ topster = new Topster(topster_size, group_limit);curated_topster = new Topster(topster_size);}@@ -213,6 +217,11 @@ private:void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);+ // reference: https://stackoverflow.com/a/27952689/131050+ uint64_t hash_combine(uint64_t lhs, uint64_t rhs) const {+ lhs ^= rhs + 0x517cc1b727220a95 + (lhs << 6) + (lhs >> 2);+ return lhs;+ }public:Index() = delete;@@ -233,7 +242,7 @@ public:const size_t per_page, const size_t page, const token_ordering token_order,const bool prefix, const size_t drop_tokens_threshold,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries,- std::vector<KV*> & raw_result_kvs, std::vector<KV*> & override_result_kvs,+ std::vector<std::vector<KV*>> & raw_result_kvs, std::vector<KV*> & override_result_kvs,const size_t typo_tokens_threshold);Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);diff --git a/include/topster.h b/include/topster.hindex bbc7f590..685e6d62 100644--- a/include/topster.h+++ b/include/topster.h@@ -117,7 +117,7 @@ struct Topster {}// if new kv score is greater than previous min heap score we sift dowm, otherwise sift up- SIFT_DOWN = is_greater_kv(kv, &old_min_heap_kv);+ SIFT_DOWN = is_greater(kv, &old_min_heap_kv);// new kv is different from old_min_heap_kv so we have to sift heapheap_op_index = old_min_heap_kv.array_index;@@ -218,12 +218,12 @@ struct Topster {if(SIFT_DOWN) {while ((2 * heap_op_index + 1) < size) {uint32_t next = (2 * heap_op_index + 1); // left child- if (next+1 < size && is_greater_kv(kvs[next], kvs[next+1])) {+ if (next+1 < size && is_greater(kvs[next], kvs[next + 1])) {// for min heap we compare with the minimum of childrennext++; // right child (2n + 2)}- if (is_greater_kv(kvs[heap_op_index], kvs[next])) {+ if (is_greater(kvs[heap_op_index], kvs[next])) {swapMe(&kvs[heap_op_index], &kvs[next]);} else {break;@@ -235,7 +235,7 @@ struct Topster {// SIFT UPwhile(heap_op_index > 0) {uint32_t parent = (heap_op_index - 1) / 2;- if (is_greater_kv(kvs[parent], kvs[heap_op_index])) {+ if (is_greater(kvs[parent], kvs[heap_op_index])) {swapMe(&kvs[heap_op_index], &kvs[parent]);heap_op_index = parent;} else {@@ -247,7 +247,7 @@ struct Topster {return true;}- static bool is_greater_kv(const struct KV* i, const struct KV* j) {+ static bool is_greater(const struct KV* i, const struct KV* j) {return std::tie(i->scores[0], i->scores[1], i->scores[2], i->key) >std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);}@@ -257,9 +257,14 @@ struct Topster {std::tie(j->scores[0], j->scores[1], j->scores[2]);}+ static bool is_greater_kv_group(const std::vector<KV*>& i, const std::vector<KV*>& j) {+ return std::tie(i[0]->scores[0], i[0]->scores[1], i[0]->scores[2], i[0]->key) >+ std::tie(j[0]->scores[0], j[0]->scores[1], j[0]->scores[2], j[0]->key);+ }+// topster must be sorted before iterated upon to remove dead array entriesvoid sort() {- std::stable_sort(kvs, kvs+size, is_greater_kv);+ std::stable_sort(kvs, kvs + size, is_greater);for(auto &group_topster: group_kv_map) {group_topster.second->sort();}diff --git a/src/collection.cpp b/src/collection.cppindex eee23762..96a1afbc 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -349,7 +349,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const std::string & highlight_full_fields,size_t typo_tokens_threshold,const std::map<std::string, size_t>& pinned_hits,- const std::vector<std::string>& hidden_hits) {+ const std::vector<std::string>& hidden_hits,+ const std::vector<std::string>& group_by_fields,+ const size_t group_limit) {std::vector<uint32_t> included_ids;std::vector<uint32_t> excluded_ids;@@ -385,6 +387,22 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}}+ // validate group by fields+ for(const std::string & field_name: group_by_fields) {+ if(search_schema.count(field_name) == 0) {+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";+ return Option<nlohmann::json>(404, error);+ }++ field search_field = search_schema.at(field_name);++ // if field is a string field then it must be a facet field as well+ if(search_field.is_string() && !search_field.is_facet()) {+ std::string error = "Field `" + field_name + "` should be a facet field.";+ return Option<nlohmann::json>(400, error);+ }+ }+// validate filter fieldsstd::vector<std::string> filter_blocks;StringUtils::split(simple_filter_query, filter_blocks, "&&");@@ -492,7 +510,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}// for a wildcard query, if filter is not specified, use default_sorting_field as a catch-all- if(query == "*" && filters.size() == 0) {+ if(query == "*" && filters.empty()) {field f = search_schema.at(default_sorting_field);std::string max_value = f.is_float() ? std::to_string(std::numeric_limits<float>::max()) :std::to_string(std::numeric_limits<int32_t>::max());@@ -608,7 +626,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const size_t max_hits = std::min((page * per_page), get_num_documents());std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results- std::vector<KV*> raw_result_kvs;+ std::vector<std::vector<KV*>> raw_result_kvs;std::vector<KV*> override_result_kvs;size_t total_found = 0;@@ -617,10 +635,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::size_t index_id = 0;for(Index* index: indices) {index->search_params = new search_args(query, search_fields, filters, facets,- index_to_included_ids[index_id], index_to_excluded_ids[index_id],- sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,- per_page, page, token_order, prefix,- drop_tokens_threshold, typo_tokens_threshold);+ index_to_included_ids[index_id], index_to_excluded_ids[index_id],+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,+ per_page, page, token_order, prefix,+ drop_tokens_threshold, typo_tokens_threshold,+ group_by_fields, group_limit);{std::lock_guard<std::mutex> lk(index->m);index->ready = true;@@ -649,9 +668,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::continue;}- for(auto & field_order_kv: index->search_params->raw_result_kvs) {- field_order_kv->query_index += searched_queries.size();- raw_result_kvs.push_back(field_order_kv);+ for(const std::vector<KV*> & kv_group: index->search_params->raw_result_kvs) {+ kv_group[0]->query_index += searched_queries.size();+ raw_result_kvs.push_back(kv_group);}for(auto & field_order_kv: index->search_params->override_result_kvs) {@@ -697,8 +716,35 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::return index_search_op;}+ Topster* aggr_topster = nullptr;++ if(group_limit > 0) {+ // group by query requires another round of topster-ing++ // needs to be atleast 1 since scoring is mandatory+ const size_t topster_size = std::max((size_t)1, max_hits);+ aggr_topster = new Topster(topster_size, group_limit);++ for(const auto& kv_group: raw_result_kvs) {+ for(KV* kv: kv_group) {+ aggr_topster->add(kv);+ }+ }++ aggr_topster->sort();++ raw_result_kvs.clear();+ raw_result_kvs.shrink_to_fit();++ for(auto &group_topster_entry: aggr_topster->group_kv_map) {+ Topster* group_topster = group_topster_entry.second;+ const std::vector<KV*> group_kvs(group_topster->kvs, group_topster->kvs+group_topster->size);+ raw_result_kvs.emplace_back(group_kvs);+ }+ }+// All fields are sorted descending- std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv);+ std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv_group);// Sort based on position in overridden liststd::sort(@@ -708,12 +754,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::});- nlohmann::json result = nlohmann::json::object();-- result["hits"] = nlohmann::json::array();- result["found"] = total_found;-- std::vector<KV*> result_kvs;+ std::vector<std::vector<KV*>> result_group_kvs;size_t override_kv_index = 0;size_t raw_results_index = 0;@@ -721,107 +762,123 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::while(override_kv_index < override_result_kvs.size() && raw_results_index < raw_result_kvs.size()) {if(override_kv_index < override_result_kvs.size() &&id_pos_map.count(override_result_kvs[override_kv_index]->key) != 0 &&- result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index]->key]) {- result_kvs.push_back(override_result_kvs[override_kv_index]);+ result_group_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index]->key]) {+ result_group_kvs.push_back({override_result_kvs[override_kv_index]});override_kv_index++;} else {- result_kvs.push_back(raw_result_kvs[raw_results_index]);+ result_group_kvs.push_back(raw_result_kvs[raw_results_index]);raw_results_index++;}}while(override_kv_index < override_result_kvs.size()) {- result_kvs.push_back(override_result_kvs[override_kv_index]);+ result_group_kvs.push_back({override_result_kvs[override_kv_index]});override_kv_index++;}while(raw_results_index < raw_result_kvs.size()) {- result_kvs.push_back(raw_result_kvs[raw_results_index]);+ result_group_kvs.push_back(raw_result_kvs[raw_results_index]);raw_results_index++;}const long start_result_index = (page - 1) * per_page;- const long end_result_index = std::min(max_hits, result_kvs.size()) - 1; // could be -1 when max_hits is 0+ const long end_result_index = std::min(max_hits, result_group_kvs.size()) - 1; // could be -1 when max_hits is 0++ nlohmann::json result = nlohmann::json::object();++ result["found"] = total_found;++ std::string hits_key = (group_limit > 1) ? "grouped_hits" : "hits";+ result[hits_key] = nlohmann::json::array();// construct results arrayfor(long result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {- const auto & field_order_kv = result_kvs[result_kvs_index];- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);+ const std::vector<KV*> & kv_group = result_group_kvs[result_kvs_index];- nlohmann::json document;- const Option<bool> & document_op = get_document_from_store(seq_id_key, document);+ nlohmann::json group_hits_array = nlohmann::json::array();+ nlohmann::json& hits_array = (group_limit > 1) ? group_hits_array : result["hits"];- if(!document_op.ok()) {- LOG(ERROR) << "Document fetch error. " << document_op.error();- continue;- }+ for(const KV* field_order_kv: kv_group) {+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);- nlohmann::json wrapper_doc;- wrapper_doc["highlights"] = nlohmann::json::array();- std::vector<highlight_t> highlights;- StringUtils string_utils;+ nlohmann::json document;+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);- // find out if fields have to be highlighted fully- std::vector<std::string> fields_highlighted_fully_vec;- spp::sparse_hash_set<std::string> fields_highlighted_fully;- StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");+ if(!document_op.ok()) {+ LOG(ERROR) << "Document fetch error. " << document_op.error();+ continue;+ }- for(std::string & highlight_full_field: fields_highlighted_fully_vec) {- StringUtils::trim(highlight_full_field);- fields_highlighted_fully.emplace(highlight_full_field);- }+ nlohmann::json wrapper_doc;+ wrapper_doc["highlights"] = nlohmann::json::array();+ std::vector<highlight_t> highlights;+ StringUtils string_utils;- for(const std::string & field_name: search_fields) {- // should not pick excluded field for highlighting- if(exclude_fields.count(field_name) > 0) {- continue;+ // find out if fields have to be highlighted fully+ std::vector<std::string> fields_highlighted_fully_vec;+ spp::sparse_hash_set<std::string> fields_highlighted_fully;+ StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");++ for(std::string & highlight_full_field: fields_highlighted_fully_vec) {+ StringUtils::trim(highlight_full_field);+ fields_highlighted_fully.emplace(highlight_full_field);}- field search_field = search_schema.at(field_name);- if(query != "*" && (search_field.type == field_types::STRING ||- search_field.type == field_types::STRING_ARRAY)) {+ for(const std::string & field_name: search_fields) {+ // should not pick excluded field for highlighting+ if(exclude_fields.count(field_name) > 0) {+ continue;+ }++ field search_field = search_schema.at(field_name);+ if(query != "*" && (search_field.type == field_types::STRING ||+ search_field.type == field_types::STRING_ARRAY)) {- bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());- highlight_t highlight;- highlight_result(search_field, searched_queries, field_order_kv, document,- string_utils, snippet_threshold, highlighted_fully, highlight);+ bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());+ highlight_t highlight;+ highlight_result(search_field, searched_queries, field_order_kv, document,+ string_utils, snippet_threshold, highlighted_fully, highlight);- if(!highlight.snippets.empty()) {- highlights.push_back(highlight);+ if(!highlight.snippets.empty()) {+ highlights.push_back(highlight);+ }}}- }- std::sort(highlights.begin(), highlights.end());+ std::sort(highlights.begin(), highlights.end());- for(const auto & highlight: highlights) {- nlohmann::json h_json = nlohmann::json::object();- h_json["field"] = highlight.field;- bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());+ for(const auto & highlight: highlights) {+ nlohmann::json h_json = nlohmann::json::object();+ h_json["field"] = highlight.field;+ bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());- if(!highlight.indices.empty()) {- h_json["indices"] = highlight.indices;- h_json["snippets"] = highlight.snippets;- if(highlight_fully) {- h_json["values"] = highlight.values;- }+ if(!highlight.indices.empty()) {+ h_json["indices"] = highlight.indices;+ h_json["snippets"] = highlight.snippets;+ if(highlight_fully) {+ h_json["values"] = highlight.values;+ }- } else {- h_json["snippet"] = highlight.snippets[0];- if(highlight_fully) {- h_json["value"] = highlight.values[0];+ } else {+ h_json["snippet"] = highlight.snippets[0];+ if(highlight_fully) {+ h_json["value"] = highlight.values[0];+ }}++ wrapper_doc["highlights"].push_back(h_json);}- wrapper_doc["highlights"].push_back(h_json);+ prune_document(document, include_fields, exclude_fields);+ wrapper_doc["document"] = document;+ wrapper_doc["text_match"] = field_order_kv->match_score;+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;+ hits_array.push_back(wrapper_doc);}- prune_document(document, include_fields, exclude_fields);- wrapper_doc["document"] = document;- wrapper_doc["text_match"] = field_order_kv->match_score;- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;-- result["hits"].push_back(wrapper_doc);+ if(group_limit > 1) {+ result["grouped_hits"].push_back(group_hits_array);+ }}result["facet_counts"] = nlohmann::json::array();@@ -923,6 +980,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::delete index->search_params;}+ delete aggr_topster;+result["request_params"] = nlohmann::json::object();;result["request_params"]["per_page"] = per_page;result["request_params"]["q"] = query;diff --git a/src/core_api.cpp b/src/core_api.cppindex 56efeec7..c987443e 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -235,6 +235,9 @@ bool get_search(http_req & req, http_res & res) {const char *FACET_QUERY = "facet_query";const char *MAX_FACET_VALUES = "max_facet_values";+ const char *GROUP_BY = "group_by";+ const char *GROUP_LIMIT = "group_limit";+const char *PER_PAGE = "per_page";const char *PAGE = "page";const char *CALLBACK = "callback";@@ -314,6 +317,18 @@ bool get_search(http_req & req, http_res & res) {req.params[EXCLUDE_FIELDS] = "";}+ if(req.params.count(GROUP_BY) == 0) {+ req.params[GROUP_BY] = "";+ }++ if(req.params.count(GROUP_LIMIT) == 0) {+ if(req.params[GROUP_BY] != "") {+ req.params[GROUP_LIMIT] = "3";+ } else {+ req.params[GROUP_LIMIT] = "0";+ }+ }+if(!StringUtils::is_uint64_t(req.params[DROP_TOKENS_THRESHOLD])) {res.set_400("Parameter `" + std::string(DROP_TOKENS_THRESHOLD) + "` must be an unsigned integer.");return false;@@ -349,6 +364,11 @@ bool get_search(http_req & req, http_res & res) {return false;}+ if(!StringUtils::is_uint64_t(req.params[GROUP_LIMIT])) {+ res.set_400("Parameter `" + std::string(GROUP_LIMIT) + "` must be an unsigned integer.");+ return false;+ }+std::string filter_str = req.params.count(FILTER) != 0 ? req.params[FILTER] : "";std::vector<std::string> search_fields;@@ -366,6 +386,9 @@ bool get_search(http_req & req, http_res & res) {spp::sparse_hash_set<std::string> include_fields(include_fields_vec.begin(), include_fields_vec.end());spp::sparse_hash_set<std::string> exclude_fields(exclude_fields_vec.begin(), exclude_fields_vec.end());+ std::vector<std::string> group_by_fields;+ StringUtils::split(req.params[GROUP_BY], group_by_fields, ",");+std::vector<sort_by> sort_fields;if(req.params.count(SORT_BY) != 0) {std::vector<std::string> sort_field_strs;@@ -455,7 +478,9 @@ bool get_search(http_req & req, http_res & res) {req.params[HIGHLIGHT_FULL_FIELDS],typo_tokens_threshold,pinned_hits,- hidden_hits+ hidden_hits,+ group_by_fields,+ static_cast<size_t>(std::stoi(req.params[GROUP_LIMIT])));uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex 1ff8f5bb..a93509d7 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1162,12 +1162,10 @@ void Index::search(Option<uint32_t> & outcome,const bool prefix, const size_t drop_tokens_threshold,size_t & all_result_ids_len,std::vector<std::vector<art_leaf*>> & searched_queries,- std::vector<KV*> & raw_result_kvs,+ std::vector<std::vector<KV*>> & raw_result_kvs,std::vector<KV*> & override_result_kvs,const size_t typo_tokens_threshold) {- const size_t num_results = (page * per_page);-// process the filtersuint32_t* filter_ids = nullptr;@@ -1221,17 +1219,33 @@ void Index::search(Option<uint32_t> & outcome,std::vector<uint32_t> dropped_ids;// loop through topster and remove elements from included and excluded id lists- for(uint32_t t = 0; t < topster->size && t < num_results; t++) {- KV* kv = topster->getKV(t);- if(ids_to_remove.count(kv->key) != 0) {- dropped_ids.push_back((uint32_t)kv->key);- } else {- raw_result_kvs.push_back(kv);+ if(topster->distinct) {+ for(auto &group_topster_entry: topster->group_kv_map) {+ Topster* group_topster = group_topster_entry.second;+ for(uint32_t t = 0; t < group_topster->size; t++) {+ KV* kv = group_topster->getKV(t);+ if(ids_to_remove.count(kv->key) != 0) {+ dropped_ids.push_back((uint32_t)kv->key);+ }+ }++ const std::vector<KV*> group_kvs(group_topster->kvs, group_topster->kvs+group_topster->size);+ raw_result_kvs.emplace_back(group_kvs);+ }+ } else {+ for(uint32_t t = 0; t < topster->size; t++) {+ KV* kv = topster->getKV(t);++ if(ids_to_remove.count(kv->key) != 0) {+ dropped_ids.push_back((uint32_t)kv->key);+ } else {+ raw_result_kvs.push_back({kv});+ }}}- for(uint32_t t = 0; t < curated_topster->size && t < num_results; t++) {+ for(uint32_t t = 0; t < curated_topster->size; t++) {KV* kv = curated_topster->getKV(t);override_result_kvs.push_back(kv);}@@ -1471,6 +1485,16 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16Match single_token_match = Match(1, 0, 0, empty_offset_diffs);const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);+ std::unordered_map<std::string, size_t> facet_to_id;++ if(search_params->group_limit > 0) {+ size_t i_facet = 0;+ for(const auto & facet: facet_schema) {+ facet_to_id[facet.first] = i_facet;+ i_facet++;+ }+ }+for(size_t i=0; i<result_size; i++) {const uint32_t seq_id = result_ids[i];@@ -1545,7 +1569,27 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}}- KV kv(field_id, query_index, seq_id, seq_id, match_score, scores);+ uint64_t distinct_id = seq_id;++ if(search_params->group_limit != 0) {+ distinct_id = 1; // some constant initial value++ // calculate hash from group_by_fields+ for(const auto& field: search_params->group_by_fields) {+ if(facet_to_id.count(field) == 0 || facet_index_v2.count(seq_id) == 0) {+ continue;+ }++ size_t facet_id = facet_to_id[field];+ const std::vector<uint64_t>& fhashes = facet_index_v2.at(seq_id)[facet_id];++ for(const auto& hash: fhashes) {+ distinct_id = hash_combine(distinct_id, hash);+ }+ }+ }++ KV kv(field_id, query_index, seq_id, distinct_id, match_score, scores);topster->add(&kv);}