Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Expose group by params.

kishorencb7dcb7367bc6
diff --git a/TODO.md b/TODO.md
index 69298247..96f2a7e3 100644
--- a/TODO.md
+++ b/TODO.md
@@ -97,6 +97,7 @@
- ~~Have a LOG(ERROR) level~~
- ~~Handle SIGTERM which is sent when process is killed~~
- ~~Use snappy compression for storage~~
+- Test for overriding result on second page
- atleast 1 token match for proceeding with drop tokens
- support wildcard query with filters
- API for optimizing on disk storage
diff --git a/include/collection.h b/include/collection.h
index 4807d091..48aa4938 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -235,7 +235,9 @@ public:
const std::string & highlight_full_fields = "",
size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,
const std::map<std::string, size_t>& pinned_hits={},
- const std::vector<std::string>& hidden_hits={});
+ const std::vector<std::string>& hidden_hits={},
+ const std::vector<std::string>& group_by_fields={},
+ const size_t group_limit = 0);
Option<nlohmann::json> get(const std::string & id);
diff --git a/include/index.h b/include/index.h
index 994aa6e5..b142c9a3 100644
--- a/include/index.h
+++ b/include/index.h
@@ -38,11 +38,13 @@ struct search_args {
bool prefix;
size_t drop_tokens_threshold;
size_t typo_tokens_threshold;
+ std::vector<std::string> group_by_fields;
+ size_t group_limit;
size_t all_result_ids_len;
std::vector<std::vector<art_leaf*>> searched_queries;
Topster* topster;
Topster* curated_topster;
- std::vector<KV*> raw_result_kvs;
+ std::vector<std::vector<KV*>> raw_result_kvs;
std::vector<KV*> override_result_kvs;
Option<uint32_t> outcome;
@@ -54,16 +56,18 @@ struct search_args {
std::vector<facet> facets, std::vector<uint32_t> included_ids, std::vector<uint32_t> excluded_ids,
std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,
size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,
- size_t drop_tokens_threshold, size_t typo_tokens_threshold):
+ size_t drop_tokens_threshold, size_t typo_tokens_threshold,
+ const std::vector<std::string>& group_by_fields, size_t group_limit):
query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),
max_facet_values(max_facet_values), per_page(per_page),
page(page), token_order(token_order), prefix(prefix),
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
+ group_by_fields(group_by_fields), group_limit(group_limit),
all_result_ids_len(0), outcome(0) {
const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
- topster = new Topster(topster_size);
+ topster = new Topster(topster_size, group_limit);
curated_topster = new Topster(topster_size);
}
@@ -213,6 +217,11 @@ private:
void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);
+ // reference: https://stackoverflow.com/a/27952689/131050
+ uint64_t hash_combine(uint64_t lhs, uint64_t rhs) const {
+ lhs ^= rhs + 0x517cc1b727220a95 + (lhs << 6) + (lhs >> 2);
+ return lhs;
+ }
public:
Index() = delete;
@@ -233,7 +242,7 @@ public:
const size_t per_page, const size_t page, const token_ordering token_order,
const bool prefix, const size_t drop_tokens_threshold,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries,
- std::vector<KV*> & raw_result_kvs, std::vector<KV*> & override_result_kvs,
+ std::vector<std::vector<KV*>> & raw_result_kvs, std::vector<KV*> & override_result_kvs,
const size_t typo_tokens_threshold);
Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
diff --git a/include/topster.h b/include/topster.h
index bbc7f590..685e6d62 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -117,7 +117,7 @@ struct Topster {
}
// if new kv score is greater than previous min heap score we sift dowm, otherwise sift up
- SIFT_DOWN = is_greater_kv(kv, &old_min_heap_kv);
+ SIFT_DOWN = is_greater(kv, &old_min_heap_kv);
// new kv is different from old_min_heap_kv so we have to sift heap
heap_op_index = old_min_heap_kv.array_index;
@@ -218,12 +218,12 @@ struct Topster {
if(SIFT_DOWN) {
while ((2 * heap_op_index + 1) < size) {
uint32_t next = (2 * heap_op_index + 1); // left child
- if (next+1 < size && is_greater_kv(kvs[next], kvs[next+1])) {
+ if (next+1 < size && is_greater(kvs[next], kvs[next + 1])) {
// for min heap we compare with the minimum of children
next++; // right child (2n + 2)
}
- if (is_greater_kv(kvs[heap_op_index], kvs[next])) {
+ if (is_greater(kvs[heap_op_index], kvs[next])) {
swapMe(&kvs[heap_op_index], &kvs[next]);
} else {
break;
@@ -235,7 +235,7 @@ struct Topster {
// SIFT UP
while(heap_op_index > 0) {
uint32_t parent = (heap_op_index - 1) / 2;
- if (is_greater_kv(kvs[parent], kvs[heap_op_index])) {
+ if (is_greater(kvs[parent], kvs[heap_op_index])) {
swapMe(&kvs[heap_op_index], &kvs[parent]);
heap_op_index = parent;
} else {
@@ -247,7 +247,7 @@ struct Topster {
return true;
}
- static bool is_greater_kv(const struct KV* i, const struct KV* j) {
+ static bool is_greater(const struct KV* i, const struct KV* j) {
return std::tie(i->scores[0], i->scores[1], i->scores[2], i->key) >
std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);
}
@@ -257,9 +257,14 @@ struct Topster {
std::tie(j->scores[0], j->scores[1], j->scores[2]);
}
+ static bool is_greater_kv_group(const std::vector<KV*>& i, const std::vector<KV*>& j) {
+ return std::tie(i[0]->scores[0], i[0]->scores[1], i[0]->scores[2], i[0]->key) >
+ std::tie(j[0]->scores[0], j[0]->scores[1], j[0]->scores[2], j[0]->key);
+ }
+
// topster must be sorted before iterated upon to remove dead array entries
void sort() {
- std::stable_sort(kvs, kvs+size, is_greater_kv);
+ std::stable_sort(kvs, kvs + size, is_greater);
for(auto &group_topster: group_kv_map) {
group_topster.second->sort();
}
diff --git a/src/collection.cpp b/src/collection.cpp
index eee23762..96a1afbc 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -349,7 +349,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const std::string & highlight_full_fields,
size_t typo_tokens_threshold,
const std::map<std::string, size_t>& pinned_hits,
- const std::vector<std::string>& hidden_hits) {
+ const std::vector<std::string>& hidden_hits,
+ const std::vector<std::string>& group_by_fields,
+ const size_t group_limit) {
std::vector<uint32_t> included_ids;
std::vector<uint32_t> excluded_ids;
@@ -385,6 +387,22 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
}
+ // validate group by fields
+ for(const std::string & field_name: group_by_fields) {
+ if(search_schema.count(field_name) == 0) {
+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";
+ return Option<nlohmann::json>(404, error);
+ }
+
+ field search_field = search_schema.at(field_name);
+
+ // if field is a string field then it must be a facet field as well
+ if(search_field.is_string() && !search_field.is_facet()) {
+ std::string error = "Field `" + field_name + "` should be a facet field.";
+ return Option<nlohmann::json>(400, error);
+ }
+ }
+
// validate filter fields
std::vector<std::string> filter_blocks;
StringUtils::split(simple_filter_query, filter_blocks, "&&");
@@ -492,7 +510,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
// for a wildcard query, if filter is not specified, use default_sorting_field as a catch-all
- if(query == "*" && filters.size() == 0) {
+ if(query == "*" && filters.empty()) {
field f = search_schema.at(default_sorting_field);
std::string max_value = f.is_float() ? std::to_string(std::numeric_limits<float>::max()) :
std::to_string(std::numeric_limits<int32_t>::max());
@@ -608,7 +626,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const size_t max_hits = std::min((page * per_page), get_num_documents());
std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results
- std::vector<KV*> raw_result_kvs;
+ std::vector<std::vector<KV*>> raw_result_kvs;
std::vector<KV*> override_result_kvs;
size_t total_found = 0;
@@ -617,10 +635,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
size_t index_id = 0;
for(Index* index: indices) {
index->search_params = new search_args(query, search_fields, filters, facets,
- index_to_included_ids[index_id], index_to_excluded_ids[index_id],
- sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
- per_page, page, token_order, prefix,
- drop_tokens_threshold, typo_tokens_threshold);
+ index_to_included_ids[index_id], index_to_excluded_ids[index_id],
+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
+ per_page, page, token_order, prefix,
+ drop_tokens_threshold, typo_tokens_threshold,
+ group_by_fields, group_limit);
{
std::lock_guard<std::mutex> lk(index->m);
index->ready = true;
@@ -649,9 +668,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
continue;
}
- for(auto & field_order_kv: index->search_params->raw_result_kvs) {
- field_order_kv->query_index += searched_queries.size();
- raw_result_kvs.push_back(field_order_kv);
+ for(const std::vector<KV*> & kv_group: index->search_params->raw_result_kvs) {
+ kv_group[0]->query_index += searched_queries.size();
+ raw_result_kvs.push_back(kv_group);
}
for(auto & field_order_kv: index->search_params->override_result_kvs) {
@@ -697,8 +716,35 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return index_search_op;
}
+ Topster* aggr_topster = nullptr;
+
+ if(group_limit > 0) {
+ // group by query requires another round of topster-ing
+
+ // needs to be atleast 1 since scoring is mandatory
+ const size_t topster_size = std::max((size_t)1, max_hits);
+ aggr_topster = new Topster(topster_size, group_limit);
+
+ for(const auto& kv_group: raw_result_kvs) {
+ for(KV* kv: kv_group) {
+ aggr_topster->add(kv);
+ }
+ }
+
+ aggr_topster->sort();
+
+ raw_result_kvs.clear();
+ raw_result_kvs.shrink_to_fit();
+
+ for(auto &group_topster_entry: aggr_topster->group_kv_map) {
+ Topster* group_topster = group_topster_entry.second;
+ const std::vector<KV*> group_kvs(group_topster->kvs, group_topster->kvs+group_topster->size);
+ raw_result_kvs.emplace_back(group_kvs);
+ }
+ }
+
// All fields are sorted descending
- std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv);
+ std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv_group);
// Sort based on position in overridden list
std::sort(
@@ -708,12 +754,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
);
- nlohmann::json result = nlohmann::json::object();
-
- result["hits"] = nlohmann::json::array();
- result["found"] = total_found;
-
- std::vector<KV*> result_kvs;
+ std::vector<std::vector<KV*>> result_group_kvs;
size_t override_kv_index = 0;
size_t raw_results_index = 0;
@@ -721,107 +762,123 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
while(override_kv_index < override_result_kvs.size() && raw_results_index < raw_result_kvs.size()) {
if(override_kv_index < override_result_kvs.size() &&
id_pos_map.count(override_result_kvs[override_kv_index]->key) != 0 &&
- result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index]->key]) {
- result_kvs.push_back(override_result_kvs[override_kv_index]);
+ result_group_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index]->key]) {
+ result_group_kvs.push_back({override_result_kvs[override_kv_index]});
override_kv_index++;
} else {
- result_kvs.push_back(raw_result_kvs[raw_results_index]);
+ result_group_kvs.push_back(raw_result_kvs[raw_results_index]);
raw_results_index++;
}
}
while(override_kv_index < override_result_kvs.size()) {
- result_kvs.push_back(override_result_kvs[override_kv_index]);
+ result_group_kvs.push_back({override_result_kvs[override_kv_index]});
override_kv_index++;
}
while(raw_results_index < raw_result_kvs.size()) {
- result_kvs.push_back(raw_result_kvs[raw_results_index]);
+ result_group_kvs.push_back(raw_result_kvs[raw_results_index]);
raw_results_index++;
}
const long start_result_index = (page - 1) * per_page;
- const long end_result_index = std::min(max_hits, result_kvs.size()) - 1; // could be -1 when max_hits is 0
+ const long end_result_index = std::min(max_hits, result_group_kvs.size()) - 1; // could be -1 when max_hits is 0
+
+ nlohmann::json result = nlohmann::json::object();
+
+ result["found"] = total_found;
+
+ std::string hits_key = (group_limit > 1) ? "grouped_hits" : "hits";
+ result[hits_key] = nlohmann::json::array();
// construct results array
for(long result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {
- const auto & field_order_kv = result_kvs[result_kvs_index];
- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);
+ const std::vector<KV*> & kv_group = result_group_kvs[result_kvs_index];
- nlohmann::json document;
- const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
+ nlohmann::json group_hits_array = nlohmann::json::array();
+ nlohmann::json& hits_array = (group_limit > 1) ? group_hits_array : result["hits"];
- if(!document_op.ok()) {
- LOG(ERROR) << "Document fetch error. " << document_op.error();
- continue;
- }
+ for(const KV* field_order_kv: kv_group) {
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);
- nlohmann::json wrapper_doc;
- wrapper_doc["highlights"] = nlohmann::json::array();
- std::vector<highlight_t> highlights;
- StringUtils string_utils;
+ nlohmann::json document;
+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
- // find out if fields have to be highlighted fully
- std::vector<std::string> fields_highlighted_fully_vec;
- spp::sparse_hash_set<std::string> fields_highlighted_fully;
- StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");
+ if(!document_op.ok()) {
+ LOG(ERROR) << "Document fetch error. " << document_op.error();
+ continue;
+ }
- for(std::string & highlight_full_field: fields_highlighted_fully_vec) {
- StringUtils::trim(highlight_full_field);
- fields_highlighted_fully.emplace(highlight_full_field);
- }
+ nlohmann::json wrapper_doc;
+ wrapper_doc["highlights"] = nlohmann::json::array();
+ std::vector<highlight_t> highlights;
+ StringUtils string_utils;
- for(const std::string & field_name: search_fields) {
- // should not pick excluded field for highlighting
- if(exclude_fields.count(field_name) > 0) {
- continue;
+ // find out if fields have to be highlighted fully
+ std::vector<std::string> fields_highlighted_fully_vec;
+ spp::sparse_hash_set<std::string> fields_highlighted_fully;
+ StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");
+
+ for(std::string & highlight_full_field: fields_highlighted_fully_vec) {
+ StringUtils::trim(highlight_full_field);
+ fields_highlighted_fully.emplace(highlight_full_field);
}
- field search_field = search_schema.at(field_name);
- if(query != "*" && (search_field.type == field_types::STRING ||
- search_field.type == field_types::STRING_ARRAY)) {
+ for(const std::string & field_name: search_fields) {
+ // should not pick excluded field for highlighting
+ if(exclude_fields.count(field_name) > 0) {
+ continue;
+ }
+
+ field search_field = search_schema.at(field_name);
+ if(query != "*" && (search_field.type == field_types::STRING ||
+ search_field.type == field_types::STRING_ARRAY)) {
- bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());
- highlight_t highlight;
- highlight_result(search_field, searched_queries, field_order_kv, document,
- string_utils, snippet_threshold, highlighted_fully, highlight);
+ bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());
+ highlight_t highlight;
+ highlight_result(search_field, searched_queries, field_order_kv, document,
+ string_utils, snippet_threshold, highlighted_fully, highlight);
- if(!highlight.snippets.empty()) {
- highlights.push_back(highlight);
+ if(!highlight.snippets.empty()) {
+ highlights.push_back(highlight);
+ }
}
}
- }
- std::sort(highlights.begin(), highlights.end());
+ std::sort(highlights.begin(), highlights.end());
- for(const auto & highlight: highlights) {
- nlohmann::json h_json = nlohmann::json::object();
- h_json["field"] = highlight.field;
- bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());
+ for(const auto & highlight: highlights) {
+ nlohmann::json h_json = nlohmann::json::object();
+ h_json["field"] = highlight.field;
+ bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());
- if(!highlight.indices.empty()) {
- h_json["indices"] = highlight.indices;
- h_json["snippets"] = highlight.snippets;
- if(highlight_fully) {
- h_json["values"] = highlight.values;
- }
+ if(!highlight.indices.empty()) {
+ h_json["indices"] = highlight.indices;
+ h_json["snippets"] = highlight.snippets;
+ if(highlight_fully) {
+ h_json["values"] = highlight.values;
+ }
- } else {
- h_json["snippet"] = highlight.snippets[0];
- if(highlight_fully) {
- h_json["value"] = highlight.values[0];
+ } else {
+ h_json["snippet"] = highlight.snippets[0];
+ if(highlight_fully) {
+ h_json["value"] = highlight.values[0];
+ }
}
+
+ wrapper_doc["highlights"].push_back(h_json);
}
- wrapper_doc["highlights"].push_back(h_json);
+ prune_document(document, include_fields, exclude_fields);
+ wrapper_doc["document"] = document;
+ wrapper_doc["text_match"] = field_order_kv->match_score;
+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;
+ hits_array.push_back(wrapper_doc);
}
- prune_document(document, include_fields, exclude_fields);
- wrapper_doc["document"] = document;
- wrapper_doc["text_match"] = field_order_kv->match_score;
- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;
-
- result["hits"].push_back(wrapper_doc);
+ if(group_limit > 1) {
+ result["grouped_hits"].push_back(group_hits_array);
+ }
}
result["facet_counts"] = nlohmann::json::array();
@@ -923,6 +980,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
delete index->search_params;
}
+ delete aggr_topster;
+
result["request_params"] = nlohmann::json::object();;
result["request_params"]["per_page"] = per_page;
result["request_params"]["q"] = query;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 56efeec7..c987443e 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -235,6 +235,9 @@ bool get_search(http_req & req, http_res & res) {
const char *FACET_QUERY = "facet_query";
const char *MAX_FACET_VALUES = "max_facet_values";
+ const char *GROUP_BY = "group_by";
+ const char *GROUP_LIMIT = "group_limit";
+
const char *PER_PAGE = "per_page";
const char *PAGE = "page";
const char *CALLBACK = "callback";
@@ -314,6 +317,18 @@ bool get_search(http_req & req, http_res & res) {
req.params[EXCLUDE_FIELDS] = "";
}
+ if(req.params.count(GROUP_BY) == 0) {
+ req.params[GROUP_BY] = "";
+ }
+
+ if(req.params.count(GROUP_LIMIT) == 0) {
+ if(req.params[GROUP_BY] != "") {
+ req.params[GROUP_LIMIT] = "3";
+ } else {
+ req.params[GROUP_LIMIT] = "0";
+ }
+ }
+
if(!StringUtils::is_uint64_t(req.params[DROP_TOKENS_THRESHOLD])) {
res.set_400("Parameter `" + std::string(DROP_TOKENS_THRESHOLD) + "` must be an unsigned integer.");
return false;
@@ -349,6 +364,11 @@ bool get_search(http_req & req, http_res & res) {
return false;
}
+ if(!StringUtils::is_uint64_t(req.params[GROUP_LIMIT])) {
+ res.set_400("Parameter `" + std::string(GROUP_LIMIT) + "` must be an unsigned integer.");
+ return false;
+ }
+
std::string filter_str = req.params.count(FILTER) != 0 ? req.params[FILTER] : "";
std::vector<std::string> search_fields;
@@ -366,6 +386,9 @@ bool get_search(http_req & req, http_res & res) {
spp::sparse_hash_set<std::string> include_fields(include_fields_vec.begin(), include_fields_vec.end());
spp::sparse_hash_set<std::string> exclude_fields(exclude_fields_vec.begin(), exclude_fields_vec.end());
+ std::vector<std::string> group_by_fields;
+ StringUtils::split(req.params[GROUP_BY], group_by_fields, ",");
+
std::vector<sort_by> sort_fields;
if(req.params.count(SORT_BY) != 0) {
std::vector<std::string> sort_field_strs;
@@ -455,7 +478,9 @@ bool get_search(http_req & req, http_res & res) {
req.params[HIGHLIGHT_FULL_FIELDS],
typo_tokens_threshold,
pinned_hits,
- hidden_hits
+ hidden_hits,
+ group_by_fields,
+ static_cast<size_t>(std::stoi(req.params[GROUP_LIMIT]))
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/src/index.cpp b/src/index.cpp
index 1ff8f5bb..a93509d7 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1162,12 +1162,10 @@ void Index::search(Option<uint32_t> & outcome,
const bool prefix, const size_t drop_tokens_threshold,
size_t & all_result_ids_len,
std::vector<std::vector<art_leaf*>> & searched_queries,
- std::vector<KV*> & raw_result_kvs,
+ std::vector<std::vector<KV*>> & raw_result_kvs,
std::vector<KV*> & override_result_kvs,
const size_t typo_tokens_threshold) {
- const size_t num_results = (page * per_page);
-
// process the filters
uint32_t* filter_ids = nullptr;
@@ -1221,17 +1219,33 @@ void Index::search(Option<uint32_t> & outcome,
std::vector<uint32_t> dropped_ids;
// loop through topster and remove elements from included and excluded id lists
- for(uint32_t t = 0; t < topster->size && t < num_results; t++) {
- KV* kv = topster->getKV(t);
- if(ids_to_remove.count(kv->key) != 0) {
- dropped_ids.push_back((uint32_t)kv->key);
- } else {
- raw_result_kvs.push_back(kv);
+ if(topster->distinct) {
+ for(auto &group_topster_entry: topster->group_kv_map) {
+ Topster* group_topster = group_topster_entry.second;
+ for(uint32_t t = 0; t < group_topster->size; t++) {
+ KV* kv = group_topster->getKV(t);
+ if(ids_to_remove.count(kv->key) != 0) {
+ dropped_ids.push_back((uint32_t)kv->key);
+ }
+ }
+
+ const std::vector<KV*> group_kvs(group_topster->kvs, group_topster->kvs+group_topster->size);
+ raw_result_kvs.emplace_back(group_kvs);
+ }
+ } else {
+ for(uint32_t t = 0; t < topster->size; t++) {
+ KV* kv = topster->getKV(t);
+
+ if(ids_to_remove.count(kv->key) != 0) {
+ dropped_ids.push_back((uint32_t)kv->key);
+ } else {
+ raw_result_kvs.push_back({kv});
+ }
}
}
- for(uint32_t t = 0; t < curated_topster->size && t < num_results; t++) {
+ for(uint32_t t = 0; t < curated_topster->size; t++) {
KV* kv = curated_topster->getKV(t);
override_result_kvs.push_back(kv);
}
@@ -1471,6 +1485,16 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
Match single_token_match = Match(1, 0, 0, empty_offset_diffs);
const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);
+ std::unordered_map<std::string, size_t> facet_to_id;
+
+ if(search_params->group_limit > 0) {
+ size_t i_facet = 0;
+ for(const auto & facet: facet_schema) {
+ facet_to_id[facet.first] = i_facet;
+ i_facet++;
+ }
+ }
+
for(size_t i=0; i<result_size; i++) {
const uint32_t seq_id = result_ids[i];
@@ -1545,7 +1569,27 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
}
- KV kv(field_id, query_index, seq_id, seq_id, match_score, scores);
+ uint64_t distinct_id = seq_id;
+
+ if(search_params->group_limit != 0) {
+ distinct_id = 1; // some constant initial value
+
+ // calculate hash from group_by_fields
+ for(const auto& field: search_params->group_by_fields) {
+ if(facet_to_id.count(field) == 0 || facet_index_v2.count(seq_id) == 0) {
+ continue;
+ }
+
+ size_t facet_id = facet_to_id[field];
+ const std::vector<uint64_t>& fhashes = facet_index_v2.at(seq_id)[facet_id];
+
+ for(const auto& hash: fhashes) {
+ distinct_id = hash_combine(distinct_id, hash);
+ }
+ }
+ }
+
+ KV kv(field_id, query_index, seq_id, distinct_id, match_score, scores);
topster->add(&kv);
}