Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

sort facet by other field for nested objects

krunale70e8d070a7b
diff --git a/include/collection.h b/include/collection.h
index 95c4b235..c79ee618 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -477,7 +477,8 @@ public:
const size_t remote_embedding_timeout_ms = 30000,
const size_t remote_embedding_num_try = 2,
const std::string& stopwords_set="",
- const std::vector<std::string>& facet_return_parent = {}) const;
+ const std::vector<std::string>& facet_return_parent = {},
+ const facet_sort_by& facet_sort_params = {}) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
@@ -505,6 +506,8 @@ public:
Option<bool> parse_facet(const std::string& facet_field, std::vector<facet>& facets) const;
+ Option<bool> validate_facet_sort_by_field(const facet_sort_by& facet_sort_params) const;
+
// Override operations
Option<uint32_t> add_override(const override_t & override, bool write_to_store = true);
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 99de5a8c..370b565f 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -184,6 +184,8 @@ public:
static bool parse_sort_by_str(std::string sort_by_str, std::vector<sort_by>& sort_fields);
+ static bool parse_facet_sort_by_str(std::string facet_sort_by_str, facet_sort_by& facet_sort_params);
+
// symlinks
Option<std::string> resolve_symlink(const std::string & symlink_name) const;
diff --git a/include/field.h b/include/field.h
index 7160762e..f7ad535b 100644
--- a/include/field.h
+++ b/include/field.h
@@ -539,6 +539,19 @@ struct sort_by {
}
};
+struct facet_sort_by {
+ std::string param;
+ std::string order;
+
+ facet_sort_by() {
+ param.clear();
+ order.clear();
+ }
+
+ facet_sort_by(const std::string& _param, const std::string& _order)
+ :param(_param), order(_order) {}
+};
+
class GeoPoint {
constexpr static const double EARTH_RADIUS = 3958.75;
constexpr static const double METER_CONVERT = 1609.00;
@@ -572,6 +585,7 @@ struct facet_count_t {
// used to fetch the actual document and value for representation
uint32_t doc_id = 0;
uint32_t array_pos = 0;
+ float sort_field_val = 0.0f;
};
struct facet_stats_t {
@@ -646,6 +660,7 @@ struct facet_value_t {
std::string value;
std::string highlighted;
uint32_t count;
+ float sort_field_val = 0.0f;
};
struct facet_hash_values_t {
diff --git a/src/collection.cpp b/src/collection.cpp
index 26d93944..09d91446 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1124,7 +1124,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const size_t remote_embedding_timeout_ms,
const size_t remote_embedding_num_try,
const std::string& stopwords_set,
- const std::vector<std::string>& facet_return_parent) const {
+ const std::vector<std::string>& facet_return_parent,
+ const facet_sort_by& facet_sort_param) const {
std::shared_lock lock(mutex);
@@ -1421,6 +1422,14 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
}
+ //validate facet_sort_by fields
+ if(!facet_sort_param.param.empty()) {
+ const auto &res = validate_facet_sort_by_field(facet_sort_param);
+ if (!res.ok()) {
+ return Option<nlohmann::json>(res.code(), res.error());
+ }
+ }
+
if(per_page > PER_PAGE_MAX) {
std::string message = "Only upto " + std::to_string(PER_PAGE_MAX) + " hits can be fetched per page.";
return Option<nlohmann::json>(422, message);
@@ -2067,6 +2076,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
bool should_return_parent = std::find(facet_return_parent.begin(), facet_return_parent.end(),
the_field.name) != facet_return_parent.end();
+ if(!facet_sort_param.param.empty()) {
+ max_facets = facet_counts.size();
+ }
+
for(size_t fi = 0; fi < max_facets; fi++) {
// remap facet value hash with actual string
auto & kv = facet_counts[fi];
@@ -2091,6 +2104,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
if(!facet_found) {
continue;
}
+ if(!facet_sort_param.param.empty()) {
+ facet_count.sort_field_val = document[facet_sort_param.param].get<float>();
+ }
}
if(the_field.nested && should_return_parent) {
@@ -2166,12 +2182,27 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
highlightedss << value[i];
i++;
}
- facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};
+ facet_value_t facet_value = {value, highlightedss.str(), facet_count.count, facet_count.sort_field_val};
facet_values.emplace_back(facet_value);
}
}
-
- std::stable_sort(facet_values.begin(), facet_values.end(), Collection::facet_count_str_compare);
+
+ if(!facet_sort_param.param.empty()) {
+ bool is_asc = facet_sort_param.order == "asc";
+ std::stable_sort(facet_values.begin(), facet_values.end(), [&](const auto& fv1, const auto& fv2) {
+ if(is_asc) {
+ return fv1.sort_field_val < fv2.sort_field_val;
+ }
+ return fv1.sort_field_val > fv2.sort_field_val;
+ });
+
+ if(facet_values.size() > max_facet_values) {
+ facet_values.erase(facet_values.begin() + max_facet_values, facet_values.end());
+ }
+
+ } else {
+ std::stable_sort(facet_values.begin(), facet_values.end(), Collection::facet_count_str_compare);
+ }
for(const auto & facet_count: facet_values) {
nlohmann::json facet_value_count = nlohmann::json::object();
@@ -4974,6 +5005,31 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector
return Option<bool>(true);
}
+Option<bool> Collection::validate_facet_sort_by_field(const facet_sort_by& facet_sort_params) const {
+ if (search_schema.count(facet_sort_params.param) == 0) {
+ std::string error = "Could not find a facet field named `" + facet_sort_params.param + "` in the schema.";
+ return Option<bool>(404, error);
+ }
+
+ const field &a_field = search_schema.at(facet_sort_params.param);
+ if (!a_field.nested) {
+ std::string error = "Field for `facet_sort_by` should be nested from same facet field";
+ return Option<bool>(400, error);
+ }
+
+ if(a_field.is_string()) {
+ std::string error = "Field for `facet_sort_by` should not be string";
+ return Option<bool>(400, error);
+ }
+
+ if ((facet_sort_params.order != "asc") && (facet_sort_params.order != "desc")) {
+ std::string error = "Sort order for field `facet_sort_by` should be asc/desc";
+ return Option<bool>(400, error);
+ }
+
+ return Option<bool>(true);
+}
+
Option<bool> Collection::populate_include_exclude_fields(const spp::sparse_hash_set<std::string>& include_fields,
const spp::sparse_hash_set<std::string>& exclude_fields,
tsl::htrie_set<char>& include_fields_full,
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 58eac68c..449387ce 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -662,6 +662,24 @@ bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<s
return true;
}
+bool CollectionManager::parse_facet_sort_by_str(std::string facet_sort_by_str, facet_sort_by& facet_sort_params) {
+ std::vector<std::string> params;
+ StringUtils::split(facet_sort_by_str, params, ":");
+
+ if(params.size() != 2) { //can have only 2 params seperated by colon
+ return false;
+ }
+
+ for(auto& param : params) {
+ std::transform(param.begin(), param.end(), param.begin(), ::tolower);
+ }
+
+ facet_sort_params.param = params[0];
+ facet_sort_params.order = params[1];
+
+ return true;
+}
+
Option<bool> add_unsigned_int_param(const std::string& param_name, const std::string& str_val, size_t* int_val) {
if(!StringUtils::is_uint32_t(str_val)) {
return Option<bool>(400, "Parameter `" + std::string(param_name) + "` must be an unsigned integer.");
@@ -806,6 +824,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *MAX_FACET_VALUES = "max_facet_values";
const char *FACET_RETURN_PARENT = "facet_return_parent";
+ const char* FACET_SORT_BY = "facet_sort_by";
const char *VECTOR_QUERY = "vector_query";
@@ -937,6 +956,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::vector<std::string> facet_return_parent;
+ facet_sort_by facet_sort_params;
+
std::string vector_query;
std::vector<std::string> include_fields_vec;
@@ -1148,6 +1169,15 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
return Option<bool>(400, "Only upto 3 sort fields are allowed.");
}
+ if(!req_params[FACET_SORT_BY].empty()) {
+ bool parse_validate_facet_sort_by = parse_facet_sort_by_str(req_params[FACET_SORT_BY],
+ facet_sort_params);
+
+ if(!parse_validate_facet_sort_by) {
+ return Option<bool>(400,std::string("Parameter `") + FACET_SORT_BY + "` is malformed.");
+ }
+ }
+
if(req_params.count(INFIX) != 0) {
std::vector<std::string> infix_strs;
StringUtils::split(req_params[INFIX], infix_strs, ",");
@@ -1224,7 +1254,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
remote_embedding_timeout_ms,
remote_embedding_num_try,
stopwords_set,
- facet_return_parent
+ facet_return_parent,
+ facet_sort_params
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 85ace93a..3e104c92 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -1875,4 +1875,121 @@ TEST_F(CollectionFacetingTest, FacetingReturnParentObject) {
ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["value"]);
ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetSortByOtherFieldVal) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "car", "type": "object", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "car": {
+ "speed": 130,
+ "brand": "Tata",
+ "type": "Hatchback"
+ }
+ })"_json;
+
+ nlohmann::json doc2 = R"({
+ "car": {
+ "speed": 140,
+ "brand": "Hyundai",
+ "type": "Sedan"
+ }
+ })"_json;
+
+ nlohmann::json doc3 = R"({
+ "car": {
+ "speed": 120,
+ "brand": "Maruti",
+ "type": "Sedan"
+ }
+ })"_json;
+
+ nlohmann::json doc4 = R"({
+ "car": {
+ "speed": 135,
+ "brand": "Toyota",
+ "type": "Hatchback"
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc3.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc4.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ //sort cars brands by top speed in asc order
+ facet_sort_by facet_sort_param{"car.speed", "asc"};
+ auto search_op = coll1->search("*", {},"", {"car.brand"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {}, facet_sort_param);
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(4, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Maruti", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("Tata", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("Toyota", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("Hyundai", results["facet_counts"][0]["counts"][3]["value"]);
+
+ //sort cars brands by top speed in desc order
+ facet_sort_param.param = "car.speed";
+ facet_sort_param.order = "desc";
+
+ search_op = coll1->search("*", {},"", {"car.brand"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {}, facet_sort_param);
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+
+ results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(4, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Hyundai", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("Toyota", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("Tata", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("Maruti", results["facet_counts"][0]["counts"][3]["value"]);
}
\ No newline at end of file