Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Range faceting (#823)

Krunal Gandhi2e8ad0f7c3eb
diff --git a/include/collection.h b/include/collection.h
index 75ad74da..72cdd76c 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -431,6 +431,8 @@ public:
bool get_enable_nested_fields();
+ Option<bool> parse_facet(const std::string& facet_field, std::vector<facet>& facets) const;
+
// Override operations
Option<uint32_t> add_override(const override_t & override);
diff --git a/include/field.h b/include/field.h
index b6057e1b..c3d284ab 100644
--- a/include/field.h
+++ b/include/field.h
@@ -677,8 +677,34 @@ struct facet {
facet_stats_t stats;
- explicit facet(const std::string& field_name): field_name(field_name) {
+ //dictionary of key=>pair(range_id, range_val)
+ std::map<int64_t, std::string> facet_range_map;
+ bool is_range_query;
+
+ bool get_range(int64_t key, std::pair<int64_t, std::string>& range_pair)
+ {
+ if(facet_range_map.empty())
+ {
+ LOG (ERROR) << "Facet range is not defined!!!";
+ }
+ auto it = facet_range_map.lower_bound(key);
+
+ if(it != facet_range_map.end())
+ {
+ range_pair.first = it->first;
+ range_pair.second = it->second;
+ return true;
+ }
+
+ return false;
+ }
+
+ explicit facet(const std::string& field_name,
+ std::map<int64_t, std::string> facet_range = {}, bool is_range_q = false)
+ :field_name(field_name){
+ facet_range_map = facet_range;
+ is_range_query = is_range_q;
}
};
diff --git a/include/string_utils.h b/include/string_utils.h
index 3472b582..51a587fe 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -21,6 +21,57 @@ struct StringUtils {
~StringUtils();
+ static size_t split_facet(const std::string& s, std::vector<std::string> & result,
+ const bool keep_empty = false, const size_t start_index = 0,
+ const size_t max_values = (std::numeric_limits<size_t>::max()-1)) {
+
+
+ std::string::const_iterator substart = s.begin()+start_index, subend;
+ size_t end_index = start_index;
+ std::string delim(""), temp("");
+ std::string current_str=s;
+ while (true) {
+ auto range_pos = current_str.find("(");
+ auto normal_pos = current_str.find(",");
+
+ if(range_pos == std::string::npos && normal_pos == std::string::npos){
+ if(!current_str.empty()){
+ result.push_back(trim(current_str));
+ }
+ break;
+ }
+ else if(range_pos < normal_pos){
+ delim="),";
+ subend = std::search(substart, s.end(), delim.begin(), delim.end());
+ temp = std::string(substart, subend + 1);
+ }
+ else{
+ delim=",";
+ subend = std::search(substart, s.end(), delim.begin(), delim.end());
+ temp = std::string(substart, subend);
+ }
+
+ end_index += temp.size() + delim.size();
+ temp = trim(temp);
+
+ if (keep_empty || !temp.empty()) {
+ result.push_back(temp);
+ }
+
+ if(result.size() == max_values) {
+ break;
+ }
+
+ if (subend == s.end()) {
+ break;
+ }
+ substart = subend + delim.size();
+ current_str = std::string(substart, s.end());
+ }
+
+ return std::min(end_index, s.size());
+ }
+
// Adapted from: http://stackoverflow.com/a/236180/131050
static size_t split(const std::string& s, std::vector<std::string> & result, const std::string& delim,
const bool keep_empty = false, const bool trim_space = true,
diff --git a/src/collection.cpp b/src/collection.cpp
index d9a35b81..a7a23736 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -19,6 +19,7 @@
const std::string override_t::MATCH_EXACT = "exact";
const std::string override_t::MATCH_CONTAINS = "contains";
+
struct sort_fields_guard_t {
std::vector<sort_by> sort_fields_std;
@@ -1066,12 +1067,12 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
}
// validate facet fields
- for(const std::string & field_name: facet_fields) {
- if(search_schema.count(field_name) == 0 || !search_schema.at(field_name).facet) {
- std::string error = "Could not find a facet field named `" + field_name + "` in the schema.";
- return Option<nlohmann::json>(404, error);
+ for(const std::string & facet_field: facet_fields) {
+
+ const auto& res = parse_facet(facet_field, facets);
+ if(!res.ok()){
+ return Option<nlohmann::json>(res.code(), res.error());
}
- facets.emplace_back(field_name);
}
// parse facet query
@@ -1651,52 +1652,64 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
}
result["facet_counts"] = nlohmann::json::array();
-
+
// populate facets
for(facet & a_facet: facets) {
nlohmann::json facet_result = nlohmann::json::object();
facet_result["field_name"] = a_facet.field_name;
facet_result["counts"] = nlohmann::json::array();
+ std::vector<facet_value_t> facet_values;
std::vector<std::pair<int64_t, facet_count_t>> facet_hash_counts;
+
for (const auto & kv : a_facet.result_map) {
facet_hash_counts.emplace_back(kv);
}
- auto the_field = search_schema.at(a_facet.field_name);
+ if(a_facet.is_range_query){
+ for(auto kv : a_facet.result_map){
+ auto facet_range_iter = a_facet.facet_range_map.find(kv.first);
+ if(facet_range_iter != a_facet.facet_range_map.end()){
+ auto & facet_count = kv.second;
+ facet_value_t facet_value = {facet_range_iter->second, std::string(), facet_count.count};
+ facet_values.emplace_back(facet_value);
+ }
+ else{
+ LOG (ERROR) << "range_id not found in result map.";
+ }
+ }
+ }
+
+ auto the_field = search_schema.at(a_facet.field_name);
// keep only top K facets
auto max_facets = std::min(max_facet_values, facet_hash_counts.size());
std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,
facet_hash_counts.end(), Collection::facet_count_compare);
+ for(size_t fi = 0; fi < max_facets; fi++) {
- std::vector<facet_value_t> facet_values;
+ if(a_facet.is_range_query){
+ break;
+ }
- for(size_t fi = 0; fi < max_facets; fi++) {
// remap facet value hash with actual string
auto & kv = facet_hash_counts[fi];
auto & facet_count = kv.second;
-
// fetch actual facet value from representative doc id
const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);
nlohmann::json document;
const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
-
if(!document_op.ok()) {
LOG(ERROR) << "Facet fetch error. " << document_op.error();
continue;
}
-
std::string value;
bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);
-
if(!facet_found) {
continue;
}
-
std::unordered_map<std::string, size_t> ftoken_pos;
std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];
-
for(size_t ti = 0; ti < ftokens.size(); ti++) {
if(the_field.is_bool()) {
if(ftokens[ti] == "1") {
@@ -1705,36 +1718,26 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
ftokens[ti] = "false";
}
}
-
const std::string& resolved_token = ftokens[ti];
ftoken_pos[resolved_token] = ti;
}
-
const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();
-
// 2 passes: first identify tokens that need to be highlighted and then construct highlighted text
-
bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);
bool normalise = is_cyrillic ? false : true;
-
Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);
-
// secondary tokenizer used for specific languages that requires transliteration
// we use 2 tokenizers so that the original text offsets are available for highlighting
Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);
-
std::string raw_token;
size_t raw_token_index = 0, tok_start = 0, tok_end = 0;
-
// need an ordered map here to ensure that it is ordered by the key (start offset)
std::map<size_t, size_t> token_offsets;
size_t prefix_token_start_index = 0;
-
while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
if(is_cyrillic) {
word_tokenizer.tokenize(raw_token);
}
-
auto token_pos_it = ftoken_pos.find(raw_token);
if(token_pos_it != ftoken_pos.end()) {
token_offsets[tok_start] = tok_end;
@@ -1743,47 +1746,39 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
}
}
}
-
auto offset_it = token_offsets.begin();
size_t i = 0;
std::stringstream highlightedss;
-
// loop until end index, accumulate token and complete highlighting
while(i < value.size()) {
if(offset_it != token_offsets.end()) {
if (i == offset_it->first) {
highlightedss << highlight_start_tag;
-
// do prefix highlighting for non-dropped last token
size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?
facet_query_last_token.size() :
(offset_it->second - i + 1);
-
if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {
token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());
} else {
token_len = (offset_it->second - i + 1);
}
-
for(size_t j = 0; j < token_len; j++) {
highlightedss << value[i + j];
}
-
highlightedss << highlight_end_tag;
offset_it++;
i += token_len;
continue;
}
}
-
highlightedss << value[i];
i++;
}
-
facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};
facet_values.emplace_back(facet_value);
}
-
+
std::stable_sort(facet_values.begin(), facet_values.end(), Collection::facet_count_str_compare);
for(const auto & facet_count: facet_values) {
@@ -4064,4 +4059,127 @@ std::string Collection::get_fallback_field_type() {
bool Collection::get_enable_nested_fields() {
return enable_nested_fields;
-};
\ No newline at end of file
+};
+
+Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector<facet>& facets) const{
+ const std::regex base_pattern("[a-z]+\\(.*\\)");
+ const std::regex range_pattern("[[a-zA-Z]+:\\[([0-9]+)\\, ([0-9]+)\\]");
+
+ if(facet_field.find(":") != std::string::npos){ //range based facet
+
+ if(!std::regex_match(facet_field, base_pattern)){
+ std::string error = "Range string base pattern not matched.";
+ return Option<bool>(400, error);
+ }
+
+ auto startpos = facet_field.find("(");
+ auto field_name = facet_field.substr(0, startpos);
+
+ const field& a_field = search_schema.at(field_name);
+ if(!a_field.is_int32() && !a_field.is_int64()){
+ std::string error = "Range facet is restricted to Numeric fields only.";
+ return Option<bool>(400, error);
+ }
+
+ facet a_facet(field_name);
+
+ //starting after "(" and excluding ")"
+ auto range_string = std::string(facet_field.begin() + startpos + 1, facet_field.end() - 1);
+
+ //split the ranges
+ std::vector<std::string> result;
+ startpos = 0;
+ int index=0;
+ int commaFound = 0, rangeFound = 0;
+ bool range_open=false;
+ while(index < range_string.size()){
+ if(range_string[index] == ']'){
+ if(range_open == true){
+ std::string range = range_string.substr(startpos, index + 1 - startpos);
+ range=StringUtils::trim(range);
+ result.emplace_back(range);
+ rangeFound++;
+ range_open=false;
+ }
+ else{
+ result.clear();
+ break;
+ }
+ }
+ else if(range_string[index] == ',' && range_open == false){
+ startpos = index+1;
+ commaFound++;
+ }
+ else if(range_string[index] == '['){
+ if((commaFound == rangeFound) && range_open==false){
+ range_open=true;
+ }
+ else{
+ result.clear();
+ break;
+ }
+ }
+
+ index++;
+ }
+
+ if((result.empty()) || (range_open==true)){
+ std::string error = "Error splitting the range string.";
+ return Option<bool>(400, error);
+ }
+
+ std::vector<std::tuple<int64_t, int64_t, std::string>> tupVec;
+
+ auto& range_map = a_facet.facet_range_map;
+ for(const auto& range : result){
+ //validate each range syntax
+ if(!std::regex_match(range, range_pattern)){
+ std::string error = "Range String range pattern not matched.";
+ return Option<bool>(400, error);
+ }
+
+ auto pos1 = range.find(":");
+ std::string range_val = range.substr(0, pos1);
+
+ auto pos2 = range.find(",");
+ auto pos3 = range.find("]");
+
+ int64_t lower_range = std::stoll(range.substr(pos1 + 2, pos2));
+ int64_t upper_range = std::stoll(range.substr(pos2 + 1, pos3));
+
+ tupVec.emplace_back(std::make_tuple(lower_range, upper_range, range_val));
+ }
+
+ //sort the range values so that we can check continuity
+ sort(tupVec.begin(), tupVec.end());
+
+ for(const auto& tup : tupVec){
+
+ int64_t lower_range = std::get<0>(tup);
+ int64_t upper_range = std::get<1>(tup);
+ std::string range_val = std::get<2>(tup);
+ //check if ranges are continous or not
+ if((!range_map.empty()) && (range_map.find(lower_range)== range_map.end())){
+ std::string error = "Ranges in range facet syntax should be continous.";
+ return Option<bool>(400, error);
+ }
+
+ range_map[upper_range] = range_val;
+ }
+
+ a_facet.is_range_query = true;
+
+ facets.emplace_back(std::move(a_facet));
+ }
+ else{//normal facet
+ facets.emplace_back(facet(facet_field));
+ }
+
+ if(search_schema.count(facets.back().field_name) == 0 || !search_schema.at(facets.back().field_name).facet) {
+ std::string error = "Could not find a facet field named `" + facets.back().field_name + "` in the schema.";
+ facets.pop_back();
+ return Option<bool>(400, error);
+ }
+
+ return Option<bool>(true);
+}
\ No newline at end of file
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 742ea395..2633d39e 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -880,7 +880,13 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
auto find_str_list_it = str_list_values.find(key);
if(find_str_list_it != str_list_values.end()) {
- StringUtils::split(val, *find_str_list_it->second, ",");
+
+ if(key == FACET_BY){
+ StringUtils::split_facet(val, *find_str_list_it->second);
+ }
+ else{
+ StringUtils::split(val, *find_str_list_it->second, ",");
+ }
continue;
}
diff --git a/src/index.cpp b/src/index.cpp
index 99ae695d..f5f7a2d1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1265,7 +1265,28 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
compute_facet_stats(a_facet, fhash, facet_field.type);
}
- if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
+ if(a_facet.is_range_query){
+ auto sort_index_it = sort_index.find(a_facet.field_name);
+
+ if(sort_index_it != sort_index.end()){
+ auto doc_id_val_map = sort_index_it->second;
+ auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);
+
+ if(doc_seq_id_it != doc_id_val_map->end()){
+
+ int64_t doc_val = doc_seq_id_it->second;
+ std::pair<int64_t, std::string> range_pair {};
+ if(a_facet.get_range(doc_val, range_pair))
+ {
+ int64_t range_id = range_pair.first;
+
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count += 1;
+ }
+ }
+ }
+ }
+ else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
facet_count_t& facet_count = a_facet.result_map[fhash];
//LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
@@ -2778,7 +2799,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::vector<std::vector<facet>> facet_batches(num_threads);
for(size_t i = 0; i < num_threads; i++) {
for(const auto& this_facet: facets) {
- facet_batches[i].emplace_back(facet(this_facet.field_name));
+ facet_batches[i].emplace_back(facet(this_facet.field_name, this_facet.facet_range_map, this_facet.is_range_query));
}
}
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index d5b07c1c..688a0512 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -979,3 +979,68 @@ TEST_F(CollectionFacetingTest, FacetByNestedIntField) {
ASSERT_EQ(2, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
ASSERT_EQ("2000", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
}
+
+TEST_F(CollectionFacetingTest, FacetParseTest){
+
+ std::vector<field> fields = {
+ field("score", field_types::INT32, true),
+ field("grade", field_types::INT32, true),
+ field("rank", field_types::INT32, true),
+ };
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ std::vector<std::string> range_facet_fields {
+ "score(fail:[0, 40], pass:[40, 100])",
+ "grade(A:[80, 100], B:[60, 80], C:[40, 60])"
+ };
+ std::vector<facet> range_facets;
+ for(const std::string & facet_field: range_facet_fields) {
+ coll1->parse_facet(facet_field, range_facets);
+ }
+ ASSERT_EQ(2, range_facets.size());
+
+ ASSERT_STREQ("score", range_facets[0].field_name.c_str());
+ ASSERT_TRUE(range_facets[0].is_range_query);
+ ASSERT_GT(range_facets[0].facet_range_map.size(), 0);
+
+ ASSERT_STREQ("grade", range_facets[1].field_name.c_str());
+ ASSERT_TRUE(range_facets[1].is_range_query);
+ ASSERT_GT(range_facets[1].facet_range_map.size(), 0);
+
+
+
+ std::vector<std::string> normal_facet_fields {
+ "score",
+ "grade"
+ };
+ std::vector<facet> normal_facets;
+ for(const std::string & facet_field: normal_facet_fields) {
+ coll1->parse_facet(facet_field, normal_facets);
+ }
+ ASSERT_EQ(2, normal_facets.size());
+
+ ASSERT_STREQ("score", normal_facets[0].field_name.c_str());
+ ASSERT_STREQ("grade", normal_facets[1].field_name.c_str());
+
+
+
+ std::vector<std::string> mixed_facet_fields {
+ "score",
+ "grade(A:[80, 100], B:[60, 80], C:[40, 60])",
+ "rank"
+ };
+ std::vector<facet> mixed_facets;
+ for(const std::string & facet_field: mixed_facet_fields) {
+ coll1->parse_facet(facet_field, mixed_facets);
+ }
+ ASSERT_EQ(3, mixed_facets.size());
+
+ ASSERT_STREQ("score", mixed_facets[0].field_name.c_str());
+
+ ASSERT_STREQ("grade", mixed_facets[1].field_name.c_str());
+ ASSERT_TRUE(mixed_facets[1].is_range_query);
+ ASSERT_GT(mixed_facets[1].facet_range_map.size(), 0);
+
+ ASSERT_STREQ("rank", mixed_facets[2].field_name.c_str());
+}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 8ad0cbc0..95ae5039 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -2919,3 +2919,238 @@ TEST_F(CollectionSpecificTest, DontHighlightPunctuation) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSpecificTest, RangeFacetTest) {
+ std::vector<field> fields = {field("place", field_types::STRING, false),
+ field("state", field_types::STRING, false),
+ field("visitors", field_types::INT32, true),};
+ Collection* coll1 = collectionManager.create_collection(
+ "coll1", 1, fields, "", 0, "", {}, {}
+ ).get();
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["place"] = "Mysore Palace";
+ doc1["state"] = "Karnataka";
+ doc1["visitors"] = 235486;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["place"] = "Hampi";
+ doc2["state"] = "Karnataka";
+ doc2["visitors"] = 187654;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["place"] = "Mahabalipuram";
+ doc3["state"] = "TamilNadu";
+ doc3["visitors"] = 174684;
+
+ nlohmann::json doc4;
+ doc4["id"] = "3";
+ doc4["place"] = "Meenakshi Amman Temple";
+ doc4["state"] = "TamilNadu";
+ doc4["visitors"] = 246676;
+
+ nlohmann::json doc5;
+ doc5["id"] = "4";
+ doc5["place"] = "Staue of Unity";
+ doc5["state"] = "Gujarat";
+ doc5["visitors"] = 345878;
+
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());
+
+ auto results = coll1->search("Karnataka", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true).get();
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_STREQ("Busy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+
+ auto results2 = coll1->search("Gujarat", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true).get();
+ ASSERT_EQ(1, results2["facet_counts"][0]["counts"].size());
+ ASSERT_EQ(1, results2["facet_counts"][0]["counts"][0]["count"].get<std::size_t>());
+ ASSERT_STREQ("VeryBusy", results2["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_TRUE(results2["facet_counts"][0]["counts"][1]["value"] == nullptr);
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificTest, RangeFacetContinuity) {
+ std::vector<field> fields = {field("place", field_types::STRING, false),
+ field("state", field_types::STRING, false),
+ field("visitors", field_types::INT32, true),};
+ Collection* coll1 = collectionManager.create_collection(
+ "coll1", 1, fields, "", 0, "", {}, {}
+ ).get();
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["place"] = "Mysore Palace";
+ doc1["state"] = "Karnataka";
+ doc1["visitors"] = 235486;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["place"] = "Hampi";
+ doc2["state"] = "Karnataka";
+ doc2["visitors"] = 187654;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["place"] = "Mahabalipuram";
+ doc3["state"] = "TamilNadu";
+ doc3["visitors"] = 174684;
+
+ nlohmann::json doc4;
+ doc4["id"] = "3";
+ doc4["place"] = "Meenakshi Amman Temple";
+ doc4["state"] = "TamilNadu";
+ doc4["visitors"] = 246676;
+
+ nlohmann::json doc5;
+ doc5["id"] = "4";
+ doc5["place"] = "Staue of Unity";
+ doc5["state"] = "Gujarat";
+ doc5["visitors"] = 345878;
+
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());
+
+ auto results = coll1->search("TamilNadu", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200001, 500000])"},
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ ASSERT_STREQ("Ranges in range facet syntax should be continous.", results.error().c_str());
+
+ auto results2 = coll1->search("TamilNadu", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:[199999, 500000])"},
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ ASSERT_STREQ("Ranges in range facet syntax should be continous.", results2.error().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificTest, RangeFacetTypo) {
+ std::vector<field> fields = {field("place", field_types::STRING, false),
+ field("state", field_types::STRING, false),
+ field("visitors", field_types::INT32, true),};
+ Collection* coll1 = collectionManager.create_collection(
+ "coll1", 1, fields, "", 0, "", {}, {}
+ ).get();
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["place"] = "Mysore Palace";
+ doc1["state"] = "Karnataka";
+ doc1["visitors"] = 235486;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["place"] = "Hampi";
+ doc2["state"] = "Karnataka";
+ doc2["visitors"] = 187654;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["place"] = "Mahabalipuram";
+ doc3["state"] = "TamilNadu";
+ doc3["visitors"] = 174684;
+
+ nlohmann::json doc4;
+ doc4["id"] = "3";
+ doc4["place"] = "Meenakshi Amman Temple";
+ doc4["state"] = "TamilNadu";
+ doc4["visitors"] = 246676;
+
+ nlohmann::json doc5;
+ doc5["id"] = "4";
+ doc5["place"] = "Staue of Unity";
+ doc5["state"] = "Gujarat";
+ doc5["visitors"] = 345878;
+
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());
+
+ auto results = coll1->search("TamilNadu", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000)"}, //missing ']' at end
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ ASSERT_STREQ("Error splitting the range string.", results.error().c_str());
+
+ auto results2 = coll1->search("TamilNadu", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:200000, 500000])"}, //missing '[' in second range
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ ASSERT_STREQ("Error splitting the range string.", results2.error().c_str());
+
+ auto results3 = coll1->search("TamilNadu", {"state"},
+ "", {"visitors(Busy:[0, 200000] VeryBusy:[200000, 500000])"}, //missing ',' between ranges
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ ASSERT_STREQ("Error splitting the range string.", results3.error().c_str());
+
+ auto results4 = coll1->search("TamilNadu", {"state"},
+ "", {"visitors(Busy:[0 200000], VeryBusy:[200000, 500000])"}, //missing ',' between first ranges values
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ ASSERT_STREQ("Range String range pattern not matched.", results4.error().c_str());
+
+ auto results5 = coll1->search("TamilNadu", {"state"},
+ "", {"visitors(Busy:[0, 200000 VeryBusy:200000, 500000])"}, //missing '],' and '['
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ ASSERT_STREQ("Range String range pattern not matched.", results5.error().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
\ No newline at end of file
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index ae5d7617..1d2e0246 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -318,6 +318,35 @@ TEST(StringUtilsTest, ContainsWord) {
ASSERT_FALSE(StringUtils::contains_word("baz foobar", "foo"));
}
+TEST(StringUtilsTest, ShouldSplitRangeFacet){
+ std::string range_facets_string = "score(fail:[0, 40], pass:[40, 100]), grade(A:[80,100], B:[60, 80], C:[40, 60])";
+ std::vector<std::string> range_facets;
+ StringUtils::split_facet(range_facets_string, range_facets);
+ ASSERT_STREQ("score(fail:[0, 40], pass:[40, 100])", range_facets[0].c_str());
+ ASSERT_STREQ("grade(A:[80,100], B:[60, 80], C:[40, 60])", range_facets[1].c_str());
+
+
+ std::string facets_string = "score, grade";
+ std::vector<std::string> facets;
+ StringUtils::split_facet(facets_string, facets);
+ ASSERT_STREQ("score", facets[0].c_str());
+ ASSERT_STREQ("grade", facets[1].c_str());
+
+
+ std::string mixed_facets_string = "score, grade(A:[80,100], B:[60, 80], C:[40, 60]), rank";
+ std::vector<std::string> mixed_facets;
+ StringUtils::split_facet(mixed_facets_string, mixed_facets);
+ ASSERT_STREQ("score", mixed_facets[0].c_str());
+ ASSERT_STREQ("grade(A:[80,100], B:[60, 80], C:[40, 60])", mixed_facets[1].c_str());
+ ASSERT_STREQ("rank", mixed_facets[2].c_str());
+
+
+ // empty string should produce empty list
+ std::vector<std::string> lines_empty;
+ StringUtils::split_facet("", lines_empty);
+ ASSERT_TRUE(lines_empty.empty());
+}
+
void tokenizeTestHelper(const std::string& filter_query, const std::vector<std::string>& tokenList) {
std::queue<std::string> tokenizeOutput;
auto tokenize_op = StringUtils::tokenize_filter_query(filter_query, tokenizeOutput);