Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow a string facet field to be filtered verbatim.

kishorencd06552eeba67
diff --git a/src/collection.cpp b/src/collection.cpp
index 121c53eb..6767e9f8 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -623,7 +623,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
f = {field_name, filter_values, EQUALS};
} else {
if(raw_value != "true" && raw_value != "false") {
- return Option<nlohmann::json>(400, "Value of filter field `" + _field.name + "`: must be `true` or `false`.");
+ return Option<nlohmann::json>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
}
std::string bool_value = (raw_value == "true") ? "1" : "0";
f = {field_name, {bool_value}, EQUALS};
@@ -631,20 +631,26 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
} else if(_field.is_string()) {
size_t filter_value_index = 0;
- NUM_COMPARATOR str_comparator = EQUALS;
+ NUM_COMPARATOR str_comparator = CONTAINS;
- if(raw_value[0] == '~') {
- // string filter should be evaluated in "contains" mode
- str_comparator = CONTAINS;
+ if(raw_value[0] == '=') {
+ if(!_field.facet) {
+ // EQUALS filtering on string is possible only on facet fields
+ return Option<nlohmann::json>(400, "To perform exact filtering, filter field `" +
+ _field.name + "` must be a facet field.");
+ }
+
+ // string filter should be evaluated in strict "equals" mode
+ str_comparator = EQUALS;
while(raw_value[++filter_value_index] == ' ');
}
if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ StringUtils::split(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values, ",");
f = {field_name, filter_values, str_comparator};
} else {
- f = {field_name, {raw_value}, str_comparator};
+ f = {field_name, {raw_value.substr(filter_value_index)}, str_comparator};
}
} else {
return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Unidentified field type.");
diff --git a/src/index.cpp b/src/index.cpp
index 94c380e8..8a0821b2 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -475,7 +475,7 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &token
float f = std::stof(token);
reinterpret_cast<float&>(hash) = f; // store as int without loss of precision
} else if(a_field.is_integer() || a_field.is_bool()) {
- hash = atol(token.c_str());
+ hash = atoll(token.c_str());
} else {
// string field
hash = StringUtils::hash_wy(token.c_str(), token.size());
@@ -983,24 +983,63 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
}
}
- if(a_filter.compare_operator == EQUALS) {
- // need to do exact match (unlike CONTAINS) by iterating the result ids and calculating scores
- if(query_suggestion.size() == str_tokens.size()) {
- uint32_t *exact_strt_ids = new uint32_t[strt_ids_size];
- size_t exact_strt_size = 0;
+ if(a_filter.compare_operator == EQUALS && f.is_facet()) {
+ // need to do exact match (unlike CONTAINS) by using the facet index
+ // field being a facet is already enforced upstream
+ uint32_t* exact_strt_ids = new uint32_t[strt_ids_size];
+ size_t exact_strt_size = 0;
+
+ size_t facet_id = facet_to_index[f.name];
+ for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {
+ uint32_t seq_id = strt_ids[strt_ids_index];
+ const std::vector<uint64_t> &fvalues = facet_index_v2[seq_id][facet_id];
+ bool found_filter = false;
+
+ if(!f.is_array()) {
+ found_filter = (str_tokens.size() == fvalues.size());
+ } else {
+ uint64_t filter_hash = 1;
+
+ for(size_t sindex=0; sindex < str_tokens.size(); sindex++) {
+ auto& str_token = str_tokens[sindex];
+ uint64_t thash = facet_token_hash(f, str_token);
+ filter_hash *= (1779033703 + 2*thash*(sindex+1));
+ }
- eq_str_filter_plain(strt_ids, strt_ids_size, query_suggestion, exact_strt_ids, exact_strt_size);
+ uint64_t all_fvalue_hash = 1;
+ size_t ftindex = 0;
+
+ for(size_t findex=0; findex < fvalues.size(); findex++) {
+ auto fhash = fvalues[findex];
+ if(fhash == FACET_ARRAY_DELIMETER) {
+ // end of array, check hash
+ if(all_fvalue_hash == filter_hash) {
+ found_filter = true;
+ break;
+ }
+ all_fvalue_hash = 1;
+ ftindex = 0;
+ } else {
+ all_fvalue_hash *= (1779033703 + 2*fhash*(ftindex + 1));
+ ftindex++;
+ }
+ }
+ }
- delete[] strt_ids;
- strt_ids = exact_strt_ids;
- strt_ids_size = exact_strt_size;
+ if(found_filter) {
+ exact_strt_ids[exact_strt_size] = seq_id;
+ exact_strt_size++;
+ }
}
- }
- // (NOT implemented) if the query is wrapped by double quotes, ensure exact match
- // bool exact_match = (filter_value.front() == '"' && filter_value.back() == '"');
+ delete[] strt_ids;
+ strt_ids = exact_strt_ids;
+ strt_ids_size = exact_strt_size;
+ }
// Otherwise, we just ensure that given record contains tokens in the filter query
+ // (NOT implemented) if the query is wrapped by double quotes, ensure phrase match
+ // bool exact_match = (filter_value.front() == '"' && filter_value.back() == '"');
uint32_t* out = nullptr;
ids_size = ArrayUtils::or_scalar(ids, ids_size, strt_ids, strt_ids_size, &out);
delete[] strt_ids;
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 7eaaa5f3..7748cbf4 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -34,6 +34,139 @@ protected:
}
};
+TEST_F(CollectionFacetingTest, FacetFieldStringFiltering) {
+ Collection *coll_str;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, true),
+ field("cast", field_types::STRING_ARRAY, false),
+ field("points", field_types::INT32, false)
+ };
+
+ std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
+
+ coll_str = collectionManager.get_collection("coll_str");
+ if(coll_str == nullptr) {
+ coll_str = collectionManager.create_collection("coll_str", 1, fields, "points").get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ nlohmann::json document = nlohmann::json::parse(json_line);
+ coll_str->add(document.dump());
+ }
+
+ infile.close();
+
+ query_fields = {"title"};
+ std::vector<std::string> facets;
+
+ // exact filter on string field must fail when single token is used
+ facets.clear();
+ facets.emplace_back("starring");
+ auto results = coll_str->search("*", query_fields, "starring:= samuel", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(0, results["found"].get<size_t>());
+
+ // same should succeed when verbatim filter is made
+ results = coll_str->search("*", query_fields, "starring:= samuel l. Jackson", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["found"].get<size_t>());
+
+ // contains filter with a single token should work as well
+ results = coll_str->search("*", query_fields, "starring: jackson", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["found"].get<size_t>());
+
+ results = coll_str->search("*", query_fields, "starring: samuel", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["found"].get<size_t>());
+
+ collectionManager.drop_collection("coll_str");
+}
+
+TEST_F(CollectionFacetingTest, FacetFieldStringArrayFiltering) {
+ Collection *coll_array_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("name_facet", field_types::STRING, true),
+ field("age", field_types::INT32, true),
+ field("years", field_types::INT32_ARRAY, true),
+ field("rating", field_types::FLOAT, true),
+ field("timestamps", field_types::INT64_ARRAY, true),
+ field("tags", field_types::STRING_ARRAY, true)};
+
+ std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
+
+ coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ if(coll_array_fields == nullptr) {
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 1, fields, "age").get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ nlohmann::json document = nlohmann::json::parse(json_line);
+ document["name_facet"] = document["name"];
+ const std::string & patched_json_line = document.dump();
+ coll_array_fields->add(patched_json_line);
+ }
+
+ infile.close();
+
+ query_fields = {"name"};
+ std::vector<std::string> facets = {"tags"};
+
+ // facet with filter on string array field must fail when exact token is used
+ facets.clear();
+ facets.push_back("tags");
+ auto results = coll_array_fields->search("Jeremy", query_fields, "tags:= PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(0, results["found"].get<size_t>());
+
+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= FINE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ // partial token filter should be made without "=" operator
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: FINE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ // to make tokens match facet value exactly, use "=" operator
+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ // don't allow exact filter on non-faceted field
+ auto res_op = coll_array_fields->search("Jeremy", query_fields, "name:= Jeremy Howard", facets, sort_fields, 0, 10, 1, FREQUENCY, false);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("To perform exact filtering, filter field `name` must be a facet field.", res_op.error().c_str());
+
+ // multi match exact query (OR condition)
+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= [Gold, bronze]", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(3, results["found"].get<size_t>());
+
+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= [Gold, bronze, fine PLATINUM]", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ(4, results["found"].get<size_t>());
+
+ // single array multi match
+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= [fine PLATINUM]", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ collectionManager.drop_collection("coll_array_fields");
+}
+
TEST_F(CollectionFacetingTest, FacetCounts) {
Collection *coll_array_fields;
@@ -140,23 +273,6 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
- // facet with filter on string array field must fail when partial token is used
- facets.clear();
- facets.push_back("tags");
- results = coll_array_fields->search("Jeremy", query_fields, "tags: PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(0, results["hits"].size());
- ASSERT_EQ(0, results["found"].get<size_t>());
-
- // partial token should be used along with ~
- results = coll_array_fields->search("Jeremy", query_fields, "tags:~ PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ(1, results["found"].get<size_t>());
-
- // or when tokens match facet value exactly
- results = coll_array_fields->search("Jeremy", query_fields, "tags: FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ(1, results["found"].get<size_t>());
-
// facet with wildcard query
facets.clear();
facets.push_back("tags");
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 57b235ef..917cece2 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -847,7 +847,7 @@ TEST_F(CollectionTest, MultipleFields) {
// filtering on unfaceted multi-valued string field
query_fields = {"title"};
- results = coll_mul_fields->search("captain", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ results = coll_mul_fields->search("captain", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(1, results["hits"].size());
ids = {"6"};
for(size_t i = 0; i < results["hits"].size(); i++) {
@@ -928,7 +928,7 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {
// filtering on facet field should be possible (supports partial word search but without typo tolerance)
query_fields = {"title"};
- result_op = coll_mul_fields->search("*", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1,
+ result_op = coll_mul_fields->search("*", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1,
FREQUENCY, false);
ASSERT_EQ(true, result_op.ok());
results = result_op.get();