Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow a string facet field to be filtered verbatim.
kishorencd06552eeba67
diff --git a/src/collection.cpp b/src/collection.cppindex 121c53eb..6767e9f8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -623,7 +623,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::f = {field_name, filter_values, EQUALS};} else {if(raw_value != "true" && raw_value != "false") {- return Option<nlohmann::json>(400, "Value of filter field `" + _field.name + "`: must be `true` or `false`.");+ return Option<nlohmann::json>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");}std::string bool_value = (raw_value == "true") ? "1" : "0";f = {field_name, {bool_value}, EQUALS};@@ -631,20 +631,26 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::} else if(_field.is_string()) {size_t filter_value_index = 0;- NUM_COMPARATOR str_comparator = EQUALS;+ NUM_COMPARATOR str_comparator = CONTAINS;- if(raw_value[0] == '~') {- // string filter should be evaluated in "contains" mode- str_comparator = CONTAINS;+ if(raw_value[0] == '=') {+ if(!_field.facet) {+ // EQUALS filtering on string is possible only on facet fields+ return Option<nlohmann::json>(400, "To perform exact filtering, filter field `" ++ _field.name + "` must be a facet field.");+ }++ // string filter should be evaluated in strict "equals" mode+ str_comparator = EQUALS;while(raw_value[++filter_value_index] == ' ');}if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");+ StringUtils::split(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values, ",");f = {field_name, filter_values, str_comparator};} else {- f = {field_name, {raw_value}, str_comparator};+ f = {field_name, {raw_value.substr(filter_value_index)}, str_comparator};}} else {return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Unidentified field type.");diff --git a/src/index.cpp b/src/index.cppindex 94c380e8..8a0821b2 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -475,7 +475,7 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &tokenfloat f = std::stof(token);reinterpret_cast<float&>(hash) = f; // store as int without loss of precision} else if(a_field.is_integer() || a_field.is_bool()) {- hash = atol(token.c_str());+ hash = atoll(token.c_str());} else {// string fieldhash = StringUtils::hash_wy(token.c_str(), token.size());@@ -983,24 +983,63 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto}}- if(a_filter.compare_operator == EQUALS) {- // need to do exact match (unlike CONTAINS) by iterating the result ids and calculating scores- if(query_suggestion.size() == str_tokens.size()) {- uint32_t *exact_strt_ids = new uint32_t[strt_ids_size];- size_t exact_strt_size = 0;+ if(a_filter.compare_operator == EQUALS && f.is_facet()) {+ // need to do exact match (unlike CONTAINS) by using the facet index+ // field being a facet is already enforced upstream+ uint32_t* exact_strt_ids = new uint32_t[strt_ids_size];+ size_t exact_strt_size = 0;++ size_t facet_id = facet_to_index[f.name];+ for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {+ uint32_t seq_id = strt_ids[strt_ids_index];+ const std::vector<uint64_t> &fvalues = facet_index_v2[seq_id][facet_id];+ bool found_filter = false;++ if(!f.is_array()) {+ found_filter = (str_tokens.size() == fvalues.size());+ } else {+ uint64_t filter_hash = 1;++ for(size_t sindex=0; sindex < str_tokens.size(); sindex++) {+ auto& str_token = str_tokens[sindex];+ uint64_t thash = facet_token_hash(f, str_token);+ filter_hash *= (1779033703 + 2*thash*(sindex+1));+ }- eq_str_filter_plain(strt_ids, strt_ids_size, query_suggestion, exact_strt_ids, exact_strt_size);+ uint64_t all_fvalue_hash = 1;+ size_t ftindex = 0;++ for(size_t findex=0; findex < fvalues.size(); findex++) {+ auto fhash = fvalues[findex];+ if(fhash == FACET_ARRAY_DELIMETER) {+ // end of array, check hash+ if(all_fvalue_hash == filter_hash) {+ found_filter = true;+ break;+ }+ all_fvalue_hash = 1;+ ftindex = 0;+ } else {+ all_fvalue_hash *= (1779033703 + 2*fhash*(ftindex + 1));+ ftindex++;+ }+ }+ }- delete[] strt_ids;- strt_ids = exact_strt_ids;- strt_ids_size = exact_strt_size;+ if(found_filter) {+ exact_strt_ids[exact_strt_size] = seq_id;+ exact_strt_size++;+ }}- }- // (NOT implemented) if the query is wrapped by double quotes, ensure exact match- // bool exact_match = (filter_value.front() == '"' && filter_value.back() == '"');+ delete[] strt_ids;+ strt_ids = exact_strt_ids;+ strt_ids_size = exact_strt_size;+ }// Otherwise, we just ensure that given record contains tokens in the filter query+ // (NOT implemented) if the query is wrapped by double quotes, ensure phrase match+ // bool exact_match = (filter_value.front() == '"' && filter_value.back() == '"');uint32_t* out = nullptr;ids_size = ArrayUtils::or_scalar(ids, ids_size, strt_ids, strt_ids_size, &out);delete[] strt_ids;diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 7eaaa5f3..7748cbf4 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -34,6 +34,139 @@ protected:}};+TEST_F(CollectionFacetingTest, FacetFieldStringFiltering) {+ Collection *coll_str;++ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, true),+ field("cast", field_types::STRING_ARRAY, false),+ field("points", field_types::INT32, false)+ };++ std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };++ coll_str = collectionManager.get_collection("coll_str");+ if(coll_str == nullptr) {+ coll_str = collectionManager.create_collection("coll_str", 1, fields, "points").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ nlohmann::json document = nlohmann::json::parse(json_line);+ coll_str->add(document.dump());+ }++ infile.close();++ query_fields = {"title"};+ std::vector<std::string> facets;++ // exact filter on string field must fail when single token is used+ facets.clear();+ facets.emplace_back("starring");+ auto results = coll_str->search("*", query_fields, "starring:= samuel", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(0, results["found"].get<size_t>());++ // same should succeed when verbatim filter is made+ results = coll_str->search("*", query_fields, "starring:= samuel l. Jackson", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(2, results["found"].get<size_t>());++ // contains filter with a single token should work as well+ results = coll_str->search("*", query_fields, "starring: jackson", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(2, results["found"].get<size_t>());++ results = coll_str->search("*", query_fields, "starring: samuel", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(2, results["found"].get<size_t>());++ collectionManager.drop_collection("coll_str");+}++TEST_F(CollectionFacetingTest, FacetFieldStringArrayFiltering) {+ Collection *coll_array_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("name_facet", field_types::STRING, true),+ field("age", field_types::INT32, true),+ field("years", field_types::INT32_ARRAY, true),+ field("rating", field_types::FLOAT, true),+ field("timestamps", field_types::INT64_ARRAY, true),+ field("tags", field_types::STRING_ARRAY, true)};++ std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };++ coll_array_fields = collectionManager.get_collection("coll_array_fields");+ if(coll_array_fields == nullptr) {+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 1, fields, "age").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ nlohmann::json document = nlohmann::json::parse(json_line);+ document["name_facet"] = document["name"];+ const std::string & patched_json_line = document.dump();+ coll_array_fields->add(patched_json_line);+ }++ infile.close();++ query_fields = {"name"};+ std::vector<std::string> facets = {"tags"};++ // facet with filter on string array field must fail when exact token is used+ facets.clear();+ facets.push_back("tags");+ auto results = coll_array_fields->search("Jeremy", query_fields, "tags:= PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(0, results["found"].get<size_t>());++ results = coll_array_fields->search("Jeremy", query_fields, "tags:= FINE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(0, results["hits"].size());++ // partial token filter should be made without "=" operator+ results = coll_array_fields->search("Jeremy", query_fields, "tags: PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<size_t>());++ results = coll_array_fields->search("Jeremy", query_fields, "tags: FINE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<size_t>());++ // to make tokens match facet value exactly, use "=" operator+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<size_t>());++ // don't allow exact filter on non-faceted field+ auto res_op = coll_array_fields->search("Jeremy", query_fields, "name:= Jeremy Howard", facets, sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("To perform exact filtering, filter field `name` must be a facet field.", res_op.error().c_str());++ // multi match exact query (OR condition)+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= [Gold, bronze]", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(3, results["found"].get<size_t>());++ results = coll_array_fields->search("Jeremy", query_fields, "tags:= [Gold, bronze, fine PLATINUM]", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ(4, results["found"].get<size_t>());++ // single array multi match+ results = coll_array_fields->search("Jeremy", query_fields, "tags:= [fine PLATINUM]", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<size_t>());++ collectionManager.drop_collection("coll_array_fields");+}+TEST_F(CollectionFacetingTest, FacetCounts) {Collection *coll_array_fields;@@ -140,23 +273,6 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());- // facet with filter on string array field must fail when partial token is used- facets.clear();- facets.push_back("tags");- results = coll_array_fields->search("Jeremy", query_fields, "tags: PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(0, results["hits"].size());- ASSERT_EQ(0, results["found"].get<size_t>());-- // partial token should be used along with ~- results = coll_array_fields->search("Jeremy", query_fields, "tags:~ PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ(1, results["found"].get<size_t>());-- // or when tokens match facet value exactly- results = coll_array_fields->search("Jeremy", query_fields, "tags: FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ(1, results["found"].get<size_t>());-// facet with wildcard queryfacets.clear();facets.push_back("tags");diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 57b235ef..917cece2 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -847,7 +847,7 @@ TEST_F(CollectionTest, MultipleFields) {// filtering on unfaceted multi-valued string fieldquery_fields = {"title"};- results = coll_mul_fields->search("captain", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ results = coll_mul_fields->search("captain", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(1, results["hits"].size());ids = {"6"};for(size_t i = 0; i < results["hits"].size(); i++) {@@ -928,7 +928,7 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {// filtering on facet field should be possible (supports partial word search but without typo tolerance)query_fields = {"title"};- result_op = coll_mul_fields->search("*", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1,+ result_op = coll_mul_fields->search("*", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1,FREQUENCY, false);ASSERT_EQ(true, result_op.ok());results = result_op.get();