Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
When a facet field is filtered upon, search should be verbatim instead of normalizing the filter_by value.
Kishore Nallane1db28a15305
diff --git a/src/index.cpp b/src/index.cppindex f7835799..6436c57a 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -301,6 +301,37 @@ Option<uint32_t> Index::batch_index(Index *index, const std::vector<std::pair<uireturn Option<>(201);}+void Index::insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,+ const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {+ for(auto & kv: token_to_offsets) {+ art_document art_doc;+ art_doc.id = seq_id;+ art_doc.score = score;+ art_doc.offsets_len = (uint32_t) kv.second.size();+ art_doc.offsets = new uint32_t[kv.second.size()];++ uint32_t num_hits = 0;++ const unsigned char *key = (const unsigned char *) kv.first.c_str();+ int key_len = (int) kv.first.length() + 1; // for the terminating \0 char++ art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);+ if(leaf != NULL) {+ num_hits = leaf->values->ids.getLength();+ }++ num_hits += 1;++ for(size_t i=0; i<kv.second.size(); i++) {+ art_doc.offsets[i] = kv.second[i];+ }++ art_insert(t, key, key_len, &art_doc, num_hits);+ delete [] art_doc.offsets;+ art_doc.offsets = nullptr;+ }+}+void Index::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {const int KEY_LEN = 8;unsigned char key[KEY_LEN];@@ -415,50 +446,22 @@ void Index::index_string_field(const std::string & text, const uint32_t score, ainsert_doc(score, t, seq_id, token_to_offsets);}-void Index::insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,- const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {- for(auto & kv: token_to_offsets) {- art_document art_doc;- art_doc.id = seq_id;- art_doc.score = score;- art_doc.offsets_len = (uint32_t) kv.second.size();- art_doc.offsets = new uint32_t[kv.second.size()];-- uint32_t num_hits = 0;-- const unsigned char *key = (const unsigned char *) kv.first.c_str();- int key_len = (int) kv.first.length() + 1; // for the terminating \0 char-- art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);- if(leaf != NULL) {- num_hits = leaf->values->ids.getLength();- }-- num_hits += 1;-- for(size_t i=0; i<kv.second.size(); i++) {- art_doc.offsets[i] = kv.second[i];- }-- art_insert(t, key, key_len, &art_doc, num_hits);- delete [] art_doc.offsets;- art_doc.offsets = nullptr;- }-}-void Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,uint32_t seq_id, const bool verbatim) const {std::unordered_map<std::string, std::unordered_map<size_t, std::vector<uint32_t>>> token_array_positions;for(size_t array_index = 0; array_index < strings.size(); array_index++) {const std::string & str = strings[array_index];-std::vector<std::string> tokens;- StringUtils::split(str, tokens, " ");+ std::string delim = verbatim ? "" : " ";++ StringUtils::split(str, tokens, delim);for(uint32_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];- string_utils.unicode_normalize(token);+ if(!verbatim) {+ string_utils.unicode_normalize(token);+ }token_array_positions[token][array_index].push_back(i);}}@@ -687,16 +690,23 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto}} else if(f.is_string()) {for(const std::string & filter_value: a_filter.values) {- // we have to tokenize the string, standardize it and then do an exact matchstd::vector<std::string> str_tokens;- StringUtils::split(filter_value, str_tokens, " ");++ bool is_facet_field = (facet_index.count(a_filter.field_name) != 0);+ std::string delim = is_facet_field ? "" : " ";+ StringUtils::split(filter_value, str_tokens, delim);uint32_t* filtered_ids = nullptr;size_t filtered_size = 0;for(size_t i = 0; i < str_tokens.size(); i++) {std::string & str_token = str_tokens[i];- string_utils.unicode_normalize(str_token);++ if(!is_facet_field) {+ // We should not normalize when the filter field is also a facet field+ string_utils.unicode_normalize(str_token);+ }+art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),str_token.length()+1);if(leaf == nullptr) {diff --git a/test/collection_test.cpp b/test/collection_test.cppindex bef2349d..1eb6abd9 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1293,9 +1293,9 @@ TEST_F(CollectionTest, FilterOnTextFields) {query_fields = {"name"};std::vector<std::string> facets;nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "tags: gold", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ(3, results["hits"].size());- std::vector<std::string> ids = {"1", "4", "0", "2"};+ std::vector<std::string> ids = {"4", "0", "2"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -1304,6 +1304,9 @@ TEST_F(CollectionTest, FilterOnTextFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}+ results = coll_array_fields->search("Jeremy", query_fields, "tags : FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+results = coll_array_fields->search("Jeremy", query_fields, "tags : bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(2, results["hits"].size());@@ -1329,19 +1332,19 @@ TEST_F(CollectionTest, FilterOnTextFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- // need not be exact matches (normalization can happen)- results = coll_array_fields->search("Jeremy", query_fields, "tags: BrONZe", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ // need to be exact matches+ results = coll_array_fields->search("Jeremy", query_fields, "tags: bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(2, results["hits"].size());- // when comparators are used, should just treat them as part of search string (special chars will be removed)+ // when comparators are used, should just treat them as part of search stringresults = coll_array_fields->search("Jeremy", query_fields, "tags:<bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(0, results["hits"].size());results = coll_array_fields->search("Jeremy", query_fields, "tags:<=BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(0, results["hits"].size());results = coll_array_fields->search("Jeremy", query_fields, "tags:>BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(0, results["hits"].size());collectionManager.drop_collection("coll_array_fields");}@@ -1442,7 +1445,7 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);ASSERT_EQ("gold", results["facet_counts"][0]["counts"][0]["value"]);- ASSERT_EQ(4, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);ASSERT_EQ("silver", results["facet_counts"][0]["counts"][1]["value"]);ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][1]["count"]);@@ -1476,12 +1479,14 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);- ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][3]["count"]);- ASSERT_EQ("gold", results["facet_counts"][0]["counts"][0]["value"]);- ASSERT_EQ("silver", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("silver", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("FINE PLATINUM", results["facet_counts"][0]["counts"][1]["value"]);ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("gold", results["facet_counts"][0]["counts"][3]["value"]);collectionManager.drop_collection("coll_array_fields");}diff --git a/test/numeric_array_documents.jsonl b/test/numeric_array_documents.jsonlindex 6c62ef00..ae659381 100644--- a/test/numeric_array_documents.jsonl+++ b/test/numeric_array_documents.jsonl@@ -1,5 +1,5 @@{"name": "Jeremy Howard", "top_3": [1.09, 1.88, 0.001], "rating": 1.09, "age": 24, "years": [2014, 2015, 2016], "timestamps": [1390354022, 1421890022, 1453426022], "tags": ["gold", "silver"]}-{"name": "Jeremy Howard", "top_3": [9.999, 8.89, 7.713], "rating": 9.999, "age": 44, "years": [2015, 2016], "timestamps": [1421890022, 1453426022], "tags": ["gold"]}+{"name": "Jeremy Howard", "top_3": [9.999, 8.89, 7.713], "rating": 9.999, "age": 44, "years": [2015, 2016], "timestamps": [1421890022, 1453426022], "tags": ["FINE PLATINUM"]}{"name": "Jeremy Howard", "top_3": [7.812, 7.770, 6.66], "rating": 7.812, "age": 21, "years": [2016], "timestamps": [1453426022], "tags": ["bronze", "gold"]}{"name": "Jeremy Howard", "top_3": [0.0, 0.0, 0.0], "rating": 0.0, "age": 63, "years": [1981, 1985], "timestamps": [348974822, 475205222], "tags": ["silver"]}{"name": "Jeremy Howard", "top_3": [5.5, 5.431, 1.001], "rating": 5.5, "age": 32, "years": [1999, 2000, 2001, 2002], "timestamps": [916968422, 948504422, 980126822, 1011662822], "tags": ["silver", "gold", "bronze"]}\ No newline at end of file