Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

When a facet field is filtered upon, search should be verbatim instead of normalizing the filter_by value.

Kishore Nallane1db28a15305
diff --git a/src/index.cpp b/src/index.cpp
index f7835799..6436c57a 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -301,6 +301,37 @@ Option<uint32_t> Index::batch_index(Index *index, const std::vector<std::pair<ui
return Option<>(201);
}
+void Index::insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,
+ const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {
+ for(auto & kv: token_to_offsets) {
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = score;
+ art_doc.offsets_len = (uint32_t) kv.second.size();
+ art_doc.offsets = new uint32_t[kv.second.size()];
+
+ uint32_t num_hits = 0;
+
+ const unsigned char *key = (const unsigned char *) kv.first.c_str();
+ int key_len = (int) kv.first.length() + 1; // for the terminating \0 char
+
+ art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
+ }
+
+ num_hits += 1;
+
+ for(size_t i=0; i<kv.second.size(); i++) {
+ art_doc.offsets[i] = kv.second[i];
+ }
+
+ art_insert(t, key, key_len, &art_doc, num_hits);
+ delete [] art_doc.offsets;
+ art_doc.offsets = nullptr;
+ }
+}
+
void Index::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
const int KEY_LEN = 8;
unsigned char key[KEY_LEN];
@@ -415,50 +446,22 @@ void Index::index_string_field(const std::string & text, const uint32_t score, a
insert_doc(score, t, seq_id, token_to_offsets);
}
-void Index::insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,
- const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {
- for(auto & kv: token_to_offsets) {
- art_document art_doc;
- art_doc.id = seq_id;
- art_doc.score = score;
- art_doc.offsets_len = (uint32_t) kv.second.size();
- art_doc.offsets = new uint32_t[kv.second.size()];
-
- uint32_t num_hits = 0;
-
- const unsigned char *key = (const unsigned char *) kv.first.c_str();
- int key_len = (int) kv.first.length() + 1; // for the terminating \0 char
-
- art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);
- if(leaf != NULL) {
- num_hits = leaf->values->ids.getLength();
- }
-
- num_hits += 1;
-
- for(size_t i=0; i<kv.second.size(); i++) {
- art_doc.offsets[i] = kv.second[i];
- }
-
- art_insert(t, key, key_len, &art_doc, num_hits);
- delete [] art_doc.offsets;
- art_doc.offsets = nullptr;
- }
-}
-
void Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
uint32_t seq_id, const bool verbatim) const {
std::unordered_map<std::string, std::unordered_map<size_t, std::vector<uint32_t>>> token_array_positions;
for(size_t array_index = 0; array_index < strings.size(); array_index++) {
const std::string & str = strings[array_index];
-
std::vector<std::string> tokens;
- StringUtils::split(str, tokens, " ");
+ std::string delim = verbatim ? "" : " ";
+
+ StringUtils::split(str, tokens, delim);
for(uint32_t i=0; i<tokens.size(); i++) {
auto & token = tokens[i];
- string_utils.unicode_normalize(token);
+ if(!verbatim) {
+ string_utils.unicode_normalize(token);
+ }
token_array_positions[token][array_index].push_back(i);
}
}
@@ -687,16 +690,23 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
}
} else if(f.is_string()) {
for(const std::string & filter_value: a_filter.values) {
- // we have to tokenize the string, standardize it and then do an exact match
std::vector<std::string> str_tokens;
- StringUtils::split(filter_value, str_tokens, " ");
+
+ bool is_facet_field = (facet_index.count(a_filter.field_name) != 0);
+ std::string delim = is_facet_field ? "" : " ";
+ StringUtils::split(filter_value, str_tokens, delim);
uint32_t* filtered_ids = nullptr;
size_t filtered_size = 0;
for(size_t i = 0; i < str_tokens.size(); i++) {
std::string & str_token = str_tokens[i];
- string_utils.unicode_normalize(str_token);
+
+ if(!is_facet_field) {
+ // We should not normalize when the filter field is also a facet field
+ string_utils.unicode_normalize(str_token);
+ }
+
art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
str_token.length()+1);
if(leaf == nullptr) {
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index bef2349d..1eb6abd9 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1293,9 +1293,9 @@ TEST_F(CollectionTest, FilterOnTextFields) {
query_fields = {"name"};
std::vector<std::string> facets;
nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "tags: gold", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ(3, results["hits"].size());
- std::vector<std::string> ids = {"1", "4", "0", "2"};
+ std::vector<std::string> ids = {"4", "0", "2"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
@@ -1304,6 +1304,9 @@ TEST_F(CollectionTest, FilterOnTextFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
+ results = coll_array_fields->search("Jeremy", query_fields, "tags : FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+
results = coll_array_fields->search("Jeremy", query_fields, "tags : bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(2, results["hits"].size());
@@ -1329,19 +1332,19 @@ TEST_F(CollectionTest, FilterOnTextFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- // need not be exact matches (normalization can happen)
- results = coll_array_fields->search("Jeremy", query_fields, "tags: BrONZe", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ // need to be exact matches
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(2, results["hits"].size());
- // when comparators are used, should just treat them as part of search string (special chars will be removed)
+ // when comparators are used, should just treat them as part of search string
results = coll_array_fields->search("Jeremy", query_fields, "tags:<bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(0, results["hits"].size());
results = coll_array_fields->search("Jeremy", query_fields, "tags:<=BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(0, results["hits"].size());
results = coll_array_fields->search("Jeremy", query_fields, "tags:>BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll_array_fields");
}
@@ -1442,7 +1445,7 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
ASSERT_EQ("gold", results["facet_counts"][0]["counts"][0]["value"]);
- ASSERT_EQ(4, (int) results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);
ASSERT_EQ("silver", results["facet_counts"][0]["counts"][1]["value"]);
ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][1]["count"]);
@@ -1476,12 +1479,14 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][1]["count"]);
+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);
+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][3]["count"]);
- ASSERT_EQ("gold", results["facet_counts"][0]["counts"][0]["value"]);
- ASSERT_EQ("silver", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("silver", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("FINE PLATINUM", results["facet_counts"][0]["counts"][1]["value"]);
ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("gold", results["facet_counts"][0]["counts"][3]["value"]);
collectionManager.drop_collection("coll_array_fields");
}
diff --git a/test/numeric_array_documents.jsonl b/test/numeric_array_documents.jsonl
index 6c62ef00..ae659381 100644
--- a/test/numeric_array_documents.jsonl
+++ b/test/numeric_array_documents.jsonl
@@ -1,5 +1,5 @@
{"name": "Jeremy Howard", "top_3": [1.09, 1.88, 0.001], "rating": 1.09, "age": 24, "years": [2014, 2015, 2016], "timestamps": [1390354022, 1421890022, 1453426022], "tags": ["gold", "silver"]}
-{"name": "Jeremy Howard", "top_3": [9.999, 8.89, 7.713], "rating": 9.999, "age": 44, "years": [2015, 2016], "timestamps": [1421890022, 1453426022], "tags": ["gold"]}
+{"name": "Jeremy Howard", "top_3": [9.999, 8.89, 7.713], "rating": 9.999, "age": 44, "years": [2015, 2016], "timestamps": [1421890022, 1453426022], "tags": ["FINE PLATINUM"]}
{"name": "Jeremy Howard", "top_3": [7.812, 7.770, 6.66], "rating": 7.812, "age": 21, "years": [2016], "timestamps": [1453426022], "tags": ["bronze", "gold"]}
{"name": "Jeremy Howard", "top_3": [0.0, 0.0, 0.0], "rating": 0.0, "age": 63, "years": [1981, 1985], "timestamps": [348974822, 475205222], "tags": ["silver"]}
{"name": "Jeremy Howard", "top_3": [5.5, 5.431, 1.001], "rating": 5.5, "age": 32, "years": [1999, 2000, 2001, 2002], "timestamps": [916968422, 948504422, 980126822, 1011662822], "tags": ["silver", "gold", "bronze"]}
\ No newline at end of file