Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Normalize facet values.

Kishore Nallan4bfa245aad8e
diff --git a/src/index.cpp b/src/index.cpp
index 02c9e878..5ec0bd95 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -937,6 +937,7 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,
std::string token;
std::string last_token;
size_t token_index = 0;
+ uint64_t facet_hash = 1;
while(tokenizer.next(token, token_index)) {
if(token.empty()) {
@@ -945,6 +946,15 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,
token_to_offsets[token].push_back(token_index + 1);
last_token = token;
+
+ if(is_facet) {
+ uint64_t token_hash = Index::facet_token_hash(a_field, token);
+ if(token_index == 0) {
+ facet_hash = token_hash;
+ } else {
+ facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
+ }
+ }
}
if(!token_to_offsets.empty()) {
@@ -953,8 +963,7 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,
}
if(is_facet) {
- uint64_t hash = Index::facet_token_hash(a_field, text);
- facet_hashes.push_back(hash);
+ facet_hashes.push_back(facet_hash);
}
}
@@ -972,6 +981,7 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st
Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);
std::string token, last_token;
size_t token_index = 0;
+ uint64_t facet_hash = 1;
// iterate and append offset positions
while(tokenizer.next(token, token_index)) {
@@ -982,6 +992,15 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st
token_to_offsets[token].push_back(token_index + 1);
token_set.insert(token);
last_token = token;
+
+ if(is_facet) {
+ uint64_t token_hash = Index::facet_token_hash(a_field, token);
+ if(token_index == 0) {
+ facet_hash = token_hash;
+ } else {
+ facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
+ }
+ }
}
//LOG(INFO) << "Str: " << str << ", last_token: " << last_token;
@@ -991,9 +1010,7 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st
}
if(is_facet) {
- uint64_t hash = facet_token_hash(a_field, str);
- //LOG(INFO) << "indexing " << token << ", hash:" << hash;
- facet_hashes.push_back(hash);
+ facet_hashes.push_back(facet_hash);
}
for(auto& the_token: token_set) {
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 680d404b..b395f659 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -575,8 +575,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());
- ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
@@ -838,3 +838,68 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringArray) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionFacetingTest, FacetValuesShouldBeNormalized) {
+ std::vector<field> fields = {field("brand", field_types::STRING, true),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"BUQU"},
+ {"Buqu"},
+ {"bu-qu"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["brand"] = records[i][0];
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("*", {},
+ "", {"brand"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+
+ // any document is chosen as representative
+ ASSERT_EQ("bu-qu", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionFacetingTest, FacetArrayValuesShouldBeNormalized) {
+ std::vector<field> fields = {field("brands", field_types::STRING_ARRAY, true),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"BUQU", "Buqu", "bu-qu"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["brands"] = nlohmann::json::array();
+
+ for(auto& str: records[i]) {
+ doc["brands"].push_back(str);
+ }
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("*", {},
+ "", {"brands"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+
+ // any document is chosen as representative
+ ASSERT_EQ("bu-qu", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}