Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Normalize facet values.
Kishore Nallan4bfa245aad8e
diff --git a/src/index.cpp b/src/index.cppindex 02c9e878..5ec0bd95 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -937,6 +937,7 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,std::string token;std::string last_token;size_t token_index = 0;+ uint64_t facet_hash = 1;while(tokenizer.next(token, token_index)) {if(token.empty()) {@@ -945,6 +946,15 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,token_to_offsets[token].push_back(token_index + 1);last_token = token;++ if(is_facet) {+ uint64_t token_hash = Index::facet_token_hash(a_field, token);+ if(token_index == 0) {+ facet_hash = token_hash;+ } else {+ facet_hash = StringUtils::hash_combine(facet_hash, token_hash);+ }+ }}if(!token_to_offsets.empty()) {@@ -953,8 +963,7 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,}if(is_facet) {- uint64_t hash = Index::facet_token_hash(a_field, text);- facet_hashes.push_back(hash);+ facet_hashes.push_back(facet_hash);}}@@ -972,6 +981,7 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& stTokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);std::string token, last_token;size_t token_index = 0;+ uint64_t facet_hash = 1;// iterate and append offset positionswhile(tokenizer.next(token, token_index)) {@@ -982,6 +992,15 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& sttoken_to_offsets[token].push_back(token_index + 1);token_set.insert(token);last_token = token;++ if(is_facet) {+ uint64_t token_hash = Index::facet_token_hash(a_field, token);+ if(token_index == 0) {+ facet_hash = token_hash;+ } else {+ facet_hash = StringUtils::hash_combine(facet_hash, token_hash);+ }+ }}//LOG(INFO) << "Str: " << str << ", last_token: " << last_token;@@ -991,9 +1010,7 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st}if(is_facet) {- uint64_t hash = facet_token_hash(a_field, str);- //LOG(INFO) << "indexing " << token << ", hash:" << hash;- facet_hashes.push_back(hash);+ facet_hashes.push_back(facet_hash);}for(auto& the_token: token_set) {diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 680d404b..b395f659 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -575,8 +575,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());- ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());collectionManager.drop_collection("coll1");@@ -838,3 +838,68 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringArray) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionFacetingTest, FacetValuesShouldBeNormalized) {+ std::vector<field> fields = {field("brand", field_types::STRING, true),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ std::vector<std::vector<std::string>> records = {+ {"BUQU"},+ {"Buqu"},+ {"bu-qu"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;+ doc["id"] = std::to_string(i);+ doc["brand"] = records[i][0];+ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("*", {},+ "", {"brand"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());++ // any document is chosen as representative+ ASSERT_EQ("bu-qu", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionFacetingTest, FacetArrayValuesShouldBeNormalized) {+ std::vector<field> fields = {field("brands", field_types::STRING_ARRAY, true),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ std::vector<std::vector<std::string>> records = {+ {"BUQU", "Buqu", "bu-qu"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;+ doc["id"] = std::to_string(i);+ doc["brands"] = nlohmann::json::array();++ for(auto& str: records[i]) {+ doc["brands"].push_back(str);+ }++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("*", {},+ "", {"brands"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());++ // any document is chosen as representative+ ASSERT_EQ("bu-qu", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());++ collectionManager.drop_collection("coll1");+}