Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Ensure that weights can fully control cross-field matching.
Kishore Nallan89a509513ab4
diff --git a/include/index.h b/include/index.hindex 65564625..37bb45e0 100644--- a/include/index.h+++ b/include/index.h@@ -35,6 +35,7 @@ struct token_candidates {struct search_field_t {std::string name;+ size_t priority;size_t weight;};diff --git a/src/collection.cpp b/src/collection.cppindex cb4ee5d9..0624cb44 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -600,21 +600,22 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::// process weights for search fieldsstd::vector<search_field_t> weighted_search_fields;+ size_t max_weight = 100;if(query_by_weights.empty()) {+ max_weight = search_fields.size();for(size_t i=1; i <= search_fields.size(); i++) {- query_by_weights.push_back(i);+ query_by_weights.push_back((max_weight - i) + 1);}} else {- auto max_weight = *std::max_element(query_by_weights.begin(), query_by_weights.end());- for(size_t i=0; i < query_by_weights.size(); i++) {- query_by_weights[i] = (max_weight - query_by_weights[i]) + 1;- }+ max_weight = *std::max_element(query_by_weights.begin(), query_by_weights.end());}for(size_t i=0; i < search_fields.size(); i++) {const auto& search_field = search_fields[i];- weighted_search_fields.push_back({search_field, query_by_weights[i]});+ const auto priority = (max_weight - query_by_weights[i]) + 1;+ const auto weight = query_by_weights[i] + 1;+ weighted_search_fields.push_back({search_field, priority, weight});}std::vector<facet> facets;diff --git a/src/index.cpp b/src/index.cppindex d1350620..6f6f4b7d 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1707,17 +1707,24 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit setuint64_t total_typos = 0, total_distances = 0, min_typos = 1000;- uint64_t verbatim_match_fields = 0; // query matching field verbatim- uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens- uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)+ uint64_t verbatim_match_fields = 0; // query matching field verbatim+ uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens+ uint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field+ uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);for(size_t i = 0; i < num_search_fields; i++) {const auto field_id = (uint8_t)(FIELD_LIMIT_NUM - i);- size_t weight = search_fields[i].weight;+ const size_t priority = search_fields[i].priority;+ const size_t weight = search_fields[i].weight;- //LOG(INFO) << "--- field index: " << i << ", weight: " << weight;+ //LOG(INFO) << "--- field index: " << i << ", priority: " << priority;++ // using `5` here because typo + prefix combo score range is: 0 - 5+ // 0 1 2+ // 0,1 2,3 4,5+ int64_t MAX_SUM_TYPOS = 5 * field_query_tokens[i].q_include_tokens.size();if(existing_field_kvs.count(field_id) != 0) {// for existing field, we will simply sum field-wise weighted scores@@ -1728,8 +1735,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint64_t tokens_found = ((match_score >> 24) & 0xFF);uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);- total_typos += (field_typos + 1) * weight;- total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * weight;+ total_typos += (field_typos + 1) * priority;+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;verbatim_match_fields += (((match_score & 0xFF)) + 1);uint64_t unique_tokens_found =@@ -1739,6 +1746,11 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,exact_match_fields++;}+ auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);+ if(weighted_tokens_match > max_weighted_tokens_match) {+ max_weighted_tokens_match = weighted_tokens_match;+ }+if(field_typos < min_typos) {min_typos = field_typos;}@@ -1746,9 +1758,9 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,total_token_matches += tokens_found;/*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))- << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * weight+ << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * priority<< ", total dist: " << (((match_score & 0xFF)))- << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * weight;*/+ << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * priority;*/continue;}@@ -1792,14 +1804,20 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint64_t tokens_found = ((match_score >> 24) & 0xFF);uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);- total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * weight;- total_typos += (field_typos + 1) * weight;+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;+ total_typos += (field_typos + 1) * priority;if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {exact_match_fields++;verbatim_match_fields++; // this is only an approximate}+ auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);++ if(weighted_tokens_match > max_weighted_tokens_match) {+ max_weighted_tokens_match = weighted_tokens_match;+ }+if(field_typos < min_typos) {min_typos = field_typos;}@@ -1814,9 +1832,11 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,total_typos = std::min<uint64_t>(255, total_typos);total_distances = std::min<uint64_t>(100, total_distances);+ max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);uint64_t aggregated_score = (- (exact_match_fields << 48) | // number of fields that contain *all tokens* in the query+ //(exact_match_fields << 48) | // number of fields that contain *all tokens* in the query+ (max_weighted_tokens_match << 48) | // weighted max number of tokens matched in a field(uniq_tokens_found << 40) | // number of unique tokens found across fields including typos((255 - min_typos) << 32) | // minimum typo cost across all fields(total_token_matches << 24) | // total matches across fields including typos@@ -1829,6 +1849,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,/*LOG(INFO) << "seq id: " << seq_id<< ", exact_match_fields: " << exact_match_fields+ << ", max_weighted_tokens_match: " << max_weighted_tokens_match<< ", uniq_tokens_found: " << uniq_tokens_found<< ", min typo score: " << (255 - min_typos)<< ", total_token_matches: " << total_token_matchesdiff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex 147dad3c..011e7527 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -21,7 +21,7 @@ std::string get_query(StringUtils & string_utils, std::string & text) {for(uint32_t i=0; i<tokens.size(); i++) {auto token = tokens[i];- string_utils.unicode_normalize(token);+ //string_utils.unicode_normalize(token);normalized_tokens.push_back(token);}@@ -87,7 +87,7 @@ void benchmark_hn_titles(char* file_path) {auto begin = std::chrono::high_resolution_clock::now();for(size_t i = 0; i < queries.size(); i++) {- auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("points", "DESC")}, 2, 10, 1, MAX_SCORE, true);+ auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("points", "DESC")}, {2}, 10, 1, MAX_SCORE, {true});if(results_op.ok() != true) {exit(2);}@@ -152,8 +152,8 @@ void benchmark_reactjs_pages(char* file_path) {auto begin = std::chrono::high_resolution_clock::now();for(size_t i = 0; i < queries.size(); i++) {- auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("dummy_sorting_field", "DESC")}, 2, 10, 1,- MAX_SCORE, true, 10, spp::sparse_hash_set<std::string>(), {"p"});+ auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("dummy_sorting_field", "DESC")}, {2}, 10, 1,+ MAX_SCORE, {true}, 10, spp::sparse_hash_set<std::string>(), {"p"});if(results_op.ok() != true) {exit(2);}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 85f4fafc..786211f1 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -169,8 +169,6 @@ TEST_F(CollectionSpecificTest, ExactSingleFieldMatch) {auto results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,1, FREQUENCY, {true, true}).get();- LOG(INFO) << results;-ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());@@ -200,11 +198,34 @@ TEST_F(CollectionSpecificTest, OrderMultiFieldFuzzyMatch) {ASSERT_TRUE(coll1->add(doc2.dump()).ok());auto results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,- 1, FREQUENCY, {true, true}).get();+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {1, 1}).get();ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+ results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {2, 1}).get();++ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ // use extreme weights to push title matching ahead++ results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {10, 1}).get();++ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+collectionManager.drop_collection("coll1");}@@ -556,3 +577,39 @@ TEST_F(CollectionSpecificTest, SingleCharMatchFullFieldHighlight) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionSpecificTest, TokensSpreadAcrossFields) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("description", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "Foo bar baz";+ doc1["description"] = "Share information with this device.";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["title"] = "Foo Random";+ doc2["description"] = "The Bar Fox";+ doc2["points"] = 250;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("foo bar", {"title", "description"}, "", {}, {}, {0}, 10,+ 1, FREQUENCY, {false, false},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {4, 1}).get();++ LOG(INFO) << results;++ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex e537b965..0e9221a2 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -859,7 +859,10 @@ TEST_F(CollectionTest, MultipleFields) {// when "starring" takes higher priority than "title"query_fields = {"starring", "title"};- results = coll_mul_fields->search("thomas", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ results = coll_mul_fields->search("thomas", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {2, 1}).get();ASSERT_EQ(4, results["hits"].size());ids = {"15", "12", "13", "14"};@@ -2925,6 +2928,17 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,"<mark>", "</mark>", {1, 4}).get();+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ // use same weights++ results = coll1->search("on a jetplane",+ {"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY,+ {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {1, 1}).get();+ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());