Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Weight all components of cross-field match score.
Kishore Nallan48ac6bb82a87
diff --git a/src/index.cpp b/src/index.cppindex 8a75f6b5..829647e5 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1716,10 +1716,10 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit setuint64_t total_typos = 0, total_distances = 0, min_typos = 1000;- uint64_t verbatim_match_fields = 0; // query matching field verbatim- uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens+ uint64_t verbatim_match_fields = 0; // query matching field verbatim+ uint64_t exact_match_fields = 0; // number of fields that contains all of query tokensuint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field- uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)+ uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);@@ -1747,13 +1747,13 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;int64_t exact_match_score = match_score & 0xFF;- verbatim_match_fields += exact_match_score;+ verbatim_match_fields += (weight * exact_match_score);uint64_t unique_tokens_found =int64_t(__builtin_popcount(existing_field_kvs[field_id]->token_bits)) - 1;if(field_typos == 0 && unique_tokens_found == field_query_tokens[i].q_include_tokens.size()) {- exact_match_fields++;+ exact_match_fields += weight;}auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);@@ -1765,7 +1765,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,min_typos = field_typos;}- total_token_matches += tokens_found;+ total_token_matches += (weight * tokens_found);/*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))<< ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * priority@@ -1817,8 +1817,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,total_typos += (field_typos + 1) * priority;if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {- exact_match_fields++;- verbatim_match_fields++; // this is only an approximate+ exact_match_fields += weight;+ verbatim_match_fields += weight; // this is only an approximate}auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);@@ -1831,7 +1831,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,min_typos = field_typos;}- total_token_matches += tokens_found;+ total_token_matches += (weight * tokens_found);//LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);}}@@ -1839,9 +1839,11 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,// num tokens present across fields including those containing typosint64_t uniq_tokens_found = int64_t(__builtin_popcount(token_bits)) - 1;+ verbatim_match_fields = std::min<uint64_t>(255, verbatim_match_fields);+ exact_match_fields = std::min<uint64_t>(255, exact_match_fields);+ max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);total_typos = std::min<uint64_t>(255, total_typos);total_distances = std::min<uint64_t>(100, total_distances);- max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);uint64_t aggregated_score = ((verbatim_match_fields << 56) | // field value *exactly* same as query tokens@@ -1857,6 +1859,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,//LOG(INFO) << "seq id: " << seq_id << ", aggregated_score: " << aggregated_score;/*LOG(INFO) << "seq id: " << seq_id+ << ", verbatim_match_fields: " << verbatim_match_fields<< ", exact_match_fields: " << exact_match_fields<< ", max_weighted_tokens_match: " << max_weighted_tokens_match<< ", uniq_tokens_found: " << uniq_tokens_found@@ -1864,7 +1867,6 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,<< ", total_token_matches: " << total_token_matches<< ", typo score: " << (255 - total_typos)<< ", distance score: " << (100 - total_distances)- << ", verbatim_match_fields: " << verbatim_match_fields<< ", aggregated_score: " << aggregated_score << ", token_bits: " << token_bits;*/kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 41006252..7a98888c 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1279,5 +1279,48 @@ TEST_F(CollectionSpecificTest, TypoCorrectionWithFaceting) {ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());+ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSpecificTest, MultiFieldMatchesShouldBeWeighted) {+ // 2 matches on low weighted fields should not overpower a single match on high weighted field+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("category", field_types::STRING, false),+ field("label", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Amazing Twin";+ doc1["category"] = "kids";+ doc1["label"] = "kids";+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Kids";+ doc2["category"] = "children";+ doc2["label"] = "children";+ doc2["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("kids", {"name", "category", "label"},+ "", {}, {}, {0, 0, 0}, 10,+ 1, FREQUENCY, {false, false, false},+ 2, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {6, 1, 1}).get();++ LOG(INFO) << results;++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++collectionManager.drop_collection("coll1");}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex e5a62584..5bef8283 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -3609,7 +3609,10 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnFieldOrder) {}auto results = coll1->search("michael jackson toxic",- {"title", "artist"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ {"title", "artist"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {1, 6}).get();ASSERT_EQ(2, results["found"].get<size_t>());ASSERT_EQ(2, results["hits"].size());