Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Further tweak exact match logic.
Kishore Nalland4bd6e67e557
diff --git a/include/index.h b/include/index.hindex d231aa0c..39dfe6aa 100644--- a/include/index.h+++ b/include/index.h@@ -334,7 +334,7 @@ public:const size_t group_limit,const std::vector<std::string> &group_by_fields, uint32_t token_bits,bool prioritize_exact_match,- bool exact_query_match,+ bool single_exact_query_token,std::vector<posting_list_t::iterator_t>& posting_lists) const;static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);diff --git a/include/posting_list.h b/include/posting_list.hindex ee09777c..8d7ea6c7 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -163,7 +163,7 @@ public:std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos);- static bool is_single_token_exact_match(+ static bool is_single_token_verbatim_match(posting_list_t::iterator_t& it,bool field_is_array);diff --git a/src/index.cpp b/src/index.cppindex d2d5c667..9fb2c182 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -943,7 +943,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);fullq << qtok << " ";}- LOG(INFO) << fullq.str();*/+ LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str();*/// Prepare excluded document IDs that we can later remove from the result setuint32_t* excluded_result_ids = nullptr;@@ -962,19 +962,11 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,// We fetch offset positions only for multi token querybool fetch_offsets = (query_suggestion.size() > 1);- bool exact_query_match = false;+ bool single_exact_query_token = false;- if(total_cost == 0 && query_suggestion.size() == query_tokens.size()) {- // does this candidate suggestion token match query exactly?- exact_query_match = true;-- for(size_t i = 0; i < query_suggestion.size(); i++) {- std::string suggestion_token((const char*) query_suggestion[i]->key, query_suggestion[i]->key_len - 1);- if(suggestion_token != query_tokens[i].value) {- exact_query_match = false;- break;- }- }+ if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {+ // does this candidate suggestion token match query token exactly?+ single_exact_query_token = true;}std::vector<uint32_t> result_id_vec;@@ -985,7 +977,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,total_cost, topster, query_suggestion, groups_processed,seq_id, sort_order, field_values, geopoint_indices,group_limit, group_by_fields, token_bits,- prioritize_exact_match, exact_query_match, its);+ prioritize_exact_match, single_exact_query_token, its);result_id_vec.push_back(seq_id);});@@ -2087,7 +2079,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const size_t group_limit, const std::vector<std::string>& group_by_fields,uint32_t token_bits,bool prioritize_exact_match,- bool exact_query_match,+ bool single_exact_query_token,std::vector<posting_list_t::iterator_t>& posting_lists) const {spp::sparse_hash_map<uint32_t, int64_t>* TEXT_MATCH_SENTINEL = &text_match_sentinel_value;@@ -2131,11 +2123,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16uint64_t match_score = 0;if (posting_lists.size() == 1) {- const uint8_t is_exact_match = uint8_t(- prioritize_exact_match && exact_query_match &&- posting_list_t::is_single_token_exact_match(posting_lists[0], field_is_array)+ const uint8_t is_verbatim_match = uint8_t(+ prioritize_exact_match && single_exact_query_token &&+ posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array));- Match single_token_match = Match(1, 0, is_exact_match);+ Match single_token_match = Match(1, 0, is_verbatim_match);match_score = single_token_match.get_match_score(total_cost);} else {uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;@@ -2169,7 +2161,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16(uint64_t(total_tokens_found) << 24) |(uint64_t(255 - total_num_typos) << 16) |(uint64_t(100 - total_distance) << 8) |- (uint64_t(total_verbatim) << 1)+ (uint64_t(total_verbatim) << 0));/*LOG(INFO) << "Match score: " << match_score << ", for seq_id: " << seq_iddiff --git a/src/posting_list.cpp b/src/posting_list.cppindex 944ac8d9..03bcc85e 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -791,7 +791,7 @@ bool posting_list_t::get_offsets(std::vector<iterator_t>& its,return true;}-bool posting_list_t::is_single_token_exact_match(posting_list_t::iterator_t& it, bool field_is_array) {+bool posting_list_t::is_single_token_verbatim_match(posting_list_t::iterator_t& it, bool field_is_array) {block_t* curr_block = it.block();uint32_t curr_index = it.index();diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex af9e4d8a..6921bee0 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -860,4 +860,51 @@ TEST_F(CollectionSpecificTest, TokenStartingWithSameLetterAsPrevToken) {ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());collectionManager.drop_collection("coll1");-}\ No newline at end of file+}++TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeDeemedAsVerbatimMatch) {+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("description", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "John";+ doc1["description"] = "Vegetable Farmer";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "John";+ doc2["description"] = "Organic Vegetable Farmer";+ doc2["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("john vegetable farmer", {"name", "description"},+ "", {}, {}, {0, 0}, 10,+ 1, FREQUENCY, {true, true},+ 2, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>").get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ results = coll1->search("john vegatable farmer", {"name", "description"},+ "", {}, {}, {1, 1}, 10,+ 1, FREQUENCY, {true, true},+ 2, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>").get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}