Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Take care of token order in phrase search.
Kishore Nallan51a5f268e2f0
diff --git a/include/match_score.h b/include/match_score.hindex 21e4a209..19ef660f 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -38,6 +38,7 @@ struct Match {uint8_t words_present;uint8_t distance;uint8_t exact_match;+ uint8_t phrase_match;std::vector<TokenOffset> offsets;Match() : words_present(0), distance(0), exact_match(0) {@@ -82,6 +83,13 @@ struct Match {}}+ template<typename T>+ void sort2(std::vector<T>& a) {+ if(a[0] < a[1]) {+ std::swap(a[0], a[1]);+ }+ }+template<typename T>void sort3(std::vector<T>& a) {if (a[0] > a[1]) {@@ -144,10 +152,16 @@ struct Match {size_t best_displacement = MAX_DISPLACEMENT;while (window.size() > 1) {- if(window.size() == 3) {- sort3<TokenOffset>(window);- } else {- std::sort(window.begin(), window.end(), std::greater<TokenOffset>()); // descending comparator+ switch(window.size()) {+ case 2:+ sort2<TokenOffset>(window);+ break;+ case 3:+ sort3<TokenOffset>(window);+ break;+ default:+ // descending comparator+ std::sort(window.begin(), window.end(), std::greater<TokenOffset>());}size_t min_offset = window.back().offset;@@ -156,6 +170,9 @@ struct Match {size_t this_num_match = 0;std::vector<TokenOffset> this_window(tokens_size);+ uint8_t prev_token_id = window[0].token_id;+ phrase_match = 1;+for (size_t i = 0; i < window.size(); i++) {if(populate_window) {this_window[window[i].token_id] = window[i];@@ -171,6 +188,12 @@ struct Match {this_window[window[i].token_id].offset = window[i].offset;}}++ if(window[i].token_id > prev_token_id) {+ phrase_match = 0;+ }++ prev_token_id = window[i].token_id;}if ( (this_num_match > best_num_match) ||@@ -215,10 +238,13 @@ struct Match {offsets = best_window;}+ if(best_num_match != tokens_size || distance != tokens_size-1) {+ phrase_match = 0;+ }+exact_match = 0;if(check_exact_match) {-if(distance > token_offsets.size()-1) {// we can exit early and don't have to care about other requirementsreturn;diff --git a/src/index.cpp b/src/index.cppindex 1750632f..3dd10e15 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3662,12 +3662,6 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &tokeLOG(INFO) << ".";*/}- // Sort ascending based on matched documents for each token for faster intersection.- // However, this causes the token order to deviate from original query's order.- sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {- return posting_t::num_ids(left->values) < posting_t::num_ids(right->values);- });-return total_cost;}diff --git a/src/posting_list.cpp b/src/posting_list.cppindex bd2a324c..a8048a69 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -1247,7 +1247,9 @@ void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool fieldfor(auto& kv: array_token_positions) {auto dist = Match(id, kv.second, false, false).distance;- if(dist == its.size() - 1) {+ bool is_phrase_match = dist == its.size() - 1;+ is_phrase_match = Match(id, kv.second, false, false).phrase_match;+ if(is_phrase_match) {phrase_ids[phrase_id_index] = ids[i];phrase_id_index++;break;diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex c16d940e..f422514a 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -2178,6 +2178,19 @@ TEST_F(CollectionSpecificTest, PhraseSearch) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ // order of tokens in phrase must be respected+ results = coll1->search(R"("train by the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search(R"("train the by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search(R"("train the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search(R"("trooper state")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(0, results["hits"].size());+// two phrasesresults = coll1->search(R"("by the" "then and")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();ASSERT_EQ(1, results["hits"].size());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 52658161..d92c1114 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -150,7 +150,7 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {ASSERT_EQ(0, results["found"].get<int>());}-TEST_F(CollectionTest, PhraseSearch) {+TEST_F(CollectionTest, MultiTokenSearch) {std::vector<std::string> facets;nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, sort_fields, {0}, 10,1, FREQUENCY,@@ -257,7 +257,7 @@ TEST_F(CollectionTest, SearchWithExcludedTokens) {ASSERT_EQ(20, results["found"].get<uint32_t>());}-TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {+TEST_F(CollectionTest, SkipUnindexedTokensDuringMultiTokenSearch) {// Tokens that are not found in the index should be skippedstd::vector<std::string> facets;nlohmann::json results = collection->search("DoesNotExist from", query_fields, "", facets, sort_fields, {0}, 10).get();@@ -346,7 +346,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {ASSERT_EQ(0, results["hits"].size());}-TEST_F(CollectionTest, PartialPhraseSearch) {+TEST_F(CollectionTest, PartialMultiTokenSearch) {std::vector<std::string> facets;nlohmann::json results = collection->search("rocket research", query_fields, "", facets,sort_fields, {0}, 10, 1, FREQUENCY, {false}, 10).get();diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex 0131f17e..34877d29 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -33,6 +33,7 @@ TEST(MatchTest, MatchScoreV2) {auto match = Match(100, token_offsets, true);ASSERT_EQ(4, match.words_present);ASSERT_EQ(3, match.distance);+ ASSERT_EQ(0, match.phrase_match);std::vector<uint16_t> expected_offsets = {25, 26, 24, 27};for(size_t i=0; i<token_offsets.size(); i++) {@@ -54,6 +55,7 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(3, match.words_present);ASSERT_EQ(2, match.distance);ASSERT_EQ(0, match.exact_match);+ ASSERT_EQ(0, match.phrase_match);expected_offsets = {170, 171, 169};for(size_t i=0; i<token_offsets.size(); i++) {@@ -69,6 +71,7 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(1, match.words_present);ASSERT_EQ(0, match.distance);ASSERT_EQ(0, match.exact_match);+ ASSERT_EQ(0, match.phrase_match);expected_offsets = {38, MAX_DISPLACEMENT, MAX_DISPLACEMENT};for(size_t i=0; i<token_offsets.size(); i++) {@@ -92,11 +95,13 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(3, match.words_present);ASSERT_EQ(2, match.distance);ASSERT_EQ(1, match.exact_match);+ ASSERT_EQ(0, match.phrase_match);match = Match(100, token_offsets, true, false);ASSERT_EQ(3, match.words_present);ASSERT_EQ(2, match.distance);ASSERT_EQ(0, match.exact_match);+ ASSERT_EQ(0, match.phrase_match);token_offsets.clear();token_offsets.push_back(token_positions_t{false, {1}});@@ -105,6 +110,7 @@ TEST(MatchTest, MatchScoreV2) {match = Match(100, token_offsets, true, true);ASSERT_EQ(0, match.exact_match);+ ASSERT_EQ(1, match.phrase_match);token_offsets.clear();token_offsets.push_back(token_positions_t{false, {0}});@@ -113,6 +119,7 @@ TEST(MatchTest, MatchScoreV2) {match = Match(100, token_offsets, true, true);ASSERT_EQ(0, match.exact_match);+ ASSERT_EQ(1, match.phrase_match);token_offsets.clear();token_offsets.push_back(token_positions_t{false, {74}});@@ -126,6 +133,23 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(expected_offsets[i], match.offsets[i].offset);}+ // check phrase match+ token_offsets.clear();+ token_offsets.push_back(token_positions_t{false, {38, 50, 187, 195, 201}});+ token_offsets.push_back(token_positions_t{false, {120, 167, 171, 196}});+ token_offsets.push_back(token_positions_t{true, {197, 250}});++ match = Match(100, token_offsets);+ ASSERT_EQ(1, match.phrase_match);++ token_offsets.clear();+ token_offsets.push_back(token_positions_t{false, {120, 167, 171, 196}});+ token_offsets.push_back(token_positions_t{false, {38, 50, 187, 195, 201}});+ token_offsets.push_back(token_positions_t{true, {197, 250}});++ match = Match(100, token_offsets);+ ASSERT_EQ(0, match.phrase_match);+/*size_t total_distance = 0, words_present = 0, offset_sum = 0;auto begin = std::chrono::high_resolution_clock::now();