Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix repeating tokens in phrase search.
Kishore Nallan10ad8162b678
diff --git a/include/match_score.h b/include/match_score.hindex 42e6bf45..629ff2f0 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -39,7 +39,6 @@ struct Match {uint8_t distance = 0;uint8_t max_offset = 0;uint8_t exact_match = 0;- uint8_t phrase_match = 0;std::vector<TokenOffset> offsets;@@ -49,7 +48,7 @@ struct Match {Match(uint8_t words_present, uint8_t distance, uint8_t max_offset, uint8_t exact_match = 0) :words_present(words_present), distance(distance), max_offset(max_offset),- exact_match(exact_match), phrase_match(0) {+ exact_match(exact_match) {}@@ -164,7 +163,6 @@ struct Match {std::vector<TokenOffset> this_window(tokens_size);uint8_t prev_token_id = window[0].token_id;- phrase_match = 1;for (size_t i = 0; i < window.size(); i++) {if(populate_window) {@@ -181,12 +179,6 @@ struct Match {this_window[window[i].token_id].offset = window[i].offset;}}-- if(window[i].token_id > prev_token_id) {- phrase_match = 0;- }-- prev_token_id = window[i].token_id;}if ( (this_num_match > best_num_match) ||@@ -232,10 +224,6 @@ struct Match {offsets = best_window;}- if(best_num_match != tokens_size || distance != tokens_size-1) {- phrase_match = 0;- }-exact_match = 0;if(check_exact_match) {diff --git a/include/posting_list.h b/include/posting_list.hindex d044333e..76dc52c8 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -188,6 +188,11 @@ public:const uint32_t* ids, const uint32_t num_ids,uint32_t*& phrase_ids, size_t& num_phrase_ids);+ static bool has_phrase_match(const std::vector<token_positions_t>& token_positions);++ static bool found_token_sequence(const std::vector<token_positions_t>& token_positions, const size_t token_index,+ const uint16_t target_pos);+static void get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,std::vector<size_t>& indices);diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 2de0fdde..8558375c 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -1218,6 +1218,35 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const boolnum_exact_ids = exact_id_index;}+bool posting_list_t::found_token_sequence(const std::vector<token_positions_t>& token_positions,+ const size_t token_index, const uint16_t target_pos) {++ if(token_index == token_positions.size()) {+ return true;+ }++ // iterate through the positions and see if `target_pos` is found in token positions+ const auto& tok_positions = token_positions[token_index].positions;+ auto pos_it = std::find(tok_positions.begin(), tok_positions.end(), target_pos);+ if(pos_it == tok_positions.end()) {+ return false;+ }++ return found_token_sequence(token_positions, token_index+1, target_pos+1);+}++bool posting_list_t::has_phrase_match(const std::vector<token_positions_t>& token_positions) {+ const auto& positions = token_positions[0].positions;++ for(auto pos: positions) {+ if(found_token_sequence(token_positions, 1, pos + 1)) {+ return true;+ }+ }++ return false;+}+void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool field_is_array, const uint32_t* ids,const uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {@@ -1241,8 +1270,8 @@ void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool fieldget_offsets(its, array_token_positions);for(auto& kv: array_token_positions) {- bool is_phrase_match = Match(id, kv.second, false, false).phrase_match;- if(is_phrase_match) {+ const auto& token_positions = kv.second;+ if(has_phrase_match(token_positions)) {phrase_ids[phrase_id_index] = ids[i];phrase_id_index++;break;diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 2c82f7ed..4cfed868 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -1543,3 +1543,51 @@ TEST_F(CollectionSpecificMoreTest, IncludeExcludeUnIndexedField) {ASSERT_EQ("Sample Title 1", res["hits"][0]["document"]["title"].get<std::string>());ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());}++TEST_F(CollectionSpecificMoreTest, PhraseMatchRepeatingTokens) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "Super easy super fast product";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["title"] = "The really easy really fast product really";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto res = coll1->search(R"("super easy super fast")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());++ res = coll1->search(R"("super easy super")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());++ res = coll1->search(R"("the really easy really fast product really")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("1", res["hits"][0]["document"]["id"].get<std::string>());++ // these should not match+ res = coll1->search(R"("the easy really really product fast really")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(0, res["hits"].size());++ res = coll1->search(R"("really the easy really fast product really")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(0, res["hits"].size());++ res = coll1->search(R"("super super easy fast")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(0, res["hits"].size());++ res = coll1->search(R"("super super easy")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(0, res["hits"].size());++ res = coll1->search(R"("product fast")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(0, res["hits"].size());+}diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex 34877d29..6306161a 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -1,6 +1,7 @@#include <chrono>#include <gtest/gtest.h>#include <match_score.h>+#include "posting_list.h"TEST(MatchTest, TokenOffsetsExceedWindowSize) {std::vector<token_positions_t> token_positions = {@@ -33,7 +34,7 @@ TEST(MatchTest, MatchScoreV2) {auto match = Match(100, token_offsets, true);ASSERT_EQ(4, match.words_present);ASSERT_EQ(3, match.distance);- ASSERT_EQ(0, match.phrase_match);+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));std::vector<uint16_t> expected_offsets = {25, 26, 24, 27};for(size_t i=0; i<token_offsets.size(); i++) {@@ -55,7 +56,7 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(3, match.words_present);ASSERT_EQ(2, match.distance);ASSERT_EQ(0, match.exact_match);- ASSERT_EQ(0, match.phrase_match);+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));expected_offsets = {170, 171, 169};for(size_t i=0; i<token_offsets.size(); i++) {@@ -71,7 +72,7 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(1, match.words_present);ASSERT_EQ(0, match.distance);ASSERT_EQ(0, match.exact_match);- ASSERT_EQ(0, match.phrase_match);+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));expected_offsets = {38, MAX_DISPLACEMENT, MAX_DISPLACEMENT};for(size_t i=0; i<token_offsets.size(); i++) {@@ -95,13 +96,13 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(3, match.words_present);ASSERT_EQ(2, match.distance);ASSERT_EQ(1, match.exact_match);- ASSERT_EQ(0, match.phrase_match);+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));match = Match(100, token_offsets, true, false);ASSERT_EQ(3, match.words_present);ASSERT_EQ(2, match.distance);ASSERT_EQ(0, match.exact_match);- ASSERT_EQ(0, match.phrase_match);+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));token_offsets.clear();token_offsets.push_back(token_positions_t{false, {1}});@@ -110,7 +111,7 @@ TEST(MatchTest, MatchScoreV2) {match = Match(100, token_offsets, true, true);ASSERT_EQ(0, match.exact_match);- ASSERT_EQ(1, match.phrase_match);+ ASSERT_TRUE(posting_list_t::has_phrase_match(token_offsets));token_offsets.clear();token_offsets.push_back(token_positions_t{false, {0}});@@ -119,7 +120,7 @@ TEST(MatchTest, MatchScoreV2) {match = Match(100, token_offsets, true, true);ASSERT_EQ(0, match.exact_match);- ASSERT_EQ(1, match.phrase_match);+ ASSERT_TRUE(posting_list_t::has_phrase_match(token_offsets));token_offsets.clear();token_offsets.push_back(token_positions_t{false, {74}});@@ -140,7 +141,7 @@ TEST(MatchTest, MatchScoreV2) {token_offsets.push_back(token_positions_t{true, {197, 250}});match = Match(100, token_offsets);- ASSERT_EQ(1, match.phrase_match);+ ASSERT_TRUE(posting_list_t::has_phrase_match(token_offsets));token_offsets.clear();token_offsets.push_back(token_positions_t{false, {120, 167, 171, 196}});@@ -148,7 +149,7 @@ TEST(MatchTest, MatchScoreV2) {token_offsets.push_back(token_positions_t{true, {197, 250}});match = Match(100, token_offsets);- ASSERT_EQ(0, match.phrase_match);+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));/*size_t total_distance = 0, words_present = 0, offset_sum = 0;auto begin = std::chrono::high_resolution_clock::now();