Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix phrase match + match scores for offset wrap around.
Kishore Nallan4651b9a6fdff
diff --git a/include/match_score.h b/include/match_score.hindex 5597fb64..7de98321 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -143,6 +143,8 @@ struct Match {size_t best_num_match = 1;size_t best_displacement = MAX_DISPLACEMENT;+ int prev_min_offset = -1;+while (window.size() > 1) {switch(window.size()) {case 2:@@ -158,6 +160,13 @@ struct Match {size_t min_offset = window.back().offset;+ if(int(min_offset) < prev_min_offset) {+ // indicates that one of the offsets are wrapping around (e.g. long document)+ break;+ }++ prev_min_offset = min_offset;+size_t this_displacement = 0;size_t this_num_match = 0;std::vector<TokenOffset> this_window(tokens_size);diff --git a/src/posting_list.cpp b/src/posting_list.cppindex b149ee60..61035fc5 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -1233,8 +1233,25 @@ bool posting_list_t::found_token_sequence(const std::vector<token_positions_t>&// iterate through the positions and see if `target_pos` is found in token positionsconst auto& tok_positions = token_positions[token_index].positions;- auto pos_it = std::find(tok_positions.begin(), tok_positions.end(), target_pos);- if(pos_it == tok_positions.end()) {+ bool found_pos = false;+ int prev_pos = -1;++ for(auto tok_pos: tok_positions) {+ if(tok_pos < prev_pos) {+ // indicates that the positions are wrapping around+ found_pos = false;+ break;+ }++ if(tok_pos == target_pos) {+ found_pos = true;+ break;+ }++ prev_pos = tok_pos;+ }++ if(!found_pos) {return false;}@@ -1243,11 +1260,19 @@ bool posting_list_t::found_token_sequence(const std::vector<token_positions_t>&bool posting_list_t::has_phrase_match(const std::vector<token_positions_t>& token_positions) {const auto& positions = token_positions[0].positions;+ int prev_pos = -1;for(auto pos: positions) {+ if(pos < prev_pos) {+ // indicates that the positions are wrapping around+ return false;+ }+if(found_token_sequence(token_positions, 1, pos + 1)) {return true;}++ prev_pos = pos;}return false;diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex 364c856d..512ccee5 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -187,7 +187,13 @@ TEST(MatchTest, MatchScoreWithOffsetWrapAround) {infile.close();+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));+auto match = Match(100, token_offsets, true, true);ASSERT_EQ(2, match.words_present);- //ASSERT_EQ(2, match.distance);+ ASSERT_EQ(2, match.distance);++ ASSERT_EQ(2, match.offsets.size());+ ASSERT_EQ(4062, match.offsets[0].offset);+ ASSERT_EQ(4060, match.offsets[1].offset);}