Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix phrase match + match scores for offset wrap around.

Kishore Nallan4651b9a6fdff
diff --git a/include/match_score.h b/include/match_score.h
index 5597fb64..7de98321 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -143,6 +143,8 @@ struct Match {
size_t best_num_match = 1;
size_t best_displacement = MAX_DISPLACEMENT;
+ int prev_min_offset = -1;
+
while (window.size() > 1) {
switch(window.size()) {
case 2:
@@ -158,6 +160,13 @@ struct Match {
size_t min_offset = window.back().offset;
+ if(int(min_offset) < prev_min_offset) {
+ // indicates that one of the offsets are wrapping around (e.g. long document)
+ break;
+ }
+
+ prev_min_offset = min_offset;
+
size_t this_displacement = 0;
size_t this_num_match = 0;
std::vector<TokenOffset> this_window(tokens_size);
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index b149ee60..61035fc5 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -1233,8 +1233,25 @@ bool posting_list_t::found_token_sequence(const std::vector<token_positions_t>&
// iterate through the positions and see if `target_pos` is found in token positions
const auto& tok_positions = token_positions[token_index].positions;
- auto pos_it = std::find(tok_positions.begin(), tok_positions.end(), target_pos);
- if(pos_it == tok_positions.end()) {
+ bool found_pos = false;
+ int prev_pos = -1;
+
+ for(auto tok_pos: tok_positions) {
+ if(tok_pos < prev_pos) {
+ // indicates that the positions are wrapping around
+ found_pos = false;
+ break;
+ }
+
+ if(tok_pos == target_pos) {
+ found_pos = true;
+ break;
+ }
+
+ prev_pos = tok_pos;
+ }
+
+ if(!found_pos) {
return false;
}
@@ -1243,11 +1260,19 @@ bool posting_list_t::found_token_sequence(const std::vector<token_positions_t>&
bool posting_list_t::has_phrase_match(const std::vector<token_positions_t>& token_positions) {
const auto& positions = token_positions[0].positions;
+ int prev_pos = -1;
for(auto pos: positions) {
+ if(pos < prev_pos) {
+ // indicates that the positions are wrapping around
+ return false;
+ }
+
if(found_token_sequence(token_positions, 1, pos + 1)) {
return true;
}
+
+ prev_pos = pos;
}
return false;
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index 364c856d..512ccee5 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -187,7 +187,13 @@ TEST(MatchTest, MatchScoreWithOffsetWrapAround) {
infile.close();
+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));
+
auto match = Match(100, token_offsets, true, true);
ASSERT_EQ(2, match.words_present);
- //ASSERT_EQ(2, match.distance);
+ ASSERT_EQ(2, match.distance);
+
+ ASSERT_EQ(2, match.offsets.size());
+ ASSERT_EQ(4062, match.offsets[0].offset);
+ ASSERT_EQ(4060, match.offsets[1].offset);
}