Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix long queries causing highlighter to misbehave and sometimes crash.

Kishore Nallanaa439c1ea6cc
diff --git a/include/match_score.h b/include/match_score.h
index c277e437..f9c1c601 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -103,7 +103,9 @@ struct Match {
static Match match(uint32_t doc_id, const std::vector<std::vector<uint16_t>> &token_offsets) {
std::priority_queue<TokenOffset, std::vector<TokenOffset>, TokenOffset> heap;
- for(uint8_t token_id=0; token_id < token_offsets.size(); token_id++) {
+ const size_t tokens_size = std::min(token_offsets.size(), WINDOW_SIZE);
+
+ for(uint8_t token_id=0; token_id < tokens_size; token_id++) {
heap.push(TokenOffset{token_id, token_offsets[token_id].front(), 0});
}
@@ -141,7 +143,7 @@ struct Match {
uint16_t num_match = 0;
uint16_t displacement = 0;
- for(size_t token_id=0; token_id<token_offsets.size(); token_id++) {
+ for(size_t token_id=0; token_id<tokens_size; token_id++) {
// If a token appeared within the window, we would have recorded its offset
if(token_offset[token_id] != MAX_DISPLACEMENT) {
num_match++;
@@ -165,7 +167,7 @@ struct Match {
if(num_match > max_match || (num_match == max_match && displacement < min_displacement)) {
min_displacement = displacement;
// record the token positions (for highlighting)
- memcpy(min_token_offset, token_offset, token_offsets.size()*sizeof(uint16_t));
+ memcpy(min_token_offset, token_offset, tokens_size*sizeof(uint16_t));
max_match = num_match;
}
@@ -181,7 +183,7 @@ struct Match {
// identify the first token which is actually present and use that as the base for run-length encoding
size_t token_index = 0;
- while(token_index < token_offsets.size()) {
+ while(token_index < tokens_size) {
if(min_token_offset[token_index] != MAX_DISPLACEMENT) {
token_start_offset = min_token_offset[token_index];
break;
@@ -190,7 +192,7 @@ struct Match {
}
const uint8_t distance = MAX_TOKENS_DISTANCE - min_displacement;
- pack_token_offsets(min_token_offset, token_offsets.size(), token_start_offset, packed_offset_diffs);
+ pack_token_offsets(min_token_offset, tokens_size, token_start_offset, packed_offset_diffs);
return Match(max_match, distance, token_start_offset, packed_offset_diffs);
}
};
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index dddfc38d..12c46574 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2064,6 +2064,18 @@ TEST_F(CollectionTest, SearchLargeTextField) {
results = res_op.get();
ASSERT_EQ(2, results["hits"].size());
+ // query whose length exceeds maximum highlight window (match score's WINDOW_SIZE)
+ res_op = coll_large_text->search(
+ "Phasellus non tristique elit Praesent non arcu id lectus accumsan venenatis at",
+ {"text"}, "", {}, sort_fields, 0, 10
+ );
+
+ ASSERT_TRUE(res_op.ok());
+ results = res_op.get();
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
collectionManager.drop_collection("coll_large_text");
}
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index 93cd7e69..6ff07c98 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -42,4 +42,16 @@ TEST(MatchTest, ShouldPackTokenOffsets) {
ASSERT_EQ(std::numeric_limits<int8_t>::max(), offset_diffs[1]);
ASSERT_EQ(0, offset_diffs[2]);
ASSERT_EQ(2, offset_diffs[3]);
+}
+
+TEST(MatchTest, TokenOffsetsExceedWindowSize) {
+ std::vector<std::vector<uint16_t>> token_positions = {
+ std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}),
+ std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}),
+ std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1})
+ };
+
+ const Match & this_match = Match::match(100, token_positions);
+
+ ASSERT_EQ(WINDOW_SIZE, this_match.words_present);
}
\ No newline at end of file