Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix highlight edge case: random first token highlight.

Kishore Nallane3a5c3d9aa92
diff --git a/include/match_score.h b/include/match_score.h
index a07d9704..575a8004 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -16,9 +16,9 @@ struct token_positions_t {
};
struct TokenOffset {
- uint8_t token_id; // token identifier
- uint16_t offset; // token's offset in the text
- uint16_t offset_index; // index of the offset in the offset vector
+ uint8_t token_id; // token identifier
+ uint16_t offset = MAX_DISPLACEMENT; // token's offset in the text
+ uint16_t offset_index; // index of the offset in the offset vector
bool operator()(const TokenOffset &a, const TokenOffset &b) {
return a.offset > b.offset;
@@ -157,24 +157,20 @@ struct Match {
for (size_t i = 0; i < window.size(); i++) {
if(populate_window) {
this_window[window[i].token_id] = window[i];
+ this_window[window[i].token_id].offset = MAX_DISPLACEMENT;
}
if ((window[i].offset - min_offset) <= WINDOW_SIZE) {
uint16_t next_offset = (i == window.size() - 1) ? window[i].offset : window[i + 1].offset;
this_displacement += window[i].offset - next_offset;
this_num_match++;
- } else {
- // to indicate that this offset should not be considered
+
if(populate_window) {
- this_window[window[i].token_id].offset = MAX_DISPLACEMENT;
+ this_window[window[i].token_id].offset = window[i].offset;
}
}
}
- if(populate_window) {
- this_window[window.back().token_id] = window.back();
- }
-
if ( (this_num_match > best_num_match) ||
(this_num_match == best_num_match && this_displacement < best_displacement)) {
best_displacement = this_displacement;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 531fcdc0..34ab0a76 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -3758,3 +3758,46 @@ TEST_F(CollectionTest, FieldSpecificNumTypos) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionTest, BadHighlightingOnText) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("text", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc;
+
+ doc["id"] = "0";
+ doc["text"] = "include destruction of natural marine and estuarine\\nhabitats, loss of productive agricultural "
+ "land,\\nand soil erosion. 90 When interviewed, multiple\\nexperts stated that inappropriate land use "
+ "and\\nmanagement is a central factor contributing to\\nenvironmental degradation in the "
+ "Castries-Gros\\nIslet Corridor. 91 The construction is placing greater\\nstress on natural resources "
+ "and biodiversity, and\\nthe capacity to produce food and retain freshwater\\nhas been diminished. "
+ "92 Moreover, increased\\nwater consumption by the tourism sector, when\\ncompounded by climate "
+ "change, is increasing food\\nand water insecurity throughout Saint Lucia, as well\\nas suppressing "
+ "long-term growth prospects. 93";
+
+ doc["points"] = 0;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("natural saint lucia", {"text"}, "", {}, {}, {1}, 10, 1, FREQUENCY,
+ true, 10).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ ASSERT_STREQ("food\\nand water insecurity throughout <mark>Saint</mark> <mark>Lucia,</mark> as well\\nas suppressing long-term",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("Saint", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
+ ASSERT_STREQ("Lucia,", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index 9aeb3619..0131f17e 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -114,6 +114,18 @@ TEST(MatchTest, MatchScoreV2) {
match = Match(100, token_offsets, true, true);
ASSERT_EQ(0, match.exact_match);
+ token_offsets.clear();
+ token_offsets.push_back(token_positions_t{false, {74}});
+ token_offsets.push_back(token_positions_t{false, {75}});
+ token_offsets.push_back(token_positions_t{false, {3, 42}});
+
+ expected_offsets = {74, 75, MAX_DISPLACEMENT};
+ match = Match(100, token_offsets, true, true);
+ ASSERT_EQ(3, match.offsets.size());
+ for(size_t i = 0; i < match.offsets.size(); i++) {
+ ASSERT_EQ(expected_offsets[i], match.offsets[i].offset);
+ }
+
/*size_t total_distance = 0, words_present = 0, offset_sum = 0;
auto begin = std::chrono::high_resolution_clock::now();