Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlight edge case: random first token highlight.
Kishore Nallane3a5c3d9aa92
diff --git a/include/match_score.h b/include/match_score.hindex a07d9704..575a8004 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -16,9 +16,9 @@ struct token_positions_t {};struct TokenOffset {- uint8_t token_id; // token identifier- uint16_t offset; // token's offset in the text- uint16_t offset_index; // index of the offset in the offset vector+ uint8_t token_id; // token identifier+ uint16_t offset = MAX_DISPLACEMENT; // token's offset in the text+ uint16_t offset_index; // index of the offset in the offset vectorbool operator()(const TokenOffset &a, const TokenOffset &b) {return a.offset > b.offset;@@ -157,24 +157,20 @@ struct Match {for (size_t i = 0; i < window.size(); i++) {if(populate_window) {this_window[window[i].token_id] = window[i];+ this_window[window[i].token_id].offset = MAX_DISPLACEMENT;}if ((window[i].offset - min_offset) <= WINDOW_SIZE) {uint16_t next_offset = (i == window.size() - 1) ? window[i].offset : window[i + 1].offset;this_displacement += window[i].offset - next_offset;this_num_match++;- } else {- // to indicate that this offset should not be considered+if(populate_window) {- this_window[window[i].token_id].offset = MAX_DISPLACEMENT;+ this_window[window[i].token_id].offset = window[i].offset;}}}- if(populate_window) {- this_window[window.back().token_id] = window.back();- }-if ( (this_num_match > best_num_match) ||(this_num_match == best_num_match && this_displacement < best_displacement)) {best_displacement = this_displacement;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 531fcdc0..34ab0a76 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -3758,3 +3758,46 @@ TEST_F(CollectionTest, FieldSpecificNumTypos) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionTest, BadHighlightingOnText) {+ Collection *coll1;++ std::vector<field> fields = {field("text", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc;++ doc["id"] = "0";+ doc["text"] = "include destruction of natural marine and estuarine\\nhabitats, loss of productive agricultural "+ "land,\\nand soil erosion. 90 When interviewed, multiple\\nexperts stated that inappropriate land use "+ "and\\nmanagement is a central factor contributing to\\nenvironmental degradation in the "+ "Castries-Gros\\nIslet Corridor. 91 The construction is placing greater\\nstress on natural resources "+ "and biodiversity, and\\nthe capacity to produce food and retain freshwater\\nhas been diminished. "+ "92 Moreover, increased\\nwater consumption by the tourism sector, when\\ncompounded by climate "+ "change, is increasing food\\nand water insecurity throughout Saint Lucia, as well\\nas suppressing "+ "long-term growth prospects. 93";++ doc["points"] = 0;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("natural saint lucia", {"text"}, "", {}, {}, {1}, 10, 1, FREQUENCY,+ true, 10).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ ASSERT_STREQ("food\\nand water insecurity throughout <mark>Saint</mark> <mark>Lucia,</mark> as well\\nas suppressing long-term",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("Saint", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());+ ASSERT_STREQ("Lucia,", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex 9aeb3619..0131f17e 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -114,6 +114,18 @@ TEST(MatchTest, MatchScoreV2) {match = Match(100, token_offsets, true, true);ASSERT_EQ(0, match.exact_match);+ token_offsets.clear();+ token_offsets.push_back(token_positions_t{false, {74}});+ token_offsets.push_back(token_positions_t{false, {75}});+ token_offsets.push_back(token_positions_t{false, {3, 42}});++ expected_offsets = {74, 75, MAX_DISPLACEMENT};+ match = Match(100, token_offsets, true, true);+ ASSERT_EQ(3, match.offsets.size());+ for(size_t i = 0; i < match.offsets.size(); i++) {+ ASSERT_EQ(expected_offsets[i], match.offsets[i].offset);+ }+/*size_t total_distance = 0, words_present = 0, offset_sum = 0;auto begin = std::chrono::high_resolution_clock::now();