Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle wraparound of token offset for large doc highlight.
Kishore Nallanf7c1678cc31a
diff --git a/src/collection.cpp b/src/collection.cppindex 7cf89ab5..132a71c9 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4093,8 +4093,12 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const// ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestionbool match_offset_found = (found_first_match && token_already_found) ||(match_offset_index <= last_valid_offset_index &&- match.offsets[match_offset_index].offset == raw_token_index &&- text_len/4 < 64000);+ match.offsets[match_offset_index].offset == raw_token_index);++ if(match_offset_found && text_len/4 > 64000) {+ // handle wrap around of token offsets: we will have to verify value of token as well+ match_offset_found = (qtoken_it != qtoken_leaves.end());+ }// Token might not appear in the best matched window, which is limited to a size of 10.// If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we willdiff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 651abeba..6358c12a 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2772,7 +2772,7 @@ TEST_F(CollectionSpecificMoreTest, DisableHighlightForLongFields) {Collection* coll1 = collectionManager.create_collection(schema).get();std::string description;- for(size_t i = 0; i < 100*1000; i++) {+ for(size_t i = 0; i < 70*1000; i++) {description += StringUtils::randstring(4) + " ";}@@ -2790,6 +2790,30 @@ TEST_F(CollectionSpecificMoreTest, DisableHighlightForLongFields) {ASSERT_TRUE(res_op.ok());ASSERT_EQ(1, res_op.get()["hits"].size());ASSERT_EQ(0, res_op.get()["hits"][0]["highlight"].size());++ // if token is found within first 64K offsets, we will highlight+ description = "";+ for(size_t i = 0; i < 1000; i++) {+ description += StringUtils::randstring(4) + " ";+ }++ description += " bazinga ";++ for(size_t i = 0; i < 70*1000; i++) {+ description += StringUtils::randstring(4) + " ";+ }++ doc["description"] = description;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ res_op = coll1->search("bazinga", {"description"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "");++ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(1, res_op.get()["hits"].size());+ ASSERT_EQ(1, res_op.get()["hits"][0]["highlight"].size());}TEST_F(CollectionSpecificMoreTest, TestStemming) {