Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle wraparound of token offset for large doc highlight.

Kishore Nallanf7c1678cc31a
diff --git a/src/collection.cpp b/src/collection.cpp
index 7cf89ab5..132a71c9 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4093,8 +4093,12 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const
// ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestion
bool match_offset_found = (found_first_match && token_already_found) ||
(match_offset_index <= last_valid_offset_index &&
- match.offsets[match_offset_index].offset == raw_token_index &&
- text_len/4 < 64000);
+ match.offsets[match_offset_index].offset == raw_token_index);
+
+ if(match_offset_found && text_len/4 > 64000) {
+ // handle wrap around of token offsets: we will have to verify value of token as well
+ match_offset_found = (qtoken_it != qtoken_leaves.end());
+ }
// Token might not appear in the best matched window, which is limited to a size of 10.
// If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we will
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 651abeba..6358c12a 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2772,7 +2772,7 @@ TEST_F(CollectionSpecificMoreTest, DisableHighlightForLongFields) {
Collection* coll1 = collectionManager.create_collection(schema).get();
std::string description;
- for(size_t i = 0; i < 100*1000; i++) {
+ for(size_t i = 0; i < 70*1000; i++) {
description += StringUtils::randstring(4) + " ";
}
@@ -2790,6 +2790,30 @@ TEST_F(CollectionSpecificMoreTest, DisableHighlightForLongFields) {
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["hits"].size());
ASSERT_EQ(0, res_op.get()["hits"][0]["highlight"].size());
+
+ // if token is found within first 64K offsets, we will highlight
+ description = "";
+ for(size_t i = 0; i < 1000; i++) {
+ description += StringUtils::randstring(4) + " ";
+ }
+
+ description += " bazinga ";
+
+ for(size_t i = 0; i < 70*1000; i++) {
+ description += StringUtils::randstring(4) + " ";
+ }
+
+ doc["description"] = description;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ res_op = coll1->search("bazinga", {"description"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "");
+
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(1, res_op.get()["hits"].size());
+ ASSERT_EQ(1, res_op.get()["hits"][0]["highlight"].size());
}
TEST_F(CollectionSpecificMoreTest, TestStemming) {