Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix stemming prefix highlight.

Kishore Nallan7e8aa01056e2
diff --git a/src/collection.cpp b/src/collection.cpp
index ddaafe2e..735aeec8 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4151,6 +4151,11 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const
}
}
+ if(num_letters < prefix_token_num_chars) {
+ // can happen in the case of stemming
+ prefix_end = tok_start + num_letters;
+ }
+
size_t char_diff = num_letters - prefix_letters;
auto new_tok_end = (char_diff <= 2 && qtoken_it.value().num_typos != 0) ? tok_end : prefix_end;
token_offsets.emplace(tok_start, new_tok_end);
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index a67b205a..8bb422c6 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2914,6 +2914,54 @@ TEST_F(CollectionSpecificMoreTest, StemmingEnglishWithCaps) {
ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());
}
+TEST_F(CollectionSpecificMoreTest, StemmingEnglishPrefixHighlight) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {
+ "facet": false,
+ "index": true,
+ "infix": true,
+ "locale": "",
+ "name": "name",
+ "optional": false,
+ "sort": false,
+ "stem": false,
+ "store": true,
+ "type": "string"
+ },
+ {
+ "facet": true,
+ "index": true,
+ "infix": true,
+ "locale": "",
+ "name": "subClass",
+ "optional": true,
+ "sort": false,
+ "stem": true,
+ "store": true,
+ "type": "string"
+ }
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["name"] = "Generic Red Onions";
+ doc["subClass"] = "ONIONS";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {};
+ auto res = coll1->search("onions", {"subClass","name"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("Generic Red <mark>Onions</mark>", res["hits"][0]["highlight"]["name"]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>ONIONS</mark>", res["hits"][0]["highlight"]["subClass"]["snippet"].get<std::string>());
+}
+
TEST_F(CollectionSpecificMoreTest, StemmingEnglishHighlights) {
nlohmann::json schema = R"({
"name": "test",