Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle special characters in prefix highlighting.

Kishore Nallancfbcbfc6fb1e
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 53a2ab13..3700a7d3 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -85,4 +85,6 @@ public:
}
void decr_token_counter();
+
+ bool should_skip_char(char c);
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index ff2a7f78..39d54ef6 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2832,6 +2832,12 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const
// group unicode code points and calculate number of actual characters
while(k <= tok_end) {
k++;
+
+ if(tokenizer.should_skip_char(text[k])) {
+ // used to handle special characters inside a tokenized word, e.g. `foo-bar`
+ continue;
+ }
+
if ((text[k] & 0xC0) == 0x80) k++;
if ((text[k] & 0xC0) == 0x80) k++;
if ((text[k] & 0xC0) == 0x80) k++;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 082db1be..8488b24a 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -334,3 +334,7 @@ void Tokenizer::decr_token_counter() {
token_counter--;
}
}
+
+bool Tokenizer::should_skip_char(char c) {
+ return is_ascii_char(c) && get_stream_mode(c) != INDEX;
+}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 553f902b..9593d33a 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -1338,6 +1338,33 @@ TEST_F(CollectionSpecificMoreTest, CopyDocHelper) {
ASSERT_EQ(1, dst.count("foo.bar"));
}
+TEST_F(CollectionSpecificMoreTest, HighlightWordWithSymbols) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection *coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "var(--icon-secondary-neutral); For components with";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto res = coll1->search("favicon", {"title"}, "", {}, {}, {2}, 10, 1,
+ FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address",
+ 20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,
+ "title").get();
+
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("<mark>var(--icon</mark>-secondary-neutral); For components with",
+ res["hits"][0]["highlight"]["title"]["snippet"].get<std::string>());
+}
+
TEST_F(CollectionSpecificMoreTest, HighlightObjectShouldBeEmptyWhenNoHighlightFieldFound) {
nlohmann::json schema = R"({
"name": "coll1",