Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix highlighting around puncutation.

Kishore Nallanbba8c5feb268
diff --git a/include/collection.h b/include/collection.h
index 7cace8c9..d230bd01 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -117,6 +117,7 @@ private:
bool is_infix_search,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
+ const uint8_t* index_symbols,
highlight_t &highlight) const;
void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
@@ -371,7 +372,9 @@ public:
const string& text, const std::map<size_t, size_t>& token_offsets,
size_t snippet_end_offset,
std::vector<std::string>& matched_tokens, std::map<size_t, size_t>::iterator& offset_it,
- std::stringstream& highlighted_text, size_t snippet_start_offset) ;
+ std::stringstream& highlighted_text,
+ const uint8_t* index_symbols,
+ size_t snippet_start_offset) ;
void process_highlight_fields(const std::vector<std::string>& search_fields,
const spp::sparse_hash_set<std::string>& exclude_fields,
diff --git a/src/collection.cpp b/src/collection.cpp
index 4726d32e..4e0275b8 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1175,6 +1175,11 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
std::string hits_key = group_limit ? "grouped_hits" : "hits";
result[hits_key] = nlohmann::json::array();
+ uint8_t index_symbols[256] = {};
+ for(char c: symbols_to_index) {
+ index_symbols[uint8_t(c)] = 1;
+ }
+
// construct results array
for(long result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {
const std::vector<KV*> & kv_group = result_group_kvs[result_kvs_index];
@@ -1217,7 +1222,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
highlight_result(raw_query, search_field, highlight_item.qtoken_leaves, q_tokens, field_order_kv,
document,string_utils, snippet_threshold, highlight_affix_num_tokens,
highlight_item.fully_highlighted, highlight_item.infix,
- highlight_start_tag, highlight_end_tag, highlight);
+ highlight_start_tag, highlight_end_tag, index_symbols, highlight);
if(!highlight.snippets.empty()) {
highlights.push_back(highlight);
}
@@ -1779,6 +1784,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
bool is_infix_search,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
+ const uint8_t* index_symbols,
highlight_t & highlight) const {
if(q_tokens.size() == 1 && q_tokens[0] == "*") {
@@ -2050,7 +2056,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
std::stringstream highlighted_text;
highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,
snippet_end_offset, matched_tokens, offset_it,
- highlighted_text, snippet_start_offset);
+ highlighted_text, index_symbols, snippet_start_offset);
highlight.snippets.push_back(highlighted_text.str());
if(search_field.type == field_types::STRING_ARRAY) {
@@ -2063,7 +2069,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
std::vector<std::string> full_matched_tokens;
highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,
text.size()-1, full_matched_tokens, offset_it,
- value_stream, 0);
+ value_stream, index_symbols, 0);
highlight.values.push_back(value_stream.str());
}
}
@@ -2081,16 +2087,29 @@ void Collection::highlight_text(const string& highlight_start_tag, const string&
size_t snippet_end_offset, std::vector<std::string>& matched_tokens,
std::map<size_t, size_t>::iterator& offset_it,
std::stringstream& highlighted_text,
+ const uint8_t* index_symbols,
size_t snippet_start_offset) {
while(snippet_start_offset <= snippet_end_offset) {
if(offset_it != token_offsets.end()) {
if (snippet_start_offset == offset_it->first) {
highlighted_text << highlight_start_tag;
- const std::string& text_token = text.substr(snippet_start_offset, (offset_it->second - snippet_start_offset) + 1);
- matched_tokens.push_back(text_token);
- size_t token_len = offset_it->second - snippet_start_offset + 1;
+ auto end_offset = offset_it->second;
+
+ // if a token ends with one or more puncutation chars, we should not highlight them
+ for(int j = end_offset; j > 0; j--) {
+ if(!std::isalnum(text[j]) && Tokenizer::is_ascii_char(text[j]) &&
+ index_symbols[uint8_t(text[j])] != 1) {
+ end_offset--;
+ } else {
+ break;
+ }
+ }
+
+ size_t token_len = end_offset - snippet_start_offset + 1;
+ const std::string& text_token = text.substr(snippet_start_offset, token_len);
+ matched_tokens.push_back(text_token);
for(size_t j = 0; j < token_len; j++) {
highlighted_text << text[snippet_start_offset + j];
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 8caa9d7c..528442b8 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -875,7 +875,7 @@ TEST_F(CollectionSpecificTest, HighlightLongFieldWithDropTokens) {
ASSERT_EQ("Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, "
"USB-C PD <mark>Charging</mark> <mark>Port</mark> & <mark>Gigabit</mark> Ethernet "
- "<mark>Port,</mark> Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for "
+ "<mark>Port</mark>, Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for "
"Notebook/Tablet PC - 2 x USB <mark>Port</mark>s - 2 x USB 3.0 - Network (RJ-45) - "
"VGA - <mark>Wired</mark>",
results["hits"][0]["highlights"][0]["value"].get<std::string>());
@@ -1228,8 +1228,8 @@ TEST_F(CollectionSpecificTest, CustomSymbolsForIndexing) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(1, results["hits"][0]["highlights"].size());
- ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);
- ASSERT_EQ("Yes, <mark>C++</mark> is great!", results["hits"][0]["highlights"][0]["snippet"]);
+ ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"].get<std::string>());
+ ASSERT_EQ("Yes, <mark>C++</mark> is great!", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
// without custom symbols, + should not be indexed, so the "C" record will show up first
@@ -2806,7 +2806,62 @@ TEST_F(CollectionSpecificTest, HighlightPrefixProperly) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("<mark>Cinderella:</mark> the story.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>Cinderella</mark>: the story.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificTest, DontHighlightPunctuation) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"??Ensure! readability, use a legible font."},
+ {"Too much clutter-- use readability.js to clean up the page."},
+ {"'DMonte Harris"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("readability", {"title"},
+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ("??Ensure! <mark>readability</mark>, use a legible font.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("Too much clutter-- use <mark>readability</mark>.js to clean up the page.", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+
+ results = coll1->search("clutter", {"title"},
+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("Too much <mark>clutter</mark>-- use readability.js to clean up the page.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ results = coll1->search("ensure", {"title"},
+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("??<mark>Ensure</mark>! readability, use a legible font.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ results = coll1->search("dmonte", {"title"},
+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("'<mark>DMonte</mark> Harris", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index b0057b7e..e6068b73 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -468,8 +468,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {
ASSERT_EQ(2, res["hits"].size());
ASSERT_EQ(2, res["found"].get<uint32_t>());
- ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
// for now we don't support synonyms on ANY prefix
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 095cfa9e..3609d5f0 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -3558,7 +3558,7 @@ TEST_F(CollectionTest, MultiFieldHighlighting) {
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
ASSERT_EQ("description", results["hits"][0]["highlights"][1]["field"].get<std::string>());
- ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger.</mark>",
+ ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger</mark>.",
results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
ASSERT_EQ("categories", results["hits"][0]["highlights"][2]["field"].get<std::string>());
@@ -4030,12 +4030,12 @@ TEST_F(CollectionTest, BadHighlightingOnText) {
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
- ASSERT_STREQ("food\\nand water insecurity throughout <mark>Saint</mark> <mark>Lucia,</mark> as well\\nas suppressing long-term",
+ ASSERT_STREQ("food\\nand water insecurity throughout <mark>Saint</mark> <mark>Lucia</mark>, as well\\nas suppressing long-term",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());
ASSERT_STREQ("Saint", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
- ASSERT_STREQ("Lucia,", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());
+ ASSERT_STREQ("Lucia", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}