Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlighting around puncutation.
Kishore Nallanbba8c5feb268
diff --git a/include/collection.h b/include/collection.hindex 7cace8c9..d230bd01 100644--- a/include/collection.h+++ b/include/collection.h@@ -117,6 +117,7 @@ private:bool is_infix_search,const std::string& highlight_start_tag,const std::string& highlight_end_tag,+ const uint8_t* index_symbols,highlight_t &highlight) const;void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);@@ -371,7 +372,9 @@ public:const string& text, const std::map<size_t, size_t>& token_offsets,size_t snippet_end_offset,std::vector<std::string>& matched_tokens, std::map<size_t, size_t>::iterator& offset_it,- std::stringstream& highlighted_text, size_t snippet_start_offset) ;+ std::stringstream& highlighted_text,+ const uint8_t* index_symbols,+ size_t snippet_start_offset) ;void process_highlight_fields(const std::vector<std::string>& search_fields,const spp::sparse_hash_set<std::string>& exclude_fields,diff --git a/src/collection.cpp b/src/collection.cppindex 4726d32e..4e0275b8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1175,6 +1175,11 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sstd::string hits_key = group_limit ? "grouped_hits" : "hits";result[hits_key] = nlohmann::json::array();+ uint8_t index_symbols[256] = {};+ for(char c: symbols_to_index) {+ index_symbols[uint8_t(c)] = 1;+ }+// construct results arrayfor(long result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {const std::vector<KV*> & kv_group = result_group_kvs[result_kvs_index];@@ -1217,7 +1222,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const shighlight_result(raw_query, search_field, highlight_item.qtoken_leaves, q_tokens, field_order_kv,document,string_utils, snippet_threshold, highlight_affix_num_tokens,highlight_item.fully_highlighted, highlight_item.infix,- highlight_start_tag, highlight_end_tag, highlight);+ highlight_start_tag, highlight_end_tag, index_symbols, highlight);if(!highlight.snippets.empty()) {highlights.push_back(highlight);}@@ -1779,6 +1784,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seabool is_infix_search,const std::string& highlight_start_tag,const std::string& highlight_end_tag,+ const uint8_t* index_symbols,highlight_t & highlight) const {if(q_tokens.size() == 1 && q_tokens[0] == "*") {@@ -2050,7 +2056,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seastd::stringstream highlighted_text;highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,snippet_end_offset, matched_tokens, offset_it,- highlighted_text, snippet_start_offset);+ highlighted_text, index_symbols, snippet_start_offset);highlight.snippets.push_back(highlighted_text.str());if(search_field.type == field_types::STRING_ARRAY) {@@ -2063,7 +2069,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seastd::vector<std::string> full_matched_tokens;highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,text.size()-1, full_matched_tokens, offset_it,- value_stream, 0);+ value_stream, index_symbols, 0);highlight.values.push_back(value_stream.str());}}@@ -2081,16 +2087,29 @@ void Collection::highlight_text(const string& highlight_start_tag, const string&size_t snippet_end_offset, std::vector<std::string>& matched_tokens,std::map<size_t, size_t>::iterator& offset_it,std::stringstream& highlighted_text,+ const uint8_t* index_symbols,size_t snippet_start_offset) {while(snippet_start_offset <= snippet_end_offset) {if(offset_it != token_offsets.end()) {if (snippet_start_offset == offset_it->first) {highlighted_text << highlight_start_tag;- const std::string& text_token = text.substr(snippet_start_offset, (offset_it->second - snippet_start_offset) + 1);- matched_tokens.push_back(text_token);- size_t token_len = offset_it->second - snippet_start_offset + 1;+ auto end_offset = offset_it->second;++ // if a token ends with one or more puncutation chars, we should not highlight them+ for(int j = end_offset; j > 0; j--) {+ if(!std::isalnum(text[j]) && Tokenizer::is_ascii_char(text[j]) &&+ index_symbols[uint8_t(text[j])] != 1) {+ end_offset--;+ } else {+ break;+ }+ }++ size_t token_len = end_offset - snippet_start_offset + 1;+ const std::string& text_token = text.substr(snippet_start_offset, token_len);+ matched_tokens.push_back(text_token);for(size_t j = 0; j < token_len; j++) {highlighted_text << text[snippet_start_offset + j];diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 8caa9d7c..528442b8 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -875,7 +875,7 @@ TEST_F(CollectionSpecificTest, HighlightLongFieldWithDropTokens) {ASSERT_EQ("Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, ""USB-C PD <mark>Charging</mark> <mark>Port</mark> & <mark>Gigabit</mark> Ethernet "- "<mark>Port,</mark> Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for "+ "<mark>Port</mark>, Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for ""Notebook/Tablet PC - 2 x USB <mark>Port</mark>s - 2 x USB 3.0 - Network (RJ-45) - ""VGA - <mark>Wired</mark>",results["hits"][0]["highlights"][0]["value"].get<std::string>());@@ -1228,8 +1228,8 @@ TEST_F(CollectionSpecificTest, CustomSymbolsForIndexing) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ(1, results["hits"][0]["highlights"].size());- ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"]);- ASSERT_EQ("Yes, <mark>C++</mark> is great!", results["hits"][0]["highlights"][0]["snippet"]);+ ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"].get<std::string>());+ ASSERT_EQ("Yes, <mark>C++</mark> is great!", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());// without custom symbols, + should not be indexed, so the "C" record will show up first@@ -2806,7 +2806,62 @@ TEST_F(CollectionSpecificTest, HighlightPrefixProperly) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());- ASSERT_EQ("<mark>Cinderella:</mark> the story.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("<mark>Cinderella</mark>: the story.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSpecificTest, DontHighlightPunctuation) {+ std::vector<field> fields = {field("title", field_types::STRING, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ std::vector<std::vector<std::string>> records = {+ {"??Ensure! readability, use a legible font."},+ {"Too much clutter-- use readability.js to clean up the page."},+ {"'DMonte Harris"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("readability", {"title"},+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ ASSERT_EQ("??Ensure! <mark>readability</mark>, use a legible font.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("Too much clutter-- use <mark>readability</mark>.js to clean up the page.", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());++ results = coll1->search("clutter", {"title"},+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("Too much <mark>clutter</mark>-- use readability.js to clean up the page.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ results = coll1->search("ensure", {"title"},+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("??<mark>Ensure</mark>! readability, use a legible font.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ results = coll1->search("dmonte", {"title"},+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("'<mark>DMonte</mark> Harris", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());collectionManager.drop_collection("coll1");}diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex b0057b7e..e6068b73 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -468,8 +468,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {ASSERT_EQ(2, res["hits"].size());ASSERT_EQ(2, res["found"].get<uint32_t>());- ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());// for now we don't support synonyms on ANY prefixdiff --git a/test/collection_test.cpp b/test/collection_test.cppindex 095cfa9e..3609d5f0 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -3558,7 +3558,7 @@ TEST_F(CollectionTest, MultiFieldHighlighting) {results["hits"][0]["highlights"][0]["snippet"].get<std::string>());ASSERT_EQ("description", results["hits"][0]["highlights"][1]["field"].get<std::string>());- ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger.</mark>",+ ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger</mark>.",results["hits"][0]["highlights"][1]["snippet"].get<std::string>());ASSERT_EQ("categories", results["hits"][0]["highlights"][2]["field"].get<std::string>());@@ -4030,12 +4030,12 @@ TEST_F(CollectionTest, BadHighlightingOnText) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());- ASSERT_STREQ("food\\nand water insecurity throughout <mark>Saint</mark> <mark>Lucia,</mark> as well\\nas suppressing long-term",+ ASSERT_STREQ("food\\nand water insecurity throughout <mark>Saint</mark> <mark>Lucia</mark>, as well\\nas suppressing long-term",results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());ASSERT_STREQ("Saint", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());- ASSERT_STREQ("Lucia,", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());+ ASSERT_STREQ("Lucia", results["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());collectionManager.drop_collection("coll1");}