Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
More fixes for highlighting.
Kishore Nallanb5e3a28ace4e
diff --git a/src/collection.cpp b/src/collection.cppindex 3f157723..f031866c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1534,6 +1534,12 @@ void Collection::highlight_result(const field &search_field,searched_queries[field_order_kv->query_indices[qindex + 1]];for (art_leaf* token_leaf : searched_query) {+ std::string token(reinterpret_cast<char*>(token_leaf->key), token_leaf->key_len - 1);++ if(query_suggestion_tokens.count(token) != 0) {+ continue;+ }+// Must search for the token string fresh on that field for the given document since `token_leaf`// is from the best matched field and need not be present in other fields of a document.Index* index = indices[field_order_kv->key % num_memory_shards];@@ -1541,30 +1547,26 @@ void Collection::highlight_result(const field &search_field,if(actual_leaf != nullptr) {query_suggestion.push_back(actual_leaf);- std::string token(reinterpret_cast<char*>(actual_leaf->key), actual_leaf->key_len - 1);- //LOG(INFO) << "field: " << search_field.name << ", key: " << token;query_suggestion_tokens.insert(token);+ //LOG(INFO) << "field: " << search_field.name << ", key: " << token;}}qindex++;} while(field_order_kv->query_indices != nullptr && qindex < field_order_kv->query_indices[0]);- if(query_suggestion.size() != q_tokens.size()) {- // can happen for compound query matched across 2 fields when some tokens are dropped- for(const std::string& q_token: q_tokens) {- if(query_suggestion_tokens.count(q_token) != 0) {- continue;- }+ for(const std::string& q_token: q_tokens) {+ if(query_suggestion_tokens.count(q_token) != 0) {+ continue;+ }- Index* index = indices[field_order_kv->key % num_memory_shards];- art_leaf *actual_leaf = index->get_token_leaf(search_field.name,- reinterpret_cast<const unsigned char *>(q_token.c_str()),- q_token.size() + 1);- if(actual_leaf != nullptr) {- query_suggestion.push_back(actual_leaf);- query_suggestion_tokens.insert(q_token);- }+ Index* index = indices[field_order_kv->key % num_memory_shards];+ art_leaf *actual_leaf = index->get_token_leaf(search_field.name,+ reinterpret_cast<const unsigned char *>(q_token.c_str()),+ q_token.size() + 1);+ if(actual_leaf != nullptr) {+ query_suggestion.push_back(actual_leaf);+ query_suggestion_tokens.insert(q_token);}}@@ -1607,9 +1609,15 @@ void Collection::highlight_result(const field &search_field,<< ", match.distance: " << size_t(this_match.distance);*/}+ const std::string& prefix_token = q_tokens.back();+if(match_indices.empty()) {// none of the tokens from the query were found on this field- return ;+ // let's try to look only for prefix matches+ Match dummy_match(0, 0);+ match_index_t match_index(dummy_match, 0, 0);+ match_indices.emplace_back(match_index);+ query_suggestion_tokens.clear();}const size_t max_array_matches = std::min((size_t)MAX_ARRAY_MATCHES, match_indices.size());@@ -1664,7 +1672,7 @@ void Collection::highlight_result(const field &search_field,snippet_start_window.push_back(tok_start);}- bool token_already_found = token_hits.count(raw_token) != 0;+ bool token_already_found = (token_hits.find(raw_token) != token_hits.end());// ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestionif ((found_first_match && token_already_found) ||@@ -1686,7 +1694,8 @@ void Collection::highlight_result(const field &search_field,found_first_match = true;- } else if(query_suggestion_tokens.find(raw_token) != query_suggestion_tokens.end()) {+ } else if(query_suggestion_tokens.find(raw_token) != query_suggestion_tokens.end() ||+ raw_token.rfind(prefix_token, 0) == 0 || prefix_token.rfind(raw_token, 0) == 0) {token_offsets.emplace(tok_start, tok_end);token_hits.insert(raw_token);}@@ -1696,9 +1705,16 @@ void Collection::highlight_result(const field &search_field,snippet_end_offset = tok_end;}- if(raw_token_index == snippet_threshold - 1 &&- raw_token_index >= last_valid_offset + highlight_affix_num_tokens) {- // since we have already crossed snippet threshold, we can break now+ // We can break early only if we have:+ // a) run out of matched indices+ // b) token_index exceeds the suffix tokens boundary+ // c) raw_token_index exceeds snippet threshold+ // d) highlight fully is not requested++ if(raw_token_index >= snippet_threshold - 1 &&+ match_offset_index == match.offsets.size() &&+ raw_token_index >= last_valid_offset + highlight_affix_num_tokens &&+ !highlighted_fully) {break;}}@@ -1713,7 +1729,7 @@ void Collection::highlight_result(const field &search_field,snippet_end_offset = text.size() - 1;}- // with `token_index_offsets` we have a list of ranges to target for highlighting+ // `token_offsets` has a list of ranges to target for highlightingauto offset_it = token_offsets.begin();std::stringstream highlighted_text;diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex bf74f410..c284d440 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -720,3 +720,83 @@ TEST_F(CollectionSpecificTest, HighlightWithDropTokens) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionSpecificTest, HighlightLongFieldWithDropTokens) {+ std::vector<field> fields = {field("description", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["description"] = "Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, USB-C PD Charging "+ "Port & Gigabit Ethernet Port, Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB "+ "Type-C - for Notebook/Tablet PC - 2 x USB Ports - 2 x USB 3.0 - "+ "Network (RJ-45) - VGA - Wired";+ doc1["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ auto results = coll1->search("wired charging gigabit port", {"description"}, "", {}, {}, {0}, 10,+ 1, FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "description", 1, {}, {}, {}, 0,+ "<mark>", "</mark>").get();++ ASSERT_EQ(1, results["hits"][0]["highlights"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ ASSERT_EQ("Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, "+ "USB-C PD <mark>Charging</mark> <mark>Port</mark> & <mark>Gigabit</mark> Ethernet "+ "<mark>Port,</mark> Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for "+ "Notebook/Tablet PC - 2 x USB <mark>Ports</mark> - 2 x USB 3.0 - Network (RJ-45) - "+ "VGA - <mark>Wired</mark>",+ results["hits"][0]["highlights"][0]["value"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSpecificTest, HighlightWithDropTokensAndPrefixSearch) {+ std::vector<field> fields = {field("username", field_types::STRING, false),+ field("name", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["username"] = "Pandaabear";+ doc1["name"] = "Panda's Basement";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["username"] = "Pandaabear";+ doc2["name"] = "Pandaabear Basic";+ doc2["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("pandaabear bas", {"username", "name"},+ "", {}, {}, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "description", 1, {}, {}, {}, 0,+ "<mark>", "</mark>").get();++ ASSERT_EQ(2, results["hits"][0]["highlights"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ ASSERT_EQ(2, results["hits"][1]["highlights"].size());++ ASSERT_EQ("<mark>Pandaabear</mark>",+ results["hits"][1]["highlights"][0]["snippet"].get<std::string>());++ ASSERT_EQ("Panda's <mark>Basement</mark>",+ results["hits"][1]["highlights"][1]["snippet"].get<std::string>());++ collectionManager.drop_collection("coll1");+}+diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 9f924bf3..465de79c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -746,23 +746,23 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());- ASSERT_STREQ("title", results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str());- ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_EQ(1, results["hits"][0]["highlights"][0]["matched_tokens"].size());- ASSERT_STREQ("Truth", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());+ ASSERT_EQ(4, results["hits"][0]["highlights"][0].size());+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["snippets"].size());+ ASSERT_STREQ("<mark>truth</mark>", results["hits"][0]["highlights"][0]["snippets"][0].get<std::string>().c_str());+ ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][0]["highlights"][0]["snippets"][1].get<std::string>().c_str());+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][0]["matched_tokens"][0][0].get<std::string>().c_str());+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][0]["matched_tokens"][1][0].get<std::string>().c_str());+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["indices"].size());+ ASSERT_EQ(1, results["hits"][0]["highlights"][0]["indices"][0]);+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["indices"][1]);- ASSERT_EQ(4, results["hits"][0]["highlights"][1].size());- ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "tags");- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["snippets"].size());- ASSERT_STREQ("<mark>truth</mark>", results["hits"][0]["highlights"][1]["snippets"][0].get<std::string>().c_str());- ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][0]["highlights"][1]["snippets"][1].get<std::string>().c_str());- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"].size());- ASSERT_EQ(1, results["hits"][0]["highlights"][1]["indices"][0]);- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"][1]);+ ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());+ ASSERT_STREQ("title", results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str());+ ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][0]["highlights"][1]["matched_tokens"].size());+ ASSERT_STREQ("Truth", results["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());ASSERT_EQ(3, results["hits"][1]["highlights"][0].size());ASSERT_STREQ("title", results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str());@@ -2500,15 +2500,18 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title, tags").get();ASSERT_EQ(2, res["hits"][0]["highlights"].size());++ ASSERT_EQ("tags", res["hits"][0]["highlights"][0]["field"]);+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["values"].size());+ ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["values"][0].get<std::string>());+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["snippets"].size());+ ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["snippets"][0].get<std::string>());+ASSERT_EQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",- res["hits"][0]["highlights"][0]["value"].get<std::string>());- ASSERT_EQ(1, res["hits"][0]["highlights"][0]["matched_tokens"].size());- ASSERT_STREQ("lazy", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());-- ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"].size());- ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>());- ASSERT_EQ(1, res["hits"][0]["highlights"][1]["snippets"].size());- ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["snippets"][0].get<std::string>());+ res["hits"][0]["highlights"][1]["value"].get<std::string>());+ ASSERT_EQ("title", res["hits"][0]["highlights"][1]["field"]);+ ASSERT_EQ(1, res["hits"][0]["highlights"][1]["matched_tokens"].size());+ ASSERT_STREQ("lazy", res["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());// excluded fields should not be returned in highlights sectionspp::sparse_hash_set<std::string> excluded_fields = {"tags"};