Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

More fixes for highlighting.

Kishore Nallanb5e3a28ace4e
diff --git a/src/collection.cpp b/src/collection.cpp
index 3f157723..f031866c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1534,6 +1534,12 @@ void Collection::highlight_result(const field &search_field,
searched_queries[field_order_kv->query_indices[qindex + 1]];
for (art_leaf* token_leaf : searched_query) {
+ std::string token(reinterpret_cast<char*>(token_leaf->key), token_leaf->key_len - 1);
+
+ if(query_suggestion_tokens.count(token) != 0) {
+ continue;
+ }
+
// Must search for the token string fresh on that field for the given document since `token_leaf`
// is from the best matched field and need not be present in other fields of a document.
Index* index = indices[field_order_kv->key % num_memory_shards];
@@ -1541,30 +1547,26 @@ void Collection::highlight_result(const field &search_field,
if(actual_leaf != nullptr) {
query_suggestion.push_back(actual_leaf);
- std::string token(reinterpret_cast<char*>(actual_leaf->key), actual_leaf->key_len - 1);
- //LOG(INFO) << "field: " << search_field.name << ", key: " << token;
query_suggestion_tokens.insert(token);
+ //LOG(INFO) << "field: " << search_field.name << ", key: " << token;
}
}
qindex++;
} while(field_order_kv->query_indices != nullptr && qindex < field_order_kv->query_indices[0]);
- if(query_suggestion.size() != q_tokens.size()) {
- // can happen for compound query matched across 2 fields when some tokens are dropped
- for(const std::string& q_token: q_tokens) {
- if(query_suggestion_tokens.count(q_token) != 0) {
- continue;
- }
+ for(const std::string& q_token: q_tokens) {
+ if(query_suggestion_tokens.count(q_token) != 0) {
+ continue;
+ }
- Index* index = indices[field_order_kv->key % num_memory_shards];
- art_leaf *actual_leaf = index->get_token_leaf(search_field.name,
- reinterpret_cast<const unsigned char *>(q_token.c_str()),
- q_token.size() + 1);
- if(actual_leaf != nullptr) {
- query_suggestion.push_back(actual_leaf);
- query_suggestion_tokens.insert(q_token);
- }
+ Index* index = indices[field_order_kv->key % num_memory_shards];
+ art_leaf *actual_leaf = index->get_token_leaf(search_field.name,
+ reinterpret_cast<const unsigned char *>(q_token.c_str()),
+ q_token.size() + 1);
+ if(actual_leaf != nullptr) {
+ query_suggestion.push_back(actual_leaf);
+ query_suggestion_tokens.insert(q_token);
}
}
@@ -1607,9 +1609,15 @@ void Collection::highlight_result(const field &search_field,
<< ", match.distance: " << size_t(this_match.distance);*/
}
+ const std::string& prefix_token = q_tokens.back();
+
if(match_indices.empty()) {
// none of the tokens from the query were found on this field
- return ;
+ // let's try to look only for prefix matches
+ Match dummy_match(0, 0);
+ match_index_t match_index(dummy_match, 0, 0);
+ match_indices.emplace_back(match_index);
+ query_suggestion_tokens.clear();
}
const size_t max_array_matches = std::min((size_t)MAX_ARRAY_MATCHES, match_indices.size());
@@ -1664,7 +1672,7 @@ void Collection::highlight_result(const field &search_field,
snippet_start_window.push_back(tok_start);
}
- bool token_already_found = token_hits.count(raw_token) != 0;
+ bool token_already_found = (token_hits.find(raw_token) != token_hits.end());
// ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestion
if ((found_first_match && token_already_found) ||
@@ -1686,7 +1694,8 @@ void Collection::highlight_result(const field &search_field,
found_first_match = true;
- } else if(query_suggestion_tokens.find(raw_token) != query_suggestion_tokens.end()) {
+ } else if(query_suggestion_tokens.find(raw_token) != query_suggestion_tokens.end() ||
+ raw_token.rfind(prefix_token, 0) == 0 || prefix_token.rfind(raw_token, 0) == 0) {
token_offsets.emplace(tok_start, tok_end);
token_hits.insert(raw_token);
}
@@ -1696,9 +1705,16 @@ void Collection::highlight_result(const field &search_field,
snippet_end_offset = tok_end;
}
- if(raw_token_index == snippet_threshold - 1 &&
- raw_token_index >= last_valid_offset + highlight_affix_num_tokens) {
- // since we have already crossed snippet threshold, we can break now
+ // We can break early only if we have:
+ // a) run out of matched indices
+ // b) token_index exceeds the suffix tokens boundary
+ // c) raw_token_index exceeds snippet threshold
+ // d) highlight fully is not requested
+
+ if(raw_token_index >= snippet_threshold - 1 &&
+ match_offset_index == match.offsets.size() &&
+ raw_token_index >= last_valid_offset + highlight_affix_num_tokens &&
+ !highlighted_fully) {
break;
}
}
@@ -1713,7 +1729,7 @@ void Collection::highlight_result(const field &search_field,
snippet_end_offset = text.size() - 1;
}
- // with `token_index_offsets` we have a list of ranges to target for highlighting
+ // `token_offsets` has a list of ranges to target for highlighting
auto offset_it = token_offsets.begin();
std::stringstream highlighted_text;
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index bf74f410..c284d440 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -720,3 +720,83 @@ TEST_F(CollectionSpecificTest, HighlightWithDropTokens) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionSpecificTest, HighlightLongFieldWithDropTokens) {
+ std::vector<field> fields = {field("description", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["description"] = "Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, USB-C PD Charging "
+ "Port & Gigabit Ethernet Port, Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB "
+ "Type-C - for Notebook/Tablet PC - 2 x USB Ports - 2 x USB 3.0 - "
+ "Network (RJ-45) - VGA - Wired";
+ doc1["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ auto results = coll1->search("wired charging gigabit port", {"description"}, "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "description", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>").get();
+
+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ("Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, "
+ "USB-C PD <mark>Charging</mark> <mark>Port</mark> & <mark>Gigabit</mark> Ethernet "
+ "<mark>Port,</mark> Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for "
+ "Notebook/Tablet PC - 2 x USB <mark>Ports</mark> - 2 x USB 3.0 - Network (RJ-45) - "
+ "VGA - <mark>Wired</mark>",
+ results["hits"][0]["highlights"][0]["value"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificTest, HighlightWithDropTokensAndPrefixSearch) {
+ std::vector<field> fields = {field("username", field_types::STRING, false),
+ field("name", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["username"] = "Pandaabear";
+ doc1["name"] = "Panda's Basement";
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["username"] = "Pandaabear";
+ doc2["name"] = "Pandaabear Basic";
+ doc2["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("pandaabear bas", {"username", "name"},
+ "", {}, {}, {2, 2}, 10,
+ 1, FREQUENCY, {true, true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "description", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>").get();
+
+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ(2, results["hits"][1]["highlights"].size());
+
+ ASSERT_EQ("<mark>Pandaabear</mark>",
+ results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+
+ ASSERT_EQ("Panda's <mark>Basement</mark>",
+ results["hits"][1]["highlights"][1]["snippet"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 9f924bf3..465de79c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -746,23 +746,23 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());
- ASSERT_STREQ("title", results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str());
- ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
- ASSERT_EQ(1, results["hits"][0]["highlights"][0]["matched_tokens"].size());
- ASSERT_STREQ("Truth", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
+ ASSERT_EQ(4, results["hits"][0]["highlights"][0].size());
+ ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "tags");
+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["snippets"].size());
+ ASSERT_STREQ("<mark>truth</mark>", results["hits"][0]["highlights"][0]["snippets"][0].get<std::string>().c_str());
+ ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][0]["highlights"][0]["snippets"][1].get<std::string>().c_str());
+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][0]["matched_tokens"][0][0].get<std::string>().c_str());
+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][0]["matched_tokens"][1][0].get<std::string>().c_str());
+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["indices"].size());
+ ASSERT_EQ(1, results["hits"][0]["highlights"][0]["indices"][0]);
+ ASSERT_EQ(2, results["hits"][0]["highlights"][0]["indices"][1]);
- ASSERT_EQ(4, results["hits"][0]["highlights"][1].size());
- ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "tags");
- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["snippets"].size());
- ASSERT_STREQ("<mark>truth</mark>", results["hits"][0]["highlights"][1]["snippets"][0].get<std::string>().c_str());
- ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][0]["highlights"][1]["snippets"][1].get<std::string>().c_str());
- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());
- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());
- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());
- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"].size());
- ASSERT_EQ(1, results["hits"][0]["highlights"][1]["indices"][0]);
- ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"][1]);
+ ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());
+ ASSERT_STREQ("title", results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str());
+ ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][0]["highlights"][1]["matched_tokens"].size());
+ ASSERT_STREQ("Truth", results["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());
ASSERT_EQ(3, results["hits"][1]["highlights"][0].size());
ASSERT_STREQ("title", results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str());
@@ -2500,15 +2500,18 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title, tags").get();
ASSERT_EQ(2, res["hits"][0]["highlights"].size());
+
+ ASSERT_EQ("tags", res["hits"][0]["highlights"][0]["field"]);
+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["values"].size());
+ ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["values"][0].get<std::string>());
+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["snippets"].size());
+ ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["snippets"][0].get<std::string>());
+
ASSERT_EQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",
- res["hits"][0]["highlights"][0]["value"].get<std::string>());
- ASSERT_EQ(1, res["hits"][0]["highlights"][0]["matched_tokens"].size());
- ASSERT_STREQ("lazy", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
-
- ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"].size());
- ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>());
- ASSERT_EQ(1, res["hits"][0]["highlights"][1]["snippets"].size());
- ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["snippets"][0].get<std::string>());
+ res["hits"][0]["highlights"][1]["value"].get<std::string>());
+ ASSERT_EQ("title", res["hits"][0]["highlights"][1]["field"]);
+ ASSERT_EQ(1, res["hits"][0]["highlights"][1]["matched_tokens"].size());
+ ASSERT_STREQ("lazy", res["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());
// excluded fields should not be returned in highlights section
spp::sparse_hash_set<std::string> excluded_fields = {"tags"};