Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Highlight field value that is a prefix of the query.

Kishore Nallan38d44a7c8ae9
diff --git a/include/index.h b/include/index.h
index 37bb45e0..b10bea9d 100644
--- a/include/index.h
+++ b/include/index.h
@@ -396,7 +396,8 @@ public:
static void transform_for_180th_meridian(GeoCoord& point, double offset);
- art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
+ void get_token_leaves(const std::string & field_name, const unsigned char* token, uint32_t token_len,
+ std::vector<art_leaf*>& leaves);
// the following methods are not synchronized because their parent calls are synchronized
diff --git a/src/collection.cpp b/src/collection.cpp
index 0624cb44..d754d66c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1501,11 +1501,12 @@ void Collection::highlight_result(const field &search_field,
// Must search for the token string fresh on that field for the given document since `token_leaf`
// is from the best matched field and need not be present in other fields of a document.
Index* index = indices[field_order_kv->key % num_memory_shards];
- art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);
- //LOG(INFO) << "field: " << search_field.name << ", key: " << token_leaf->key;
+ std::vector<art_leaf*> leaves;
+ index->get_token_leaves(search_field.name, &token_leaf->key[0], token_leaf->key_len, leaves);
- if(actual_leaf != nullptr) {
+ for(const auto actual_leaf: leaves) {
+ //LOG(INFO) << "field: " << search_field.name << ", key: " << token_leaf->key;
query_suggestion.push_back(actual_leaf);
std::string token(reinterpret_cast<char*>(actual_leaf->key), actual_leaf->key_len-1);
query_suggestion_tokens.insert(token);
@@ -1525,10 +1526,11 @@ void Collection::highlight_result(const field &search_field,
}
Index* index = indices[field_order_kv->key % num_memory_shards];
- art_leaf *actual_leaf = index->get_token_leaf(search_field.name,
- reinterpret_cast<const unsigned char *>(q_token.c_str()),
- q_token.size() + 1);
- if(actual_leaf != nullptr) {
+ std::vector<art_leaf*> leaves;
+ index->get_token_leaves(search_field.name, reinterpret_cast<const unsigned char*>(q_token.c_str()),
+ q_token.size() + 1, leaves);
+
+ for(const auto actual_leaf: leaves) {
std::vector<uint16_t> positions;
uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);
if(doc_index != actual_leaf->values->ids.getLength()) {
diff --git a/src/index.cpp b/src/index.cpp
index 6f6f4b7d..1b8227d2 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2639,10 +2639,13 @@ void Index::tokenize_string_field(const nlohmann::json& document, const field& s
}
}
-art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {
+void Index::get_token_leaves(const std::string & field_name, const unsigned char* token, uint32_t token_len,
+ std::vector<art_leaf*>& leaves) {
std::shared_lock lock(mutex);
- const art_tree *t = search_index.at(field_name);
- return (art_leaf*) art_search(t, token, (int) token_len);
+ art_tree *t = search_index.at(field_name);
+ token_len = (token_len == 0) ? 0 : token_len-1;
+ art_fuzzy_search(t, token, token_len, 0, 0, 2, token_ordering::MAX_SCORE,
+ true, nullptr, 0, leaves);
}
const spp::sparse_hash_map<std::string, art_tree *> &Index::_get_search_index() const {
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 786211f1..d316b230 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -606,10 +606,49 @@ TEST_F(CollectionSpecificTest, TokensSpreadAcrossFields) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {4, 1}).get();
- LOG(INFO) << results;
-
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionSpecificTest, HighlightSecondaryFieldWithPrefixMatch) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("description", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["title"] = "Functions and Equations";
+ doc1["description"] = "Use a function to solve an equation.";
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["title"] = "Function of effort";
+ doc2["description"] = "Learn all about it.";
+ doc2["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("function", {"title", "description"}, "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {true, true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 1}).get();
+
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());
+
+ ASSERT_EQ("<mark>Functions</mark> and Equations",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ ASSERT_EQ("Use a <mark>function</mark> to solve an equation.",
+ results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 0e9221a2..d82982d0 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -3317,7 +3317,7 @@ TEST_F(CollectionTest, MultiFieldHighlighting) {
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_EQ(2, results["hits"][0]["highlights"].size());
+ ASSERT_EQ(3, results["hits"][0]["highlights"].size());
ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"].get<std::string>());
ASSERT_EQ("Best Wireless Vehicle <mark>Charger</mark>",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
@@ -3326,6 +3326,10 @@ TEST_F(CollectionTest, MultiFieldHighlighting) {
ASSERT_EQ("Easily replenish your cell phone with this wireless <mark>charger.</mark>",
results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
+ ASSERT_EQ("categories", results["hits"][0]["highlights"][2]["field"].get<std::string>());
+ ASSERT_EQ("Car <mark>Chargers</mark>",
+ results["hits"][0]["highlights"][2]["snippets"][0].get<std::string>());
+
results = coll1->search("John With Denver",
{"description"}, "", {}, {}, {0}, 10, 1, FREQUENCY,
{true}, 1, spp::sparse_hash_set<std::string>(),