Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlighting for nested array of string.
Kishore Nallan99e11064a69e
diff --git a/include/collection.h b/include/collection.hindex 8ebd0e88..fb0b6fd4 100644--- a/include/collection.h+++ b/include/collection.h@@ -288,7 +288,7 @@ private:void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type,const size_t total_tokens) const;- bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,+ bool handle_highlight_text(std::string& text, bool normalise, const field &search_field, const bool is_arr_obj_ele,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,highlight_t& highlight, StringUtils & string_utils, bool use_word_tokenizer,const size_t highlight_affix_num_tokens,diff --git a/src/collection.cpp b/src/collection.cppindex ba521d9f..dfac82f7 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3072,7 +3072,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,index_symbols[uint8_t(c)] = 1;}- handle_highlight_text(value, normalise, the_field, symbols_to_index, token_separators,+ handle_highlight_text(value, normalise, the_field, false, symbols_to_index, token_separators,highlight, string_utils, use_word_tokenizer,highlight_affix_num_tokens, qtoken_leaves, last_valid_offset_index,prefix_token_num_chars, false, snippet_threshold, false, ftokens,@@ -4071,7 +4071,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seastd::string text = h_obj.get<std::string>();h_obj = nlohmann::json::object();- handle_highlight_text(text, normalise, search_field, symbols_to_index,+ handle_highlight_text(text, normalise, search_field, is_arr_obj_ele, symbols_to_index,token_separators, array_highlight, string_utils, use_word_tokenizer,highlight_affix_num_tokens,qtoken_leaves, last_valid_offset_index,@@ -4165,7 +4165,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seatext = document[search_field.name][match_index.index];}- handle_highlight_text(text, normalise, search_field, symbols_to_index, token_separators,+ handle_highlight_text(text, normalise, search_field, false, symbols_to_index, token_separators,highlight, string_utils, use_word_tokenizer, highlight_affix_num_tokens,qtoken_leaves, last_valid_offset_index, prefix_token_num_chars,highlight_fully, snippet_threshold, is_infix_search, raw_query_tokens,@@ -4195,7 +4195,8 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea}bool Collection::handle_highlight_text(std::string& text, bool normalise, const field &search_field,- const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,+ const bool is_arr_obj_ele, const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators,highlight_t& highlight, StringUtils & string_utils, bool use_word_tokenizer,const size_t highlight_affix_num_tokens,const tsl::htrie_map<char, token_leaf>& qtoken_leaves, int last_valid_offset_index,@@ -4267,8 +4268,10 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const// Token might not appear in the best matched window, which is limited to a size of 10.// If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we will- // locate all tokens that appear in the query / query candidates- bool raw_token_found = !match_offset_found && (highlight_fully || text_len < snippet_threshold * 6) &&+ // locate all tokens that appear in the query / query candidates. Likewise, for text within nested array of+ // objects have to be exhaustively looked for highlight tokens.+ bool raw_token_found = !match_offset_found &&+ (highlight_fully || is_arr_obj_ele || text_len < snippet_threshold * 6) &&qtoken_leaves.find(raw_token) != qtoken_leaves.end();if (match_offset_found || raw_token_found) {@@ -4325,6 +4328,12 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, constsnippet_start_offset = snippet_start_window.front();}+ found_first_match = true;+ } else if(raw_token_found && is_arr_obj_ele) {+ if(!found_first_match) {+ snippet_start_offset = snippet_start_window.front();+ }+found_first_match = true;}} else if(is_infix_search && text.size() < 100 &&@@ -4333,7 +4342,7 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, consttoken_hits.insert(raw_token);}- if(raw_token_index >= last_valid_offset + highlight_affix_num_tokens) {+ if(last_valid_offset_index != -1 && raw_token_index >= last_valid_offset + highlight_affix_num_tokens) {// register end of highlight snippetif(snippet_end_offset == text.size() - 1) {snippet_end_offset = tok_end;@@ -4349,7 +4358,7 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, constif(raw_token_index >= snippet_threshold &&match_offset_index > last_valid_offset_index &&raw_token_index >= last_valid_offset + highlight_affix_num_tokens &&- !highlight_fully) {+ !is_arr_obj_ele && !highlight_fully) {break;}}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 503f9034..dd8f37c2 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -1487,6 +1487,39 @@ TEST_F(CollectionNestedFieldsTest, ExplicitSchemaForNestedArrayTypeValidation) {"Hint: field inside an array of objects must be an array type as well.", add_op.error());}+TEST_F(CollectionNestedFieldsTest, NestedStringArrayHighlight) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "passages", "type": "object[]"},+ {"name": "passages.text", "type": "string[]"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc_str = std::string (R"({"passages": [{"text": "In January 1880, two of Tesla's uncles put together enough money to help him )") ++ "leave Gospić for Prague, where he was to study. He arrived too late to enroll at Charles-Ferdinand " ++ "University; he had never studied Greek, a required subject; and he was illiterate in Czech, another " ++ "required subject. Tesla did, however, attend lectures in philosophy at the university as an auditor " ++ "but he did not receive grades for the courses." + R"("}]})";++ auto doc1 = nlohmann::json::parse(doc_str);+ coll1->add(doc1.dump(), CREATE);++ auto results = coll1->search("grades", {"passages.text"},+ "", {}, {}, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ("he did not receive <mark>grades</mark> for the courses.",+ results["hits"][0]["highlight"]["passages"][0]["text"]["snippet"].get<std::string>());+}+TEST_F(CollectionNestedFieldsTest, OptionalNestedOptionalOjectArrStringField) {nlohmann::json schema = R"({"name": "coll1",