Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Reuse text highlighting for facet query highlight.
Kishore Nallanbf01fbb773d9
diff --git a/src/collection.cpp b/src/collection.cppindex 422b4214..24b3d331 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2402,7 +2402,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,// remap facet value hash with actual stringauto & facet_count = facet_counts[fi];std::string value;- nlohmann::json document, parent;+ nlohmann::json document;if(should_fetch_doc_from_store) {const std::string &seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);@@ -2425,81 +2425,84 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}}- if(the_field.nested && should_return_parent) {- parent = get_facet_parent(the_field.name, document);- }+ highlight_t highlight;- std::unordered_map<std::string, size_t> ftoken_pos;- std::vector<string>& ftokens = a_facet.is_intersected ? a_facet.fvalue_tokens[facet_count.fvalue] :- a_facet.hash_tokens[facet_count.fhash];- //LOG(INFO) << "working on hash_tokens for hash " << kv.first << " with size " << ftokens.size();- for(size_t ti = 0; ti < ftokens.size(); ti++) {- if(the_field.is_bool()) {- if(ftokens[ti] == "1") {- ftokens[ti] = "true";- } else {- ftokens[ti] = "false";- }- }- const std::string& resolved_token = ftokens[ti];- ftoken_pos[resolved_token] = ti;- }- const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();- // 2 passes: first identify tokens that need to be highlighted and then construct highlighted text- bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);- bool normalise = is_cyrillic ? false : true;- Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);- // secondary tokenizer used for specific languages that requires transliteration- // we use 2 tokenizers so that the original text offsets are available for highlighting- Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);- std::string raw_token;- size_t raw_token_index = 0, tok_start = 0, tok_end = 0;- // need an ordered map here to ensure that it is ordered by the key (start offset)- std::map<size_t, size_t> token_offsets;- size_t prefix_token_start_index = 0;- while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {- if(is_cyrillic) {- word_tokenizer.tokenize(raw_token);- }- auto token_pos_it = ftoken_pos.find(raw_token);- if(token_pos_it != ftoken_pos.end()) {- token_offsets[tok_start] = tok_end;- if(raw_token == last_full_q_token) {- prefix_token_start_index = tok_start;- }+ if(!facet_query.query.empty()) {+ bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);+ bool use_word_tokenizer = the_field.locale == "th" || the_field.locale == "ja" ||+ Tokenizer::is_cyrillic(the_field.locale);+ bool normalise = !use_word_tokenizer;++ std::vector<std::string> fquery_tokens;+ Tokenizer(facet_query.query, true, false, the_field.locale, symbols_to_index,+ token_separators).tokenize(fquery_tokens);++ if(fquery_tokens.empty()) {+ continue;}- }- //LOG(INFO) << "token_offsets size " << token_offsets.size();- auto offset_it = token_offsets.begin();- size_t i = 0;- std::stringstream highlightedss;- // loop until end index, accumulate token and complete highlighting- while(i < value.size()) {- if(offset_it != token_offsets.end()) {- if (i == offset_it->first) {- highlightedss << highlight_start_tag;- // do prefix highlighting for non-dropped last token- size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?- facet_query_last_token.size() :- (offset_it->second - i + 1);- if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {- token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());++ std::vector<string>& ftokens = a_facet.is_intersected ? a_facet.fvalue_tokens[facet_count.fvalue] :+ a_facet.hash_tokens[facet_count.fhash];++ tsl::htrie_map<char, token_leaf> qtoken_leaves;++ //LOG(INFO) << "working on hash_tokens for hash " << kv.first << " with size " << ftokens.size();+ for(size_t ti = 0; ti < ftokens.size(); ti++) {+ if(the_field.is_bool()) {+ if(ftokens[ti] == "1") {+ ftokens[ti] = "true";} else {- token_len = (offset_it->second - i + 1);+ ftokens[ti] = "false";}- for(size_t j = 0; j < token_len; j++) {- highlightedss << value[i + j];- }- highlightedss << highlight_end_tag;- offset_it++;- i += token_len;- continue;}++ Tokenizer(facet_query.query, true, false, the_field.locale, symbols_to_index,+ token_separators).tokenize(ftokens[ti]);++ const std::string& resolved_token = ftokens[ti];+ size_t root_len = (fquery_tokens.size() == ftokens.size()) ?+ fquery_tokens[ti].size() :+ resolved_token.size();++ token_leaf leaf(nullptr, root_len, 0, (ti == ftokens.size()-1));+ qtoken_leaves.emplace(resolved_token, leaf);}- highlightedss << value[i];- i++;++ std::vector<std::string> raw_fquery_tokens;+ Tokenizer(facet_query.query, normalise, false, the_field.locale, symbols_to_index,+ token_separators).tokenize(raw_fquery_tokens);++ if(raw_fquery_tokens.empty()) {+ continue;+ }++ size_t prefix_token_num_chars = StringUtils::get_num_chars(raw_fquery_tokens.back());++ StringUtils string_utils;+ size_t last_valid_offset = 0;+ int last_valid_offset_index = -1;+ match_index_t match_index(Match(), 0, 0);++ uint8_t index_symbols[256] = {};+ for(char c: symbols_to_index) {+ index_symbols[uint8_t(c)] = 1;+ }++ handle_highlight_text(value, normalise, the_field, symbols_to_index, token_separators,+ highlight, string_utils, use_word_tokenizer,+ highlight_affix_num_tokens, qtoken_leaves, last_valid_offset_index,+ prefix_token_num_chars, false, snippet_threshold, false, ftokens,+ last_valid_offset, highlight_start_tag, highlight_end_tag,+ index_symbols, match_index);+ }++ nlohmann::json parent;+ if(the_field.nested && should_return_parent) {+ parent = get_facet_parent(the_field.name, document);}- facet_value_t facet_value = {value, highlightedss.str(), facet_count.count,++ const auto& highlighted_text = highlight.snippets.empty() ? value : highlight.snippets[0];+ facet_value_t facet_value = {value, highlighted_text, facet_count.count,facet_count.sort_field_val, parent};facet_values.emplace_back(facet_value);}diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 17c5739f..36b20b37 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -854,7 +854,7 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringWithColon) {auto results = res_op.get();ASSERT_STREQ("foo:bar", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>foo:b</mark>ar", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>foo:ba</mark>r", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());results = coll1->search("*", {}, "", {"title"}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cppindex bc40b063..71ad6527 100644--- a/test/collection_optimized_faceting_test.cpp+++ b/test/collection_optimized_faceting_test.cpp@@ -2221,6 +2221,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetQueryWithDifferentLocale) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());ASSERT_EQ("çapeta", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("<mark>ç</mark>apeta", results["facet_counts"][0]["counts"][0]["highlighted"]);search_op = coll1->search("*", query_fields, "", {"phone(sort_by:_alpha:desc)"},sort_fields, {0}, 10, 1, FREQUENCY,{false},@@ -2239,6 +2240,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetQueryWithDifferentLocale) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());ASSERT_EQ("teléfono justo", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("<mark>telé</mark>fono justo", results["facet_counts"][0]["counts"][0]["highlighted"]);}TEST_F(CollectionOptimizedFacetingTest, ValueIndexStatsMinMax) {