Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Prioritize records whose fields match exactly with query.
Kishore Nallan25f6fe06147d
diff --git a/include/index.h b/include/index.hindex 478da8ec..180b84a7 100644--- a/include/index.h+++ b/include/index.h@@ -19,6 +19,7 @@#include "string_utils.h"#include "num_tree.h"#include "magic_enum.hpp"+#include "match_score.h"struct token_t {size_t position;@@ -223,7 +224,8 @@ private:size_t & all_result_ids_len,size_t& field_num_results,const size_t typo_tokens_threshold,- const size_t group_limit, const std::vector<std::string>& group_by_fields) const;+ const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const std::vector<token_t>& query_tokens) const;void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;@@ -303,7 +305,7 @@ public:spp::sparse_hash_set<uint64_t>& groups_processed,const uint32_t *result_ids, const size_t result_size,const size_t group_limit, const std::vector<std::string>& group_by_fields,- uint32_t token_bits) const;+ uint32_t token_bits, const std::vector<token_t>& query_tokens) const;static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);@@ -366,8 +368,13 @@ public:static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,const std::vector<uint32_t*>& leaf_to_indices,const size_t result_index,- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);+ /*static bool is_exact_token_match(const art_leaf* leaf,+ const std::vector<uint32_t*>& leaf_to_indices,+ const size_t result_index,+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);+*/static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);static double transform_for_180th_meridian(Geofence& poly);diff --git a/include/match_score.h b/include/match_score.hindex c5d69733..3a5d0d97 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -18,6 +18,11 @@const size_t WINDOW_SIZE = 10;const uint16_t MAX_DISPLACEMENT = std::numeric_limits<uint16_t>::max();+struct token_positions_t {+ bool last_token = false;+ std::vector<uint16_t> positions;+};+struct TokenOffset {uint8_t token_id; // token identifieruint16_t offset; // token's offset in the text@@ -39,22 +44,24 @@ struct TokenOffset {struct Match {uint8_t words_present;uint8_t distance;+ uint8_t exact_match;std::vector<TokenOffset> offsets;- Match() : words_present(0), distance(0) {+ Match() : words_present(0), distance(0), exact_match(0) {}- Match(uint8_t words_present, uint8_t distance) : words_present(words_present), distance(distance) {+ Match(uint8_t words_present, uint8_t distance) : words_present(words_present), distance(distance), exact_match(0) {}// Explicit construction of match scorestatic inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance) {uint64_t match_score = (- (int64_t(words_present) << 16) |- (int64_t(255 - total_cost) << 8) |- (int64_t(100 - distance))+ (int64_t(words_present) << 24) |+ (int64_t(255 - total_cost) << 16) |+ (int64_t(100 - distance) << 8) |+ (int64_t(0) << 1));return match_score;@@ -63,9 +70,10 @@ struct Match {// Construct a single match score from individual components (for multi-field sort)inline uint64_t get_match_score(const uint32_t total_cost) const {uint64_t match_score = (- (int64_t(words_present) << 16) |- (int64_t(255 - total_cost) << 8) |- (int64_t(100 - distance))+ (int64_t(words_present) << 24) |+ (int64_t(255 - total_cost) << 16) |+ (int64_t(100 - distance) << 8) |+ (int64_t(exact_match) << 1));return match_score;@@ -123,13 +131,13 @@ struct Match {Until queue size is 1.*/- Match(uint32_t doc_id, const std::vector<std::vector<uint16_t>> &token_offsets, bool populate_window=true) {+ Match(uint32_t doc_id, const std::vector<token_positions_t>& token_offsets, bool populate_window=true) {// in case if number of tokens in query is greater than max windowconst size_t tokens_size = std::min(token_offsets.size(), WINDOW_SIZE);std::vector<TokenOffset> window(tokens_size);for (size_t token_id = 0; token_id < tokens_size; token_id++) {- window[token_id] = TokenOffset{static_cast<uint8_t>(token_id), token_offsets[token_id][0], 0};+ window[token_id] = TokenOffset{static_cast<uint8_t>(token_id), token_offsets[token_id].positions[0], 0};}std::vector<TokenOffset> best_window;@@ -193,7 +201,7 @@ struct Match {window.pop_back();const uint8_t token_id = smallest_offset.token_id;- const std::vector<uint16_t> &this_token_offsets = token_offsets[token_id];+ const std::vector<uint16_t>& this_token_offsets = token_offsets[token_id].positions;if (smallest_offset.offset == this_token_offsets.back()) {// no more offsets for this token@@ -215,5 +223,24 @@ struct Match {if(populate_window) {offsets = best_window;}++ int last_token_index = -1;+ size_t total_offsets = 0;+ exact_match = 0;++ for(const auto& token_positions: token_offsets) {+ if(token_positions.last_token && !token_positions.positions.empty()) {+ last_token_index = token_positions.positions.back();+ }+ total_offsets += token_positions.positions.size();+ if(total_offsets > token_offsets.size()) {+ break;+ }+ }++ if(last_token_index == int(token_offsets.size())-1 &&+ total_offsets == token_offsets.size() && distance == token_offsets.size()-1) {+ exact_match = 1;+ }}};diff --git a/src/collection.cpp b/src/collection.cppindex 2677b153..5e39fe52 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1449,14 +1449,16 @@ void Collection::highlight_result(const field &search_field,return ;}+ //LOG(INFO) << "Document ID: " << document["id"];+// positions in the field of each token in the query- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);std::vector<match_index_t> match_indices;for(const auto& kv: array_token_positions) {- const std::vector<std::vector<uint16_t>>& token_positions = kv.second;+ const std::vector<token_positions_t>& token_positions = kv.second;size_t array_index = kv.first;if(token_positions.empty()) {diff --git a/src/index.cpp b/src/index.cppindex f7861763..2f7030d1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -534,7 +534,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, arstd::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale);- std::string token;+ std::string token, last_token;size_t token_index = 0;std::vector<uint64_t> facet_hashes;@@ -549,9 +549,13 @@ void Index::index_string_field(const std::string & text, const int64_t score, arfacet_hashes.push_back(hash);}- token_to_offsets[token].push_back(token_index);+ token_to_offsets[token].push_back(token_index + 1);+ last_token = token;}+ // push 0 for the last occurring token (used for exact match ranking)+ token_to_offsets[last_token].push_back(0);+/*if(seq_id == 0) {LOG(INFO) << "field name: " << a_field.name;}*/@@ -581,7 +585,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cstd::set<std::string> token_set; // required to deal with repeating tokensTokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale);- std::string token;+ std::string token, last_token;size_t token_index = 0;// iterate and append offset positions@@ -596,23 +600,25 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c//LOG(INFO) << "indexing " << token << ", hash:" << hash;}- token_positions[token].push_back(token_index);+ token_positions[token].push_back(token_index + 1);token_set.insert(token);+ last_token = token;}if(is_facet) {facet_hashes.push_back(FACET_ARRAY_DELIMETER); // as a delimiter}- // repeat last element to indicate end of offsets for this array index- for(auto & the_token: token_set) {+ for(auto& the_token: token_set) {+ // repeat last element to indicate end of offsets for this array indextoken_positions[the_token].push_back(token_positions[the_token].back());- }- // iterate and append this array index to all tokens- for(auto & the_token: token_set) {+ // iterate and append this array index to all tokenstoken_positions[the_token].push_back(array_index);}++ // push 0 for the last occurring token (used for exact match ranking)+ token_positions[last_token].push_back(0);}if(is_facet) {@@ -863,7 +869,10 @@ void Index::search_candidates(const uint8_t & field_id,uint32_t** all_result_ids, size_t & all_result_ids_len,size_t& field_num_results,const size_t typo_tokens_threshold,- const size_t group_limit, const std::vector<std::string>& group_by_fields) const {+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ const std::vector<token_t>& query_tokens) const {+const long long combination_limit = 10;auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };@@ -948,7 +957,7 @@ void Index::search_candidates(const uint8_t & field_id,// go through each matching document id and calculate match scorescore_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,groups_processed, filtered_result_ids, filtered_results_size,- group_limit, group_by_fields, token_bits);+ group_limit, group_by_fields, token_bits, query_tokens);field_num_results += filtered_results_size;@@ -966,7 +975,7 @@ void Index::search_candidates(const uint8_t & field_id,}*/score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits);+ groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits, query_tokens);field_num_results += result_size;@@ -1362,16 +1371,16 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,}for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);// iterate array_token_positions and compute hashfor(const auto& kv: array_token_positions) {- const std::vector<std::vector<uint16_t>>& token_positions = kv.second;+ const std::vector<token_positions_t>& token_positions = kv.second;size_t this_hash = 1;for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {- auto& positions = token_positions[token_index];+ auto& positions = token_positions[token_index].positions;for(auto pos: positions) {this_hash *= (1779033703 + 2*(token_index+1)*(pos+1));}@@ -1592,7 +1601,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint32_t token_bits = 255;score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},- groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits);+ groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {});collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);all_result_ids_len = filter_ids_length;@@ -1679,7 +1688,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,}uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set- uint64_t total_typos = 0, total_distances = 0;+ uint64_t total_typos = 0, total_distances = 0, exact_matches = 0;uint64_t num_exact_matches = 0;//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);@@ -1696,11 +1705,12 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,//LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);int64_t match_score = existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];- total_distances += ((100 - (match_score & 0xFF)) + 1) * weight;- uint64_t tokens_found = ((match_score >> 16) & 0xFF);- int64_t field_typos = 255 - ((match_score >> 8) & 0xFF);+ uint64_t tokens_found = ((match_score >> 24) & 0xFF);+ int64_t field_typos = 255 - ((match_score >> 16) & 0xFF);total_typos += (field_typos + 1) * weight;+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * weight;+ exact_matches += (((match_score & 0xFF)) + 1) * weight;if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {num_exact_matches++;@@ -1934,7 +1944,7 @@ void Index::search_field(const uint8_t & field_id,search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,groups_processed, all_result_ids, all_result_ids_len, field_num_results,- typo_tokens_threshold, group_limit, group_by_fields);+ typo_tokens_threshold, group_limit, group_by_fields, query_tokens);}resume_typo_loop:@@ -2004,17 +2014,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16spp::sparse_hash_set<uint64_t>& groups_processed,const uint32_t *result_ids, const size_t result_size,const size_t group_limit, const std::vector<std::string>& group_by_fields,- uint32_t token_bits) const {-- std::vector<uint32_t *> leaf_to_indices;- for (art_leaf *token_leaf: query_suggestion) {- uint32_t *indices = new uint32_t[result_size];- token_leaf->values->ids.indexOf(result_ids, result_size, indices);- leaf_to_indices.push_back(indices);- }-- Match single_token_match = Match(1, 0);- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);+ uint32_t token_bits,+ const std::vector<token_t>& query_tokens) const {int sort_order[3]; // 1 or -1 based on DESC or ASC respectivelyspp::sparse_hash_map<uint32_t, int64_t>* field_values[3];@@ -2052,21 +2053,37 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}}+ std::vector<uint32_t *> leaf_to_indices;+ for (art_leaf *token_leaf: query_suggestion) {+ uint32_t *indices = new uint32_t[result_size];+ token_leaf->values->ids.indexOf(result_ids, result_size, indices);+ leaf_to_indices.push_back(indices);+ }++ Match single_token_match = Match(1, 0);+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);+//auto begin = std::chrono::high_resolution_clock::now();+ //const std::string first_token((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1);++ // We will have to be judicious about computing full match score: only when token does not match exact query+ bool use_single_token_score = (query_suggestion.size() == 1) &&+ (query_suggestion.size() == query_tokens.size()) &&+ ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));for (size_t i = 0; i < result_size; i++) {const uint32_t seq_id = result_ids[i];uint64_t match_score = 0;- if (query_suggestion.size() <= 1) {+ if (use_single_token_score) {match_score = single_token_match_score;} else {- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);for (const auto& kv: array_token_positions) {- const std::vector<std::vector<uint16_t>> &token_positions = kv.second;+ const std::vector<token_positions_t>& token_positions = kv.second;if (token_positions.empty()) {continue;}@@ -2190,7 +2207,7 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,const std::vector<uint32_t*>& leaf_to_indices,const size_t result_index,- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>>& array_token_positions) {+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions) {if(query_suggestion.empty()) {return ;}@@ -2209,9 +2226,14 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggescontinue;}- // Array offset storage format:- // a) last element is array_index b) second and third last elements will be largest offset- // (last element is repeated to indicate end of offsets for a given array index)+ //LOG(INFO) << "key: " << token_leaf->key;++ // Plain string format:+ // offset1, offset2, ... , 0 (if token is the last offset for the document)++ // Array string format:+ // offset1, ... , offsetn, offsetn, array_index, 0 (if token is the last offset for the document)+ // (last offset is repeated to indicate end of offsets for a given array index)/*uint32_t* offsets = token_leaf->values->offsets.uncompress();for(size_t ii=0; ii < token_leaf->values->offsets.getLength(); ii++) {@@ -2232,30 +2254,49 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggesstd::vector<uint16_t> positions;int prev_pos = -1;+ bool is_last_token = false;while(start_offset < end_offset) {int pos = token_leaf->values->offsets.at(start_offset);start_offset++;+ if(pos == 0) {+ // indicates that token is the last token on the doc+ is_last_token = true;+ start_offset++;+ continue;+ }+if(pos == prev_pos) { // indicates end of array indexif(!positions.empty()) {size_t array_index = (size_t) token_leaf->values->offsets.at(start_offset);- array_token_positions[array_index].push_back(positions);+ is_last_token = false;++ if(start_offset+1 < end_offset) {+ size_t next_offset = (size_t) token_leaf->values->offsets.at(start_offset + 1);+ if(next_offset == 0) {+ // indicates that token is the last token on the doc+ is_last_token = true;+ start_offset++;+ }+ }++ array_token_positions[array_index].push_back(token_positions_t{is_last_token, positions});positions.clear();}- start_offset++; // skip current value which is the array index+ start_offset++; // skip current value which is the array index or flag for last indexprev_pos = -1;continue;}prev_pos = pos;- positions.push_back((uint16_t)pos);+ positions.push_back((uint16_t)pos - 1);}if(!positions.empty()) {// for plain string fields- array_token_positions[0].push_back(positions);+ array_token_positions[0].push_back(token_positions_t{is_last_token, positions});}}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 29d18de2..7cfefd40 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -718,7 +718,7 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {false, 0).get();ASSERT_EQ(2, results["hits"].size());- ids = {"0", "1"};+ ids = {"1", "0"};for (size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -766,26 +766,23 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {ASSERT_EQ(0, results["hits"][0]["highlights"][1]["indices"][0]);ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"][1]);- ASSERT_EQ(3, results["hits"][1]["highlights"][0].size());- ASSERT_STREQ("title", results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str());- ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_EQ(1, results["hits"][1]["highlights"][0]["matched_tokens"].size());- ASSERT_STREQ("Truth", results["hits"][1]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());-- ASSERT_EQ(4, results["hits"][1]["highlights"][1].size());- ASSERT_STREQ(results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str(), "tags");-- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["snippets"].size());- ASSERT_STREQ("<mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][0].get<std::string>().c_str());- ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][1].get<std::string>().c_str());-- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["matched_tokens"].size());- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());-- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"].size());- ASSERT_EQ(1, results["hits"][1]["highlights"][1]["indices"][0]);- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"][1]);+ ASSERT_EQ(4, results["hits"][1]["highlights"][0].size());+ ASSERT_STREQ(results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str(), "tags");+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["snippets"].size());+ ASSERT_STREQ("<mark>truth</mark>", results["hits"][1]["highlights"][0]["snippets"][0].get<std::string>().c_str());+ ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][1]["highlights"][0]["snippets"][1].get<std::string>().c_str());+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("truth", results["hits"][1]["highlights"][0]["matched_tokens"][0][0].get<std::string>().c_str());+ ASSERT_STREQ("truth", results["hits"][1]["highlights"][0]["matched_tokens"][1][0].get<std::string>().c_str());+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["indices"].size());+ ASSERT_EQ(1, results["hits"][1]["highlights"][0]["indices"][0]);+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["indices"][1]);++ ASSERT_EQ(3, results["hits"][1]["highlights"][1].size());+ ASSERT_STREQ("title", results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str());+ ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][1]["highlights"][1]["snippet"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][1]["highlights"][1]["matched_tokens"].size());+ ASSERT_STREQ("Truth", results["hits"][1]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());// highlight fields must be ordered based on match scoreresults = coll_array_text->search("amazing movie", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,@@ -2476,14 +2473,15 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title, tags").get();ASSERT_EQ(2, res["hits"][0]["highlights"].size());- ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",- res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());+ ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["values"][0].get<std::string>());+ ASSERT_EQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",+ res["hits"][0]["highlights"][1]["value"].get<std::string>());- ASSERT_EQ(1, res["hits"][0]["highlights"][0]["matched_tokens"].size());- ASSERT_STREQ("lazy", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());+ ASSERT_EQ(1, res["hits"][0]["highlights"][1]["matched_tokens"].size());+ ASSERT_STREQ("lazy", res["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());- ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"][0].size());- ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>().c_str());+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["values"][0].size());+ ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["values"][0].get<std::string>().c_str());// excluded fields should not be returned in highlights sectionspp::sparse_hash_set<std::string> excluded_fields = {"tags"};@@ -3087,6 +3085,59 @@ TEST_F(CollectionTest, MultiFieldRelevance5) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionTest, ExactMatch) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Alpha", "DJ"},+ {"Alpha Beta", "DJ"},+ {"Alpha Beta Gamma", "DJ"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("alpha beta",+ {"title"}, "", {}, {}, 2, 10, 1, FREQUENCY,+ true, 10).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("2", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ results = coll1->search("alpha", {"title"}, "", {}, {}, 2, 10, 1, FREQUENCY, true, 10).get();++ LOG(INFO) << results;++ ASSERT_EQ(3, results["found"].get<size_t>());+ ASSERT_EQ(3, results["hits"].size());++ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("2", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][2]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionTest, MultiFieldHighlighting) {Collection *coll1;diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex 0be9931a..41459a87 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -3,10 +3,19 @@#include <match_score.h>TEST(MatchTest, TokenOffsetsExceedWindowSize) {- std::vector<std::vector<uint16_t>> token_positions = {- std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}),- std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}),- std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1})+ std::vector<token_positions_t> token_positions = {+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})},+ token_positions_t{false, std::vector<uint16_t>({1})}};const Match & this_match = Match(100, token_positions);@@ -15,11 +24,11 @@ TEST(MatchTest, TokenOffsetsExceedWindowSize) {}TEST(MatchTest, MatchScoreV2) {- std::vector<std::vector<uint16_t>> token_offsets;- token_offsets.push_back({25});- token_offsets.push_back({26});- token_offsets.push_back({11, 18, 24, 60});- token_offsets.push_back({14, 27, 63});+ std::vector<token_positions_t> token_offsets;+ token_offsets.push_back(token_positions_t{false, {25}});+ token_offsets.push_back(token_positions_t{false, {26}});+ token_offsets.push_back(token_positions_t{false, {11, 18, 24, 60}});+ token_offsets.push_back(token_positions_t{false, {14, 27, 63}});auto match = Match(100, token_offsets, true);ASSERT_EQ(4, match.words_present);@@ -37,9 +46,9 @@ TEST(MatchTest, MatchScoreV2) {ASSERT_EQ(0, match.offsets.size());token_offsets.clear();- token_offsets.push_back({38, 50, 170, 187, 195, 222});- token_offsets.push_back({39, 140, 171, 189, 223});- token_offsets.push_back({169, 180});+ token_offsets.push_back(token_positions_t{false, {38, 50, 170, 187, 195, 222}});+ token_offsets.push_back(token_positions_t{false, {39, 140, 171, 189, 223}});+ token_offsets.push_back(token_positions_t{false, {169, 180}});match = Match(100, token_offsets, true);ASSERT_EQ(3, match.words_present);@@ -51,9 +60,9 @@ TEST(MatchTest, MatchScoreV2) {}token_offsets.clear();- token_offsets.push_back({38, 50, 187, 195, 201});- token_offsets.push_back({120, 167, 171, 223});- token_offsets.push_back({240, 250});+ token_offsets.push_back(token_positions_t{false, {38, 50, 187, 195, 201}});+ token_offsets.push_back(token_positions_t{false, {120, 167, 171, 223}});+ token_offsets.push_back(token_positions_t{false, {240, 250}});match = Match(100, token_offsets, true);ASSERT_EQ(1, match.words_present);