Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Match query tokens across multiple fields effectively.
kishorenc7e2b0fcdcbff
diff --git a/TODO.md b/TODO.mdindex b9d4b2d2..a4f13b61 100644--- a/TODO.md+++ b/TODO.md@@ -136,6 +136,8 @@ e) Async import- Use bitmap index instead of compressed array for doc list?- Primary_rank_scores and secondary_rank_scores hashmaps should be combined?- d-ary heap?+- ~~topster: reject min heap value compare only when field is same~~+- ~~match index instead of match score~~**API**diff --git a/include/collection.h b/include/collection.hindex 7a77699b..eac329c0 100644--- a/include/collection.h+++ b/include/collection.h@@ -100,6 +100,8 @@ struct doc_seq_id_t {class Collection {private:+ const uint8_t CURATED_RECORD_IDENTIFIER = 100;+struct highlight_t {std::string field;std::vector<std::string> snippets;diff --git a/include/index.h b/include/index.hindex c74fb4dd..97a8b06c 100644--- a/include/index.h+++ b/include/index.h@@ -149,7 +149,10 @@ private:void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const uint32_t* result_ids, size_t results_size);- void search_field(const uint8_t & field_id, const std::string & query,+ void search_field(const uint8_t & field_id,+ std::vector<std::string>& query_tokens,+ std::vector<std::string>& search_tokens,+ size_t& num_tokens_dropped,const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,const std::vector<uint32_t>& curated_ids,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,diff --git a/include/topster.h b/include/topster.hindex bad3ea30..26b15e7b 100644--- a/include/topster.h+++ b/include/topster.h@@ -8,17 +8,18 @@struct KV {uint8_t field_id{};+ uint8_t match_score_index{};uint16_t query_index{};uint16_t array_index{};uint64_t key{};uint64_t distinct_key{};- uint64_t match_score{};int64_t scores[3]{}; // match score + 2 custom attributesKV(uint8_t fieldId, uint16_t queryIndex, uint64_t key, uint64_t distinct_key,- uint64_t match_score, const int64_t *scores):- field_id(fieldId), query_index(queryIndex), array_index(0), key(key),- distinct_key(distinct_key), match_score(match_score) {+ uint8_t match_score_index, const int64_t *scores):+ field_id(fieldId), match_score_index(match_score_index),+ query_index(queryIndex), array_index(0), key(key),+ distinct_key(distinct_key) {this->scores[0] = scores[0];this->scores[1] = scores[1];this->scores[2] = scores[2];@@ -54,11 +55,11 @@ struct Topster {for(size_t i=0; i<capacity; i++) {data[i].field_id = 0;+ data[i].match_score_index = 0;data[i].query_index = 0;data[i].array_index = i;data[i].key = 0;data[i].distinct_key = 0;- data[i].match_score = 0;kvs[i] = &data[i];}}@@ -92,10 +93,10 @@ struct Topster {LOG(INFO) << "kv key: " << kv.first << " => " << kv.second->match_score;}*/- bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller_equal(kv, kvs[0]);+ bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller(kv, kvs[0]);size_t heap_op_index = 0;- if(!distinct && less_than_min_heap) {+ if(!distinct && less_than_min_heap && kv->field_id == kvs[0]->field_id) {// for non-distinct, if incoming value is smaller than min-heap ignorereturn false;}@@ -180,10 +181,15 @@ struct Topster {KV* existing_kv = found_it->second;//LOG(INFO) << "existing_kv: " << existing_kv->key << " -> " << existing_kv->match_score;- if(is_smaller_equal(kv, existing_kv)) {+ if(is_smaller(kv, existing_kv) && kv->field_id == existing_kv->field_id) {return false;}+ if(kv->field_id != existing_kv->field_id) {+ // allows a record to be matched across different fields (aggregated matching)+ kv->scores[0] += existing_kv->scores[0];+ }+SIFT_DOWN = true;// replace existing kv and sift down@@ -252,8 +258,8 @@ struct Topster {std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);}- static bool is_smaller_equal(const struct KV* i, const struct KV* j) {- return std::tie(i->scores[0], i->scores[1], i->scores[2]) <=+ static bool is_smaller(const struct KV* i, const struct KV* j) {+ return std::tie(i->scores[0], i->scores[1], i->scores[2]) <std::tie(j->scores[0], j->scores[1], j->scores[2]);}diff --git a/src/collection.cpp b/src/collection.cppindex e220d920..e618386b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -868,7 +868,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::size_t result_position = result_group_kvs.size() + 1;uint64_t override_position = override_result_kvs[override_kv_index][0]->distinct_key;if(result_position == override_position) {- override_result_kvs[override_kv_index][0]->match_score = 0; // to identify curated result+ override_result_kvs[override_kv_index][0]->match_score_index = CURATED_RECORD_IDENTIFIER;result_group_kvs.push_back(override_result_kvs[override_kv_index]);override_kv_index++;continue;@@ -880,7 +880,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}while(override_kv_index < override_result_kvs.size()) {- override_result_kvs[override_kv_index][0]->match_score = 0; // to identify curated result+ override_result_kvs[override_kv_index][0]->match_score_index = CURATED_RECORD_IDENTIFIER;result_group_kvs.push_back({override_result_kvs[override_kv_index]});override_kv_index++;}@@ -983,10 +983,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::prune_document(document, include_fields, exclude_fields);wrapper_doc["document"] = document;- wrapper_doc["text_match"] = field_order_kv->match_score;- if(field_order_kv->match_score == 0) {+ if(field_order_kv->match_score_index == CURATED_RECORD_IDENTIFIER) {wrapper_doc["curated"] = true;+ } else {+ wrapper_doc["text_match"] = field_order_kv->scores[field_order_kv->match_score_index];}hits_array.push_back(wrapper_doc);diff --git a/src/index.cpp b/src/index.cppindex 3a5258a4..989e9c1c 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -948,6 +948,12 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, siresult_ids = excluded_result_ids;}+ LOG(INFO) << "n: " << n;+ for(size_t i=0; i < query_suggestion.size(); i++) {+ LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "+ << query_suggestion[i]->values->ids.getLength();+ }+if(filter_ids != nullptr) {// intersect once again with filter idsuint32_t* filtered_result_ids = nullptr;@@ -1322,7 +1328,7 @@ void Index::collate_included_ids(const std::string & query, const std::string &scores[1] = int64_t(1);scores[2] = int64_t(1);- KV kv(field_id, searched_queries.size(), seq_id, distinct_id, match_score, scores);+ KV kv(field_id, searched_queries.size(), seq_id, distinct_id, 0, scores);curated_topster->add(&kv);}}@@ -1428,13 +1434,21 @@ void Index::search(Option<uint32_t> & outcome,filter_ids = nullptr;} else {const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);++ std::vector<std::string> tokens;+ Tokenizer(query, false, true).tokenize(tokens);+for(size_t i = 0; i < num_search_fields; i++) {// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.empty() || filter_ids_length > 0) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort resultsconst std::string & field = search_fields[i];- search_field(field_id, query, field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,+ std::vector<std::string> query_tokens = tokens;+ std::vector<std::string> search_tokens = tokens;+ size_t num_tokens_dropped = 0;++ search_field(field_id, query_tokens, search_tokens, num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);collate_included_ids(query, field, field_id, included_ids_map, curated_topster, searched_queries);@@ -1469,7 +1483,11 @@ void Index::search(Option<uint32_t> & outcome,4. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/-void Index::search_field(const uint8_t & field_id, const std::string & query, const std::string & field,+void Index::search_field(const uint8_t & field_id,+ std::vector<std::string>& query_tokens,+ std::vector<std::string>& search_tokens,+ size_t& num_tokens_dropped,+ const std::string & field,uint32_t *filter_ids, size_t filter_ids_length,const std::vector<uint32_t>& curated_ids,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,@@ -1484,17 +1502,11 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co// To prevent us from doing ART search repeatedly as we iterate through possible correctionsspp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;- // Used to drop the least occurring token(s) for partial searches- std::unordered_map<std::string, uint32_t> token_to_count;-std::vector<std::vector<int>> token_to_costs;- Tokenizer tokenizer(query, false, true);- std::string token;- size_t token_index = 0;- std::vector<std::string> tokens;+ for(size_t stoken_index=0; stoken_index < search_tokens.size(); stoken_index++) {+ const std::string& token = search_tokens[stoken_index];- while(tokenizer.next(token, token_index)) {std::vector<int> all_costs;// This ensures that we don't end up doing a cost of 1 for a single char etc.int bounded_cost = get_bounded_typo_cost(max_cost, token.length());@@ -1504,7 +1516,6 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co}token_to_costs.push_back(all_costs);- tokens.push_back(token);}// stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"@@ -1528,19 +1539,19 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, cotoken_candidates_vec.clear();size_t token_index = 0;- while(token_index < tokens.size()) {+ while(token_index < search_tokens.size()) {// For each token, look up the generated cost for this iteration and search using that cost- std::string token = tokens[token_index];+ std::string token = search_tokens[token_index];const std::string token_cost_hash = token + std::to_string(costs[token_index]);std::vector<art_leaf*> leaves;- //LOG(INFO) << "\nSearching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];+ LOG(INFO) << "\nSearching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];if(token_cost_cache.count(token_cost_hash) != 0) {leaves = token_cost_cache[token_cost_hash];} else {// prefix should apply only for last token- const bool prefix_search = prefix && (token_index == tokens.size()-1);+ const bool prefix_search = prefix && (token_index == search_tokens.size()-1);const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;// If this is a prefix search, look for more candidates and do a union of those document IDs@@ -1556,7 +1567,6 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, coif(!leaves.empty()) {//log_leaves(costs[token_index], token, leaves);token_candidates_vec.push_back(token_candidates{token, costs[token_index], leaves});- token_to_count[token] = std::max(token_to_count[token], leaves.at(0)->values->ids.getLength());} else {// No result at `cost = costs[token_index]`. Remove costs until `cost` for token and re-do combinationsauto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);@@ -1572,7 +1582,8 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co// otherwise, we try to drop the token and search with remaining tokensif(token_to_costs[token_index].empty()) {token_to_costs.erase(token_to_costs.begin()+token_index);- tokens.erase(tokens.begin()+token_index);+ search_tokens.erase(search_tokens.begin()+token_index);+ query_tokens.erase(query_tokens.begin()+token_index);costs.erase(costs.begin()+token_index);}}@@ -1586,7 +1597,7 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, cotoken_index++;}- if(!token_candidates_vec.empty() && token_candidates_vec.size() == tokens.size()) {+ if(!token_candidates_vec.empty() && token_candidates_vec.size() == search_tokens.size()) {// If all tokens were found, go ahead and search for candidates with what we have so farsearch_candidates(field_id, filter_ids, filter_ids_length, curated_ids, sort_fields, token_candidates_vec,searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,@@ -1602,27 +1613,30 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co}// When there are not enough overall results and atleast one token has results- if(all_result_ids_len < drop_tokens_threshold && token_to_count.size() > 1) {- // Drop token with least hits and try searching again- std::string truncated_query;-- std::vector<std::pair<std::string, uint32_t>> token_count_pairs;- for (auto itr = token_to_count.begin(); itr != token_to_count.end(); ++itr) {- token_count_pairs.push_back(*itr);- }-- std::sort(token_count_pairs.begin(), token_count_pairs.end(), [=]- (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {- return a.second > b.second;+ if(all_result_ids_len < drop_tokens_threshold && !query_tokens.empty() && num_tokens_dropped < query_tokens.size()) {+ // Drop tokens from right until (len/2 + 1), and then from left until (len/2 + 1)++ std::vector<std::string> truncated_tokens;+ num_tokens_dropped++;++ size_t mid_index = (query_tokens.size() / 2);+ if(num_tokens_dropped <= mid_index) {+ // drop from right+ size_t end_index = (query_tokens.size() - 1) - num_tokens_dropped;+ for(size_t i=0; i <= end_index; i++) {+ truncated_tokens.push_back(query_tokens[i]);+ }+ } else {+ // drop from left+ size_t start_index = (num_tokens_dropped - mid_index);+ for(size_t i=start_index; i<query_tokens.size(); i++) {+ truncated_tokens.push_back(query_tokens[i]);}- );-- for(uint32_t i = 0; i < token_count_pairs.size()-1; i++) {- // iterate till last but one- truncated_query += " " + token_count_pairs.at(i).first;}- return search_field(field_id, truncated_query, field, filter_ids, filter_ids_length, curated_ids,++ return search_field(field_id, query_tokens, truncated_tokens, num_tokens_dropped, field,+ filter_ids, filter_ids_length, curated_ids,facets, sort_fields, num_typos,searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,token_order, prefix);@@ -1719,6 +1733,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const int64_t default_score = 0;int64_t scores[3] = {0};+ size_t match_score_index = 0;// avoiding loopif(sort_fields.size() > 0) {@@ -1727,6 +1742,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16scores[0] = (it == field_values[0]->end()) ? default_score : it->second;} else {scores[0] = int64_t(match_score);+ match_score_index = 0;}if (sort_order[0] == -1) {scores[0] = -scores[0];@@ -1740,6 +1756,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16scores[1] = (it == field_values[1]->end()) ? default_score : it->second;} else {scores[1] = int64_t(match_score);+ match_score_index = 1;}if (sort_order[1] == -1) {scores[1] = -scores[1];@@ -1753,6 +1770,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16scores[2] = (it == field_values[2]->end()) ? default_score : it->second;} else {scores[2] = int64_t(match_score);+ match_score_index = 2;}if(sort_order[2] == -1) {scores[2] = -scores[2];@@ -1766,7 +1784,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16groups_processed.emplace(distinct_id);}- KV kv(field_id, query_index, seq_id, distinct_id, match_score, scores);+ KV kv(field_id, query_index, seq_id, distinct_id, match_score_index, scores);topster->add(&kv);}diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex fb8a12ed..223ccab0 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -248,6 +248,71 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {ASSERT_STREQ("Value of `group_limit` must be between 1 and 99.", res_op.error().c_str());}+TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("artist", field_types::STRING, false),+ field("genre", field_types::STRING, true),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Train or Highway", "Gord Downie", "rock"},+ {"Down There by the Train", "Dustin Kensrue", "pop"},+ {"In the Train", "Dustin Kensrue", "pop"},+ {"State Trooper", "Dustin Kensrue", "country"},+ {"Down There Somewhere", "Dustin Kensrue", "pop"},+ {"Down There by the Train", "Gord Downie", "rock"},+ {"Down and Outside", "Gord Downie", "rock"},+ {"Let it be", "Downie Kensrue", "country"},+ {"There was a Train", "Gord Kensrue", "country"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["genre"] = records[i][2];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("Dustin Kensrue Down There by the Train",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"genre"}, 2).get();++ ASSERT_EQ(3, results["found"].get<size_t>());+ ASSERT_EQ(3, results["grouped_hits"].size());++ ASSERT_STREQ("pop", results["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());+ ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());+ ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("2", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());+ ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());+ ASSERT_STREQ("5", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());+ ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());+ ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionGroupingTest, GroupingWithGropLimitOfOne) {auto res = coll_group->search("*", {}, "", {"brand"}, {}, 0, 50, 1, FREQUENCY,false, Index::DROP_TOKENS_THRESHOLD,@@ -264,8 +329,8 @@ TEST_F(CollectionGroupingTest, GroupingWithGropLimitOfOne) {ASSERT_EQ(1, res["grouped_hits"][i]["hits"].size());}- ASSERT_STREQ("4", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("10", res["grouped_hits"][3]["hits"][0]["document"]["id"].get<std::string>().c_str()); // unbrandedASSERT_STREQ("9", res["grouped_hits"][4]["hits"][0]["document"]["id"].get<std::string>().c_str());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex b61e2d12..3f546efa 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -246,7 +246,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// should not try to drop tokens to expand queryresults.clear();results = collection->search("the a", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false, 10).get();- ASSERT_EQ(8, results["hits"].size());+ ASSERT_EQ(9, results["hits"].size());results.clear();results = collection->search("the a", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false, 0).get();@@ -294,7 +294,7 @@ TEST_F(CollectionTest, QueryWithTypo) {nlohmann::json results = collection->search("kind biologcal", query_fields, "", facets, sort_fields, 2, 3).get();ASSERT_EQ(3, results["hits"].size());- std::vector<std::string> ids = {"19", "20", "21"};+ std::vector<std::string> ids = {"19", "3", "20"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -382,7 +382,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {std::vector<std::string> facets;nlohmann::json results = collection->search("ISX what", query_fields, "", facets, sort_fields, 1, 4, 1, FREQUENCY, false).get();ASSERT_EQ(4, results["hits"].size());- ASSERT_EQ(9, results["found"].get<uint32_t>());+ ASSERT_EQ(13, results["found"].get<uint32_t>());std::vector<std::string> ids = {"8", "19", "6", "21"};@@ -2520,3 +2520,81 @@ TEST_F(CollectionTest, CreateCollectionInvalidFieldType) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionTest, MultiFieldRelevance) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Down There by the Train", "Dustin Kensrue"},+ {"Down There by the Train", "Gord Downie"},+ {"State Trooper", "Dustin Kensrue"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("Dustin Kensrue Down There by the Train",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(3, results["found"].get<size_t>());+ ASSERT_EQ(3, results["hits"].size());++ std::vector<size_t> expected_ids = {0, 1, 2};++ for(size_t i=0; i<expected_ids.size(); i++) {+ ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));+ }++ // remove documents, reindex in another order and search again+ for(size_t i=0; i<expected_ids.size(); i++) {+ coll1->remove_if_found(i, true);+ }++ records = {+ {"State Trooper", "Dustin Kensrue"},+ {"Down There by the Train", "Gord Downie"},+ {"Down There by the Train", "Dustin Kensrue"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ results = coll1->search("Dustin Kensrue Down There by the Train",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(3, results["found"].get<size_t>());+ ASSERT_EQ(3, results["hits"].size());++ expected_ids = {2, 1, 0};++ for(size_t i=0; i<expected_ids.size(); i++) {+ ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));+ }++ collectionManager.drop_collection("coll1");+}diff --git a/test/topster_test.cpp b/test/topster_test.cppindex 104e941e..2bff8732 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -36,7 +36,7 @@ TEST(TopsterTest, MaxIntValues) {scores[1] = data[i].primary_attr;scores[2] = data[i].secondary_attr;- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, data[i].match_score, scores);+ KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);topster.add(&kv);}@@ -48,11 +48,11 @@ TEST(TopsterTest, MaxIntValues) {EXPECT_EQ(ids[i], topster.getKeyAt(i));if(ids[i] == 1) {- EXPECT_EQ(12, (int) topster.getKV(i)->match_score);+ EXPECT_EQ(12, (int) topster.getKV(i)->scores[topster.getKV(i)->match_score_index]);}if(ids[i] == 5) {- EXPECT_EQ(10, (int) topster.getKV(i)->match_score);+ EXPECT_EQ(10, (int) topster.getKV(i)->scores[topster.getKV(i)->match_score_index]);}}}@@ -88,7 +88,7 @@ TEST(TopsterTest, MaxFloatValues) {scores[1] = Index::float_to_in64_t(data[i].primary_attr);scores[2] = data[i].secondary_attr;- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, data[i].match_score, scores);+ KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);topster.add(&kv);}@@ -134,29 +134,29 @@ TEST(TopsterTest, DistinctIntValues) {scores[1] = data[i].primary_attr;scores[2] = data[i].secondary_attr;- KV kv(data[i].field_id, data[i].query_index, i+100, data[i].distinct_key, data[i].match_score, scores);+ KV kv(data[i].field_id, data[i].query_index, i+100, data[i].distinct_key, 0, scores);dist_topster.add(&kv);}dist_topster.sort();- std::vector<uint64_t> distinct_ids = {4, 1, 5, 8, 9};+ std::vector<uint64_t> distinct_ids = {4, 1, 8, 5, 9};for(uint32_t i = 0; i < dist_topster.size; i++) {EXPECT_EQ(distinct_ids[i], dist_topster.getDistinctKeyAt(i));if(distinct_ids[i] == 1) {- EXPECT_EQ(12, (int) dist_topster.getKV(i)->match_score);+ EXPECT_EQ(12, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);- EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->match_score);- EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->match_score);+ EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);+ EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);}if(distinct_ids[i] == 5) {- EXPECT_EQ(10, (int) dist_topster.getKV(i)->match_score);+ EXPECT_EQ(9, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);- EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->match_score);- EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->match_score);+ EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);+ EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);}}}\ No newline at end of file