Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Consider tokens matching across fields during ranking.
kishorencbc1d88f1eb4c
diff --git a/include/index.h b/include/index.hindex 4ad2f1d6..5d1fb23e 100644--- a/include/index.h+++ b/include/index.h@@ -17,8 +17,13 @@#include "string_utils.h"#include "num_tree.h"+struct token_t {+ size_t position;+ std::string value;+};+struct token_candidates {- std::string token;+ token_t token;size_t cost;std::vector<art_leaf*> candidates;};@@ -156,7 +161,8 @@ private:static inline uint32_t next_suggestion(const std::vector<token_candidates> &token_candidates_vec,long long int n,std::vector<art_leaf *>& actual_query_suggestion,- std::vector<art_leaf *>& query_suggestion);+ std::vector<art_leaf *>& query_suggestion,+ uint32_t& token_bits);void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;@@ -164,8 +170,8 @@ private:const uint32_t* result_ids, size_t results_size);void search_field(const uint8_t & field_id,- std::vector<std::string>& query_tokens,- std::vector<std::string>& search_tokens,+ std::vector<token_t>& query_tokens,+ std::vector<token_t>& search_tokens,const uint32_t* exclude_token_ids,size_t exclude_token_ids_size,size_t& num_tokens_dropped,@@ -264,7 +270,7 @@ public:void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const uint32_t *result_ids, const size_t result_size);+ const uint32_t *result_ids, const size_t result_size, uint32_t token_bits);static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);diff --git a/include/match_score.h b/include/match_score.hindex 6243c3cb..8cdcec07 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -46,10 +46,8 @@ struct Match {}// Explicit construction of match score- static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance,- const uint8_t field_id) {+ static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance) {uint64_t match_score = (- //(int64_t(field_id) << 24) |(int64_t(words_present) << 16) |(int64_t(255 - total_cost) << 8) |(int64_t(distance))@@ -59,9 +57,8 @@ struct Match {}// Construct a single match score from individual components (for multi-field sort)- inline uint64_t get_match_score(const uint32_t total_cost, const uint8_t field_id) const {+ inline uint64_t get_match_score(const uint32_t total_cost) const {uint64_t match_score = (- //(int64_t(field_id) << 24) |(int64_t(words_present) << 16) |(int64_t(255 - total_cost) << 8) |(int64_t(distance))diff --git a/include/topster.h b/include/topster.hindex f4ed57a2..5550fbd0 100644--- a/include/topster.h+++ b/include/topster.h@@ -11,14 +11,15 @@ struct KV {uint8_t match_score_index{};uint16_t query_index{};uint16_t array_index{};+ uint32_t token_bits{};uint64_t key{};uint64_t distinct_key{};int64_t scores[3]{}; // match score + 2 custom attributes- KV(uint8_t fieldId, uint16_t queryIndex, uint64_t key, uint64_t distinct_key,+ KV(uint8_t field_id, uint16_t queryIndex, uint32_t token_bits, uint64_t key, uint64_t distinct_key,uint8_t match_score_index, const int64_t *scores):- field_id(fieldId), match_score_index(match_score_index),- query_index(queryIndex), array_index(0), key(key),+ field_id(field_id), match_score_index(match_score_index),+ query_index(queryIndex), array_index(0), token_bits(token_bits), key(key),distinct_key(distinct_key) {this->scores[0] = scores[0];this->scores[1] = scores[1];@@ -58,6 +59,7 @@ struct Topster {data[i].match_score_index = 0;data[i].query_index = 0;data[i].array_index = i;+ data[i].token_bits = 0;data[i].key = 0;data[i].distinct_key = 0;kvs[i] = &data[i];diff --git a/src/collection.cpp b/src/collection.cppindex 630e8bfc..8191ff10 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1351,7 +1351,7 @@ void Collection::highlight_result(const field &search_field,}const Match & this_match = Match(field_order_kv->key, token_positions);- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);+ uint64_t this_match_score = this_match.get_match_score(1);match_indices.emplace_back(this_match, this_match_score, array_index);/*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)diff --git a/src/index.cpp b/src/index.cppindex 0ddd5006..377c532f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -842,13 +842,15 @@ void Index::search_candidates(const uint8_t & field_id,// actual query suggestion preserves original order of tokens in querystd::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());- uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion, query_suggestion);+ uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set+ uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,+ query_suggestion, token_bits);- /*LOG(INFO) << "n: " << n;+ LOG(INFO) << "n: " << n;for(size_t i=0; i < query_suggestion.size(); i++) {LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "<< query_suggestion[i]->values->ids.getLength() << ", total_cost: " << total_cost;- }*/+ }// initialize results with the starting element (for further intersection)size_t result_size = query_suggestion[0]->values->ids.getLength();@@ -911,7 +913,7 @@ void Index::search_candidates(const uint8_t & field_id,// go through each matching document id and calculate match scorescore_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,- groups_processed, filtered_result_ids, filtered_results_size);+ groups_processed, filtered_result_ids, filtered_results_size, token_bits);field_num_results += filtered_results_size;@@ -925,7 +927,7 @@ void Index::search_candidates(const uint8_t & field_id,*all_result_ids = new_all_result_ids;score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,- groups_processed, result_ids, result_size);+ groups_processed, result_ids, result_size, token_bits);/*if(result_size != 0) {LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size;@@ -1266,7 +1268,8 @@ void Index::collate_included_ids(const std::vector<std::string>& q_included_tokescores[1] = int64_t(1);scores[2] = int64_t(1);- KV kv(field_id, searched_queries.size(), seq_id, distinct_id, 0, scores);+ uint32_t token_bits = (uint32_t(1) << 31);+ KV kv(field_id, searched_queries.size(), token_bits, seq_id, distinct_id, 0, scores);curated_topster->add(&kv);}}@@ -1416,8 +1419,9 @@ void Index::search(Option<uint32_t> & outcome,filter_ids = excluded_result_ids;}+ uint32_t token_bits = 255;score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},- groups_processed, filter_ids, filter_ids_length);+ groups_processed, filter_ids, filter_ids_length, token_bits);collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);all_result_ids_len = filter_ids_length;@@ -1426,6 +1430,20 @@ void Index::search(Option<uint32_t> & outcome,} else {spp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;+ std::vector<token_t> q_include_pos_tokens;+ for(size_t i=0; i < q_include_tokens.size(); i++) {+ q_include_pos_tokens.push_back({i, q_include_tokens[i]});+ }++ std::vector<std::vector<token_t>> q_pos_synonyms;+ for(const auto& q_syn_vec: q_synonyms) {+ std::vector<token_t> q_pos_syn;+ for(size_t i=0; i < q_syn_vec.size(); i++) {+ q_pos_syn.push_back({i, q_syn_vec[i]});+ }+ q_pos_synonyms.emplace_back(q_pos_syn);+ }+// non-wildcardfor(size_t i = 0; i < num_search_fields; i++) {// proceed to query search only when no filters are provided or when filtering produces results@@ -1433,8 +1451,8 @@ void Index::search(Option<uint32_t> & outcome,const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);const std::string& field = search_fields[i].name;- std::vector<std::string> query_tokens = q_include_tokens;- std::vector<std::string> search_tokens = q_include_tokens;+ std::vector<token_t> query_tokens = q_include_pos_tokens;+ std::vector<token_t> search_tokens = q_include_pos_tokens;size_t num_tokens_dropped = 0;//LOG(INFO) << "searching field! " << field;@@ -1450,7 +1468,7 @@ void Index::search(Option<uint32_t> & outcome,token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);// do synonym based searches- for(const auto& syn_tokens: q_synonyms) {+ for(const auto& syn_tokens: q_pos_synonyms) {num_tokens_dropped = 0;query_tokens = search_tokens = syn_tokens;@@ -1484,22 +1502,30 @@ void Index::search(Option<uint32_t> & outcome,}int64_t aggregated_score = 0;+ uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set++ //LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);for(size_t i = 0; i < num_search_fields && num_search_fields > 1; i++) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);-- // produces weights that are powers of 2size_t weight = search_fields[i].weight;+ //LOG(INFO) << "--- field index: " << i;+if(field_id == kvs[0]->field_id) {+ token_bits |= kvs[0]->token_bits;+ //LOG(INFO) << "kvs[0]->field_id pop count: " << __builtin_popcount(token_bits);aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] * weight);continue;}if(existing_field_kvs.count(field_id) != 0) {// for existing field, we will simply sum field-wise weighted scores+ token_bits |= existing_field_kvs[field_id]->token_bits;+ //LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);+aggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]- * weight);+ * weight);continue;}@@ -1528,6 +1554,9 @@ void Index::search(Option<uint32_t> & outcome,continue;}+ token_bits |= 1UL << token_index; // sets nth bit+ //LOG(INFO) << "token_index: " << token_index << ", pop count: " << __builtin_popcount(token_bits);+uint32_t start_offset = leaves[0]->values->offset_index.at(doc_index);uint32_t end_offset = (doc_index == leaves[0]->values->ids.getLength() - 1) ?leaves[0]->values->offsets.getLength() :@@ -1541,11 +1570,16 @@ void Index::search(Option<uint32_t> & outcome,}if(words_present != 0) {- uint64_t match_score = Match::get_match_score(words_present, 0, 100, field_id);+ uint64_t match_score = Match::get_match_score(words_present, 0, 100);aggregated_score += (match_score * weight);}}+ //LOG(INFO) << "seq id: " << seq_id << ", pop count: " << __builtin_popcount(token_bits);++ //aggregated_score *= std::max(1, __builtin_popcount(field_id));+ aggregated_score *= __builtin_popcount(token_bits);+//LOG(INFO) << "kvs[0].key: " << kvs[0]->key;kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;topster->add(kvs[0]);@@ -1584,8 +1618,8 @@ void Index::search(Option<uint32_t> & outcome,5. Sort the docs based on some ranking criteria*/void Index::search_field(const uint8_t & field_id,- std::vector<std::string>& query_tokens,- std::vector<std::string>& search_tokens,+ std::vector<token_t>& query_tokens,+ std::vector<token_t>& search_tokens,const uint32_t* exclude_token_ids,size_t exclude_token_ids_size,size_t& num_tokens_dropped,@@ -1610,7 +1644,7 @@ void Index::search_field(const uint8_t & field_id,std::vector<std::vector<int>> token_to_costs;for(size_t stoken_index=0; stoken_index < search_tokens.size(); stoken_index++) {- const std::string& token = search_tokens[stoken_index];+ const std::string& token = search_tokens[stoken_index].value;std::vector<int> all_costs;// This ensures that we don't end up doing a cost of 1 for a single char etc.@@ -1646,7 +1680,7 @@ void Index::search_field(const uint8_t & field_id,while(token_index < search_tokens.size()) {// For each token, look up the generated cost for this iteration and search using that cost- const std::string& token = search_tokens[token_index];+ const std::string& token = search_tokens[token_index].value;const std::string token_cost_hash = token + std::to_string(costs[token_index]);std::vector<art_leaf*> leaves;@@ -1671,7 +1705,7 @@ void Index::search_field(const uint8_t & field_id,if(!leaves.empty()) {//log_leaves(costs[token_index], token, leaves);- token_candidates_vec.push_back(token_candidates{token, costs[token_index], leaves});+ token_candidates_vec.push_back(token_candidates{search_tokens[token_index], costs[token_index], leaves});} else {// No result at `cost = costs[token_index]`. Remove `cost` for token and re-do combinationsauto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);@@ -1719,7 +1753,7 @@ void Index::search_field(const uint8_t & field_id,if(!query_tokens.empty() && num_tokens_dropped < query_tokens.size()) {// Drop tokens from right until (len/2 + 1), and then from left until (len/2 + 1)- std::vector<std::string> truncated_tokens;+ std::vector<token_t> truncated_tokens;num_tokens_dropped++;size_t mid_index = (query_tokens.size() / 2);@@ -1727,13 +1761,13 @@ void Index::search_field(const uint8_t & field_id,// drop from rightsize_t end_index = (query_tokens.size() - 1) - num_tokens_dropped;for(size_t i=0; i <= end_index; i++) {- truncated_tokens.push_back(query_tokens[i]);+ truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});}} else {// drop from leftsize_t start_index = (num_tokens_dropped - mid_index);for(size_t i=start_index; i<query_tokens.size(); i++) {- truncated_tokens.push_back(query_tokens[i]);+ truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});}}@@ -1768,7 +1802,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const uint8_t & field_id, const uint32_t total_cost, Topster* topster,const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const uint32_t *result_ids, const size_t result_size) {+ const uint32_t *result_ids, const size_t result_size, uint32_t token_bits) {std::vector<uint32_t*> leaf_to_indices;for (art_leaf *token_leaf: query_suggestion) {@@ -1781,7 +1815,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16get_facet_to_index(facet_to_index);Match single_token_match = Match(1, 0);- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);int sort_order[3]; // 1 or -1 based on DESC or ASC respectivelyspp::sparse_hash_map<uint32_t, int64_t>* field_values[3];@@ -1816,7 +1850,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16continue;}const Match & match = Match(seq_id, token_positions, false);- uint64_t this_match_score = match.get_match_score(total_cost, field_id);+ uint64_t this_match_score = match.get_match_score(total_cost);match_score += this_match_score;@@ -1883,7 +1917,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16groups_processed.emplace(distinct_id);}- KV kv(field_id, query_index, seq_id, distinct_id, match_score_index, scores);+ KV kv(field_id, query_index, token_bits, seq_id, distinct_id, match_score_index, scores);topster->add(&kv);}@@ -1982,18 +2016,21 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggesinline uint32_t Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,long long int n,std::vector<art_leaf *>& actual_query_suggestion,- std::vector<art_leaf *>& query_suggestion) {+ std::vector<art_leaf *>& query_suggestion,+ uint32_t& token_bits) {uint32_t total_cost = 0;// generate the next combination from `token_leaves` and store it in `query_suggestion`ldiv_t q { n, 0 };for(long long i = 0 ; i < (long long) token_candidates_vec.size(); i++) {- size_t token_size = token_candidates_vec[i].token.size();+ size_t token_size = token_candidates_vec[i].token.value.size();q = ldiv(q.quot, token_candidates_vec[i].candidates.size());actual_query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];total_cost += token_candidates_vec[i].cost;+ token_bits |= 1UL << token_candidates_vec[i].token.position; // sets n-th bit+if(actual_query_suggestion[i]->key_len != token_size+1) {total_cost++;}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex d811a153..c12d3298 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2790,13 +2790,11 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,"<mark>", "</mark>", {1, 1}).get();- /*LOG(INFO) << results;-ASSERT_EQ(2, results["found"].get<size_t>());ASSERT_EQ(2, results["hits"].size());ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());*/+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}@@ -2926,8 +2924,8 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnFieldOrder) {ASSERT_EQ(2, results["found"].get<size_t>());ASSERT_EQ(2, results["hits"].size());- ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}diff --git a/test/topster_test.cpp b/test/topster_test.cppindex 2bff8732..304b5a8a 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -9,25 +9,26 @@ TEST(TopsterTest, MaxIntValues) {struct {uint8_t field_id;uint16_t query_index;+ uint32_t token_bits;uint64_t key;uint64_t match_score;int64_t primary_attr;int64_t secondary_attr;} data[14] = {- {1, 0, 1, 11, 20, 30},- {1, 0, 1, 12, 20, 32},- {1, 0, 2, 4, 20, 30},- {1, 2, 3, 7, 20, 30},- {1, 0, 4, 14, 20, 30},- {1, 1, 5, 9, 20, 30},- {1, 1, 5, 10, 20, 32},- {1, 1, 5, 9, 20, 30},- {1, 0, 6, 6, 20, 30},- {1, 2, 7, 6, 22, 30},- {1, 2, 7, 6, 22, 30},- {1, 1, 8, 9, 20, 30},- {1, 0, 9, 8, 20, 30},- {1, 3, 10, 5, 20, 30},+ {1, 0, 255, 1, 11, 20, 30},+ {1, 0, 255, 1, 12, 20, 32},+ {1, 0, 255, 2, 4, 20, 30},+ {1, 2, 255, 3, 7, 20, 30},+ {1, 0, 255, 4, 14, 20, 30},+ {1, 1, 255, 5, 9, 20, 30},+ {1, 1, 255, 5, 10, 20, 32},+ {1, 1, 255, 5, 9, 20, 30},+ {1, 0, 255, 6, 6, 20, 30},+ {1, 2, 255, 7, 6, 22, 30},+ {1, 2, 255, 7, 6, 22, 30},+ {1, 1, 255, 8, 9, 20, 30},+ {1, 0, 255, 9, 8, 20, 30},+ {1, 3, 255, 10, 5, 20, 30},};for(int i = 0; i < 14; i++) {@@ -36,7 +37,7 @@ TEST(TopsterTest, MaxIntValues) {scores[1] = data[i].primary_attr;scores[2] = data[i].secondary_attr;- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);+ KV kv(data[i].field_id, data[i].query_index, data[i].token_bits, data[i].key, data[i].key, 0, scores);topster.add(&kv);}@@ -63,23 +64,24 @@ TEST(TopsterTest, MaxFloatValues) {struct {uint8_t field_id;uint16_t query_index;+ uint32_t token_bits;uint64_t key;uint64_t match_score;float primary_attr;int64_t secondary_attr;} data[12] = {- {1, 0, 1, 11, 1.09, 30},- {1, 0, 2, 11, -20, 30},- {1, 2, 3, 11, -20, 30},- {1, 0, 4, 11, 7.812, 30},- {1, 0, 4, 11, 7.912, 30},- {1, 1, 5, 11, 0.0, 34},- {1, 0, 6, 11, -22, 30},- {1, 2, 7, 11, -22, 30},- {1, 1, 8, 11, -9.998, 30},- {1, 1, 8, 11, -9.998, 30},- {1, 0, 9, 11, -9.999, 30},- {1, 3, 10, 11, -20, 30},+ {1, 0, 255, 1, 11, 1.09, 30},+ {1, 0, 255, 2, 11, -20, 30},+ {1, 2, 255, 3, 11, -20, 30},+ {1, 0, 255, 4, 11, 7.812, 30},+ {1, 0, 255, 4, 11, 7.912, 30},+ {1, 1, 255, 5, 11, 0.0, 34},+ {1, 0, 255, 6, 11, -22, 30},+ {1, 2, 255, 7, 11, -22, 30},+ {1, 1, 255, 8, 11, -9.998, 30},+ {1, 1, 255, 8, 11, -9.998, 30},+ {1, 0, 255, 9, 11, -9.999, 30},+ {1, 3, 255, 10, 11, -20, 30},};for(int i = 0; i < 12; i++) {@@ -88,7 +90,7 @@ TEST(TopsterTest, MaxFloatValues) {scores[1] = Index::float_to_in64_t(data[i].primary_attr);scores[2] = data[i].secondary_attr;- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);+ KV kv(data[i].field_id, data[i].query_index, data[i].token_bits, data[i].key, data[i].key, 0, scores);topster.add(&kv);}@@ -107,25 +109,26 @@ TEST(TopsterTest, DistinctIntValues) {struct {uint8_t field_id;uint16_t query_index;+ uint32_t token_bits;uint64_t distinct_key;uint64_t match_score;int64_t primary_attr;int64_t secondary_attr;} data[14] = {- {1, 0, 1, 11, 20, 30},- {1, 0, 1, 12, 20, 32},- {1, 0, 2, 4, 20, 30},- {1, 2, 3, 7, 20, 30},- {1, 0, 4, 14, 20, 30},- {1, 1, 5, 9, 20, 30},- {1, 1, 5, 10, 20, 32},- {1, 1, 5, 9, 20, 30},- {1, 0, 6, 6, 20, 30},- {1, 2, 7, 6, 22, 30},- {1, 2, 7, 6, 22, 30},- {1, 1, 8, 9, 20, 30},- {1, 0, 9, 8, 20, 30},- {1, 3, 10, 5, 20, 30},+ {1, 0, 255, 1, 11, 20, 30},+ {1, 0, 255, 1, 12, 20, 32},+ {1, 0, 255, 2, 4, 20, 30},+ {1, 2, 255, 3, 7, 20, 30},+ {1, 0, 255, 4, 14, 20, 30},+ {1, 1, 255, 5, 9, 20, 30},+ {1, 1, 255, 5, 10, 20, 32},+ {1, 1, 255, 5, 9, 20, 30},+ {1, 0, 255, 6, 6, 20, 30},+ {1, 2, 255, 7, 6, 22, 30},+ {1, 2, 255, 7, 6, 22, 30},+ {1, 1, 255, 8, 9, 20, 30},+ {1, 0, 255, 9, 8, 20, 30},+ {1, 3, 255, 10, 5, 20, 30},};for(int i = 0; i < 14; i++) {@@ -134,7 +137,7 @@ TEST(TopsterTest, DistinctIntValues) {scores[1] = data[i].primary_attr;scores[2] = data[i].secondary_attr;- KV kv(data[i].field_id, data[i].query_index, i+100, data[i].distinct_key, 0, scores);+ KV kv(data[i].field_id, data[i].query_index, data[i].token_bits, i+100, data[i].distinct_key, 0, scores);dist_topster.add(&kv);}