Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Consider tokens matching across fields during ranking.

kishorencbc1d88f1eb4c
diff --git a/include/index.h b/include/index.h
index 4ad2f1d6..5d1fb23e 100644
--- a/include/index.h
+++ b/include/index.h
@@ -17,8 +17,13 @@
#include "string_utils.h"
#include "num_tree.h"
+struct token_t {
+ size_t position;
+ std::string value;
+};
+
struct token_candidates {
- std::string token;
+ token_t token;
size_t cost;
std::vector<art_leaf*> candidates;
};
@@ -156,7 +161,8 @@ private:
static inline uint32_t next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
long long int n,
std::vector<art_leaf *>& actual_query_suggestion,
- std::vector<art_leaf *>& query_suggestion);
+ std::vector<art_leaf *>& query_suggestion,
+ uint32_t& token_bits);
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
@@ -164,8 +170,8 @@ private:
const uint32_t* result_ids, size_t results_size);
void search_field(const uint8_t & field_id,
- std::vector<std::string>& query_tokens,
- std::vector<std::string>& search_tokens,
+ std::vector<token_t>& query_tokens,
+ std::vector<token_t>& search_tokens,
const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size,
size_t& num_tokens_dropped,
@@ -264,7 +270,7 @@ public:
void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const uint32_t *result_ids, const size_t result_size);
+ const uint32_t *result_ids, const size_t result_size, uint32_t token_bits);
static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);
diff --git a/include/match_score.h b/include/match_score.h
index 6243c3cb..8cdcec07 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -46,10 +46,8 @@ struct Match {
}
// Explicit construction of match score
- static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance,
- const uint8_t field_id) {
+ static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance) {
uint64_t match_score = (
- //(int64_t(field_id) << 24) |
(int64_t(words_present) << 16) |
(int64_t(255 - total_cost) << 8) |
(int64_t(distance))
@@ -59,9 +57,8 @@ struct Match {
}
// Construct a single match score from individual components (for multi-field sort)
- inline uint64_t get_match_score(const uint32_t total_cost, const uint8_t field_id) const {
+ inline uint64_t get_match_score(const uint32_t total_cost) const {
uint64_t match_score = (
- //(int64_t(field_id) << 24) |
(int64_t(words_present) << 16) |
(int64_t(255 - total_cost) << 8) |
(int64_t(distance))
diff --git a/include/topster.h b/include/topster.h
index f4ed57a2..5550fbd0 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -11,14 +11,15 @@ struct KV {
uint8_t match_score_index{};
uint16_t query_index{};
uint16_t array_index{};
+ uint32_t token_bits{};
uint64_t key{};
uint64_t distinct_key{};
int64_t scores[3]{}; // match score + 2 custom attributes
- KV(uint8_t fieldId, uint16_t queryIndex, uint64_t key, uint64_t distinct_key,
+ KV(uint8_t field_id, uint16_t queryIndex, uint32_t token_bits, uint64_t key, uint64_t distinct_key,
uint8_t match_score_index, const int64_t *scores):
- field_id(fieldId), match_score_index(match_score_index),
- query_index(queryIndex), array_index(0), key(key),
+ field_id(field_id), match_score_index(match_score_index),
+ query_index(queryIndex), array_index(0), token_bits(token_bits), key(key),
distinct_key(distinct_key) {
this->scores[0] = scores[0];
this->scores[1] = scores[1];
@@ -58,6 +59,7 @@ struct Topster {
data[i].match_score_index = 0;
data[i].query_index = 0;
data[i].array_index = i;
+ data[i].token_bits = 0;
data[i].key = 0;
data[i].distinct_key = 0;
kvs[i] = &data[i];
diff --git a/src/collection.cpp b/src/collection.cpp
index 630e8bfc..8191ff10 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1351,7 +1351,7 @@ void Collection::highlight_result(const field &search_field,
}
const Match & this_match = Match(field_order_kv->key, token_positions);
- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);
+ uint64_t this_match_score = this_match.get_match_score(1);
match_indices.emplace_back(this_match, this_match_score, array_index);
/*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)
diff --git a/src/index.cpp b/src/index.cpp
index 0ddd5006..377c532f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -842,13 +842,15 @@ void Index::search_candidates(const uint8_t & field_id,
// actual query suggestion preserves original order of tokens in query
std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());
- uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion, query_suggestion);
+ uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set
+ uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,
+ query_suggestion, token_bits);
- /*LOG(INFO) << "n: " << n;
+ LOG(INFO) << "n: " << n;
for(size_t i=0; i < query_suggestion.size(); i++) {
LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "
<< query_suggestion[i]->values->ids.getLength() << ", total_cost: " << total_cost;
- }*/
+ }
// initialize results with the starting element (for further intersection)
size_t result_size = query_suggestion[0]->values->ids.getLength();
@@ -911,7 +913,7 @@ void Index::search_candidates(const uint8_t & field_id,
// go through each matching document id and calculate match score
score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
- groups_processed, filtered_result_ids, filtered_results_size);
+ groups_processed, filtered_result_ids, filtered_results_size, token_bits);
field_num_results += filtered_results_size;
@@ -925,7 +927,7 @@ void Index::search_candidates(const uint8_t & field_id,
*all_result_ids = new_all_result_ids;
score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
- groups_processed, result_ids, result_size);
+ groups_processed, result_ids, result_size, token_bits);
/*if(result_size != 0) {
LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size;
@@ -1266,7 +1268,8 @@ void Index::collate_included_ids(const std::vector<std::string>& q_included_toke
scores[1] = int64_t(1);
scores[2] = int64_t(1);
- KV kv(field_id, searched_queries.size(), seq_id, distinct_id, 0, scores);
+ uint32_t token_bits = (uint32_t(1) << 31);
+ KV kv(field_id, searched_queries.size(), token_bits, seq_id, distinct_id, 0, scores);
curated_topster->add(&kv);
}
}
@@ -1416,8 +1419,9 @@ void Index::search(Option<uint32_t> & outcome,
filter_ids = excluded_result_ids;
}
+ uint32_t token_bits = 255;
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
- groups_processed, filter_ids, filter_ids_length);
+ groups_processed, filter_ids, filter_ids_length, token_bits);
collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
all_result_ids_len = filter_ids_length;
@@ -1426,6 +1430,20 @@ void Index::search(Option<uint32_t> & outcome,
} else {
spp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;
+ std::vector<token_t> q_include_pos_tokens;
+ for(size_t i=0; i < q_include_tokens.size(); i++) {
+ q_include_pos_tokens.push_back({i, q_include_tokens[i]});
+ }
+
+ std::vector<std::vector<token_t>> q_pos_synonyms;
+ for(const auto& q_syn_vec: q_synonyms) {
+ std::vector<token_t> q_pos_syn;
+ for(size_t i=0; i < q_syn_vec.size(); i++) {
+ q_pos_syn.push_back({i, q_syn_vec[i]});
+ }
+ q_pos_synonyms.emplace_back(q_pos_syn);
+ }
+
// non-wildcard
for(size_t i = 0; i < num_search_fields; i++) {
// proceed to query search only when no filters are provided or when filtering produces results
@@ -1433,8 +1451,8 @@ void Index::search(Option<uint32_t> & outcome,
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
const std::string& field = search_fields[i].name;
- std::vector<std::string> query_tokens = q_include_tokens;
- std::vector<std::string> search_tokens = q_include_tokens;
+ std::vector<token_t> query_tokens = q_include_pos_tokens;
+ std::vector<token_t> search_tokens = q_include_pos_tokens;
size_t num_tokens_dropped = 0;
//LOG(INFO) << "searching field! " << field;
@@ -1450,7 +1468,7 @@ void Index::search(Option<uint32_t> & outcome,
token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
// do synonym based searches
- for(const auto& syn_tokens: q_synonyms) {
+ for(const auto& syn_tokens: q_pos_synonyms) {
num_tokens_dropped = 0;
query_tokens = search_tokens = syn_tokens;
@@ -1484,22 +1502,30 @@ void Index::search(Option<uint32_t> & outcome,
}
int64_t aggregated_score = 0;
+ uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set
+
+ //LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);
for(size_t i = 0; i < num_search_fields && num_search_fields > 1; i++) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
-
- // produces weights that are powers of 2
size_t weight = search_fields[i].weight;
+ //LOG(INFO) << "--- field index: " << i;
+
if(field_id == kvs[0]->field_id) {
+ token_bits |= kvs[0]->token_bits;
+ //LOG(INFO) << "kvs[0]->field_id pop count: " << __builtin_popcount(token_bits);
aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] * weight);
continue;
}
if(existing_field_kvs.count(field_id) != 0) {
// for existing field, we will simply sum field-wise weighted scores
+ token_bits |= existing_field_kvs[field_id]->token_bits;
+ //LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);
+
aggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]
- * weight);
+ * weight);
continue;
}
@@ -1528,6 +1554,9 @@ void Index::search(Option<uint32_t> & outcome,
continue;
}
+ token_bits |= 1UL << token_index; // sets nth bit
+ //LOG(INFO) << "token_index: " << token_index << ", pop count: " << __builtin_popcount(token_bits);
+
uint32_t start_offset = leaves[0]->values->offset_index.at(doc_index);
uint32_t end_offset = (doc_index == leaves[0]->values->ids.getLength() - 1) ?
leaves[0]->values->offsets.getLength() :
@@ -1541,11 +1570,16 @@ void Index::search(Option<uint32_t> & outcome,
}
if(words_present != 0) {
- uint64_t match_score = Match::get_match_score(words_present, 0, 100, field_id);
+ uint64_t match_score = Match::get_match_score(words_present, 0, 100);
aggregated_score += (match_score * weight);
}
}
+ //LOG(INFO) << "seq id: " << seq_id << ", pop count: " << __builtin_popcount(token_bits);
+
+ //aggregated_score *= std::max(1, __builtin_popcount(field_id));
+ aggregated_score *= __builtin_popcount(token_bits);
+
//LOG(INFO) << "kvs[0].key: " << kvs[0]->key;
kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;
topster->add(kvs[0]);
@@ -1584,8 +1618,8 @@ void Index::search(Option<uint32_t> & outcome,
5. Sort the docs based on some ranking criteria
*/
void Index::search_field(const uint8_t & field_id,
- std::vector<std::string>& query_tokens,
- std::vector<std::string>& search_tokens,
+ std::vector<token_t>& query_tokens,
+ std::vector<token_t>& search_tokens,
const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size,
size_t& num_tokens_dropped,
@@ -1610,7 +1644,7 @@ void Index::search_field(const uint8_t & field_id,
std::vector<std::vector<int>> token_to_costs;
for(size_t stoken_index=0; stoken_index < search_tokens.size(); stoken_index++) {
- const std::string& token = search_tokens[stoken_index];
+ const std::string& token = search_tokens[stoken_index].value;
std::vector<int> all_costs;
// This ensures that we don't end up doing a cost of 1 for a single char etc.
@@ -1646,7 +1680,7 @@ void Index::search_field(const uint8_t & field_id,
while(token_index < search_tokens.size()) {
// For each token, look up the generated cost for this iteration and search using that cost
- const std::string& token = search_tokens[token_index];
+ const std::string& token = search_tokens[token_index].value;
const std::string token_cost_hash = token + std::to_string(costs[token_index]);
std::vector<art_leaf*> leaves;
@@ -1671,7 +1705,7 @@ void Index::search_field(const uint8_t & field_id,
if(!leaves.empty()) {
//log_leaves(costs[token_index], token, leaves);
- token_candidates_vec.push_back(token_candidates{token, costs[token_index], leaves});
+ token_candidates_vec.push_back(token_candidates{search_tokens[token_index], costs[token_index], leaves});
} else {
// No result at `cost = costs[token_index]`. Remove `cost` for token and re-do combinations
auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
@@ -1719,7 +1753,7 @@ void Index::search_field(const uint8_t & field_id,
if(!query_tokens.empty() && num_tokens_dropped < query_tokens.size()) {
// Drop tokens from right until (len/2 + 1), and then from left until (len/2 + 1)
- std::vector<std::string> truncated_tokens;
+ std::vector<token_t> truncated_tokens;
num_tokens_dropped++;
size_t mid_index = (query_tokens.size() / 2);
@@ -1727,13 +1761,13 @@ void Index::search_field(const uint8_t & field_id,
// drop from right
size_t end_index = (query_tokens.size() - 1) - num_tokens_dropped;
for(size_t i=0; i <= end_index; i++) {
- truncated_tokens.push_back(query_tokens[i]);
+ truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});
}
} else {
// drop from left
size_t start_index = (num_tokens_dropped - mid_index);
for(size_t i=start_index; i<query_tokens.size(); i++) {
- truncated_tokens.push_back(query_tokens[i]);
+ truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});
}
}
@@ -1768,7 +1802,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
const uint8_t & field_id, const uint32_t total_cost, Topster* topster,
const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const uint32_t *result_ids, const size_t result_size) {
+ const uint32_t *result_ids, const size_t result_size, uint32_t token_bits) {
std::vector<uint32_t*> leaf_to_indices;
for (art_leaf *token_leaf: query_suggestion) {
@@ -1781,7 +1815,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
get_facet_to_index(facet_to_index);
Match single_token_match = Match(1, 0);
- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);
+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);
int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
spp::sparse_hash_map<uint32_t, int64_t>* field_values[3];
@@ -1816,7 +1850,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
continue;
}
const Match & match = Match(seq_id, token_positions, false);
- uint64_t this_match_score = match.get_match_score(total_cost, field_id);
+ uint64_t this_match_score = match.get_match_score(total_cost);
match_score += this_match_score;
@@ -1883,7 +1917,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
groups_processed.emplace(distinct_id);
}
- KV kv(field_id, query_index, seq_id, distinct_id, match_score_index, scores);
+ KV kv(field_id, query_index, token_bits, seq_id, distinct_id, match_score_index, scores);
topster->add(&kv);
}
@@ -1982,18 +2016,21 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges
inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
long long int n,
std::vector<art_leaf *>& actual_query_suggestion,
- std::vector<art_leaf *>& query_suggestion) {
+ std::vector<art_leaf *>& query_suggestion,
+ uint32_t& token_bits) {
uint32_t total_cost = 0;
// generate the next combination from `token_leaves` and store it in `query_suggestion`
ldiv_t q { n, 0 };
for(long long i = 0 ; i < (long long) token_candidates_vec.size(); i++) {
- size_t token_size = token_candidates_vec[i].token.size();
+ size_t token_size = token_candidates_vec[i].token.value.size();
q = ldiv(q.quot, token_candidates_vec[i].candidates.size());
actual_query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];
query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];
total_cost += token_candidates_vec[i].cost;
+ token_bits |= 1UL << token_candidates_vec[i].token.position; // sets n-th bit
+
if(actual_query_suggestion[i]->key_len != token_size+1) {
total_cost++;
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index d811a153..c12d3298 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2790,13 +2790,11 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 1}).get();
- /*LOG(INFO) << results;
-
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());*/
+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
@@ -2926,8 +2924,8 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnFieldOrder) {
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
- ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index 2bff8732..304b5a8a 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -9,25 +9,26 @@ TEST(TopsterTest, MaxIntValues) {
struct {
uint8_t field_id;
uint16_t query_index;
+ uint32_t token_bits;
uint64_t key;
uint64_t match_score;
int64_t primary_attr;
int64_t secondary_attr;
} data[14] = {
- {1, 0, 1, 11, 20, 30},
- {1, 0, 1, 12, 20, 32},
- {1, 0, 2, 4, 20, 30},
- {1, 2, 3, 7, 20, 30},
- {1, 0, 4, 14, 20, 30},
- {1, 1, 5, 9, 20, 30},
- {1, 1, 5, 10, 20, 32},
- {1, 1, 5, 9, 20, 30},
- {1, 0, 6, 6, 20, 30},
- {1, 2, 7, 6, 22, 30},
- {1, 2, 7, 6, 22, 30},
- {1, 1, 8, 9, 20, 30},
- {1, 0, 9, 8, 20, 30},
- {1, 3, 10, 5, 20, 30},
+ {1, 0, 255, 1, 11, 20, 30},
+ {1, 0, 255, 1, 12, 20, 32},
+ {1, 0, 255, 2, 4, 20, 30},
+ {1, 2, 255, 3, 7, 20, 30},
+ {1, 0, 255, 4, 14, 20, 30},
+ {1, 1, 255, 5, 9, 20, 30},
+ {1, 1, 255, 5, 10, 20, 32},
+ {1, 1, 255, 5, 9, 20, 30},
+ {1, 0, 255, 6, 6, 20, 30},
+ {1, 2, 255, 7, 6, 22, 30},
+ {1, 2, 255, 7, 6, 22, 30},
+ {1, 1, 255, 8, 9, 20, 30},
+ {1, 0, 255, 9, 8, 20, 30},
+ {1, 3, 255, 10, 5, 20, 30},
};
for(int i = 0; i < 14; i++) {
@@ -36,7 +37,7 @@ TEST(TopsterTest, MaxIntValues) {
scores[1] = data[i].primary_attr;
scores[2] = data[i].secondary_attr;
- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);
+ KV kv(data[i].field_id, data[i].query_index, data[i].token_bits, data[i].key, data[i].key, 0, scores);
topster.add(&kv);
}
@@ -63,23 +64,24 @@ TEST(TopsterTest, MaxFloatValues) {
struct {
uint8_t field_id;
uint16_t query_index;
+ uint32_t token_bits;
uint64_t key;
uint64_t match_score;
float primary_attr;
int64_t secondary_attr;
} data[12] = {
- {1, 0, 1, 11, 1.09, 30},
- {1, 0, 2, 11, -20, 30},
- {1, 2, 3, 11, -20, 30},
- {1, 0, 4, 11, 7.812, 30},
- {1, 0, 4, 11, 7.912, 30},
- {1, 1, 5, 11, 0.0, 34},
- {1, 0, 6, 11, -22, 30},
- {1, 2, 7, 11, -22, 30},
- {1, 1, 8, 11, -9.998, 30},
- {1, 1, 8, 11, -9.998, 30},
- {1, 0, 9, 11, -9.999, 30},
- {1, 3, 10, 11, -20, 30},
+ {1, 0, 255, 1, 11, 1.09, 30},
+ {1, 0, 255, 2, 11, -20, 30},
+ {1, 2, 255, 3, 11, -20, 30},
+ {1, 0, 255, 4, 11, 7.812, 30},
+ {1, 0, 255, 4, 11, 7.912, 30},
+ {1, 1, 255, 5, 11, 0.0, 34},
+ {1, 0, 255, 6, 11, -22, 30},
+ {1, 2, 255, 7, 11, -22, 30},
+ {1, 1, 255, 8, 11, -9.998, 30},
+ {1, 1, 255, 8, 11, -9.998, 30},
+ {1, 0, 255, 9, 11, -9.999, 30},
+ {1, 3, 255, 10, 11, -20, 30},
};
for(int i = 0; i < 12; i++) {
@@ -88,7 +90,7 @@ TEST(TopsterTest, MaxFloatValues) {
scores[1] = Index::float_to_in64_t(data[i].primary_attr);
scores[2] = data[i].secondary_attr;
- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);
+ KV kv(data[i].field_id, data[i].query_index, data[i].token_bits, data[i].key, data[i].key, 0, scores);
topster.add(&kv);
}
@@ -107,25 +109,26 @@ TEST(TopsterTest, DistinctIntValues) {
struct {
uint8_t field_id;
uint16_t query_index;
+ uint32_t token_bits;
uint64_t distinct_key;
uint64_t match_score;
int64_t primary_attr;
int64_t secondary_attr;
} data[14] = {
- {1, 0, 1, 11, 20, 30},
- {1, 0, 1, 12, 20, 32},
- {1, 0, 2, 4, 20, 30},
- {1, 2, 3, 7, 20, 30},
- {1, 0, 4, 14, 20, 30},
- {1, 1, 5, 9, 20, 30},
- {1, 1, 5, 10, 20, 32},
- {1, 1, 5, 9, 20, 30},
- {1, 0, 6, 6, 20, 30},
- {1, 2, 7, 6, 22, 30},
- {1, 2, 7, 6, 22, 30},
- {1, 1, 8, 9, 20, 30},
- {1, 0, 9, 8, 20, 30},
- {1, 3, 10, 5, 20, 30},
+ {1, 0, 255, 1, 11, 20, 30},
+ {1, 0, 255, 1, 12, 20, 32},
+ {1, 0, 255, 2, 4, 20, 30},
+ {1, 2, 255, 3, 7, 20, 30},
+ {1, 0, 255, 4, 14, 20, 30},
+ {1, 1, 255, 5, 9, 20, 30},
+ {1, 1, 255, 5, 10, 20, 32},
+ {1, 1, 255, 5, 9, 20, 30},
+ {1, 0, 255, 6, 6, 20, 30},
+ {1, 2, 255, 7, 6, 22, 30},
+ {1, 2, 255, 7, 6, 22, 30},
+ {1, 1, 255, 8, 9, 20, 30},
+ {1, 0, 255, 9, 8, 20, 30},
+ {1, 3, 255, 10, 5, 20, 30},
};
for(int i = 0; i < 14; i++) {
@@ -134,7 +137,7 @@ TEST(TopsterTest, DistinctIntValues) {
scores[1] = data[i].primary_attr;
scores[2] = data[i].secondary_attr;
- KV kv(data[i].field_id, data[i].query_index, i+100, data[i].distinct_key, 0, scores);
+ KV kv(data[i].field_id, data[i].query_index, data[i].token_bits, i+100, data[i].distinct_key, 0, scores);
dist_topster.add(&kv);
}