Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Match query tokens across multiple fields effectively.

kishorenc7e2b0fcdcbff
diff --git a/TODO.md b/TODO.md
index b9d4b2d2..a4f13b61 100644
--- a/TODO.md
+++ b/TODO.md
@@ -136,6 +136,8 @@ e) Async import
- Use bitmap index instead of compressed array for doc list?
- Primary_rank_scores and secondary_rank_scores hashmaps should be combined?
- d-ary heap?
+- ~~topster: reject min heap value compare only when field is same~~
+- ~~match index instead of match score~~
**API**
diff --git a/include/collection.h b/include/collection.h
index 7a77699b..eac329c0 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -100,6 +100,8 @@ struct doc_seq_id_t {
class Collection {
private:
+ const uint8_t CURATED_RECORD_IDENTIFIER = 100;
+
struct highlight_t {
std::string field;
std::vector<std::string> snippets;
diff --git a/include/index.h b/include/index.h
index c74fb4dd..97a8b06c 100644
--- a/include/index.h
+++ b/include/index.h
@@ -149,7 +149,10 @@ private:
void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
const uint32_t* result_ids, size_t results_size);
- void search_field(const uint8_t & field_id, const std::string & query,
+ void search_field(const uint8_t & field_id,
+ std::vector<std::string>& query_tokens,
+ std::vector<std::string>& search_tokens,
+ size_t& num_tokens_dropped,
const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
const std::vector<uint32_t>& curated_ids,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,
diff --git a/include/topster.h b/include/topster.h
index bad3ea30..26b15e7b 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -8,17 +8,18 @@
struct KV {
uint8_t field_id{};
+ uint8_t match_score_index{};
uint16_t query_index{};
uint16_t array_index{};
uint64_t key{};
uint64_t distinct_key{};
- uint64_t match_score{};
int64_t scores[3]{}; // match score + 2 custom attributes
KV(uint8_t fieldId, uint16_t queryIndex, uint64_t key, uint64_t distinct_key,
- uint64_t match_score, const int64_t *scores):
- field_id(fieldId), query_index(queryIndex), array_index(0), key(key),
- distinct_key(distinct_key), match_score(match_score) {
+ uint8_t match_score_index, const int64_t *scores):
+ field_id(fieldId), match_score_index(match_score_index),
+ query_index(queryIndex), array_index(0), key(key),
+ distinct_key(distinct_key) {
this->scores[0] = scores[0];
this->scores[1] = scores[1];
this->scores[2] = scores[2];
@@ -54,11 +55,11 @@ struct Topster {
for(size_t i=0; i<capacity; i++) {
data[i].field_id = 0;
+ data[i].match_score_index = 0;
data[i].query_index = 0;
data[i].array_index = i;
data[i].key = 0;
data[i].distinct_key = 0;
- data[i].match_score = 0;
kvs[i] = &data[i];
}
}
@@ -92,10 +93,10 @@ struct Topster {
LOG(INFO) << "kv key: " << kv.first << " => " << kv.second->match_score;
}*/
- bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller_equal(kv, kvs[0]);
+ bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller(kv, kvs[0]);
size_t heap_op_index = 0;
- if(!distinct && less_than_min_heap) {
+ if(!distinct && less_than_min_heap && kv->field_id == kvs[0]->field_id) {
// for non-distinct, if incoming value is smaller than min-heap ignore
return false;
}
@@ -180,10 +181,15 @@ struct Topster {
KV* existing_kv = found_it->second;
//LOG(INFO) << "existing_kv: " << existing_kv->key << " -> " << existing_kv->match_score;
- if(is_smaller_equal(kv, existing_kv)) {
+ if(is_smaller(kv, existing_kv) && kv->field_id == existing_kv->field_id) {
return false;
}
+ if(kv->field_id != existing_kv->field_id) {
+ // allows a record to be matched across different fields (aggregated matching)
+ kv->scores[0] += existing_kv->scores[0];
+ }
+
SIFT_DOWN = true;
// replace existing kv and sift down
@@ -252,8 +258,8 @@ struct Topster {
std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);
}
- static bool is_smaller_equal(const struct KV* i, const struct KV* j) {
- return std::tie(i->scores[0], i->scores[1], i->scores[2]) <=
+ static bool is_smaller(const struct KV* i, const struct KV* j) {
+ return std::tie(i->scores[0], i->scores[1], i->scores[2]) <
std::tie(j->scores[0], j->scores[1], j->scores[2]);
}
diff --git a/src/collection.cpp b/src/collection.cpp
index e220d920..e618386b 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -868,7 +868,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
size_t result_position = result_group_kvs.size() + 1;
uint64_t override_position = override_result_kvs[override_kv_index][0]->distinct_key;
if(result_position == override_position) {
- override_result_kvs[override_kv_index][0]->match_score = 0; // to identify curated result
+ override_result_kvs[override_kv_index][0]->match_score_index = CURATED_RECORD_IDENTIFIER;
result_group_kvs.push_back(override_result_kvs[override_kv_index]);
override_kv_index++;
continue;
@@ -880,7 +880,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
while(override_kv_index < override_result_kvs.size()) {
- override_result_kvs[override_kv_index][0]->match_score = 0; // to identify curated result
+ override_result_kvs[override_kv_index][0]->match_score_index = CURATED_RECORD_IDENTIFIER;
result_group_kvs.push_back({override_result_kvs[override_kv_index]});
override_kv_index++;
}
@@ -983,10 +983,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
prune_document(document, include_fields, exclude_fields);
wrapper_doc["document"] = document;
- wrapper_doc["text_match"] = field_order_kv->match_score;
- if(field_order_kv->match_score == 0) {
+ if(field_order_kv->match_score_index == CURATED_RECORD_IDENTIFIER) {
wrapper_doc["curated"] = true;
+ } else {
+ wrapper_doc["text_match"] = field_order_kv->scores[field_order_kv->match_score_index];
}
hits_array.push_back(wrapper_doc);
diff --git a/src/index.cpp b/src/index.cpp
index 3a5258a4..989e9c1c 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -948,6 +948,12 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
result_ids = excluded_result_ids;
}
+ LOG(INFO) << "n: " << n;
+ for(size_t i=0; i < query_suggestion.size(); i++) {
+ LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "
+ << query_suggestion[i]->values->ids.getLength();
+ }
+
if(filter_ids != nullptr) {
// intersect once again with filter ids
uint32_t* filtered_result_ids = nullptr;
@@ -1322,7 +1328,7 @@ void Index::collate_included_ids(const std::string & query, const std::string &
scores[1] = int64_t(1);
scores[2] = int64_t(1);
- KV kv(field_id, searched_queries.size(), seq_id, distinct_id, match_score, scores);
+ KV kv(field_id, searched_queries.size(), seq_id, distinct_id, 0, scores);
curated_topster->add(&kv);
}
}
@@ -1428,13 +1434,21 @@ void Index::search(Option<uint32_t> & outcome,
filter_ids = nullptr;
} else {
const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);
+
+ std::vector<std::string> tokens;
+ Tokenizer(query, false, true).tokenize(tokens);
+
for(size_t i = 0; i < num_search_fields; i++) {
// proceed to query search only when no filters are provided or when filtering produces results
if(filters.empty() || filter_ids_length > 0) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort results
const std::string & field = search_fields[i];
- search_field(field_id, query, field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
+ std::vector<std::string> query_tokens = tokens;
+ std::vector<std::string> search_tokens = tokens;
+ size_t num_tokens_dropped = 0;
+
+ search_field(field_id, query_tokens, search_tokens, num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,
token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
collate_included_ids(query, field, field_id, included_ids_map, curated_topster, searched_queries);
@@ -1469,7 +1483,11 @@ void Index::search(Option<uint32_t> & outcome,
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-void Index::search_field(const uint8_t & field_id, const std::string & query, const std::string & field,
+void Index::search_field(const uint8_t & field_id,
+ std::vector<std::string>& query_tokens,
+ std::vector<std::string>& search_tokens,
+ size_t& num_tokens_dropped,
+ const std::string & field,
uint32_t *filter_ids, size_t filter_ids_length,
const std::vector<uint32_t>& curated_ids,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,
@@ -1484,17 +1502,11 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
- // Used to drop the least occurring token(s) for partial searches
- std::unordered_map<std::string, uint32_t> token_to_count;
-
std::vector<std::vector<int>> token_to_costs;
- Tokenizer tokenizer(query, false, true);
- std::string token;
- size_t token_index = 0;
- std::vector<std::string> tokens;
+ for(size_t stoken_index=0; stoken_index < search_tokens.size(); stoken_index++) {
+ const std::string& token = search_tokens[stoken_index];
- while(tokenizer.next(token, token_index)) {
std::vector<int> all_costs;
// This ensures that we don't end up doing a cost of 1 for a single char etc.
int bounded_cost = get_bounded_typo_cost(max_cost, token.length());
@@ -1504,7 +1516,6 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
}
token_to_costs.push_back(all_costs);
- tokens.push_back(token);
}
// stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"
@@ -1528,19 +1539,19 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
token_candidates_vec.clear();
size_t token_index = 0;
- while(token_index < tokens.size()) {
+ while(token_index < search_tokens.size()) {
// For each token, look up the generated cost for this iteration and search using that cost
- std::string token = tokens[token_index];
+ std::string token = search_tokens[token_index];
const std::string token_cost_hash = token + std::to_string(costs[token_index]);
std::vector<art_leaf*> leaves;
- //LOG(INFO) << "\nSearching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];
+ LOG(INFO) << "\nSearching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];
if(token_cost_cache.count(token_cost_hash) != 0) {
leaves = token_cost_cache[token_cost_hash];
} else {
// prefix should apply only for last token
- const bool prefix_search = prefix && (token_index == tokens.size()-1);
+ const bool prefix_search = prefix && (token_index == search_tokens.size()-1);
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
// If this is a prefix search, look for more candidates and do a union of those document IDs
@@ -1556,7 +1567,6 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
if(!leaves.empty()) {
//log_leaves(costs[token_index], token, leaves);
token_candidates_vec.push_back(token_candidates{token, costs[token_index], leaves});
- token_to_count[token] = std::max(token_to_count[token], leaves.at(0)->values->ids.getLength());
} else {
// No result at `cost = costs[token_index]`. Remove costs until `cost` for token and re-do combinations
auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
@@ -1572,7 +1582,8 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
// otherwise, we try to drop the token and search with remaining tokens
if(token_to_costs[token_index].empty()) {
token_to_costs.erase(token_to_costs.begin()+token_index);
- tokens.erase(tokens.begin()+token_index);
+ search_tokens.erase(search_tokens.begin()+token_index);
+ query_tokens.erase(query_tokens.begin()+token_index);
costs.erase(costs.begin()+token_index);
}
}
@@ -1586,7 +1597,7 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
token_index++;
}
- if(!token_candidates_vec.empty() && token_candidates_vec.size() == tokens.size()) {
+ if(!token_candidates_vec.empty() && token_candidates_vec.size() == search_tokens.size()) {
// If all tokens were found, go ahead and search for candidates with what we have so far
search_candidates(field_id, filter_ids, filter_ids_length, curated_ids, sort_fields, token_candidates_vec,
searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,
@@ -1602,27 +1613,30 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
}
// When there are not enough overall results and atleast one token has results
- if(all_result_ids_len < drop_tokens_threshold && token_to_count.size() > 1) {
- // Drop token with least hits and try searching again
- std::string truncated_query;
-
- std::vector<std::pair<std::string, uint32_t>> token_count_pairs;
- for (auto itr = token_to_count.begin(); itr != token_to_count.end(); ++itr) {
- token_count_pairs.push_back(*itr);
- }
-
- std::sort(token_count_pairs.begin(), token_count_pairs.end(), [=]
- (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {
- return a.second > b.second;
+ if(all_result_ids_len < drop_tokens_threshold && !query_tokens.empty() && num_tokens_dropped < query_tokens.size()) {
+ // Drop tokens from right until (len/2 + 1), and then from left until (len/2 + 1)
+
+ std::vector<std::string> truncated_tokens;
+ num_tokens_dropped++;
+
+ size_t mid_index = (query_tokens.size() / 2);
+ if(num_tokens_dropped <= mid_index) {
+ // drop from right
+ size_t end_index = (query_tokens.size() - 1) - num_tokens_dropped;
+ for(size_t i=0; i <= end_index; i++) {
+ truncated_tokens.push_back(query_tokens[i]);
+ }
+ } else {
+ // drop from left
+ size_t start_index = (num_tokens_dropped - mid_index);
+ for(size_t i=start_index; i<query_tokens.size(); i++) {
+ truncated_tokens.push_back(query_tokens[i]);
}
- );
-
- for(uint32_t i = 0; i < token_count_pairs.size()-1; i++) {
- // iterate till last but one
- truncated_query += " " + token_count_pairs.at(i).first;
}
- return search_field(field_id, truncated_query, field, filter_ids, filter_ids_length, curated_ids,
+
+ return search_field(field_id, query_tokens, truncated_tokens, num_tokens_dropped, field,
+ filter_ids, filter_ids_length, curated_ids,
facets, sort_fields, num_typos,
searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,
token_order, prefix);
@@ -1719,6 +1733,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
const int64_t default_score = 0;
int64_t scores[3] = {0};
+ size_t match_score_index = 0;
// avoiding loop
if(sort_fields.size() > 0) {
@@ -1727,6 +1742,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
scores[0] = (it == field_values[0]->end()) ? default_score : it->second;
} else {
scores[0] = int64_t(match_score);
+ match_score_index = 0;
}
if (sort_order[0] == -1) {
scores[0] = -scores[0];
@@ -1740,6 +1756,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
scores[1] = (it == field_values[1]->end()) ? default_score : it->second;
} else {
scores[1] = int64_t(match_score);
+ match_score_index = 1;
}
if (sort_order[1] == -1) {
scores[1] = -scores[1];
@@ -1753,6 +1770,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
scores[2] = (it == field_values[2]->end()) ? default_score : it->second;
} else {
scores[2] = int64_t(match_score);
+ match_score_index = 2;
}
if(sort_order[2] == -1) {
scores[2] = -scores[2];
@@ -1766,7 +1784,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
groups_processed.emplace(distinct_id);
}
- KV kv(field_id, query_index, seq_id, distinct_id, match_score, scores);
+ KV kv(field_id, query_index, seq_id, distinct_id, match_score_index, scores);
topster->add(&kv);
}
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index fb8a12ed..223ccab0 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -248,6 +248,71 @@ TEST_F(CollectionGroupingTest, GroupingCompoundKey) {
ASSERT_STREQ("Value of `group_limit` must be between 1 and 99.", res_op.error().c_str());
}
+TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false),
+ field("genre", field_types::STRING, true),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Train or Highway", "Gord Downie", "rock"},
+ {"Down There by the Train", "Dustin Kensrue", "pop"},
+ {"In the Train", "Dustin Kensrue", "pop"},
+ {"State Trooper", "Dustin Kensrue", "country"},
+ {"Down There Somewhere", "Dustin Kensrue", "pop"},
+ {"Down There by the Train", "Gord Downie", "rock"},
+ {"Down and Outside", "Gord Downie", "rock"},
+ {"Let it be", "Downie Kensrue", "country"},
+ {"There was a Train", "Gord Kensrue", "country"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["genre"] = records[i][2];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("Dustin Kensrue Down There by the Train",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,
+ false, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"genre"}, 2).get();
+
+ ASSERT_EQ(3, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["grouped_hits"].size());
+
+ ASSERT_STREQ("pop", results["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
+ ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());
+ ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("2", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());
+ ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());
+ ASSERT_STREQ("5", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());
+ ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());
+ ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionGroupingTest, GroupingWithGropLimitOfOne) {
auto res = coll_group->search("*", {}, "", {"brand"}, {}, 0, 50, 1, FREQUENCY,
false, Index::DROP_TOKENS_THRESHOLD,
@@ -264,8 +329,8 @@ TEST_F(CollectionGroupingTest, GroupingWithGropLimitOfOne) {
ASSERT_EQ(1, res["grouped_hits"][i]["hits"].size());
}
- ASSERT_STREQ("4", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("10", res["grouped_hits"][3]["hits"][0]["document"]["id"].get<std::string>().c_str()); // unbranded
ASSERT_STREQ("9", res["grouped_hits"][4]["hits"][0]["document"]["id"].get<std::string>().c_str());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index b61e2d12..3f546efa 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -246,7 +246,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
// should not try to drop tokens to expand query
results.clear();
results = collection->search("the a", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false, 10).get();
- ASSERT_EQ(8, results["hits"].size());
+ ASSERT_EQ(9, results["hits"].size());
results.clear();
results = collection->search("the a", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false, 0).get();
@@ -294,7 +294,7 @@ TEST_F(CollectionTest, QueryWithTypo) {
nlohmann::json results = collection->search("kind biologcal", query_fields, "", facets, sort_fields, 2, 3).get();
ASSERT_EQ(3, results["hits"].size());
- std::vector<std::string> ids = {"19", "20", "21"};
+ std::vector<std::string> ids = {"19", "3", "20"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
@@ -382,7 +382,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
std::vector<std::string> facets;
nlohmann::json results = collection->search("ISX what", query_fields, "", facets, sort_fields, 1, 4, 1, FREQUENCY, false).get();
ASSERT_EQ(4, results["hits"].size());
- ASSERT_EQ(9, results["found"].get<uint32_t>());
+ ASSERT_EQ(13, results["found"].get<uint32_t>());
std::vector<std::string> ids = {"8", "19", "6", "21"};
@@ -2520,3 +2520,81 @@ TEST_F(CollectionTest, CreateCollectionInvalidFieldType) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionTest, MultiFieldRelevance) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Down There by the Train", "Dustin Kensrue"},
+ {"Down There by the Train", "Gord Downie"},
+ {"State Trooper", "Dustin Kensrue"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("Dustin Kensrue Down There by the Train",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(3, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"].size());
+
+ std::vector<size_t> expected_ids = {0, 1, 2};
+
+ for(size_t i=0; i<expected_ids.size(); i++) {
+ ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));
+ }
+
+ // remove documents, reindex in another order and search again
+ for(size_t i=0; i<expected_ids.size(); i++) {
+ coll1->remove_if_found(i, true);
+ }
+
+ records = {
+ {"State Trooper", "Dustin Kensrue"},
+ {"Down There by the Train", "Gord Downie"},
+ {"Down There by the Train", "Dustin Kensrue"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ results = coll1->search("Dustin Kensrue Down There by the Train",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(3, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"].size());
+
+ expected_ids = {2, 1, 0};
+
+ for(size_t i=0; i<expected_ids.size(); i++) {
+ ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));
+ }
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index 104e941e..2bff8732 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -36,7 +36,7 @@ TEST(TopsterTest, MaxIntValues) {
scores[1] = data[i].primary_attr;
scores[2] = data[i].secondary_attr;
- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, data[i].match_score, scores);
+ KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);
topster.add(&kv);
}
@@ -48,11 +48,11 @@ TEST(TopsterTest, MaxIntValues) {
EXPECT_EQ(ids[i], topster.getKeyAt(i));
if(ids[i] == 1) {
- EXPECT_EQ(12, (int) topster.getKV(i)->match_score);
+ EXPECT_EQ(12, (int) topster.getKV(i)->scores[topster.getKV(i)->match_score_index]);
}
if(ids[i] == 5) {
- EXPECT_EQ(10, (int) topster.getKV(i)->match_score);
+ EXPECT_EQ(10, (int) topster.getKV(i)->scores[topster.getKV(i)->match_score_index]);
}
}
}
@@ -88,7 +88,7 @@ TEST(TopsterTest, MaxFloatValues) {
scores[1] = Index::float_to_in64_t(data[i].primary_attr);
scores[2] = data[i].secondary_attr;
- KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, data[i].match_score, scores);
+ KV kv(data[i].field_id, data[i].query_index, data[i].key, data[i].key, 0, scores);
topster.add(&kv);
}
@@ -134,29 +134,29 @@ TEST(TopsterTest, DistinctIntValues) {
scores[1] = data[i].primary_attr;
scores[2] = data[i].secondary_attr;
- KV kv(data[i].field_id, data[i].query_index, i+100, data[i].distinct_key, data[i].match_score, scores);
+ KV kv(data[i].field_id, data[i].query_index, i+100, data[i].distinct_key, 0, scores);
dist_topster.add(&kv);
}
dist_topster.sort();
- std::vector<uint64_t> distinct_ids = {4, 1, 5, 8, 9};
+ std::vector<uint64_t> distinct_ids = {4, 1, 8, 5, 9};
for(uint32_t i = 0; i < dist_topster.size; i++) {
EXPECT_EQ(distinct_ids[i], dist_topster.getDistinctKeyAt(i));
if(distinct_ids[i] == 1) {
- EXPECT_EQ(12, (int) dist_topster.getKV(i)->match_score);
+ EXPECT_EQ(12, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);
EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);
- EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->match_score);
- EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->match_score);
+ EXPECT_EQ(12, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);
+ EXPECT_EQ(11, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);
}
if(distinct_ids[i] == 5) {
- EXPECT_EQ(10, (int) dist_topster.getKV(i)->match_score);
+ EXPECT_EQ(9, (int) dist_topster.getKV(i)->scores[dist_topster.getKV(i)->match_score_index]);
EXPECT_EQ(2, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->size);
- EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->match_score);
- EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->match_score);
+ EXPECT_EQ(10, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(0)->scores[0]);
+ EXPECT_EQ(9, dist_topster.group_kv_map[dist_topster.getDistinctKeyAt(i)]->getKV(1)->scores[0]);
}
}
}
\ No newline at end of file