Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Prioritize records whose fields match exactly with query.

Kishore Nallan25f6fe06147d
diff --git a/include/index.h b/include/index.h
index 478da8ec..180b84a7 100644
--- a/include/index.h
+++ b/include/index.h
@@ -19,6 +19,7 @@
#include "string_utils.h"
#include "num_tree.h"
#include "magic_enum.hpp"
+#include "match_score.h"
struct token_t {
size_t position;
@@ -223,7 +224,8 @@ private:
size_t & all_result_ids_len,
size_t& field_num_results,
const size_t typo_tokens_threshold,
- const size_t group_limit, const std::vector<std::string>& group_by_fields) const;
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const std::vector<token_t>& query_tokens) const;
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
@@ -303,7 +305,7 @@ public:
spp::sparse_hash_set<uint64_t>& groups_processed,
const uint32_t *result_ids, const size_t result_size,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
- uint32_t token_bits) const;
+ uint32_t token_bits, const std::vector<token_t>& query_tokens) const;
static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);
@@ -366,8 +368,13 @@ public:
static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
const std::vector<uint32_t*>& leaf_to_indices,
const size_t result_index,
- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);
+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);
+ /*static bool is_exact_token_match(const art_leaf* leaf,
+ const std::vector<uint32_t*>& leaf_to_indices,
+ const size_t result_index,
+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);
+*/
static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);
static double transform_for_180th_meridian(Geofence& poly);
diff --git a/include/match_score.h b/include/match_score.h
index c5d69733..3a5d0d97 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -18,6 +18,11 @@
const size_t WINDOW_SIZE = 10;
const uint16_t MAX_DISPLACEMENT = std::numeric_limits<uint16_t>::max();
+struct token_positions_t {
+ bool last_token = false;
+ std::vector<uint16_t> positions;
+};
+
struct TokenOffset {
uint8_t token_id; // token identifier
uint16_t offset; // token's offset in the text
@@ -39,22 +44,24 @@ struct TokenOffset {
struct Match {
uint8_t words_present;
uint8_t distance;
+ uint8_t exact_match;
std::vector<TokenOffset> offsets;
- Match() : words_present(0), distance(0) {
+ Match() : words_present(0), distance(0), exact_match(0) {
}
- Match(uint8_t words_present, uint8_t distance) : words_present(words_present), distance(distance) {
+ Match(uint8_t words_present, uint8_t distance) : words_present(words_present), distance(distance), exact_match(0) {
}
// Explicit construction of match score
static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance) {
uint64_t match_score = (
- (int64_t(words_present) << 16) |
- (int64_t(255 - total_cost) << 8) |
- (int64_t(100 - distance))
+ (int64_t(words_present) << 24) |
+ (int64_t(255 - total_cost) << 16) |
+ (int64_t(100 - distance) << 8) |
+ (int64_t(0) << 1)
);
return match_score;
@@ -63,9 +70,10 @@ struct Match {
// Construct a single match score from individual components (for multi-field sort)
inline uint64_t get_match_score(const uint32_t total_cost) const {
uint64_t match_score = (
- (int64_t(words_present) << 16) |
- (int64_t(255 - total_cost) << 8) |
- (int64_t(100 - distance))
+ (int64_t(words_present) << 24) |
+ (int64_t(255 - total_cost) << 16) |
+ (int64_t(100 - distance) << 8) |
+ (int64_t(exact_match) << 1)
);
return match_score;
@@ -123,13 +131,13 @@ struct Match {
Until queue size is 1.
*/
- Match(uint32_t doc_id, const std::vector<std::vector<uint16_t>> &token_offsets, bool populate_window=true) {
+ Match(uint32_t doc_id, const std::vector<token_positions_t>& token_offsets, bool populate_window=true) {
// in case if number of tokens in query is greater than max window
const size_t tokens_size = std::min(token_offsets.size(), WINDOW_SIZE);
std::vector<TokenOffset> window(tokens_size);
for (size_t token_id = 0; token_id < tokens_size; token_id++) {
- window[token_id] = TokenOffset{static_cast<uint8_t>(token_id), token_offsets[token_id][0], 0};
+ window[token_id] = TokenOffset{static_cast<uint8_t>(token_id), token_offsets[token_id].positions[0], 0};
}
std::vector<TokenOffset> best_window;
@@ -193,7 +201,7 @@ struct Match {
window.pop_back();
const uint8_t token_id = smallest_offset.token_id;
- const std::vector<uint16_t> &this_token_offsets = token_offsets[token_id];
+ const std::vector<uint16_t>& this_token_offsets = token_offsets[token_id].positions;
if (smallest_offset.offset == this_token_offsets.back()) {
// no more offsets for this token
@@ -215,5 +223,24 @@ struct Match {
if(populate_window) {
offsets = best_window;
}
+
+ int last_token_index = -1;
+ size_t total_offsets = 0;
+ exact_match = 0;
+
+ for(const auto& token_positions: token_offsets) {
+ if(token_positions.last_token && !token_positions.positions.empty()) {
+ last_token_index = token_positions.positions.back();
+ }
+ total_offsets += token_positions.positions.size();
+ if(total_offsets > token_offsets.size()) {
+ break;
+ }
+ }
+
+ if(last_token_index == int(token_offsets.size())-1 &&
+ total_offsets == token_offsets.size() && distance == token_offsets.size()-1) {
+ exact_match = 1;
+ }
}
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 2677b153..5e39fe52 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1449,14 +1449,16 @@ void Collection::highlight_result(const field &search_field,
return ;
}
+ //LOG(INFO) << "Document ID: " << document["id"];
+
// positions in the field of each token in the query
- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;
+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
Index::populate_token_positions(query_suggestion, leaf_to_indices, 0, array_token_positions);
std::vector<match_index_t> match_indices;
for(const auto& kv: array_token_positions) {
- const std::vector<std::vector<uint16_t>>& token_positions = kv.second;
+ const std::vector<token_positions_t>& token_positions = kv.second;
size_t array_index = kv.first;
if(token_positions.empty()) {
diff --git a/src/index.cpp b/src/index.cpp
index f7861763..2f7030d1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -534,7 +534,7 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale);
- std::string token;
+ std::string token, last_token;
size_t token_index = 0;
std::vector<uint64_t> facet_hashes;
@@ -549,9 +549,13 @@ void Index::index_string_field(const std::string & text, const int64_t score, ar
facet_hashes.push_back(hash);
}
- token_to_offsets[token].push_back(token_index);
+ token_to_offsets[token].push_back(token_index + 1);
+ last_token = token;
}
+ // push 0 for the last occurring token (used for exact match ranking)
+ token_to_offsets[last_token].push_back(0);
+
/*if(seq_id == 0) {
LOG(INFO) << "field name: " << a_field.name;
}*/
@@ -581,7 +585,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
std::set<std::string> token_set; // required to deal with repeating tokens
Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale);
- std::string token;
+ std::string token, last_token;
size_t token_index = 0;
// iterate and append offset positions
@@ -596,23 +600,25 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
//LOG(INFO) << "indexing " << token << ", hash:" << hash;
}
- token_positions[token].push_back(token_index);
+ token_positions[token].push_back(token_index + 1);
token_set.insert(token);
+ last_token = token;
}
if(is_facet) {
facet_hashes.push_back(FACET_ARRAY_DELIMETER); // as a delimiter
}
- // repeat last element to indicate end of offsets for this array index
- for(auto & the_token: token_set) {
+ for(auto& the_token: token_set) {
+ // repeat last element to indicate end of offsets for this array index
token_positions[the_token].push_back(token_positions[the_token].back());
- }
- // iterate and append this array index to all tokens
- for(auto & the_token: token_set) {
+ // iterate and append this array index to all tokens
token_positions[the_token].push_back(array_index);
}
+
+ // push 0 for the last occurring token (used for exact match ranking)
+ token_positions[last_token].push_back(0);
}
if(is_facet) {
@@ -863,7 +869,10 @@ void Index::search_candidates(const uint8_t & field_id,
uint32_t** all_result_ids, size_t & all_result_ids_len,
size_t& field_num_results,
const size_t typo_tokens_threshold,
- const size_t group_limit, const std::vector<std::string>& group_by_fields) const {
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const std::vector<token_t>& query_tokens) const {
+
const long long combination_limit = 10;
auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };
@@ -948,7 +957,7 @@ void Index::search_candidates(const uint8_t & field_id,
// go through each matching document id and calculate match score
score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
groups_processed, filtered_result_ids, filtered_results_size,
- group_limit, group_by_fields, token_bits);
+ group_limit, group_by_fields, token_bits, query_tokens);
field_num_results += filtered_results_size;
@@ -966,7 +975,7 @@ void Index::search_candidates(const uint8_t & field_id,
}*/
score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits);
+ groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits, query_tokens);
field_num_results += result_size;
@@ -1362,16 +1371,16 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
}
for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {
- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;
+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);
// iterate array_token_positions and compute hash
for(const auto& kv: array_token_positions) {
- const std::vector<std::vector<uint16_t>>& token_positions = kv.second;
+ const std::vector<token_positions_t>& token_positions = kv.second;
size_t this_hash = 1;
for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {
- auto& positions = token_positions[token_index];
+ auto& positions = token_positions[token_index].positions;
for(auto pos: positions) {
this_hash *= (1779033703 + 2*(token_index+1)*(pos+1));
}
@@ -1592,7 +1601,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
uint32_t token_bits = 255;
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
- groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits);
+ groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {});
collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
all_result_ids_len = filter_ids_length;
@@ -1679,7 +1688,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
}
uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set
- uint64_t total_typos = 0, total_distances = 0;
+ uint64_t total_typos = 0, total_distances = 0, exact_matches = 0;
uint64_t num_exact_matches = 0;
//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);
@@ -1696,11 +1705,12 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
//LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);
int64_t match_score = existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];
- total_distances += ((100 - (match_score & 0xFF)) + 1) * weight;
- uint64_t tokens_found = ((match_score >> 16) & 0xFF);
- int64_t field_typos = 255 - ((match_score >> 8) & 0xFF);
+ uint64_t tokens_found = ((match_score >> 24) & 0xFF);
+ int64_t field_typos = 255 - ((match_score >> 16) & 0xFF);
total_typos += (field_typos + 1) * weight;
+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * weight;
+ exact_matches += (((match_score & 0xFF)) + 1) * weight;
if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {
num_exact_matches++;
@@ -1934,7 +1944,7 @@ void Index::search_field(const uint8_t & field_id,
search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,
curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,
groups_processed, all_result_ids, all_result_ids_len, field_num_results,
- typo_tokens_threshold, group_limit, group_by_fields);
+ typo_tokens_threshold, group_limit, group_by_fields, query_tokens);
}
resume_typo_loop:
@@ -2004,17 +2014,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
spp::sparse_hash_set<uint64_t>& groups_processed,
const uint32_t *result_ids, const size_t result_size,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
- uint32_t token_bits) const {
-
- std::vector<uint32_t *> leaf_to_indices;
- for (art_leaf *token_leaf: query_suggestion) {
- uint32_t *indices = new uint32_t[result_size];
- token_leaf->values->ids.indexOf(result_ids, result_size, indices);
- leaf_to_indices.push_back(indices);
- }
-
- Match single_token_match = Match(1, 0);
- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);
+ uint32_t token_bits,
+ const std::vector<token_t>& query_tokens) const {
int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
spp::sparse_hash_map<uint32_t, int64_t>* field_values[3];
@@ -2052,21 +2053,37 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
}
+ std::vector<uint32_t *> leaf_to_indices;
+ for (art_leaf *token_leaf: query_suggestion) {
+ uint32_t *indices = new uint32_t[result_size];
+ token_leaf->values->ids.indexOf(result_ids, result_size, indices);
+ leaf_to_indices.push_back(indices);
+ }
+
+ Match single_token_match = Match(1, 0);
+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);
+
//auto begin = std::chrono::high_resolution_clock::now();
+ //const std::string first_token((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1);
+
+ // We will have to be judicious about computing full match score: only when token does not match exact query
+ bool use_single_token_score = (query_suggestion.size() == 1) &&
+ (query_suggestion.size() == query_tokens.size()) &&
+ ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));
for (size_t i = 0; i < result_size; i++) {
const uint32_t seq_id = result_ids[i];
uint64_t match_score = 0;
- if (query_suggestion.size() <= 1) {
+ if (use_single_token_score) {
match_score = single_token_match_score;
} else {
- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> array_token_positions;
+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);
for (const auto& kv: array_token_positions) {
- const std::vector<std::vector<uint16_t>> &token_positions = kv.second;
+ const std::vector<token_positions_t>& token_positions = kv.second;
if (token_positions.empty()) {
continue;
}
@@ -2190,7 +2207,7 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,
const std::vector<uint32_t*>& leaf_to_indices,
const size_t result_index,
- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>>& array_token_positions) {
+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions) {
if(query_suggestion.empty()) {
return ;
}
@@ -2209,9 +2226,14 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges
continue;
}
- // Array offset storage format:
- // a) last element is array_index b) second and third last elements will be largest offset
- // (last element is repeated to indicate end of offsets for a given array index)
+ //LOG(INFO) << "key: " << token_leaf->key;
+
+ // Plain string format:
+ // offset1, offset2, ... , 0 (if token is the last offset for the document)
+
+ // Array string format:
+ // offset1, ... , offsetn, offsetn, array_index, 0 (if token is the last offset for the document)
+ // (last offset is repeated to indicate end of offsets for a given array index)
/*uint32_t* offsets = token_leaf->values->offsets.uncompress();
for(size_t ii=0; ii < token_leaf->values->offsets.getLength(); ii++) {
@@ -2232,30 +2254,49 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges
std::vector<uint16_t> positions;
int prev_pos = -1;
+ bool is_last_token = false;
while(start_offset < end_offset) {
int pos = token_leaf->values->offsets.at(start_offset);
start_offset++;
+ if(pos == 0) {
+ // indicates that token is the last token on the doc
+ is_last_token = true;
+ start_offset++;
+ continue;
+ }
+
if(pos == prev_pos) { // indicates end of array index
if(!positions.empty()) {
size_t array_index = (size_t) token_leaf->values->offsets.at(start_offset);
- array_token_positions[array_index].push_back(positions);
+ is_last_token = false;
+
+ if(start_offset+1 < end_offset) {
+ size_t next_offset = (size_t) token_leaf->values->offsets.at(start_offset + 1);
+ if(next_offset == 0) {
+ // indicates that token is the last token on the doc
+ is_last_token = true;
+ start_offset++;
+ }
+ }
+
+ array_token_positions[array_index].push_back(token_positions_t{is_last_token, positions});
positions.clear();
}
- start_offset++; // skip current value which is the array index
+ start_offset++; // skip current value which is the array index or flag for last index
prev_pos = -1;
continue;
}
prev_pos = pos;
- positions.push_back((uint16_t)pos);
+ positions.push_back((uint16_t)pos - 1);
}
if(!positions.empty()) {
// for plain string fields
- array_token_positions[0].push_back(positions);
+ array_token_positions[0].push_back(token_positions_t{is_last_token, positions});
}
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 29d18de2..7cfefd40 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -718,7 +718,7 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
false, 0).get();
ASSERT_EQ(2, results["hits"].size());
- ids = {"0", "1"};
+ ids = {"1", "0"};
for (size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
@@ -766,26 +766,23 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
ASSERT_EQ(0, results["hits"][0]["highlights"][1]["indices"][0]);
ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"][1]);
- ASSERT_EQ(3, results["hits"][1]["highlights"][0].size());
- ASSERT_STREQ("title", results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str());
- ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
- ASSERT_EQ(1, results["hits"][1]["highlights"][0]["matched_tokens"].size());
- ASSERT_STREQ("Truth", results["hits"][1]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
-
- ASSERT_EQ(4, results["hits"][1]["highlights"][1].size());
- ASSERT_STREQ(results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str(), "tags");
-
- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["snippets"].size());
- ASSERT_STREQ("<mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][0].get<std::string>().c_str());
- ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][1].get<std::string>().c_str());
-
- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["matched_tokens"].size());
- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());
- ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());
-
- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"].size());
- ASSERT_EQ(1, results["hits"][1]["highlights"][1]["indices"][0]);
- ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"][1]);
+ ASSERT_EQ(4, results["hits"][1]["highlights"][0].size());
+ ASSERT_STREQ(results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str(), "tags");
+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["snippets"].size());
+ ASSERT_STREQ("<mark>truth</mark>", results["hits"][1]["highlights"][0]["snippets"][0].get<std::string>().c_str());
+ ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][1]["highlights"][0]["snippets"][1].get<std::string>().c_str());
+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["matched_tokens"].size());
+ ASSERT_STREQ("truth", results["hits"][1]["highlights"][0]["matched_tokens"][0][0].get<std::string>().c_str());
+ ASSERT_STREQ("truth", results["hits"][1]["highlights"][0]["matched_tokens"][1][0].get<std::string>().c_str());
+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["indices"].size());
+ ASSERT_EQ(1, results["hits"][1]["highlights"][0]["indices"][0]);
+ ASSERT_EQ(2, results["hits"][1]["highlights"][0]["indices"][1]);
+
+ ASSERT_EQ(3, results["hits"][1]["highlights"][1].size());
+ ASSERT_STREQ("title", results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str());
+ ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][1]["highlights"][1]["snippet"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][1]["highlights"][1]["matched_tokens"].size());
+ ASSERT_STREQ("Truth", results["hits"][1]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());
// highlight fields must be ordered based on match score
results = coll_array_text->search("amazing movie", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
@@ -2476,14 +2473,15 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title, tags").get();
ASSERT_EQ(2, res["hits"][0]["highlights"].size());
- ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",
- res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());
+ ASSERT_EQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["values"][0].get<std::string>());
+ ASSERT_EQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",
+ res["hits"][0]["highlights"][1]["value"].get<std::string>());
- ASSERT_EQ(1, res["hits"][0]["highlights"][0]["matched_tokens"].size());
- ASSERT_STREQ("lazy", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());
+ ASSERT_EQ(1, res["hits"][0]["highlights"][1]["matched_tokens"].size());
+ ASSERT_STREQ("lazy", res["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());
- ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"][0].size());
- ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>().c_str());
+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["values"][0].size());
+ ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][0]["values"][0].get<std::string>().c_str());
// excluded fields should not be returned in highlights section
spp::sparse_hash_set<std::string> excluded_fields = {"tags"};
@@ -3087,6 +3085,59 @@ TEST_F(CollectionTest, MultiFieldRelevance5) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionTest, ExactMatch) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Alpha", "DJ"},
+ {"Alpha Beta", "DJ"},
+ {"Alpha Beta Gamma", "DJ"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("alpha beta",
+ {"title"}, "", {}, {}, 2, 10, 1, FREQUENCY,
+ true, 10).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("2", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ results = coll1->search("alpha", {"title"}, "", {}, {}, 2, 10, 1, FREQUENCY, true, 10).get();
+
+ LOG(INFO) << results;
+
+ ASSERT_EQ(3, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"].size());
+
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("2", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][2]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionTest, MultiFieldHighlighting) {
Collection *coll1;
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index 0be9931a..41459a87 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -3,10 +3,19 @@
#include <match_score.h>
TEST(MatchTest, TokenOffsetsExceedWindowSize) {
- std::vector<std::vector<uint16_t>> token_positions = {
- std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}),
- std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}),
- std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1}), std::vector<uint16_t>({1})
+ std::vector<token_positions_t> token_positions = {
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})},
+ token_positions_t{false, std::vector<uint16_t>({1})}
};
const Match & this_match = Match(100, token_positions);
@@ -15,11 +24,11 @@ TEST(MatchTest, TokenOffsetsExceedWindowSize) {
}
TEST(MatchTest, MatchScoreV2) {
- std::vector<std::vector<uint16_t>> token_offsets;
- token_offsets.push_back({25});
- token_offsets.push_back({26});
- token_offsets.push_back({11, 18, 24, 60});
- token_offsets.push_back({14, 27, 63});
+ std::vector<token_positions_t> token_offsets;
+ token_offsets.push_back(token_positions_t{false, {25}});
+ token_offsets.push_back(token_positions_t{false, {26}});
+ token_offsets.push_back(token_positions_t{false, {11, 18, 24, 60}});
+ token_offsets.push_back(token_positions_t{false, {14, 27, 63}});
auto match = Match(100, token_offsets, true);
ASSERT_EQ(4, match.words_present);
@@ -37,9 +46,9 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(0, match.offsets.size());
token_offsets.clear();
- token_offsets.push_back({38, 50, 170, 187, 195, 222});
- token_offsets.push_back({39, 140, 171, 189, 223});
- token_offsets.push_back({169, 180});
+ token_offsets.push_back(token_positions_t{false, {38, 50, 170, 187, 195, 222}});
+ token_offsets.push_back(token_positions_t{false, {39, 140, 171, 189, 223}});
+ token_offsets.push_back(token_positions_t{false, {169, 180}});
match = Match(100, token_offsets, true);
ASSERT_EQ(3, match.words_present);
@@ -51,9 +60,9 @@ TEST(MatchTest, MatchScoreV2) {
}
token_offsets.clear();
- token_offsets.push_back({38, 50, 187, 195, 201});
- token_offsets.push_back({120, 167, 171, 223});
- token_offsets.push_back({240, 250});
+ token_offsets.push_back(token_positions_t{false, {38, 50, 187, 195, 201}});
+ token_offsets.push_back(token_positions_t{false, {120, 167, 171, 223}});
+ token_offsets.push_back(token_positions_t{false, {240, 250}});
match = Match(100, token_offsets, true);
ASSERT_EQ(1, match.words_present);