Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Introduce option to rank text by token position.

Kishore Nallan223b104823be
diff --git a/include/collection.h b/include/collection.h
index 0ec47c93..2844488f 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -48,6 +48,7 @@ private:
const size_t DEFAULT_TOPSTER_SIZE = 250;
struct highlight_t {
+ size_t field_index;
std::string field;
std::vector<std::string> snippets;
std::vector<std::string> values;
@@ -60,7 +61,7 @@ private:
}
bool operator<(const highlight_t& a) const {
- return match_score > a.match_score;
+ return std::tie(match_score, field_index) > std::tie(a.match_score, field_index);
}
};
@@ -107,6 +108,7 @@ private:
void highlight_result(const std::string& raw_query,
const field &search_field,
+ const size_t search_field_index,
const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
const std::vector<std::string>& q_tokens,
const KV* field_order_kv, const nlohmann::json &document,
@@ -330,7 +332,8 @@ public:
const size_t max_extra_prefix = INT16_MAX,
const size_t max_extra_suffix = INT16_MAX,
const size_t facet_query_num_typos = 2,
- const size_t filter_curated_hits_option = 2) const;
+ const size_t filter_curated_hits_option = 2,
+ const bool prioritize_token_position = false) const;
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/include/index.h b/include/index.h
index 3b5f8a49..ee667353 100644
--- a/include/index.h
+++ b/include/index.h
@@ -331,6 +331,7 @@ struct search_args {
size_t group_limit;
std::string default_sorting_field;
bool prioritize_exact_match;
+ bool prioritize_token_position;
size_t all_result_ids_len;
bool exhaustive_search;
size_t concurrency;
@@ -360,7 +361,8 @@ struct search_args {
size_t max_facet_values, size_t max_hits, size_t per_page, size_t page, token_ordering token_order,
const std::vector<bool>& prefixes, size_t drop_tokens_threshold, size_t typo_tokens_threshold,
const std::vector<std::string>& group_by_fields, size_t group_limit,
- const string& default_sorting_field, bool prioritize_exact_match, bool exhaustive_search,
+ const string& default_sorting_field, bool prioritize_exact_match,
+ const bool prioritize_token_position, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms,
size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<infix_t>& infixes,
const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,
@@ -372,8 +374,8 @@ struct search_args {
page(page), token_order(token_order), prefixes(prefixes),
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),
- prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),
- exhaustive_search(exhaustive_search), concurrency(concurrency),
+ prioritize_exact_match(prioritize_exact_match), prioritize_token_position(prioritize_token_position),
+ all_result_ids_len(0), exhaustive_search(exhaustive_search), concurrency(concurrency),
search_cutoff_ms(search_cutoff_ms),
min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo), max_candidates(max_candidates),
infixes(infixes), max_extra_prefix(max_extra_prefix), max_extra_suffix(max_extra_suffix),
@@ -590,6 +592,7 @@ private:
const std::vector<uint32_t>& num_typos,
const std::vector<bool>& prefixes,
bool prioritize_exact_match,
+ const bool prioritize_token_position,
const bool exhaustive_search,
const size_t max_candidates,
int syn_orig_num_tokens,
@@ -715,6 +718,7 @@ public:
const uint32_t seq_id, const int sort_order[3],
const bool prioritize_exact_match,
const bool single_exact_query_token,
+ const bool prioritize_token_position,
size_t num_query_tokens,
int syn_orig_num_tokens,
const std::vector<posting_list_t::iterator_t>& posting_lists) const;
@@ -780,7 +784,8 @@ public:
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields,
- const string& default_sorting_field, bool prioritize_exact_match, bool exhaustive_search,
+ const string& default_sorting_field, bool prioritize_exact_match,
+ const bool prioritize_token_position, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,
size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, const size_t facet_query_num_typos,
@@ -899,6 +904,7 @@ public:
const token_ordering& token_order,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const bool prioritize_token_position,
const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo,
size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
@@ -934,6 +940,7 @@ public:
uint32_t*& all_result_ids, size_t& all_result_ids_len,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
bool prioritize_exact_match,
+ const bool prioritize_token_position,
std::set<uint64>& query_hashes,
const token_ordering token_order,
const std::vector<bool>& prefixes,
@@ -969,7 +976,9 @@ public:
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const std::vector<std::string>& group_by_fields,
+ bool prioritize_exact_match,
+ const bool search_all_candidates,
const uint32_t* filter_ids, uint32_t filter_ids_length,
const uint32_t total_cost,
const int syn_orig_num_tokens,
diff --git a/include/match_score.h b/include/match_score.h
index 873aab80..f3db9b7f 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -37,27 +37,31 @@ struct TokenOffset {
struct Match {
uint8_t words_present;
uint8_t distance;
+ uint8_t max_offset;
uint8_t exact_match;
uint8_t phrase_match;
+
std::vector<TokenOffset> offsets;
- Match() : words_present(0), distance(0), exact_match(0) {
+ Match() : words_present(0), distance(0), exact_match(0), max_offset(0) {
}
- Match(uint8_t words_present, uint8_t distance, uint8_t exact_match = 0) :
- words_present(words_present), distance(distance), exact_match(exact_match) {
+ Match(uint8_t words_present, uint8_t distance, uint8_t max_offset, uint8_t exact_match = 0) :
+ words_present(words_present), distance(distance), max_offset(max_offset),
+ exact_match(exact_match), phrase_match(0) {
}
// Construct a single match score from individual components (for multi-field sort)
inline uint64_t get_match_score(const uint32_t total_cost, const uint32_t unique_words) const {
uint64_t match_score = (
- (int64_t(unique_words) << 32) |
- (int64_t(words_present) << 24) |
- (int64_t(255 - total_cost) << 16) |
- (int64_t(100 - distance) << 8) |
- (int64_t(exact_match) << 0)
+ (int64_t(unique_words) << 40) |
+ (int64_t(words_present) << 32) |
+ (int64_t(255 - total_cost) << 24) |
+ (int64_t(100 - distance) << 16) |
+ (int64_t(exact_match) << 8) |
+ (int64_t(255 - max_offset) << 0)
);
return match_score;
@@ -189,6 +193,7 @@ struct Match {
(this_num_match == best_num_match && this_displacement < best_displacement)) {
best_displacement = this_displacement;
best_num_match = this_num_match;
+ max_offset = std::min((uint16_t)255, window.front().offset);
if(populate_window) {
best_window = this_window;
}
diff --git a/include/posting.h b/include/posting.h
index d2595058..0e7595f8 100644
--- a/include/posting.h
+++ b/include/posting.h
@@ -103,7 +103,7 @@ public:
static void get_array_token_positions(
uint32_t id,
const std::vector<void*>& posting_lists,
- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions
+ std::map<size_t, std::vector<token_positions_t>>& array_token_positions
);
static void get_exact_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,
diff --git a/include/posting_list.h b/include/posting_list.h
index 93460e5d..7500fb5c 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -172,7 +172,7 @@ public:
static bool get_offsets(
const std::vector<iterator_t>& its,
- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos
+ std::map<size_t, std::vector<token_positions_t>>& array_token_pos
);
static bool is_single_token_verbatim_match(const posting_list_t::iterator_t& it, bool field_is_array);
@@ -187,6 +187,8 @@ public:
static void get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,
std::vector<size_t>& indices);
+
+ static size_t get_last_offset(const posting_list_t::iterator_t& it, bool field_is_array);
};
template<class T>
diff --git a/src/collection.cpp b/src/collection.cpp
index 0b3a0894..18447daa 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -751,7 +751,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
const size_t max_extra_prefix,
const size_t max_extra_suffix,
const size_t facet_query_num_typos,
- const size_t filter_curated_hits_option) const {
+ const size_t filter_curated_hits_option,
+ const bool prioritize_token_position) const {
std::shared_lock lock(mutex);
@@ -1097,7 +1098,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
per_page, page, token_order, prefixes,
drop_tokens_threshold, typo_tokens_threshold,
group_by_fields, group_limit, default_sorting_field,
- prioritize_exact_match,
+ prioritize_exact_match, prioritize_token_position,
exhaustive_search, 4,
search_stop_millis,
min_len_1typo, min_len_2typo, max_candidates, infixes,
@@ -1278,7 +1279,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
std::vector<highlight_t> highlights;
StringUtils string_utils;
- for(auto& highlight_item: highlight_items) {
+ for(size_t i = 0; i < highlight_items.size(); i++) {
+ auto& highlight_item = highlight_items[i];
const std::string& field_name = highlight_item.name;
if(search_schema.count(field_name) == 0) {
continue;
@@ -1290,7 +1292,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
search_field.type == field_types::STRING_ARRAY)) {
highlight_t highlight;
- highlight_result(raw_query, search_field, highlight_item.qtoken_leaves, q_tokens, field_order_kv,
+ highlight_result(raw_query, search_field, i, highlight_item.qtoken_leaves, q_tokens, field_order_kv,
document,string_utils, snippet_threshold, highlight_affix_num_tokens,
highlight_item.fully_highlighted, highlight_item.infix,
highlight_start_tag, highlight_end_tag, index_symbols, highlight);
@@ -1845,6 +1847,7 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t
}
void Collection::highlight_result(const std::string& raw_query, const field &search_field,
+ const size_t search_field_index,
const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
const std::vector<std::string>& q_tokens,
const KV* field_order_kv, const nlohmann::json & document,
@@ -1889,11 +1892,13 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
array_len = document[search_field.name].size();
}
+ const std::vector<token_positions_t> empty_offsets;
+
for(size_t i = 0; i < array_len; i++) {
std::string text = field_is_array ? document[search_field.name][i] : document[search_field.name];
StringUtils::tolowercase(text);
if(text.size() < 100 && text.find(raw_query_tokens.front()) != std::string::npos) {
- const Match & this_match = Match(field_order_kv->key, {}, false, false);
+ const Match & this_match = Match(field_order_kv->key, empty_offsets, false, false);
uint64_t this_match_score = this_match.get_match_score(0, 1);
match_indices.emplace_back(this_match, this_match_score, i);
}
@@ -1913,7 +1918,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
posting_lists.push_back(token_leaf.leaf->values);
}
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;
posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);
for(const auto& kv: array_token_positions) {
@@ -1928,9 +1933,10 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());
match_indices.emplace_back(this_match, this_match_score, array_index);
- /*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)
- << ", match_score: " << this_match_score
- << ", match.distance: " << size_t(this_match.distance);*/
+ /*LOG(INFO) << "doc_id: " << document["id"] << ", search_field: " << search_field.name
+ << ", words_present: " << size_t(this_match.words_present)
+ << ", match_score: " << this_match_score
+ << ", match.distance: " << size_t(this_match.distance);*/
}
}
@@ -2146,6 +2152,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
highlight.field = search_field.name;
+ highlight.field_index = search_field_index;
if(!match_indices.empty()) {
highlight.match_score = match_indices[0].match_score;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 2b98aa8c..8f66ddaa 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -637,6 +637,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *HIGHLIGHT_END_TAG = "highlight_end_tag";
const char *PRIORITIZE_EXACT_MATCH = "prioritize_exact_match";
+ const char *PRIORITIZE_TOKEN_POSITION = "prioritize_token_position";
const char *PRE_SEGMENTED_QUERY = "pre_segmented_query";
const char *SEARCH_CUTOFF_MS = "search_cutoff_ms";
@@ -705,6 +706,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::vector<uint32_t> query_by_weights;
size_t limit_hits = UINT32_MAX;
bool prioritize_exact_match = true;
+ bool prioritize_token_position = false;
bool pre_segmented_query = false;
bool enable_overrides = true;
size_t filter_curated_hits_option = 2;
@@ -750,6 +752,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::unordered_map<std::string, bool*> bool_values = {
{PRIORITIZE_EXACT_MATCH, &prioritize_exact_match},
+ {PRIORITIZE_TOKEN_POSITION, &prioritize_token_position},
{PRE_SEGMENTED_QUERY, &pre_segmented_query},
{EXHAUSTIVE_SEARCH, &exhaustive_search},
{SPLIT_JOIN_TOKENS, &split_join_tokens},
@@ -903,7 +906,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
max_extra_prefix,
max_extra_suffix,
facet_query_num_typos,
- filter_curated_hits_option
+ filter_curated_hits_option,
+ prioritize_token_position
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/src/index.cpp b/src/index.cpp
index 63fd7fb9..6162a3ef 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1155,6 +1155,7 @@ void Index::search_all_candidates(const size_t num_search_fields,
const std::vector<uint32_t>& num_typos,
const std::vector<bool>& prefixes,
bool prioritize_exact_match,
+ const bool prioritize_token_position,
const bool exhaustive_search,
const size_t max_candidates,
int syn_orig_num_tokens,
@@ -1279,7 +1280,8 @@ void Index::search_all_candidates(const size_t num_search_fields,
search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields,
sort_fields, topster,groups_processed,
- searched_queries, qtoken_set, group_limit, group_by_fields, prioritize_exact_match,
+ searched_queries, qtoken_set, group_limit, group_by_fields,
+ prioritize_exact_match, prioritize_token_position,
filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,
exclude_token_ids, exclude_token_ids_size,
sort_order, field_values, geopoint_indices,
@@ -1843,6 +1845,7 @@ void Index::run_search(search_args* search_params) {
search_params->group_limit, search_params->group_by_fields,
search_params->default_sorting_field,
search_params->prioritize_exact_match,
+ search_params->prioritize_token_position,
search_params->exhaustive_search,
search_params->concurrency,
search_params->search_cutoff_ms,
@@ -2270,7 +2273,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields,
- const string& default_sorting_field, bool prioritize_exact_match, bool exhaustive_search,
+ const string& default_sorting_field, bool prioritize_exact_match,
+ const bool prioritize_token_position, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,
size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, const size_t facet_query_num_typos,
@@ -2387,7 +2391,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
- query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,
+ prioritize_token_position, query_hashes, token_order, prefixes,
+ typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,
field_values, geopoint_indices);
@@ -2423,14 +2428,15 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
- query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,
+ prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
}
}
// do synonym based searches
do_synonym_search(the_fields, filters, included_ids_map, sort_fields_std, curated_topster, token_order,
- 0, group_limit, group_by_fields, prioritize_exact_match, exhaustive_search, concurrency,
+ 0, group_limit, group_by_fields, prioritize_exact_match, prioritize_token_position,
+ exhaustive_search, concurrency,
min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
excluded_result_ids, excluded_result_ids_size, topster, q_pos_synonyms, syn_orig_num_tokens,
groups_processed, searched_queries, all_result_ids, all_result_ids_len,
@@ -2478,7 +2484,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
- query_hashes, token_order, prefixes, typo_tokens_threshold,
+ prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold,
exhaustive_search, max_candidates, min_len_1typo,
min_len_2typo, -1, sort_order, field_values, geopoint_indices);
@@ -2709,6 +2715,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
uint32_t*& all_result_ids, size_t & all_result_ids_len,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
bool prioritize_exact_match,
+ const bool prioritize_token_position,
std::set<uint64>& query_hashes,
const token_ordering token_order,
const std::vector<bool>& prefixes,
@@ -2874,7 +2881,8 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
sort_fields, token_candidates_vec, searched_queries, qtoken_set, topster,
groups_processed, all_result_ids, all_result_ids_len,
typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
- num_typos, prefixes, prioritize_exact_match, exhaustive_search, max_candidates,
+ num_typos, prefixes, prioritize_exact_match, prioritize_token_position,
+ exhaustive_search, max_candidates,
syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
query_hashes, id_buff);
@@ -2999,7 +3007,9 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
std::vector<std::vector<art_leaf*>>& searched_queries,
tsl::htrie_map<char, token_leaf>& qtoken_set,
const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const std::vector<std::string>& group_by_fields,
+ const bool prioritize_exact_match,
+ const bool prioritize_token_position,
const uint32_t* filter_ids, uint32_t filter_ids_length,
const uint32_t total_cost, const int syn_orig_num_tokens,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
@@ -3119,7 +3129,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
score_results2(sort_fields, searched_queries.size(), fi, field_is_array,
total_cost, field_match_score,
seq_id, sort_order,
- prioritize_exact_match, single_exact_query_token,
+ prioritize_exact_match, single_exact_query_token, prioritize_token_position,
query_tokens.size(), syn_orig_num_tokens, token_postings);
if(field_match_score > max_field_match_score) {
@@ -3148,7 +3158,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
// NOTE: `query_len` is total tokens matched across fields.
// Within a field, only a subset can match
- uint64_t aggregated_score = (int64_t(query_len) << 48) |
+ uint64_t aggregated_score = (int64_t(query_len) << 56) |
(int64_t(max_field_match_score) << 8) |
(int64_t(the_fields[max_field_match_index].weight) << 0);
@@ -3470,6 +3480,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
const token_ordering& token_order,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const bool prioritize_token_position,
const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo,
size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
@@ -3493,7 +3504,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
- query_hashes, token_order, {0}, typo_tokens_threshold,
+ prioritize_token_position, query_hashes, token_order, {0}, typo_tokens_threshold,
exhaustive_search, max_candidates, min_len_1typo,
min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
}
@@ -3558,7 +3569,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
int64_t match_score = 0;
score_results2(sort_fields, searched_queries.size(), field_id, field_is_array,
- 0, match_score, seq_id, sort_order, false, false, 1, -1, {});
+ 0, match_score, seq_id, sort_order, false, false, false, 1, -1, {});
int64_t scores[3] = {0};
int64_t match_score_index = 0;
@@ -3884,7 +3895,7 @@ void Index::search_wildcard(const std::vector<filter>& filters,
int64_t match_score = 0;
score_results2(sort_fields, (uint16_t) searched_queries.size(), 0, false, 0,
- match_score, seq_id, sort_order, false, false, 1, -1, plists);
+ match_score, seq_id, sort_order, false, false, false, 1, -1, plists);
int64_t scores[3] = {0};
int64_t match_score_index = 0;
@@ -4187,6 +4198,7 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui
const uint32_t seq_id, const int sort_order[3],
const bool prioritize_exact_match,
const bool single_exact_query_token,
+ const bool prioritize_token_position,
size_t num_query_tokens,
int syn_orig_num_tokens,
const std::vector<posting_list_t::iterator_t>& posting_lists) const {
@@ -4201,10 +4213,29 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui
);
size_t words_present = (num_query_tokens == 1 && syn_orig_num_tokens != -1) ? syn_orig_num_tokens : 1;
size_t distance = (num_query_tokens == 1 && syn_orig_num_tokens != -1) ? syn_orig_num_tokens-1 : 0;
- Match single_token_match = Match(words_present, distance, is_verbatim_match);
+ size_t max_offset = prioritize_token_position ? posting_list_t::get_last_offset(posting_lists[0],
+ field_is_array) : 255;
+ Match single_token_match = Match(words_present, distance, max_offset, is_verbatim_match);
match_score = single_token_match.get_match_score(total_cost, words_present);
+
+ /*auto this_words_present = ((match_score >> 32) & 0xFF);
+ auto unique_words = ((match_score >> 40) & 0xFF);
+ auto typo_score = ((match_score >> 24) & 0xFF);
+ auto proximity = ((match_score >> 16) & 0xFF);
+ auto verbatim = ((match_score >> 8) & 0xFF);
+ auto offset_score = ((match_score >> 0) & 0xFF);
+
+ LOG(INFO) << "seq_id: " << seq_id
+ << ", words_present: " << this_words_present
+ << ", unique_words: " << unique_words
+ << ", typo_score: " << typo_score
+ << ", proximity: " << proximity
+ << ", verbatim: " << verbatim
+ << ", offset_score: " << offset_score
+ << ", match_score: " << match_score;*/
+
} else {
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;
posting_list_t::get_offsets(posting_lists, array_token_positions);
for (const auto& kv: array_token_positions) {
@@ -4219,11 +4250,12 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui
// Within a field, only a subset of query tokens can match (unique_words), but even a smaller set
// might be available within the window used for proximity calculation (this_words_present)
- auto this_words_present = ((this_match_score >> 24) & 0xFF);
- auto unique_words = field_is_array ? this_words_present : ((this_match_score >> 32) & 0xFF);
- auto typo_score = ((this_match_score >> 16) & 0xFF);
- auto proximity = ((this_match_score >> 8) & 0xFF);
- auto verbatim = (this_match_score & 0xFF);
+ auto this_words_present = ((this_match_score >> 32) & 0xFF);
+ auto unique_words = field_is_array ? this_words_present : ((this_match_score >> 40) & 0xFF);
+ auto typo_score = ((this_match_score >> 24) & 0xFF);
+ auto proximity = ((this_match_score >> 16) & 0xFF);
+ auto verbatim = ((this_match_score >> 8) & 0xFF);
+ auto offset_score = prioritize_token_position ? ((this_match_score >> 0) & 0xFF) : 0;
if(syn_orig_num_tokens != -1 && num_query_tokens == posting_lists.size()) {
unique_words = syn_orig_num_tokens;
@@ -4232,11 +4264,12 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui
}
uint64_t mod_match_score = (
- (int64_t(this_words_present) << 32) |
- (int64_t(unique_words) << 24) |
- (int64_t(typo_score) << 16) |
- (int64_t(proximity) << 8) |
- (int64_t(verbatim) << 0)
+ (int64_t(this_words_present) << 40) |
+ (int64_t(unique_words) << 32) |
+ (int64_t(typo_score) << 24) |
+ (int64_t(proximity) << 16) |
+ (int64_t(verbatim) << 8) |
+ (int64_t(offset_score) << 0)
);
if(mod_match_score > match_score) {
@@ -4250,6 +4283,7 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui
<< ", typo_score: " << typo_score
<< ", proximity: " << proximity
<< ", verbatim: " << verbatim
+ << ", offset_score: " << offset_score
<< ", mod_match_score: " << mod_match_score
<< ", token_positions: " << token_positions.size()
<< ", num_query_tokens: " << num_query_tokens
@@ -4348,7 +4382,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
Match single_token_match = Match(words_present, distance, is_verbatim_match);
match_score = single_token_match.get_match_score(total_cost, words_present);
} else {
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;
posting_list_t::get_offsets(posting_lists, array_token_positions);
// NOTE: tokens found returned by matcher is only within the best matched window, so we have to still consider
diff --git a/src/posting.cpp b/src/posting.cpp
index 4480b2ae..e6be0a9e 100644
--- a/src/posting.cpp
+++ b/src/posting.cpp
@@ -433,7 +433,7 @@ void posting_t::destroy_list(void*& obj) {
}
void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>& raw_posting_lists,
- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions) {
+ std::map<size_t, std::vector<token_positions_t>>& array_token_positions) {
std::vector<posting_list_t*> plists;
std::vector<posting_list_t*> expanded_plists;
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 66fbfd9d..02c3c488 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -772,14 +772,15 @@ bool posting_list_t::take_id(result_iter_state_t& istate, uint32_t id) {
}
bool posting_list_t::get_offsets(const std::vector<iterator_t>& its,
- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos) {
+ std::map<size_t, std::vector<token_positions_t>>& array_token_pos) {
// Plain string format:
// offset1, offset2, ... , 0 (if token is the last offset for the document)
// Array string format:
// offset1, ... , offsetn, offsetn, array_index, 0 (if token is the last offset for the document)
- // (last offset is repeated to indicate end of offsets for a given array index)
+ // NOTE 1: last offset is repeated to indicate end of offsets for a given array index)
+ // NOTE 2: offsets are 1-index based (since 0 is used as last offset marking)
// For each result ID and for each block it is contained in, calculate offsets
@@ -901,7 +902,7 @@ bool posting_list_t::is_single_token_verbatim_match(const posting_list_t::iterat
return false;
} else if((end_offset - start_offset) == 2 && offsets[end_offset-1] == 0) {
- // already checked for `offsets[start_offset] == 1` first
+ // we've already checked for `offsets[start_offset] == 1` earlier
return true;
}
@@ -1242,7 +1243,7 @@ void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool field
it.skip_to(id);
}
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;
get_offsets(its, array_token_positions);
for(auto& kv: array_token_positions) {
@@ -1331,6 +1332,60 @@ bool posting_list_t::all_ended2(const std::vector<posting_list_t::iterator_t>& i
return !its[0].valid() && !its[1].valid();
}
+size_t posting_list_t::get_last_offset(const posting_list_t::iterator_t& it, bool field_is_array) {
+ block_t* curr_block = it.block();
+ uint32_t curr_index = it.index();
+ uint32_t* offsets = it.offsets;
+
+ if(curr_block == nullptr || curr_index == UINT32_MAX) {
+ return 0;
+ }
+
+ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?
+ curr_block->offsets.getLength() :
+ it.offset_index[curr_index + 1];
+
+ if(field_is_array) {
+ uint32_t start_offset = it.offset_index[curr_index];
+ int prev_pos = -1;
+ size_t max_offset = 0;
+
+ while(start_offset < end_offset) {
+ int pos = offsets[start_offset];
+ start_offset++;
+
+ if(pos > max_offset) {
+ max_offset = pos;
+ }
+
+ if(pos == prev_pos) { // indicates end of array index
+ size_t array_index = (size_t) offsets[start_offset];
+
+ if(start_offset+1 < end_offset) {
+ size_t next_offset = (size_t) offsets[start_offset + 1];
+ if(next_offset == 0) {
+ // indicates that token is the last token on the doc
+ start_offset++;
+ }
+ }
+
+ start_offset++; // skip current value which is the array index or flag for last index
+ prev_pos = -1;
+ continue;
+ }
+
+ prev_pos = pos;
+ }
+
+ return max_offset;
+
+ } else {
+ return offsets[end_offset-1] == 0 ? offsets[end_offset-2] : offsets[end_offset-1];
+ }
+
+ return 0;
+}
+
/* iterator_t operations */
posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* start, posting_list_t::block_t* end,
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index f9175b84..3763b9c6 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -1729,10 +1729,10 @@ TEST_F(CollectionSortingTest, RepeatingTokenRanking) {
ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
- ASSERT_EQ(4284743939, results["hits"][0]["text_match"].get<uint32_t>());
- ASSERT_EQ(4284743683, results["hits"][1]["text_match"].get<uint32_t>());
- ASSERT_EQ(4284743683, results["hits"][2]["text_match"].get<uint32_t>());
- ASSERT_EQ(4284743683, results["hits"][3]["text_match"].get<uint32_t>());
+ ASSERT_EQ(144681433946980355, results["hits"][0]["text_match"].get<size_t>());
+ ASSERT_EQ(144681433946914819, results["hits"][1]["text_match"].get<size_t>());
+ ASSERT_EQ(144681433946914819, results["hits"][2]["text_match"].get<size_t>());
+ ASSERT_EQ(144681433946914819, results["hits"][3]["text_match"].get<size_t>());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 55d030f4..abe03519 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -87,7 +87,6 @@ TEST_F(CollectionSpecificMoreTest, PrefixExpansionWhenExactMatchExists) {
}
TEST_F(CollectionSpecificMoreTest, PrefixExpansionOnSingleField) {
- // when no default sorting field is provided, tokens must be ordered on frequency
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false)};
@@ -451,4 +450,132 @@ TEST_F(CollectionSpecificMoreTest, WrongTypoCorrection) {
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {
+ Collection *coll1;
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::string> tokens = {
+ "Alpha Beta Gamma", "Omega Alpha Theta", "Omega Theta Alpha", "Indigo Omega Theta Alpha"
+ };
+
+ for(size_t i = 0; i < tokens.size(); i++) {
+ std::string title = tokens[i];
+ nlohmann::json doc;
+ doc["title"] = title;
+ doc["points"] = i;
+ coll1->add(doc.dump());
+ }
+
+ auto results = coll1->search("alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {off}, 32767, 32767, 2,
+ false, true).get();
+
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
+ ASSERT_EQ("3", results["hits"][3]["document"]["id"].get<std::string>());
+
+ results = coll1->search("alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {off}, 32767, 32767, 2,
+ false, false).get();
+
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][3]["document"]["id"].get<std::string>());
+
+ results = coll1->search("theta alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {off}, 32767, 32767, 2,
+ false, false).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+
+ results = coll1->search("theta alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {off}, 32767, 32767, 2,
+ false, true).get();
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("3", results["hits"][2]["document"]["id"].get<std::string>());
+}
+
+TEST_F(CollectionSpecificMoreTest, PositionalTokenRankingWithArray) {
+ Collection *coll1;
+ std::vector<field> fields = {field("tags", field_types::STRING_ARRAY, false),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["tags"] = {"alpha foo", "gamma", "beta alpha"};
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["tags"] = {"omega", "omega beta alpha"};
+ doc2["points"] = 200;
+
+ coll1->add(doc1.dump());
+ coll1->add(doc2.dump());
+
+ auto results = coll1->search("alpha", {"tags"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {off}, 32767, 32767, 2,
+ false, false).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ results = coll1->search("alpha", {"tags"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {off}, 32767, 32767, 2,
+ false, true).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
}
\ No newline at end of file