Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Introduce option to rank text by token position.
Kishore Nallan223b104823be
diff --git a/include/collection.h b/include/collection.hindex 0ec47c93..2844488f 100644--- a/include/collection.h+++ b/include/collection.h@@ -48,6 +48,7 @@ private:const size_t DEFAULT_TOPSTER_SIZE = 250;struct highlight_t {+ size_t field_index;std::string field;std::vector<std::string> snippets;std::vector<std::string> values;@@ -60,7 +61,7 @@ private:}bool operator<(const highlight_t& a) const {- return match_score > a.match_score;+ return std::tie(match_score, field_index) > std::tie(a.match_score, field_index);}};@@ -107,6 +108,7 @@ private:void highlight_result(const std::string& raw_query,const field &search_field,+ const size_t search_field_index,const tsl::htrie_map<char, token_leaf>& qtoken_leaves,const std::vector<std::string>& q_tokens,const KV* field_order_kv, const nlohmann::json &document,@@ -330,7 +332,8 @@ public:const size_t max_extra_prefix = INT16_MAX,const size_t max_extra_suffix = INT16_MAX,const size_t facet_query_num_typos = 2,- const size_t filter_curated_hits_option = 2) const;+ const size_t filter_curated_hits_option = 2,+ const bool prioritize_token_position = false) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/include/index.h b/include/index.hindex 3b5f8a49..ee667353 100644--- a/include/index.h+++ b/include/index.h@@ -331,6 +331,7 @@ struct search_args {size_t group_limit;std::string default_sorting_field;bool prioritize_exact_match;+ bool prioritize_token_position;size_t all_result_ids_len;bool exhaustive_search;size_t concurrency;@@ -360,7 +361,8 @@ struct search_args {size_t max_facet_values, size_t max_hits, size_t per_page, size_t page, token_ordering token_order,const std::vector<bool>& prefixes, size_t drop_tokens_threshold, size_t typo_tokens_threshold,const std::vector<std::string>& group_by_fields, size_t group_limit,- const string& default_sorting_field, bool prioritize_exact_match, bool exhaustive_search,+ const string& default_sorting_field, bool prioritize_exact_match,+ const bool prioritize_token_position, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms,size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<infix_t>& infixes,const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,@@ -372,8 +374,8 @@ struct search_args {page(page), token_order(token_order), prefixes(prefixes),drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),- prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),- exhaustive_search(exhaustive_search), concurrency(concurrency),+ prioritize_exact_match(prioritize_exact_match), prioritize_token_position(prioritize_token_position),+ all_result_ids_len(0), exhaustive_search(exhaustive_search), concurrency(concurrency),search_cutoff_ms(search_cutoff_ms),min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo), max_candidates(max_candidates),infixes(infixes), max_extra_prefix(max_extra_prefix), max_extra_suffix(max_extra_suffix),@@ -590,6 +592,7 @@ private:const std::vector<uint32_t>& num_typos,const std::vector<bool>& prefixes,bool prioritize_exact_match,+ const bool prioritize_token_position,const bool exhaustive_search,const size_t max_candidates,int syn_orig_num_tokens,@@ -715,6 +718,7 @@ public:const uint32_t seq_id, const int sort_order[3],const bool prioritize_exact_match,const bool single_exact_query_token,+ const bool prioritize_token_position,size_t num_query_tokens,int syn_orig_num_tokens,const std::vector<posting_list_t::iterator_t>& posting_lists) const;@@ -780,7 +784,8 @@ public:std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,const size_t typo_tokens_threshold, const size_t group_limit,const std::vector<std::string>& group_by_fields,- const string& default_sorting_field, bool prioritize_exact_match, bool exhaustive_search,+ const string& default_sorting_field, bool prioritize_exact_match,+ const bool prioritize_token_position, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,const size_t max_extra_suffix, const size_t facet_query_num_typos,@@ -899,6 +904,7 @@ public:const token_ordering& token_order,const size_t typo_tokens_threshold, const size_t group_limit,const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const bool prioritize_token_position,const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo,size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,@@ -934,6 +940,7 @@ public:uint32_t*& all_result_ids, size_t& all_result_ids_len,const size_t group_limit, const std::vector<std::string>& group_by_fields,bool prioritize_exact_match,+ const bool prioritize_token_position,std::set<uint64>& query_hashes,const token_ordering token_order,const std::vector<bool>& prefixes,@@ -969,7 +976,9 @@ public:std::vector<std::vector<art_leaf*>>& searched_queries,tsl::htrie_map<char, token_leaf>& qtoken_set,const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const std::vector<std::string>& group_by_fields,+ bool prioritize_exact_match,+ const bool search_all_candidates,const uint32_t* filter_ids, uint32_t filter_ids_length,const uint32_t total_cost,const int syn_orig_num_tokens,diff --git a/include/match_score.h b/include/match_score.hindex 873aab80..f3db9b7f 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -37,27 +37,31 @@ struct TokenOffset {struct Match {uint8_t words_present;uint8_t distance;+ uint8_t max_offset;uint8_t exact_match;uint8_t phrase_match;+std::vector<TokenOffset> offsets;- Match() : words_present(0), distance(0), exact_match(0) {+ Match() : words_present(0), distance(0), exact_match(0), max_offset(0) {}- Match(uint8_t words_present, uint8_t distance, uint8_t exact_match = 0) :- words_present(words_present), distance(distance), exact_match(exact_match) {+ Match(uint8_t words_present, uint8_t distance, uint8_t max_offset, uint8_t exact_match = 0) :+ words_present(words_present), distance(distance), max_offset(max_offset),+ exact_match(exact_match), phrase_match(0) {}// Construct a single match score from individual components (for multi-field sort)inline uint64_t get_match_score(const uint32_t total_cost, const uint32_t unique_words) const {uint64_t match_score = (- (int64_t(unique_words) << 32) |- (int64_t(words_present) << 24) |- (int64_t(255 - total_cost) << 16) |- (int64_t(100 - distance) << 8) |- (int64_t(exact_match) << 0)+ (int64_t(unique_words) << 40) |+ (int64_t(words_present) << 32) |+ (int64_t(255 - total_cost) << 24) |+ (int64_t(100 - distance) << 16) |+ (int64_t(exact_match) << 8) |+ (int64_t(255 - max_offset) << 0));return match_score;@@ -189,6 +193,7 @@ struct Match {(this_num_match == best_num_match && this_displacement < best_displacement)) {best_displacement = this_displacement;best_num_match = this_num_match;+ max_offset = std::min((uint16_t)255, window.front().offset);if(populate_window) {best_window = this_window;}diff --git a/include/posting.h b/include/posting.hindex d2595058..0e7595f8 100644--- a/include/posting.h+++ b/include/posting.h@@ -103,7 +103,7 @@ public:static void get_array_token_positions(uint32_t id,const std::vector<void*>& posting_lists,- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions+ std::map<size_t, std::vector<token_positions_t>>& array_token_positions);static void get_exact_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,diff --git a/include/posting_list.h b/include/posting_list.hindex 93460e5d..7500fb5c 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -172,7 +172,7 @@ public:static bool get_offsets(const std::vector<iterator_t>& its,- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos+ std::map<size_t, std::vector<token_positions_t>>& array_token_pos);static bool is_single_token_verbatim_match(const posting_list_t::iterator_t& it, bool field_is_array);@@ -187,6 +187,8 @@ public:static void get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,std::vector<size_t>& indices);++ static size_t get_last_offset(const posting_list_t::iterator_t& it, bool field_is_array);};template<class T>diff --git a/src/collection.cpp b/src/collection.cppindex 0b3a0894..18447daa 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -751,7 +751,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sconst size_t max_extra_prefix,const size_t max_extra_suffix,const size_t facet_query_num_typos,- const size_t filter_curated_hits_option) const {+ const size_t filter_curated_hits_option,+ const bool prioritize_token_position) const {std::shared_lock lock(mutex);@@ -1097,7 +1098,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sper_page, page, token_order, prefixes,drop_tokens_threshold, typo_tokens_threshold,group_by_fields, group_limit, default_sorting_field,- prioritize_exact_match,+ prioritize_exact_match, prioritize_token_position,exhaustive_search, 4,search_stop_millis,min_len_1typo, min_len_2typo, max_candidates, infixes,@@ -1278,7 +1279,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sstd::vector<highlight_t> highlights;StringUtils string_utils;- for(auto& highlight_item: highlight_items) {+ for(size_t i = 0; i < highlight_items.size(); i++) {+ auto& highlight_item = highlight_items[i];const std::string& field_name = highlight_item.name;if(search_schema.count(field_name) == 0) {continue;@@ -1290,7 +1292,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const ssearch_field.type == field_types::STRING_ARRAY)) {highlight_t highlight;- highlight_result(raw_query, search_field, highlight_item.qtoken_leaves, q_tokens, field_order_kv,+ highlight_result(raw_query, search_field, i, highlight_item.qtoken_leaves, q_tokens, field_order_kv,document,string_utils, snippet_threshold, highlight_affix_num_tokens,highlight_item.fully_highlighted, highlight_item.infix,highlight_start_tag, highlight_end_tag, index_symbols, highlight);@@ -1845,6 +1847,7 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t}void Collection::highlight_result(const std::string& raw_query, const field &search_field,+ const size_t search_field_index,const tsl::htrie_map<char, token_leaf>& qtoken_leaves,const std::vector<std::string>& q_tokens,const KV* field_order_kv, const nlohmann::json & document,@@ -1889,11 +1892,13 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaarray_len = document[search_field.name].size();}+ const std::vector<token_positions_t> empty_offsets;+for(size_t i = 0; i < array_len; i++) {std::string text = field_is_array ? document[search_field.name][i] : document[search_field.name];StringUtils::tolowercase(text);if(text.size() < 100 && text.find(raw_query_tokens.front()) != std::string::npos) {- const Match & this_match = Match(field_order_kv->key, {}, false, false);+ const Match & this_match = Match(field_order_kv->key, empty_offsets, false, false);uint64_t this_match_score = this_match.get_match_score(0, 1);match_indices.emplace_back(this_match, this_match_score, i);}@@ -1913,7 +1918,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaposting_lists.push_back(token_leaf.leaf->values);}- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);for(const auto& kv: array_token_positions) {@@ -1928,9 +1933,10 @@ void Collection::highlight_result(const std::string& raw_query, const field &seauint64_t this_match_score = this_match.get_match_score(1, token_positions.size());match_indices.emplace_back(this_match, this_match_score, array_index);- /*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)- << ", match_score: " << this_match_score- << ", match.distance: " << size_t(this_match.distance);*/+ /*LOG(INFO) << "doc_id: " << document["id"] << ", search_field: " << search_field.name+ << ", words_present: " << size_t(this_match.words_present)+ << ", match_score: " << this_match_score+ << ", match.distance: " << size_t(this_match.distance);*/}}@@ -2146,6 +2152,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea}highlight.field = search_field.name;+ highlight.field_index = search_field_index;if(!match_indices.empty()) {highlight.match_score = match_indices[0].match_score;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 2b98aa8c..8f66ddaa 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -637,6 +637,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *HIGHLIGHT_END_TAG = "highlight_end_tag";const char *PRIORITIZE_EXACT_MATCH = "prioritize_exact_match";+ const char *PRIORITIZE_TOKEN_POSITION = "prioritize_token_position";const char *PRE_SEGMENTED_QUERY = "pre_segmented_query";const char *SEARCH_CUTOFF_MS = "search_cutoff_ms";@@ -705,6 +706,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::vector<uint32_t> query_by_weights;size_t limit_hits = UINT32_MAX;bool prioritize_exact_match = true;+ bool prioritize_token_position = false;bool pre_segmented_query = false;bool enable_overrides = true;size_t filter_curated_hits_option = 2;@@ -750,6 +752,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::unordered_map<std::string, bool*> bool_values = {{PRIORITIZE_EXACT_MATCH, &prioritize_exact_match},+ {PRIORITIZE_TOKEN_POSITION, &prioritize_token_position},{PRE_SEGMENTED_QUERY, &pre_segmented_query},{EXHAUSTIVE_SEARCH, &exhaustive_search},{SPLIT_JOIN_TOKENS, &split_join_tokens},@@ -903,7 +906,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& remax_extra_prefix,max_extra_suffix,facet_query_num_typos,- filter_curated_hits_option+ filter_curated_hits_option,+ prioritize_token_position);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex 63fd7fb9..6162a3ef 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1155,6 +1155,7 @@ void Index::search_all_candidates(const size_t num_search_fields,const std::vector<uint32_t>& num_typos,const std::vector<bool>& prefixes,bool prioritize_exact_match,+ const bool prioritize_token_position,const bool exhaustive_search,const size_t max_candidates,int syn_orig_num_tokens,@@ -1279,7 +1280,8 @@ void Index::search_all_candidates(const size_t num_search_fields,search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields,sort_fields, topster,groups_processed,- searched_queries, qtoken_set, group_limit, group_by_fields, prioritize_exact_match,+ searched_queries, qtoken_set, group_limit, group_by_fields,+ prioritize_exact_match, prioritize_token_position,filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,exclude_token_ids, exclude_token_ids_size,sort_order, field_values, geopoint_indices,@@ -1843,6 +1845,7 @@ void Index::run_search(search_args* search_params) {search_params->group_limit, search_params->group_by_fields,search_params->default_sorting_field,search_params->prioritize_exact_match,+ search_params->prioritize_token_position,search_params->exhaustive_search,search_params->concurrency,search_params->search_cutoff_ms,@@ -2270,7 +2273,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vstd::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,const size_t typo_tokens_threshold, const size_t group_limit,const std::vector<std::string>& group_by_fields,- const string& default_sorting_field, bool prioritize_exact_match, bool exhaustive_search,+ const string& default_sorting_field, bool prioritize_exact_match,+ const bool prioritize_token_position, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,const size_t max_extra_suffix, const size_t facet_query_num_typos,@@ -2387,7 +2391,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vexcluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,- query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,+ prioritize_token_position, query_hashes, token_order, prefixes,+ typo_tokens_threshold, exhaustive_search,max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,field_values, geopoint_indices);@@ -2423,14 +2428,15 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vexcluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,- query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,+ prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);}}// do synonym based searchesdo_synonym_search(the_fields, filters, included_ids_map, sort_fields_std, curated_topster, token_order,- 0, group_limit, group_by_fields, prioritize_exact_match, exhaustive_search, concurrency,+ 0, group_limit, group_by_fields, prioritize_exact_match, prioritize_token_position,+ exhaustive_search, concurrency,min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,excluded_result_ids, excluded_result_ids_size, topster, q_pos_synonyms, syn_orig_num_tokens,groups_processed, searched_queries, all_result_ids, all_result_ids_len,@@ -2478,7 +2484,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vexcluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,- query_hashes, token_order, prefixes, typo_tokens_threshold,+ prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold,exhaustive_search, max_candidates, min_len_1typo,min_len_2typo, -1, sort_order, field_values, geopoint_indices);@@ -2709,6 +2715,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,uint32_t*& all_result_ids, size_t & all_result_ids_len,const size_t group_limit, const std::vector<std::string>& group_by_fields,bool prioritize_exact_match,+ const bool prioritize_token_position,std::set<uint64>& query_hashes,const token_ordering token_order,const std::vector<bool>& prefixes,@@ -2874,7 +2881,8 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,sort_fields, token_candidates_vec, searched_queries, qtoken_set, topster,groups_processed, all_result_ids, all_result_ids_len,typo_tokens_threshold, group_limit, group_by_fields, query_tokens,- num_typos, prefixes, prioritize_exact_match, exhaustive_search, max_candidates,+ num_typos, prefixes, prioritize_exact_match, prioritize_token_position,+ exhaustive_search, max_candidates,syn_orig_num_tokens, sort_order, field_values, geopoint_indices,query_hashes, id_buff);@@ -2999,7 +3007,9 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,std::vector<std::vector<art_leaf*>>& searched_queries,tsl::htrie_map<char, token_leaf>& qtoken_set,const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const std::vector<std::string>& group_by_fields,+ const bool prioritize_exact_match,+ const bool prioritize_token_position,const uint32_t* filter_ids, uint32_t filter_ids_length,const uint32_t total_cost, const int syn_orig_num_tokens,const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,@@ -3119,7 +3129,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,score_results2(sort_fields, searched_queries.size(), fi, field_is_array,total_cost, field_match_score,seq_id, sort_order,- prioritize_exact_match, single_exact_query_token,+ prioritize_exact_match, single_exact_query_token, prioritize_token_position,query_tokens.size(), syn_orig_num_tokens, token_postings);if(field_match_score > max_field_match_score) {@@ -3148,7 +3158,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,// NOTE: `query_len` is total tokens matched across fields.// Within a field, only a subset can match- uint64_t aggregated_score = (int64_t(query_len) << 48) |+ uint64_t aggregated_score = (int64_t(query_len) << 56) |(int64_t(max_field_match_score) << 8) |(int64_t(the_fields[max_field_match_index].weight) << 0);@@ -3470,6 +3480,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,const token_ordering& token_order,const size_t typo_tokens_threshold, const size_t group_limit,const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const bool prioritize_token_position,const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo,size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,@@ -3493,7 +3504,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,- query_hashes, token_order, {0}, typo_tokens_threshold,+ prioritize_token_position, query_hashes, token_order, {0}, typo_tokens_threshold,exhaustive_search, max_candidates, min_len_1typo,min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);}@@ -3558,7 +3569,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<seint64_t match_score = 0;score_results2(sort_fields, searched_queries.size(), field_id, field_is_array,- 0, match_score, seq_id, sort_order, false, false, 1, -1, {});+ 0, match_score, seq_id, sort_order, false, false, false, 1, -1, {});int64_t scores[3] = {0};int64_t match_score_index = 0;@@ -3884,7 +3895,7 @@ void Index::search_wildcard(const std::vector<filter>& filters,int64_t match_score = 0;score_results2(sort_fields, (uint16_t) searched_queries.size(), 0, false, 0,- match_score, seq_id, sort_order, false, false, 1, -1, plists);+ match_score, seq_id, sort_order, false, false, false, 1, -1, plists);int64_t scores[3] = {0};int64_t match_score_index = 0;@@ -4187,6 +4198,7 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const uiconst uint32_t seq_id, const int sort_order[3],const bool prioritize_exact_match,const bool single_exact_query_token,+ const bool prioritize_token_position,size_t num_query_tokens,int syn_orig_num_tokens,const std::vector<posting_list_t::iterator_t>& posting_lists) const {@@ -4201,10 +4213,29 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui);size_t words_present = (num_query_tokens == 1 && syn_orig_num_tokens != -1) ? syn_orig_num_tokens : 1;size_t distance = (num_query_tokens == 1 && syn_orig_num_tokens != -1) ? syn_orig_num_tokens-1 : 0;- Match single_token_match = Match(words_present, distance, is_verbatim_match);+ size_t max_offset = prioritize_token_position ? posting_list_t::get_last_offset(posting_lists[0],+ field_is_array) : 255;+ Match single_token_match = Match(words_present, distance, max_offset, is_verbatim_match);match_score = single_token_match.get_match_score(total_cost, words_present);++ /*auto this_words_present = ((match_score >> 32) & 0xFF);+ auto unique_words = ((match_score >> 40) & 0xFF);+ auto typo_score = ((match_score >> 24) & 0xFF);+ auto proximity = ((match_score >> 16) & 0xFF);+ auto verbatim = ((match_score >> 8) & 0xFF);+ auto offset_score = ((match_score >> 0) & 0xFF);++ LOG(INFO) << "seq_id: " << seq_id+ << ", words_present: " << this_words_present+ << ", unique_words: " << unique_words+ << ", typo_score: " << typo_score+ << ", proximity: " << proximity+ << ", verbatim: " << verbatim+ << ", offset_score: " << offset_score+ << ", match_score: " << match_score;*/+} else {- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;posting_list_t::get_offsets(posting_lists, array_token_positions);for (const auto& kv: array_token_positions) {@@ -4219,11 +4250,12 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui// Within a field, only a subset of query tokens can match (unique_words), but even a smaller set// might be available within the window used for proximity calculation (this_words_present)- auto this_words_present = ((this_match_score >> 24) & 0xFF);- auto unique_words = field_is_array ? this_words_present : ((this_match_score >> 32) & 0xFF);- auto typo_score = ((this_match_score >> 16) & 0xFF);- auto proximity = ((this_match_score >> 8) & 0xFF);- auto verbatim = (this_match_score & 0xFF);+ auto this_words_present = ((this_match_score >> 32) & 0xFF);+ auto unique_words = field_is_array ? this_words_present : ((this_match_score >> 40) & 0xFF);+ auto typo_score = ((this_match_score >> 24) & 0xFF);+ auto proximity = ((this_match_score >> 16) & 0xFF);+ auto verbatim = ((this_match_score >> 8) & 0xFF);+ auto offset_score = prioritize_token_position ? ((this_match_score >> 0) & 0xFF) : 0;if(syn_orig_num_tokens != -1 && num_query_tokens == posting_lists.size()) {unique_words = syn_orig_num_tokens;@@ -4232,11 +4264,12 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui}uint64_t mod_match_score = (- (int64_t(this_words_present) << 32) |- (int64_t(unique_words) << 24) |- (int64_t(typo_score) << 16) |- (int64_t(proximity) << 8) |- (int64_t(verbatim) << 0)+ (int64_t(this_words_present) << 40) |+ (int64_t(unique_words) << 32) |+ (int64_t(typo_score) << 24) |+ (int64_t(proximity) << 16) |+ (int64_t(verbatim) << 8) |+ (int64_t(offset_score) << 0));if(mod_match_score > match_score) {@@ -4250,6 +4283,7 @@ int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const ui<< ", typo_score: " << typo_score<< ", proximity: " << proximity<< ", verbatim: " << verbatim+ << ", offset_score: " << offset_score<< ", mod_match_score: " << mod_match_score<< ", token_positions: " << token_positions.size()<< ", num_query_tokens: " << num_query_tokens@@ -4348,7 +4382,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16Match single_token_match = Match(words_present, distance, is_verbatim_match);match_score = single_token_match.get_match_score(total_cost, words_present);} else {- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;posting_list_t::get_offsets(posting_lists, array_token_positions);// NOTE: tokens found returned by matcher is only within the best matched window, so we have to still considerdiff --git a/src/posting.cpp b/src/posting.cppindex 4480b2ae..e6be0a9e 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -433,7 +433,7 @@ void posting_t::destroy_list(void*& obj) {}void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>& raw_posting_lists,- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions) {+ std::map<size_t, std::vector<token_positions_t>>& array_token_positions) {std::vector<posting_list_t*> plists;std::vector<posting_list_t*> expanded_plists;diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 66fbfd9d..02c3c488 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -772,14 +772,15 @@ bool posting_list_t::take_id(result_iter_state_t& istate, uint32_t id) {}bool posting_list_t::get_offsets(const std::vector<iterator_t>& its,- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos) {+ std::map<size_t, std::vector<token_positions_t>>& array_token_pos) {// Plain string format:// offset1, offset2, ... , 0 (if token is the last offset for the document)// Array string format:// offset1, ... , offsetn, offsetn, array_index, 0 (if token is the last offset for the document)- // (last offset is repeated to indicate end of offsets for a given array index)+ // NOTE 1: last offset is repeated to indicate end of offsets for a given array index)+ // NOTE 2: offsets are 1-index based (since 0 is used as last offset marking)// For each result ID and for each block it is contained in, calculate offsets@@ -901,7 +902,7 @@ bool posting_list_t::is_single_token_verbatim_match(const posting_list_t::iteratreturn false;} else if((end_offset - start_offset) == 2 && offsets[end_offset-1] == 0) {- // already checked for `offsets[start_offset] == 1` first+ // we've already checked for `offsets[start_offset] == 1` earlierreturn true;}@@ -1242,7 +1243,7 @@ void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool fieldit.skip_to(id);}- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;get_offsets(its, array_token_positions);for(auto& kv: array_token_positions) {@@ -1331,6 +1332,60 @@ bool posting_list_t::all_ended2(const std::vector<posting_list_t::iterator_t>& ireturn !its[0].valid() && !its[1].valid();}+size_t posting_list_t::get_last_offset(const posting_list_t::iterator_t& it, bool field_is_array) {+ block_t* curr_block = it.block();+ uint32_t curr_index = it.index();+ uint32_t* offsets = it.offsets;++ if(curr_block == nullptr || curr_index == UINT32_MAX) {+ return 0;+ }++ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?+ curr_block->offsets.getLength() :+ it.offset_index[curr_index + 1];++ if(field_is_array) {+ uint32_t start_offset = it.offset_index[curr_index];+ int prev_pos = -1;+ size_t max_offset = 0;++ while(start_offset < end_offset) {+ int pos = offsets[start_offset];+ start_offset++;++ if(pos > max_offset) {+ max_offset = pos;+ }++ if(pos == prev_pos) { // indicates end of array index+ size_t array_index = (size_t) offsets[start_offset];++ if(start_offset+1 < end_offset) {+ size_t next_offset = (size_t) offsets[start_offset + 1];+ if(next_offset == 0) {+ // indicates that token is the last token on the doc+ start_offset++;+ }+ }++ start_offset++; // skip current value which is the array index or flag for last index+ prev_pos = -1;+ continue;+ }++ prev_pos = pos;+ }++ return max_offset;++ } else {+ return offsets[end_offset-1] == 0 ? offsets[end_offset-2] : offsets[end_offset-1];+ }++ return 0;+}+/* iterator_t operations */posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* start, posting_list_t::block_t* end,diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex f9175b84..3763b9c6 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -1729,10 +1729,10 @@ TEST_F(CollectionSortingTest, RepeatingTokenRanking) {ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());- ASSERT_EQ(4284743939, results["hits"][0]["text_match"].get<uint32_t>());- ASSERT_EQ(4284743683, results["hits"][1]["text_match"].get<uint32_t>());- ASSERT_EQ(4284743683, results["hits"][2]["text_match"].get<uint32_t>());- ASSERT_EQ(4284743683, results["hits"][3]["text_match"].get<uint32_t>());+ ASSERT_EQ(144681433946980355, results["hits"][0]["text_match"].get<size_t>());+ ASSERT_EQ(144681433946914819, results["hits"][1]["text_match"].get<size_t>());+ ASSERT_EQ(144681433946914819, results["hits"][2]["text_match"].get<size_t>());+ ASSERT_EQ(144681433946914819, results["hits"][3]["text_match"].get<size_t>());collectionManager.drop_collection("coll1");}diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 55d030f4..abe03519 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -87,7 +87,6 @@ TEST_F(CollectionSpecificMoreTest, PrefixExpansionWhenExactMatchExists) {}TEST_F(CollectionSpecificMoreTest, PrefixExpansionOnSingleField) {- // when no default sorting field is provided, tokens must be ordered on frequencyCollection *coll1;std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false)};@@ -451,4 +450,132 @@ TEST_F(CollectionSpecificMoreTest, WrongTypoCorrection) {ASSERT_EQ(0, results["hits"].size());collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {+ Collection *coll1;+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::string> tokens = {+ "Alpha Beta Gamma", "Omega Alpha Theta", "Omega Theta Alpha", "Indigo Omega Theta Alpha"+ };++ for(size_t i = 0; i < tokens.size(); i++) {+ std::string title = tokens[i];+ nlohmann::json doc;+ doc["title"] = title;+ doc["points"] = i;+ coll1->add(doc.dump());+ }++ auto results = coll1->search("alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {off}, 32767, 32767, 2,+ false, true).get();++ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());+ ASSERT_EQ("3", results["hits"][3]["document"]["id"].get<std::string>());++ results = coll1->search("alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {off}, 32767, 32767, 2,+ false, false).get();++ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][3]["document"]["id"].get<std::string>());++ results = coll1->search("theta alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {off}, 32767, 32767, 2,+ false, false).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());++ results = coll1->search("theta alpha", {"title"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {off}, 32767, 32767, 2,+ false, true).get();+ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("3", results["hits"][2]["document"]["id"].get<std::string>());+}++TEST_F(CollectionSpecificMoreTest, PositionalTokenRankingWithArray) {+ Collection *coll1;+ std::vector<field> fields = {field("tags", field_types::STRING_ARRAY, false),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["tags"] = {"alpha foo", "gamma", "beta alpha"};+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["tags"] = {"omega", "omega beta alpha"};+ doc2["points"] = 200;++ coll1->add(doc1.dump());+ coll1->add(doc2.dump());++ auto results = coll1->search("alpha", {"tags"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {off}, 32767, 32767, 2,+ false, false).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ results = coll1->search("alpha", {"tags"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {off}, 32767, 32767, 2,+ false, true).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());}\ No newline at end of file