Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

OR based token intersection.

Kishore Nallan4889fba17943
diff --git a/include/art.h b/include/art.h
index 04739e65..7455c413 100644
--- a/include/art.h
+++ b/include/art.h
@@ -104,6 +104,17 @@ typedef struct {
unsigned char key[];
} art_leaf;
+struct token_leaf {
+ art_leaf* leaf;
+ uint32_t root_len;
+ uint32_t num_typos;
+
+ token_leaf(art_leaf* leaf, uint32_t root_len, uint32_t num_typos) : leaf(leaf), root_len(root_len),
+ num_typos(num_typos) {
+
+ }
+};
+
/**
* Main struct, points to root.
*/
@@ -261,7 +272,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, ar
int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,
const int max_words, const token_ordering token_order, const bool prefix,
const uint32_t *filter_ids, size_t filter_ids_length,
- std::vector<art_leaf *> &results, const std::set<art_leaf *>& exclude_leaves = {});
+ std::vector<art_leaf *> &results, const std::set<std::string>& exclude_leaves = {});
void encode_int32(int32_t n, unsigned char *chars);
diff --git a/include/collection.h b/include/collection.h
index fca21351..441bbdeb 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -17,6 +17,7 @@
#include <json.hpp>
#include <field.h>
#include <option.h>
+#include <tsl/htrie_map.h>
#include "tokenizer.h"
#include "synonym_index.h"
@@ -29,6 +30,7 @@ struct highlight_field_t {
std::string name;
bool fully_highlighted;
bool infix;
+ tsl::htrie_map<char, token_leaf> qtoken_leaves;
highlight_field_t(const std::string& name, bool fully_highlighted, bool infix):
name(name), fully_highlighted(fully_highlighted), infix(infix) {
@@ -104,13 +106,14 @@ private:
std::string get_seq_id_key(uint32_t seq_id) const;
void highlight_result(const std::string& raw_query,
- const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,
+ const field &search_field,
+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
const std::vector<std::string>& q_tokens,
const KV* field_order_kv, const nlohmann::json &document,
StringUtils & string_utils,
const size_t snippet_threshold,
const size_t highlight_affix_num_tokens,
- bool highlighted_fully,
+ bool highlight_fully,
bool is_infix_search,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
@@ -279,7 +282,7 @@ public:
const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
size_t per_page = 10, size_t page = 1,
- token_ordering token_order = FREQUENCY, const std::vector<bool>& prefixes = {false},
+ token_ordering token_order = FREQUENCY, const std::vector<bool>& prefixes = {true},
size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),
const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),
@@ -358,9 +361,9 @@ public:
// highlight ops
- static void highlight_text(const string& highlight_start_tag, const string& highlight_end_tag, const string& last_raw_q_token,
+ static void highlight_text(const string& highlight_start_tag, const string& highlight_end_tag,
const string& text, const std::map<size_t, size_t>& token_offsets,
- const std::map<size_t, std::string>& prefix_start_offsets, size_t snippet_end_offset,
+ size_t snippet_end_offset,
std::vector<std::string>& matched_tokens, std::map<size_t, size_t>::iterator& offset_it,
std::stringstream& highlighted_text, size_t snippet_start_offset) ;
@@ -370,6 +373,8 @@ public:
const string& highlight_fields,
const std::string& highlight_full_fields,
const std::vector<infix_t>& infixes,
+ std::vector<std::string>& q_tokens,
+ const tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<highlight_field_t>& highlight_items) const;
Option<bool> alter(nlohmann::json& alter_payload);
diff --git a/include/id_list.h b/include/id_list.h
index 65c2387f..234231ea 100644
--- a/include/id_list.h
+++ b/include/id_list.h
@@ -6,9 +6,6 @@
typedef uint32_t last_id_t;
-#define FOR_ELE_SIZE sizeof(uint32_t)
-#define METADATA_OVERHEAD 5
-
/*
Compressed chain of blocks that store the document IDs and offsets of a given token.
Offsets of singular and multi-valued fields are encoded differently.
diff --git a/include/index.h b/include/index.h
index a9284c90..813bdb6f 100644
--- a/include/index.h
+++ b/include/index.h
@@ -23,6 +23,7 @@
#include "threadpool.h"
#include "adi_tree.h"
#include "tsl/htrie_set.h"
+#include <tsl/htrie_map.h>
#include "id_list.h"
#include "synonym_index.h"
@@ -36,13 +37,21 @@ using array_mapped_infix_t = std::vector<tsl::htrie_set<char>*>;
struct token_t {
size_t position;
std::string value;
- bool prefix;
- token_t(size_t position, const std::string& value, bool prefix): position(position), value(value), prefix(prefix) {
+ bool is_prefix_searched;
+ uint32_t root_len; // if prefix searched, length of the root token
+ uint32_t num_typos;
+
+ token_t() {};
+
+ token_t(size_t position, const std::string& value, bool is_prefix_searched, uint32_t root_len, uint32_t num_typos):
+ position(position), value(value), is_prefix_searched(is_prefix_searched),
+ root_len(root_len), num_typos(num_typos) {
}
};
+// FIXME: deprecated
struct token_candidates {
token_t token;
size_t cost;
@@ -50,6 +59,13 @@ struct token_candidates {
std::vector<art_leaf*> candidates;
};
+struct tok_candidates {
+ token_t token;
+ size_t cost;
+ bool prefix_search;
+ std::vector<std::string> candidates;
+};
+
struct search_field_t {
std::string name;
size_t priority;
@@ -318,6 +334,7 @@ struct search_args {
const size_t facet_query_num_typos;
const bool filter_curated_hits;
const bool split_join_tokens;
+ tsl::htrie_map<char, token_leaf> qtoken_set;
spp::sparse_hash_set<uint64_t> groups_processed;
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -481,6 +498,13 @@ private:
// Internal utility functions
+ static inline uint32_t next_suggestion2(const std::vector<tok_candidates>& token_candidates_vec,
+ long long int n,
+ std::vector<token_t>& query_suggestion,
+ int syn_orig_num_tokens,
+ uint32_t& token_bits,
+ uint64& qhash);
+
static inline uint32_t next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
long long int n,
std::vector<art_leaf *>& actual_query_suggestion,
@@ -540,6 +564,32 @@ private:
size_t min_len_2typo,
size_t max_candidates) const;
+ void search_all_candidates(const size_t num_search_fields,
+ const std::vector<search_field_t>& the_fields,
+ const uint32_t* filter_ids, size_t filter_ids_length,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<tok_candidates>& token_candidates_vec,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ bool prioritize_exact_match,
+ const bool exhaustive_search,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff) const;
+
void search_candidates(const uint8_t & field_id,
bool field_is_array,
const uint32_t* filter_ids, size_t filter_ids_length,
@@ -581,7 +631,6 @@ private:
std::vector<uint64_t>& facet_hashes);
void collate_included_ids(const std::vector<token_t>& q_included_tokens,
- const std::string & field, const uint8_t field_id,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries) const;
@@ -650,6 +699,16 @@ public:
static void concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids);
+ int64_t score_results2(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,
+ const bool field_is_array, const uint32_t total_cost,
+ int64_t& match_score,
+ const uint32_t seq_id, const int sort_order[3],
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool prioritize_exact_match,
+ const bool single_exact_query_token,
+ int syn_orig_num_tokens,
+ const std::vector<posting_list_t::iterator_t>& posting_lists) const;
+
void score_results(const std::vector<sort_by> &sort_fields, const uint16_t &query_index, const uint8_t &field_id,
bool field_is_array, const uint32_t total_cost,
Topster *topster, const std::vector<art_leaf *> &query_suggestion,
@@ -703,7 +762,9 @@ public:
const size_t per_page,
const size_t page, const token_ordering token_order, const std::vector<bool>& prefixes,
const size_t drop_tokens_threshold, size_t& all_result_ids_len,
- spp::sparse_hash_set<uint64_t>& groups_processed, std::vector<std::vector<art_leaf*>>& searched_queries,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const std::vector<const override_t*>& filter_overrides,
@@ -765,14 +826,17 @@ public:
void search_wildcard(const std::vector<filter>& filters,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,
+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
spp::sparse_hash_set<uint64_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size, const uint8_t field_id, const string& field,
uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
- uint32_t filter_ids_length, const size_t concurrency) const;
+ uint32_t filter_ids_length, const size_t concurrency,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const;
void search_infix(const std::string& query, const std::string& field_name, std::vector<uint32_t>& ids,
size_t max_extra_prefix, size_t max_extra_suffix) const;
@@ -802,41 +866,96 @@ public:
const std::vector<search_field_t>& search_fields, uint32_t*& exclude_token_ids,
size_t& exclude_token_ids_size) const;
- void do_infix_search(const std::vector<sort_by>& sort_fields_std,
+ void do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
+ const std::vector<infix_t>& infixes,
+ const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
- const size_t max_extra_suffix, const infix_t& field_infix, const uint8_t field_id,
- const string& field_name, const std::vector<token_t>& query_tokens, Topster* actual_topster,
+ const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,
const uint32_t *filter_ids, size_t filter_ids_length,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
const std::vector<uint32_t>& curated_ids_sorted,
- size_t field_num_results, uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
spp::sparse_hash_set<uint64_t>& groups_processed) const;
- void do_synonym_search(const std::vector<filter>& filters,
+ void do_synonym_search(const std::vector<search_field_t>& the_fields,
+ const std::vector<filter>& filters,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
- const token_ordering& token_order, const size_t drop_tokens_threshold,
+ const token_ordering& token_order,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo,
size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size, size_t i, uint32_t actual_filter_ids_length,
- int field_num_typos,
- bool field_prefix, const uint8_t field_id, const string& field_name,
- const std::unordered_map<string, field>::const_iterator& field_it,
- std::vector<token_t>& query_tokens,
- size_t num_tokens_dropped, Topster* actual_topster, size_t field_num_results,
- std::vector<query_tokens_t>& field_query_tokens, size_t& all_result_ids_len,
+ size_t exclude_token_ids_size,
+ Topster* actual_topster,
+ std::vector<query_tokens_t>& field_query_tokens,
spp::sparse_hash_set<uint64_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries, uint32_t*& all_result_ids,
- uint32_t*& actual_filter_ids, std::set<uint64>& query_hashes) const;
-
- void
- do_phrase_search(const uint32_t* filter_ids, uint32_t filter_ids_length, const string& field_name, bool is_array,
- uint32_t* phrase_match_ids, const std::vector<std::vector<std::string>>& field_phrases,
- uint32_t*& actual_filter_ids, uint32_t& actual_filter_ids_length,
- size_t& phrase_match_ids_size) const;
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const uint32_t* filter_ids, uint32_t filter_ids_length,
+ std::set<uint64>& query_hashes,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ std::vector<std::vector<token_t>>& all_queries) const;
+
+ void do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
+ std::vector<query_tokens_t>& field_query_tokens,
+ uint32_t*& filter_ids, uint32_t& filter_ids_length) const;
+
+ void fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
+ const std::vector<token_t>& query_tokens,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const uint32_t* filter_ids, size_t filter_ids_length,
+ const std::vector<uint32_t>& curated_ids,
+ const std::vector<sort_by>& sort_fields,
+ const std::vector<uint32_t>& num_typos,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster, spp::sparse_hash_set<uint64_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ bool prioritize_exact_match,
+ std::set<uint64>& query_hashes,
+ const token_ordering token_order,
+ const std::vector<bool>& prefixes,
+ const size_t typo_tokens_threshold,
+ const bool exhaustive_search,
+ size_t min_len_1typo,
+ size_t min_len_2typo,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const;
+
+ void search_across_fields(const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ const std::vector<search_field_t>& the_fields,
+ const size_t num_search_fields,
+ const std::vector<sort_by>& sort_fields,
+ Topster* topster,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const uint32_t* filter_ids, uint32_t filter_ids_length,
+ const uint32_t total_cost,
+ const int syn_orig_num_tokens,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ uint32_t*& all_result_ids,
+ size_t& all_result_ids_len) const;
void
search_fields(const std::vector<filter>& filters,
@@ -860,15 +979,16 @@ public:
uint32_t*& all_result_ids,
spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const;
- void aggregate_and_score_fields(const std::vector<query_tokens_t>& field_query_tokens,
- const std::vector<search_field_t>& the_fields, Topster* topster,
- const size_t num_search_fields,
- spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const;
-
void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
std::vector<std::string>& query_tokens,
token_ordering token_order,
std::vector<filter>& filters) const;
+
+ void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices, uint32_t seq_id,
+ int64_t max_field_match_score,
+ int64_t* scores, int64_t& match_score_index) const;
};
template<class T>
diff --git a/include/or_iterator.h b/include/or_iterator.h
new file mode 100644
index 00000000..7307b523
--- /dev/null
+++ b/include/or_iterator.h
@@ -0,0 +1,103 @@
+#pragma once
+
+#include <vector>
+#include "posting_list.h"
+
+/*
+ * Takes a list of posting list iterators and returns an unique OR sequence of elements lazily
+ */
+class or_iterator_t {
+private:
+ std::vector<posting_list_t::iterator_t> its;
+ int curr_index = 0;
+
+ void advance_smallest();
+
+public:
+ explicit or_iterator_t(std::vector<posting_list_t::iterator_t>& its);
+
+ or_iterator_t(or_iterator_t&& rhs) noexcept;
+ or_iterator_t& operator=(or_iterator_t&& rhs) noexcept;
+ ~or_iterator_t() noexcept;
+
+ // utility methods for manipulating groups of iterators
+
+ static bool at_end(const std::vector<or_iterator_t>& its);
+ static bool at_end2(const std::vector<or_iterator_t>& its);
+
+ static bool equals(std::vector<or_iterator_t>& its);
+ static bool equals2(std::vector<or_iterator_t>& its);
+
+ static void advance_all(std::vector<or_iterator_t>& its);
+ static void advance_all2(std::vector<or_iterator_t>& its);
+
+ static void advance_non_largest(std::vector<or_iterator_t>& its);
+ static void advance_non_largest2(std::vector<or_iterator_t>& its);
+
+ // actual iterator operations
+
+ [[nodiscard]] bool valid() const;
+
+ bool next();
+
+ bool skip_to(uint32_t id);
+
+ [[nodiscard]] uint32_t id() const;
+
+ [[nodiscard]] const std::vector<posting_list_t::iterator_t>& get_its() const;
+
+ static bool take_id(result_iter_state_t& istate, uint32_t id);
+
+ template<class T>
+ static bool intersect(std::vector<or_iterator_t>& its, result_iter_state_t& istate, T func);
+};
+
+template<class T>
+bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_state_t& istate, T func) {
+ size_t it_size = its.size();
+
+ switch (its.size()) {
+ case 0:
+ break;
+ case 1:
+ while(its.size() == it_size && its[0].valid()) {
+ auto id = its[0].id();
+ if(take_id(istate, id)) {
+ func(id, its);
+ }
+
+ its[0].next();
+ }
+ break;
+ case 2:
+ while(its.size() == it_size && !at_end2(its)) {
+ if(equals2(its)) {
+ auto id = its[0].id();
+ if(take_id(istate, id)) {
+ func(id, its);
+ }
+
+ advance_all2(its);
+ } else {
+ advance_non_largest2(its);
+ }
+ }
+ break;
+ default:
+ while(its.size() == it_size && !at_end(its)) {
+ if(equals(its)) {
+ //LOG(INFO) << its[0].id();
+ auto id = its[0].id();
+ if(take_id(istate, its[0].id())) {
+ func(id, its);
+ }
+
+ advance_all(its);
+ } else {
+ advance_non_largest(its);
+ }
+ }
+ }
+
+ return false;
+}
diff --git a/include/posting.h b/include/posting.h
index d2685131..d2595058 100644
--- a/include/posting.h
+++ b/include/posting.h
@@ -40,11 +40,6 @@ struct compact_posting_list_t {
};
class posting_t {
-private:
-
- static void to_expanded_plists(const std::vector<void*>& raw_posting_lists, std::vector<posting_list_t*>& plists,
- std::vector<posting_list_t*>& expanded_plists);
-
public:
static constexpr size_t COMPACT_LIST_THRESHOLD_LENGTH = 64;
static constexpr size_t MAX_BLOCK_ELEMENTS = 256;
@@ -52,12 +47,12 @@ public:
struct block_intersector_t {
std::vector<posting_list_t*> plists;
std::vector<posting_list_t*> expanded_plists;
- posting_list_t::result_iter_state_t& iter_state;
+ result_iter_state_t& iter_state;
ThreadPool* thread_pool;
size_t parallelize_min_ids;
block_intersector_t(const std::vector<void*>& raw_posting_lists,
- posting_list_t::result_iter_state_t& iter_state,
+ result_iter_state_t& iter_state,
ThreadPool* thread_pool,
size_t parallelize_min_ids = 1):
iter_state(iter_state), thread_pool(thread_pool),
@@ -84,6 +79,9 @@ public:
void split_lists(size_t concurrency, std::vector<std::vector<posting_list_t::iterator_t>>& partial_its_vec);
};
+ static void to_expanded_plists(const std::vector<void*>& raw_posting_lists, std::vector<posting_list_t*>& plists,
+ std::vector<posting_list_t*>& expanded_plists);
+
static void upsert(void*& obj, uint32_t id, const std::vector<uint32_t>& offsets);
static void erase(void*& obj, uint32_t id);
diff --git a/include/posting_list.h b/include/posting_list.h
index d8ebcc4b..93460e5d 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -8,8 +8,24 @@
typedef uint32_t last_id_t;
-#define FOR_ELE_SIZE sizeof(uint32_t)
-#define METADATA_OVERHEAD 5
+struct result_iter_state_t {
+ const uint32_t* excluded_result_ids = nullptr;
+ const size_t excluded_result_ids_size = 0;
+
+ const uint32_t* filter_ids = nullptr;
+ const size_t filter_ids_length = 0;
+
+ size_t excluded_result_ids_index = 0;
+ size_t filter_ids_index = 0;
+ size_t index = 0;
+
+ result_iter_state_t() = default;
+
+ result_iter_state_t(const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
+ const uint32_t* filter_ids, const size_t filter_ids_length) : excluded_result_ids(excluded_result_ids),
+ excluded_result_ids_size(excluded_result_ids_size),
+ filter_ids(filter_ids), filter_ids_length(filter_ids_length) {}
+};
/*
Compressed chain of blocks that store the document IDs and offsets of a given token.
@@ -48,18 +64,24 @@ public:
private:
block_t* curr_block;
uint32_t curr_index;
-
block_t* end_block;
+ bool auto_destroy;
+ uint32_t field_id;
+
public:
// uncompressed data structures for performance
uint32_t* ids = nullptr;
uint32_t* offset_index = nullptr;
uint32_t* offsets = nullptr;
- explicit iterator_t(block_t* start, block_t* end);
- iterator_t(iterator_t&& rhs) noexcept;
+ explicit iterator_t(block_t* start, block_t* end, bool auto_destroy = true, uint32_t field_id = 0);
~iterator_t();
+
+ iterator_t(iterator_t&& rhs) noexcept;
+ iterator_t& operator=(iterator_t&& rhs) noexcept;
+
+ void destroy();
[[nodiscard]] bool valid() const;
void next();
void skip_to(uint32_t id);
@@ -67,28 +89,12 @@ public:
[[nodiscard]] uint32_t id() const;
[[nodiscard]] inline uint32_t index() const;
[[nodiscard]] inline block_t* block() const;
- };
-
- struct result_iter_state_t {
- const uint32_t* excluded_result_ids = nullptr;
- const size_t excluded_result_ids_size = 0;
-
- const uint32_t* filter_ids = nullptr;
- const size_t filter_ids_length = 0;
-
- size_t excluded_result_ids_index = 0;
- size_t filter_ids_index = 0;
- size_t index = 0;
+ [[nodiscard]] uint32_t get_field_id() const;
- result_iter_state_t() = default;
-
- result_iter_state_t(uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
- const uint32_t* filter_ids, const size_t filter_ids_length) : excluded_result_ids(excluded_result_ids),
- excluded_result_ids_size(excluded_result_ids_size),
- filter_ids(filter_ids), filter_ids_length(filter_ids_length) {}
+ posting_list_t::iterator_t clone() const;
};
-private:
+public:
// maximum number of IDs (and associated offsets) to store in each block before another block is created
const uint16_t BLOCK_MAX_ELEMENTS;
@@ -104,6 +110,9 @@ private:
static bool at_end(const std::vector<posting_list_t::iterator_t>& its);
static bool at_end2(const std::vector<posting_list_t::iterator_t>& its);
+ static bool all_ended(const std::vector<posting_list_t::iterator_t>& its);
+ static bool all_ended2(const std::vector<posting_list_t::iterator_t>& its);
+
static bool equals(std::vector<posting_list_t::iterator_t>& its);
static bool equals2(std::vector<posting_list_t::iterator_t>& its);
@@ -116,8 +125,6 @@ private:
static uint32_t advance_smallest(std::vector<posting_list_t::iterator_t>& its);
static uint32_t advance_smallest2(std::vector<posting_list_t::iterator_t>& its);
-public:
-
posting_list_t() = delete;
explicit posting_list_t(uint16_t max_block_elements);
@@ -148,7 +155,7 @@ public:
bool contains_atleast_one(const uint32_t* target_ids, size_t target_ids_size);
- iterator_t new_iterator(block_t* start_block = nullptr, block_t* end_block = nullptr);
+ iterator_t new_iterator(block_t* start_block = nullptr, block_t* end_block = nullptr, uint32_t field_id = 0);
static void merge(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids);
diff --git a/src/art.cpp b/src/art.cpp
index c37d01d3..0b47efa2 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -942,8 +942,8 @@ void* art_delete(art_tree *t, const unsigned char *key, int key_len) {
int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_results,
const uint32_t* filter_ids, size_t filter_ids_length,
- const std::set<art_leaf*>& exclude_leaves, const art_leaf* exact_leaf,
- std::vector<art_leaf *> &results) {
+ const std::set<std::string>& exclude_leaves, const art_leaf* exact_leaf,
+ std::vector<art_leaf *>& results) {
printf("INSIDE art_topk_iter: root->type: %d\n", root->type);
@@ -957,8 +957,6 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
q.push(root);
- size_t num_large_lists = 0;
-
while(!q.empty() && results.size() < max_results*4) {
art_node *n = (art_node *) q.top();
q.pop();
@@ -973,22 +971,22 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
if (!n) continue;
if (IS_LEAF(n)) {
art_leaf *l = (art_leaf *) LEAF_RAW(n);
-
//LOG(INFO) << "END LEAF SCORE: " << l->max_score;
- if(exclude_leaves.count(l) != 0 || l == exact_leaf) {
- continue;
- }
if(filter_ids_length == 0) {
+ std::string tok(reinterpret_cast<char*>(l->key), l->key_len - 1);
+ if(exclude_leaves.count(tok) != 0 || l == exact_leaf) {
+ continue;
+ }
results.push_back(l);
} else {
// we will push leaf only if filter matches with leaf IDs
- if(!IS_COMPACT_POSTING(l->values)) {
- num_large_lists++;
- }
-
bool found_atleast_one = posting_t::contains_atleast_one(l->values, filter_ids, filter_ids_length);
if(found_atleast_one) {
+ std::string tok(reinterpret_cast<char*>(l->key), l->key_len - 1);
+ if(exclude_leaves.count(tok) != 0 || l == exact_leaf) {
+ continue;
+ }
results.push_back(l);
}
}
@@ -1491,7 +1489,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,
const int max_words, const token_ordering token_order, const bool prefix,
const uint32_t *filter_ids, size_t filter_ids_length,
- std::vector<art_leaf *> &results, const std::set<art_leaf *>& exclude_leaves) {
+ std::vector<art_leaf *> &results, const std::set<std::string>& exclude_leaves) {
std::vector<const art_node*> nodes;
int irow[term_len + 1];
diff --git a/src/collection.cpp b/src/collection.cpp
index 481640af..eb3fcd47 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -984,7 +984,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
false);
for(size_t i = 0; i < q_include_tokens.size(); i++) {
auto& q_include_token = q_include_tokens[i];
- field_query_tokens[0].q_include_tokens.emplace_back(i, q_include_token, (i == q_include_tokens.size()-1));
+ field_query_tokens[0].q_include_tokens.emplace_back(i, q_include_token, (i == q_include_tokens.size() - 1),
+ q_include_token.size(), 0);
}
} else {
field_query_tokens.emplace_back(query_tokens_t{});
@@ -1000,7 +1001,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
for(size_t i = 0; i < q_include_tokens.size(); i++) {
auto& q_include_token = q_include_tokens[i];
q_tokens.push_back(q_include_token);
- field_query_tokens[0].q_include_tokens.emplace_back(i, q_include_token, (i == q_include_tokens.size()-1));
+ field_query_tokens[0].q_include_tokens.emplace_back(i, q_include_token, (i == q_include_tokens.size() - 1),
+ q_include_token.size(), 0);
}
for(auto& phrase: field_query_tokens[0].q_phrases) {
@@ -1158,8 +1160,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
std::vector<highlight_field_t> highlight_items;
if(query != "*") {
- process_highlight_fields(search_fields, include_fields, exclude_fields,
- highlight_fields, highlight_full_fields, infixes, highlight_items);
+ process_highlight_fields(search_fields, include_fields, exclude_fields, highlight_fields, highlight_full_fields,
+ infixes, q_tokens, search_params->qtoken_set, highlight_items);
}
nlohmann::json result = nlohmann::json::object();
@@ -1212,8 +1214,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
search_field.type == field_types::STRING_ARRAY)) {
highlight_t highlight;
- highlight_result(raw_query, search_field, searched_queries, q_tokens, field_order_kv, document,
- string_utils, snippet_threshold, highlight_affix_num_tokens,
+ highlight_result(raw_query, search_field, highlight_item.qtoken_leaves, q_tokens, field_order_kv,
+ document,string_utils, snippet_threshold, highlight_affix_num_tokens,
highlight_item.fully_highlighted, highlight_item.infix,
highlight_start_tag, highlight_end_tag, highlight);
if(!highlight.snippets.empty()) {
@@ -1468,6 +1470,8 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
const string& highlight_fields,
const std::string& highlight_full_fields,
const std::vector<infix_t>& infixes,
+ std::vector<std::string>& q_tokens,
+ const tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<highlight_field_t>& highlight_items) const {
// identify full highlight fields
@@ -1512,11 +1516,42 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
StringUtils::split(highlight_fields, highlight_field_names, ",");
for(size_t i = 0; i < highlight_field_names.size(); i++) {
+ if(search_schema.count(highlight_field_names[i]) == 0) {
+ // ignore fields not part of schema
+ continue;
+ }
bool fully_highlighted = (fields_highlighted_fully_set.count(highlight_field_names[i]) != 0);
bool infixed = (fields_infixed_set.count(highlight_field_names[i]) != 0);
highlight_items.emplace_back(highlight_field_names[i], fully_highlighted, infixed);
}
}
+
+ std::string qtoken;
+ for(auto it = qtoken_set.begin(); it != qtoken_set.end(); ++it) {
+ it.key(qtoken);
+
+ for(auto& highlight_item: highlight_items) {
+ const auto& field_name = highlight_item.name;
+ art_leaf* leaf = index->get_token_leaf(field_name, (const unsigned char*) qtoken.c_str(), qtoken.size()+1);
+ if(leaf) {
+ highlight_item.qtoken_leaves.insert(qtoken, token_leaf(leaf, it.value().root_len, it.value().num_typos));
+ }
+ }
+ }
+
+ // We will also add tokens from the query if they are not already added.
+ // This helps handle highlighting of tokens which were dropped from the query to return results.
+ for(auto& q_token: q_tokens) {
+ if(qtoken_set.find(q_token) == qtoken_set.end()) {
+ for(auto& highlight_item: highlight_items) {
+ const auto& field_name = highlight_item.name;
+ art_leaf* leaf = index->get_token_leaf(field_name, (const unsigned char*) q_token.c_str(), q_token.size()+1);
+ if(leaf) {
+ highlight_item.qtoken_leaves.insert(q_token, token_leaf(leaf, q_token.size(), 0));
+ }
+ }
+ }
+ }
}
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
@@ -1732,13 +1767,13 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t
}
void Collection::highlight_result(const std::string& raw_query, const field &search_field,
- const std::vector<std::vector<art_leaf *>> &searched_queries,
+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
const std::vector<std::string>& q_tokens,
const KV* field_order_kv, const nlohmann::json & document,
StringUtils & string_utils,
const size_t snippet_threshold,
const size_t highlight_affix_num_tokens,
- bool highlighted_fully,
+ bool highlight_fully,
bool is_infix_search,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
@@ -1748,134 +1783,88 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
return;
}
- std::vector<art_leaf*> query_suggestion;
- std::set<std::string> query_suggestion_tokens;
-
bool is_cyrillic = Tokenizer::is_cyrillic(search_field.locale);
bool normalise = is_cyrillic ? false : true;
std::vector<std::string> raw_query_tokens;
Tokenizer(raw_query, normalise, false, search_field.locale, symbols_to_index, token_separators).tokenize(raw_query_tokens);
+
const std::string& last_raw_q_token = raw_query_tokens.back();
const std::string& last_q_token = q_tokens.back();
std::set<std::string> last_full_q_tokens;
- size_t qindex = 0;
-
- do {
- if(searched_queries.size() <= field_order_kv->query_index) {
- // in filter based overrides with matched tokens removal, we could have no queries but still might
- // want to highlight fields from actual query tokens
- break;
- }
-
- if(field_order_kv->query_indices != nullptr &&
- searched_queries.size() <= field_order_kv->query_indices[qindex + 1]) {
- LOG(ERROR) << "Query indices exceed searched queries length, searched_queries.size(): "
- << searched_queries.size() << ", field_order_kv->query_indices[qindex + 1]: "
- << field_order_kv->query_indices[qindex + 1] << ", qindex: " << qindex
- << "field_order_kv->query_indices[0]: " << field_order_kv->query_indices[0];
- break;
- }
-
- auto searched_query =
- (field_order_kv->query_indices == nullptr) ? searched_queries[field_order_kv->query_index] :
- searched_queries[field_order_kv->query_indices[qindex + 1]];
-
- for (size_t i = 0; i < searched_query.size(); i++) {
- art_leaf* token_leaf = searched_query[i];
- std::string token(reinterpret_cast<char*>(token_leaf->key), token_leaf->key_len - 1);
-
- if(query_suggestion_tokens.count(token) != 0) {
- continue;
- }
+ if(qtoken_leaves.empty() && !is_infix_search) {
+ // none of the tokens from the query were found on this field
+ return ;
+ }
- // Must search for the token string fresh on that field for the given document since `token_leaf`
- // is from the best matched field and need not be present in other fields of a document.
- art_leaf* actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);
+ std::vector<match_index_t> match_indices;
- if(actual_leaf != nullptr && posting_t::contains(actual_leaf->values, field_order_kv->key)) {
- query_suggestion.push_back(actual_leaf);
- query_suggestion_tokens.insert(token);
- //LOG(INFO) << "field: " << search_field.name << ", key: " << token;
- if(i == searched_query.size()-1 &&
- (q_tokens.size() == searched_query.size() || token.rfind(last_q_token, 0) == 0)) {
- last_full_q_tokens.insert(token);
+ if(is_infix_search) {
+ // could be an optional field
+ if(document.contains(search_field.name)) {
+ size_t array_len = 1;
+ bool field_is_array = document[search_field.name].is_array();
+ if(field_is_array) {
+ array_len = document[search_field.name].size();
+ }
+
+ for(size_t i = 0; i < array_len; i++) {
+ std::string text = field_is_array ? document[search_field.name][i] : document[search_field.name];
+ StringUtils::tolowercase(text);
+ if(text.size() < 100 && text.find(raw_query_tokens.front()) != std::string::npos) {
+ const Match & this_match = Match(field_order_kv->key, {}, false, false);
+ uint64_t this_match_score = this_match.get_match_score(0, 1);
+ match_indices.emplace_back(this_match, this_match_score, i);
}
}
}
- qindex++;
- } while(field_order_kv->query_indices != nullptr && qindex < field_order_kv->query_indices[0]);
+ } else {
- for(size_t i = 0; i < q_tokens.size(); i++) {
- const std::string& q_token = q_tokens[i];
+ /*std::string qtok_buff;
+ for(auto it = qtoken_leaves.begin(); it != qtoken_leaves.end(); ++it) {
+ it.key(qtok_buff);
+ LOG(INFO) << "Token: " << qtok_buff << ", root_len: " << it.value().root_len;
+ }*/
- if(query_suggestion_tokens.count(q_token) != 0) {
- continue;
+ std::vector<void*> posting_lists;
+ for(auto token_leaf: qtoken_leaves) {
+ posting_lists.push_back(token_leaf.leaf->values);
}
- art_leaf *actual_leaf = index->get_token_leaf(search_field.name,
- reinterpret_cast<const unsigned char *>(q_token.c_str()),
- q_token.size() + 1);
+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);
- if(actual_leaf != nullptr && posting_t::contains(actual_leaf->values, field_order_kv->key)) {
- query_suggestion.push_back(actual_leaf);
- query_suggestion_tokens.insert(q_token);
- } else if(i == q_tokens.size()-1) {
- // we will copy the last token for highlighting prefix matches
- query_suggestion_tokens.insert(q_token);
- }
- }
+ for(const auto& kv: array_token_positions) {
+ const std::vector<token_positions_t>& token_positions = kv.second;
+ size_t array_index = kv.first;
- if(query_suggestion_tokens.empty()) {
- // none of the tokens from the query were found on this field
- return ;
- }
-
- std::vector<void*> posting_lists;
- for(art_leaf* leaf: query_suggestion) {
- posting_lists.push_back(leaf->values);
- }
-
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
- posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);
-
- std::vector<match_index_t> match_indices;
+ if(token_positions.empty()) {
+ continue;
+ }
- for(const auto& kv: array_token_positions) {
- const std::vector<token_positions_t>& token_positions = kv.second;
- size_t array_index = kv.first;
+ const Match & this_match = Match(field_order_kv->key, token_positions, true, true);
+ uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());
+ match_indices.emplace_back(this_match, this_match_score, array_index);
- if(token_positions.empty()) {
- continue;
+ /*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)
+ << ", match_score: " << this_match_score
+ << ", match.distance: " << size_t(this_match.distance);*/
}
-
- const Match & this_match = Match(field_order_kv->key, token_positions, true, true);
- uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());
- match_indices.emplace_back(this_match, this_match_score, array_index);
-
- /*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)
- << ", match_score: " << this_match_score
- << ", match.distance: " << size_t(this_match.distance);*/
}
if(match_indices.empty()) {
- // none of the tokens from the query were found on this field
- // let's try to look only for prefix matches
- Match dummy_match(0, 0);
- match_index_t match_index(dummy_match, 0, 0);
- match_indices.emplace_back(match_index);
- query_suggestion_tokens.clear();
+ return ;
}
const size_t max_array_matches = std::min((size_t)MAX_ARRAY_MATCHES, match_indices.size());
std::partial_sort(match_indices.begin(), match_indices.begin()+max_array_matches, match_indices.end());
- for(size_t index = 0; index < max_array_matches; index++) {
- std::sort(match_indices[index].match.offsets.begin(), match_indices[index].match.offsets.end());
- const auto& match_index = match_indices[index];
+ for(size_t array_i = 0; array_i < max_array_matches; array_i++) {
+ std::sort(match_indices[array_i].match.offsets.begin(), match_indices[array_i].match.offsets.end());
+ const auto& match_index = match_indices[array_i];
const Match& match = match_index.match;
size_t last_valid_offset = 0;
@@ -1924,7 +1913,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
// need an ordered map here to ensure that it is ordered by the key (start offset)
std::map<size_t, size_t> token_offsets;
- std::map<size_t, std::string> prefix_start_offsets;
int match_offset_index = 0;
std::string raw_token;
@@ -1958,13 +1946,27 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
bool token_already_found = (token_hits.find(raw_token) != token_hits.end());
+ auto qtoken_it = qtoken_leaves.find(raw_token);
// ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestion
if ((found_first_match && token_already_found) ||
(match_offset_index <= last_valid_offset_index &&
match.offsets[match_offset_index].offset == raw_token_index)) {
- token_offsets.emplace(tok_start, tok_end);
+ // check if the matched token is a prefix of this found token
+ if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().root_len < raw_token.size()) {
+ // need to ensure that only the prefix portion is highlighted
+ // if length diff is within 2, we still might not want to highlight partially in some cases
+ // e.g. "samsng" vs "samsung" -> full highlight is preferred, unless it's a full prefix match
+ //size_t char_diff = raw_token.size() - qtoken_it.value().root_len;
+ size_t char_diff = (tok_end - tok_start + 1) - last_raw_q_token.size();
+ auto new_tok_end = (char_diff <= 2 && qtoken_it.value().num_typos != 0) ?
+ tok_end : (tok_end - char_diff);
+ token_offsets.emplace(tok_start, new_tok_end);
+ } else {
+ token_offsets.emplace(tok_start, tok_end);
+ }
+
token_hits.insert(raw_token);
// to skip over duplicate tokens in the query
@@ -1979,9 +1981,23 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
found_first_match = true;
- } else if(query_suggestion_tokens.find(raw_token) != query_suggestion_tokens.end() ||
- raw_token.rfind(last_raw_q_token, 0) == 0) {
- token_offsets.emplace(tok_start, tok_end);
+ } else if((highlight_fully || text.size() < snippet_threshold * 6) &&
+ qtoken_leaves.find(raw_token) != qtoken_leaves.end()) {
+
+ // Token might not appear in the best matched window, which is limited to a size of 10.
+ // If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we will
+ // locate all tokens that appear in the query / query candidates
+
+ if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().root_len < raw_token.size()) {
+ // need to ensure that only the prefix portion is highlighted
+ size_t char_diff = (tok_end - tok_start + 1) - last_raw_q_token.size();
+ auto new_tok_end = (char_diff <= 2 && qtoken_it.value().num_typos != 0) ?
+ tok_end : (tok_end - char_diff);
+ token_offsets.emplace(tok_start, new_tok_end);
+ } else {
+ token_offsets.emplace(tok_start, tok_end);
+ }
+
token_hits.insert(raw_token);
} else if(is_infix_search && text.size() < 100 &&
raw_token.find(raw_query_tokens.front()) != std::string::npos) {
@@ -1989,10 +2005,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
token_hits.insert(raw_token);
}
- if(last_full_q_tokens.find(raw_token) != last_full_q_tokens.end()) {
- prefix_start_offsets.emplace(tok_start, raw_token);
- }
-
if(raw_token_index >= last_valid_offset + highlight_affix_num_tokens) {
// register end of highlight snippet
if(snippet_end_offset == text.size() - 1) {
@@ -2009,7 +2021,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
if(raw_token_index >= snippet_threshold &&
match_offset_index > last_valid_offset_index &&
raw_token_index >= last_valid_offset + highlight_affix_num_tokens &&
- !highlighted_fully) {
+ !highlight_fully) {
break;
}
}
@@ -2032,8 +2044,8 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
std::stringstream highlighted_text;
- highlight_text(highlight_start_tag, highlight_end_tag, last_raw_q_token, text, token_offsets,
- prefix_start_offsets, snippet_end_offset, matched_tokens, offset_it,
+ highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,
+ snippet_end_offset, matched_tokens, offset_it,
highlighted_text, snippet_start_offset);
highlight.snippets.push_back(highlighted_text.str());
@@ -2041,25 +2053,27 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
highlight.indices.push_back(match_index.index);
}
- if(highlighted_fully) {
+ if(highlight_fully) {
std::stringstream value_stream;
offset_it = token_offsets.begin();
std::vector<std::string> full_matched_tokens;
- highlight_text(highlight_start_tag, highlight_end_tag, last_raw_q_token, text, token_offsets,
- prefix_start_offsets, text.size()-1, full_matched_tokens, offset_it,
+ highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,
+ text.size()-1, full_matched_tokens, offset_it,
value_stream, 0);
highlight.values.push_back(value_stream.str());
}
}
highlight.field = search_field.name;
- highlight.match_score = match_indices[0].match_score;
+
+ if(!match_indices.empty()) {
+ highlight.match_score = match_indices[0].match_score;
+ }
}
void Collection::highlight_text(const string& highlight_start_tag, const string& highlight_end_tag,
- const string& last_raw_q_token, const string& text,
+ const string& text,
const std::map<size_t, size_t>& token_offsets,
- const std::map<size_t, std::string>& prefix_start_offsets,
size_t snippet_end_offset, std::vector<std::string>& matched_tokens,
std::map<size_t, size_t>::iterator& offset_it,
std::stringstream& highlighted_text,
@@ -2074,25 +2088,6 @@ void Collection::highlight_text(const string& highlight_start_tag, const string&
size_t token_len = offset_it->second - snippet_start_offset + 1;
- if(prefix_start_offsets.find(offset_it->first) != prefix_start_offsets.end() &&
- last_raw_q_token.size() < token_len) {
- // if length diff is within 2, we still might not want to highlight partially in some cases
- // e.g. "samsng" vs "samsung" -> full highlight is preferred
- bool within_two_chars = (abs(int64_t(last_raw_q_token.size()) - (int64_t)token_len) <= 2);
-
- if(!within_two_chars || last_raw_q_token.back() == text_token[last_raw_q_token.size()-1]) {
- // also account for presence ascii symbols in the text
- size_t num_symbols = 0;
- for(size_t j = 0; j < text_token.size(); j++) {
- char c = text_token[j];
- if(Tokenizer::is_ascii_char(c) && !isalnum(c)) {
- num_symbols++;
- }
- }
- token_len = std::min(token_len, last_raw_q_token.size() + num_symbols);
- }
- }
-
for(size_t j = 0; j < token_len; j++) {
highlighted_text << text[snippet_start_offset + j];
}
diff --git a/src/index.cpp b/src/index.cpp
index 7b81963b..448f7556 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -19,6 +19,7 @@
#include <posting.h>
#include <thread_local_vars.h>
#include <unordered_set>
+#include <or_iterator.h>
#include "logger.h"
#define RETURN_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\
@@ -38,6 +39,16 @@ spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;
+struct token_posting_t {
+ uint32_t token_id;
+ const posting_list_t::iterator_t& posting;
+
+ token_posting_t(uint32_t token_id, const posting_list_t::iterator_t& posting):
+ token_id(token_id), posting(posting) {
+
+ }
+};
+
Index::Index(const std::string& name, const uint32_t collection_id, const Store* store,
SynonymIndex* synonym_index, ThreadPool* thread_pool,
const std::unordered_map<std::string, field> & search_schema,
@@ -1108,6 +1119,76 @@ void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {
}
}
+void Index::search_all_candidates(const size_t num_search_fields,
+ const std::vector<search_field_t>& the_fields,
+ const uint32_t* filter_ids, size_t filter_ids_length,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<tok_candidates>& token_candidates_vec,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ bool prioritize_exact_match,
+ const bool exhaustive_search,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff) const {
+
+ auto product = []( long long a, tok_candidates & b ) { return a*b.candidates.size(); };
+ long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);
+
+ long long combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
+
+ for(long long n = 0; n < N && n < combination_limit; ++n) {
+ RETURN_CIRCUIT_BREAKER
+
+ // every element in `query_suggestion` contains a token and its associated hits
+ std::vector<token_t> query_suggestion(token_candidates_vec.size());
+
+ uint64 qhash;
+
+ uint32_t token_bits = 0;
+ uint32_t total_cost = next_suggestion2(token_candidates_vec, n, query_suggestion, syn_orig_num_tokens,
+ token_bits, qhash);
+
+ if(query_hashes.find(qhash) != query_hashes.end()) {
+ // skip this query since it has already been processed before
+ continue;
+ }
+
+ query_hashes.insert(qhash);
+
+ //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
+ //LOG(INFO) << "n: " << n;
+
+ std::stringstream fullq;
+ for(const auto& qtok : query_suggestion) {
+ fullq << qtok.value << " ";
+ }
+
+ /*LOG(INFO) << "query: " << fullq.str() << ", total_cost: " << total_cost;*/
+
+ search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields,
+ sort_fields, topster,groups_processed,
+ searched_queries, qtoken_set, group_limit, group_by_fields, prioritize_exact_match,
+ filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,
+ exclude_token_ids, exclude_token_ids_size,
+ sort_order, field_values, geopoint_indices,
+ all_result_ids, all_result_ids_len);
+ }
+}
+
void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
const uint32_t* filter_ids, size_t filter_ids_length,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
@@ -1183,7 +1264,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
posting_lists.push_back(query_leaf->values);
}
- posting_list_t::result_iter_state_t iter_state(
+ result_iter_state_t iter_state(
excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length
);
@@ -1626,6 +1707,11 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
}
}
+ if(filter_ids_length == 0) {
+ delete [] filter_ids;
+ filter_ids = nullptr;
+ }
+
/*long long int timeMillis =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
@@ -1650,6 +1736,7 @@ void Index::run_search(search_args* search_params) {
search_params->prefixes, search_params->drop_tokens_threshold,
search_params->all_result_ids_len, search_params->groups_processed,
search_params->searched_queries,
+ search_params->qtoken_set,
search_params->raw_result_kvs, search_params->override_result_kvs,
search_params->typo_tokens_threshold,
search_params->group_limit, search_params->group_by_fields,
@@ -1671,7 +1758,6 @@ void Index::run_search(search_args* search_params) {
}
void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
- const std::string & field, const uint8_t field_id,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
Topster* curated_topster,
std::vector<std::vector<art_leaf*>> & searched_queries) const {
@@ -1680,26 +1766,6 @@ void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
return;
}
- // created searched queries so that curated results can be highlighted
-
- std::vector<art_leaf *> override_query;
-
- for(const token_t& token: q_included_tokens) {
- if(token.value == "*") {
- continue;
- }
-
- const size_t token_len = token.value.size() + 1;
-
- std::vector<art_leaf*> leaves;
- art_fuzzy_search(search_index.at(field), (const unsigned char *) token.value.c_str(), token_len,
- 0, 0, 1, token_ordering::MAX_SCORE, false, nullptr, 0, leaves);
-
- if(!leaves.empty()) {
- override_query.push_back(leaves[0]);
- }
- }
-
for(const auto& pos_ids: included_ids_map) {
const size_t outer_pos = pos_ids.first;
@@ -1718,12 +1784,10 @@ void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
scores[2] = int64_t(1);
uint32_t token_bits = 0;
- KV kv(field_id, searched_queries.size(), token_bits, seq_id, distinct_id, 0, scores);
+ KV kv(0, searched_queries.size(), token_bits, seq_id, distinct_id, 0, scores);
curated_topster->add(&kv);
}
}
-
- searched_queries.push_back(override_query);
}
void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) {
@@ -1938,7 +2002,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&
std::set<std::string> window_tokens_set;
for (size_t i = start_index; i < start_index + window_len; i++) {
bool is_prefix = (i == (start_index + window_len - 1));
- window_tokens.emplace_back(i, tokens[i], is_prefix);
+ window_tokens.emplace_back(i, tokens[i], is_prefix, tokens[i].size(), 0);
window_tokens_set.emplace(tokens[i]);
}
@@ -2055,6 +2119,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
const size_t drop_tokens_threshold, size_t& all_result_ids_len,
spp::sparse_hash_set<uint64_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields,
@@ -2078,6 +2143,10 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
do_filtering(filter_ids, filter_ids_length, filters, true);
+ if(!filters.empty() && filter_ids_length == 0) {
+ return ;
+ }
+
std::vector<uint32_t> included_ids_vec;
for(const auto& seq_id_pos: included_ids) {
included_ids_vec.push_back(seq_id_pos.first);
@@ -2142,123 +2211,169 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
uint32_t* all_result_ids = nullptr;
const size_t num_search_fields = std::min(the_fields.size(), (size_t) FIELD_LIMIT_NUM);
- uint32_t* exclude_token_ids = nullptr;
- size_t exclude_token_ids_size = 0;
+
+ // handle phrase searches
+ if(!field_query_tokens[0].q_phrases.empty()) {
+ do_phrase_search(num_search_fields, the_fields, field_query_tokens, filter_ids, filter_ids_length);
+ if(filter_ids_length == 0) {
+ return ;
+ }
+ }
// handle exclusion of tokens/phrases
+ uint32_t* exclude_token_ids = nullptr;
+ size_t exclude_token_ids_size = 0;
handle_exclusion(num_search_fields, field_query_tokens, the_fields, exclude_token_ids, exclude_token_ids_size);
- std::vector<Topster*> ftopsters;
+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;
+ std::vector<size_t> geopoint_indices;
+ populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);
+
+ // Prepare excluded document IDs that we can later remove from the result set
+ uint32_t* excluded_result_ids = nullptr;
+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
+ &curated_ids_sorted[0], curated_ids_sorted.size(),
+ &excluded_result_ids);
auto is_wildcard_query = !field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&
field_query_tokens[0].q_include_tokens[0].value == "*";
// for phrase query, parser will set field_query_tokens to "*", need to handle that
- if (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {
+ if (is_wildcard_query) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
const std::string& field = the_fields[0].name;
- curate_filtered_ids(filters, curated_ids, exclude_token_ids,
- exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);
+ curate_filtered_ids(filters, curated_ids, excluded_result_ids,
+ excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);
search_wildcard(filters, included_ids_map, sort_fields_std, topster,
curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
curated_ids, curated_ids_sorted,
- exclude_token_ids, exclude_token_ids_size, field_id, field,
- all_result_ids, all_result_ids_len, filter_ids, filter_ids_length, concurrency);
+ excluded_result_ids, excluded_result_ids_size, field_id, field,
+ all_result_ids, all_result_ids_len, filter_ids, filter_ids_length, concurrency,
+ sort_order, field_values, geopoint_indices);
} else {
// Non-wildcard
// In multi-field searches, a record can be matched across different fields, so we use this for aggregation
- spp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;
//begin = std::chrono::high_resolution_clock::now();
- for(size_t i = 0; i < num_search_fields; i++) {
- Topster* ftopster = new Topster(topster->MAX_SIZE, topster->distinct);
- ftopsters.push_back(ftopster);
- }
-
- // We do progressive typo relaxation here so that results with minimal typos are fetched first,
- // and further results are fetched only if `typo_tokens_threshold` is not satisfied.
- for(int min_typo = 0; min_typo <= 2; min_typo++) {
- search_fields(filters, included_ids_map, sort_fields_std, min_typo, num_typos, topster,
- curated_topster, token_order, prefixes, drop_tokens_threshold, groups_processed,
- searched_queries, typo_tokens_threshold, group_limit, group_by_fields,
- prioritize_exact_match, exhaustive_search, concurrency, min_len_1typo, min_len_2typo,
- max_candidates, infixes, max_extra_prefix, max_extra_suffix,
- filter_ids, filter_ids_length, curated_ids, curated_ids_sorted, num_search_fields,
- exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, split_join_tokens,
- field_query_tokens, the_fields, all_result_ids_len, all_result_ids, topster_ids);
-
- if (!exhaustive_search && all_result_ids_len >= typo_tokens_threshold) {
- break;
- }
- }
-
- // now try to drop tokens
- size_t num_tokens_dropped = 0;
- std::vector<token_t> orig_tokens = field_query_tokens[0].q_include_tokens;
- std::vector<query_tokens_t> truncated_field_query_tokens = field_query_tokens;
+ // FIXME: needed?
+ std::set<uint64> query_hashes;
- while(exhaustive_search || all_result_ids_len < drop_tokens_threshold) {
- // When atleast two tokens from the query are available we can drop one
- std::vector<token_t> truncated_tokens;
+ fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, excluded_result_ids,
+ excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
+ sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
+ query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search, min_len_1typo,
+ min_len_2typo, -1, sort_order, field_values, geopoint_indices);
- if(orig_tokens.size() > 1 && num_tokens_dropped < 2*(orig_tokens.size()-1)) {
- bool prefix_search = false;
+ // try split/joining tokens if no results are found
+ if(all_result_ids_len == 0 && split_join_tokens) {
+ std::vector<std::vector<std::string>> space_resolved_queries;
- if(num_tokens_dropped < orig_tokens.size()-1) {
- // drop from right
- size_t truncated_len = orig_tokens.size() - num_tokens_dropped - 1;
- for(size_t i=0; i < truncated_len; i++) {
- truncated_tokens.emplace_back(orig_tokens[i]);
- }
- } else {
- // drop from left
- prefix_search = true;
- size_t start_index = (num_tokens_dropped + 1) - orig_tokens.size() + 1;
- for(size_t i = start_index; i < orig_tokens.size(); i++) {
- truncated_tokens.emplace_back(orig_tokens[i]);
- }
+ for(size_t i = 0; i < num_search_fields; i++) {
+ std::vector<std::string> q_include_tokens;
+ for(auto& q_include_token: field_query_tokens[i].q_include_tokens) {
+ q_include_tokens.push_back(q_include_token.value);
}
- num_tokens_dropped++;
+ resolve_space_as_typos(q_include_tokens, the_fields[i].name,space_resolved_queries);
- for(size_t i = 0; i < num_search_fields; i++) {
- truncated_field_query_tokens[i].q_include_tokens = truncated_tokens;
+ if(!space_resolved_queries.empty()) {
+ break;
}
+ }
- std::vector<bool> drop_token_prefixes;
+ // only one query is resolved for now, so just use that
+ if(!space_resolved_queries.empty()) {
+ const auto& resolved_query = space_resolved_queries[0];
+ std::vector<token_t> resolved_tokens;
- for(const auto p: prefixes) {
- drop_token_prefixes.push_back(p && prefix_search);
+ for(size_t j=0; j < resolved_query.size(); j++) {
+ bool is_prefix = (j == resolved_query.size()-1);
+ resolved_tokens.emplace_back(j, space_resolved_queries[0][j], is_prefix,
+ space_resolved_queries[0][j].size(), 0);
}
- for(int min_typo = 0; min_typo <= 2; min_typo++) {
- search_fields(filters, included_ids_map, sort_fields_std, min_typo, num_typos, topster,
- curated_topster, token_order, drop_token_prefixes, drop_tokens_threshold, groups_processed,
- searched_queries, typo_tokens_threshold, group_limit, group_by_fields,
- prioritize_exact_match, exhaustive_search, concurrency, min_len_1typo, min_len_2typo,
- max_candidates, infixes, max_extra_prefix, max_extra_suffix,
- filter_ids, filter_ids_length, curated_ids, curated_ids_sorted, num_search_fields,
- exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, false,
- truncated_field_query_tokens, the_fields, all_result_ids_len, all_result_ids, topster_ids);
-
- if (!exhaustive_search && all_result_ids_len >= typo_tokens_threshold) {
+ fuzzy_search_fields(the_fields, resolved_tokens, excluded_result_ids,
+ excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
+ sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
+ query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search, min_len_1typo,
+ min_len_2typo, -1, sort_order, field_values, geopoint_indices);
+ }
+ }
+
+ // do synonym based searches
+ std::vector<std::vector<token_t>> all_queries = {field_query_tokens[0].q_include_tokens};
+
+ do_synonym_search(the_fields, filters, included_ids_map, sort_fields_std, curated_topster, token_order,
+ 0, group_limit, group_by_fields, prioritize_exact_match, exhaustive_search, concurrency,
+ min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
+ excluded_result_ids, excluded_result_ids_size, topster, field_query_tokens,
+ groups_processed, searched_queries, all_result_ids, all_result_ids_len,
+ filter_ids, filter_ids_length, query_hashes,
+ sort_order, field_values, geopoint_indices,
+ qtoken_set, all_queries);
+
+ if(all_result_ids_len < drop_tokens_threshold) {
+ // gather up both original query and synonym queries and do drop tokens
+ for(size_t qi = 0; qi < all_queries.size(); qi++) {
+ auto& orig_tokens = all_queries[qi];
+ size_t num_tokens_dropped = 0;
+ int syn_orig_num_token = (qi == 0) ? -1 : all_queries[0].size();
+
+ while(exhaustive_search || all_result_ids_len < drop_tokens_threshold) {
+ // When atleast two tokens from the query are available we can drop one
+ std::vector<token_t> truncated_tokens;
+
+ if(orig_tokens.size() > 1 && num_tokens_dropped < 2*(orig_tokens.size()-1)) {
+ bool prefix_search = false;
+
+ if(num_tokens_dropped < orig_tokens.size()-1) {
+ // drop from right
+ size_t truncated_len = orig_tokens.size() - num_tokens_dropped - 1;
+ for(size_t i=0; i < truncated_len; i++) {
+ truncated_tokens.emplace_back(orig_tokens[i]);
+ }
+ } else {
+ // drop from left
+ prefix_search = true;
+ size_t start_index = (num_tokens_dropped + 1) - orig_tokens.size() + 1;
+ for(size_t i = start_index; i < orig_tokens.size(); i++) {
+ truncated_tokens.emplace_back(orig_tokens[i]);
+ }
+ }
+
+ num_tokens_dropped++;
+ std::vector<bool> drop_token_prefixes;
+
+ for(const auto p: prefixes) {
+ drop_token_prefixes.push_back(p && prefix_search);
+ }
+
+ fuzzy_search_fields(the_fields, truncated_tokens, excluded_result_ids,
+ excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
+ sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
+ query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search, min_len_1typo,
+ min_len_2typo, syn_orig_num_token, sort_order, field_values, geopoint_indices);
+
+ } else {
break;
}
}
- } else {
- break;
}
}
- //auto begin0 = std::chrono::high_resolution_clock::now();
- /*size_t total_q_tokens = field_query_tokens[0].q_include_tokens.size();
- for(const auto& phrase: field_query_tokens[0].q_phrases) {
- total_q_tokens += phrase.size();
- }*/
-
- aggregate_and_score_fields(field_query_tokens, the_fields, topster, num_search_fields, topster_ids);
+ do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std, searched_queries,
+ group_limit, group_by_fields,
+ max_extra_prefix, max_extra_suffix,
+ field_query_tokens[0].q_include_tokens,
+ topster, filter_ids, filter_ids_length,
+ sort_order, field_values, geopoint_indices,
+ curated_ids_sorted, all_result_ids, all_result_ids_len, groups_processed);
/*auto timeMillis0 = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin0).count();
@@ -2269,6 +2384,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
//LOG(INFO) << "topster size: " << topster->size;
delete [] exclude_token_ids;
+ delete [] excluded_result_ids;
if(!facets.empty()) {
const size_t num_threads = std::min(concurrency, all_result_ids_len);
@@ -2373,523 +2489,770 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
delete [] filter_ids;
delete [] all_result_ids;
- for(Topster* ftopster: ftopsters) {
- delete ftopster;
- }
-
//LOG(INFO) << "all_result_ids_len " << all_result_ids_len << " for index " << name;
//long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";
}
-void Index::aggregate_and_score_fields(const std::vector<query_tokens_t>& field_query_tokens,
- const std::vector<search_field_t>& the_fields, Topster* topster,
- const size_t num_search_fields,
- spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const {
- for(auto& seq_id_kvs: topster_ids) {
- const uint64_t seq_id = seq_id_kvs.first;
- auto& kvs = seq_id_kvs.second; // each `kv` can be from a different field
+void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
+ const std::vector<token_t>& query_tokens,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const uint32_t* filter_ids, size_t filter_ids_length,
+ const std::vector<uint32_t>& curated_ids,
+ const std::vector<sort_by> & sort_fields,
+ const std::vector<uint32_t>& num_typos,
+ std::vector<std::vector<art_leaf*>> & searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster, spp::sparse_hash_set<uint64_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t & all_result_ids_len,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ bool prioritize_exact_match,
+ std::set<uint64>& query_hashes,
+ const token_ordering token_order,
+ const std::vector<bool>& prefixes,
+ const size_t typo_tokens_threshold,
+ const bool exhaustive_search,
+ size_t min_len_1typo,
+ size_t min_len_2typo,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const {
+
+ // NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
+
+ // To prevent us from doing ART search repeatedly as we iterate through possible corrections
+ spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
- std::sort(kvs.begin(), kvs.end(), Topster::is_greater);
+ std::vector<std::vector<int>> token_to_costs;
- // kvs[0] will store query indices of the kv group (across fields)
- kvs[0]->query_indices = new uint64_t[kvs.size() + 1];
- kvs[0]->query_indices[0] = kvs.size();
+ for(size_t stoken_index=0; stoken_index < query_tokens.size(); stoken_index++) {
+ const std::string& token = query_tokens[stoken_index].value;
- //LOG(INFO) << "DOC ID: " << seq_id << ", score: " << kvs[0]->scores[kvs[0]->match_score_index];
+ std::vector<int> all_costs;
+ // This ensures that we don't end up doing a cost of 1 for a single char etc.
+ int bounded_cost = get_bounded_typo_cost(2, token.length(), min_len_1typo, min_len_2typo);
- // to calculate existing aggregate scores across best matching fields
- spp::sparse_hash_map<uint8_t, KV*> existing_field_kvs;
- for(size_t kv_i = 0; kv_i < kvs.size(); kv_i++) {
- existing_field_kvs.emplace(kvs[kv_i]->field_id, kvs[kv_i]);
- kvs[0]->query_indices[kv_i+1] = kvs[kv_i]->query_index;
- /*LOG(INFO) << "kv_i: " << kv_i << ", kvs[kv_i]->query_index: " << kvs[kv_i]->query_index << ", "
- << "searched_query: " << searched_queries[kvs[kv_i]->query_index][0];*/
+ for(int cost = 0; cost <= bounded_cost; cost++) {
+ all_costs.push_back(cost);
}
- uint32_t token_bits = 0;
- int64_t max_field_match_score = 0;
- size_t max_field_match_index = 0;
+ token_to_costs.push_back(all_costs);
+ }
- //LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);
+ // stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"
+ std::vector<tok_candidates> token_candidates_vec;
+ std::set<std::string> unique_tokens;
- for(size_t i = 0; i < num_search_fields; i++) {
- const auto field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
- //LOG(INFO) << "--- field index: " << i << ", priority: " << priority;
+ const size_t num_search_fields = std::min(the_fields.size(), (size_t) FIELD_LIMIT_NUM);
+
+ auto product = []( long long a, std::vector<int>& b ) { return a*b.size(); };
+ long long n = 0;
+ long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
+
+ const long long combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
+ const size_t num_fuzzy_candidates = exhaustive_search ? 10000 : 4;
- if(existing_field_kvs.count(field_id) != 0) {
- // for existing field, we will simply sum field-wise weighted scores
- int64_t match_score = existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];
- token_bits |= existing_field_kvs[field_id]->token_bits;
+ while(n < N && n < combination_limit) {
+ RETURN_CIRCUIT_BREAKER
- // we will reassemble match score to use unique tokens for cross-field matches
+ // Outerloop generates combinations of [cost to max_cost] for each token
+ // For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.
+ std::vector<uint32_t> costs(token_to_costs.size());
+ ldiv_t q { n, 0 };
+ for(long long i = (token_to_costs.size() - 1); 0 <= i ; --i ) {
+ q = ldiv(q.quot, token_to_costs[i].size());
+ costs[i] = token_to_costs[i][q.rem];
+ }
- uint32_t unique_tokens_found = ((match_score >> 32) & 0xFF);
- uint64_t tokens_found = ((match_score >> 24) & 0xFF);
- uint64_t typo_score = ((match_score >> 16) & 0xFF);
- uint64_t proximity_score = ((match_score >> 8) & 0xFF);
- int64_t exact_match_score = match_score & 0xFF;
+ unique_tokens.clear();
+ token_candidates_vec.clear();
+ size_t token_index = 0;
- // exclude dropped-token cases
- if(unique_tokens_found != field_query_tokens[0].q_include_tokens.size()) {
- exact_match_score = 0;
- }
+ while(token_index < query_tokens.size()) {
+ // For each token, look up the generated cost for this iteration and search using that cost
+ const std::string& token = query_tokens[token_index].value;
+ const std::string token_cost_hash = token + std::to_string(costs[token_index]);
- // NOTE: unique_tokens_found can be > than tokens_found as `tokens_found` is counted only within matching
- // window of X tokens.
+ std::vector<art_leaf*> leaves;
- int64_t multi_field_match_score = (int64_t(unique_tokens_found) << 32) |
- (int64_t(tokens_found) << 24) |
- (int64_t(typo_score) << 16) |
- (int64_t(proximity_score) << 8) |
- (int64_t(exact_match_score) << 0);
+ if(token_cost_cache.count(token_cost_hash) != 0) {
+ leaves = token_cost_cache[token_cost_hash];
+ } else {
+ //auto begin = std::chrono::high_resolution_clock::now();
- if(multi_field_match_score > max_field_match_score) {
- max_field_match_score = multi_field_match_score;
- max_field_match_index = i;
- }
+ for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
+ auto& the_field = the_fields[field_id];
+ const bool field_prefix = (field_id < prefixes.size()) ? prefixes[field_id] : prefixes[0];;
+ const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;
+ const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
- /*LOG(INFO) << "seq_id: " << seq_id << ", tokens_found: " << tokens_found
- << ", typo_score: " << typo_score
- << ", proximity_score: " << proximity_score
- << ", exact_match_score: " << exact_match_score
- << ", unique_tokens_found: " << unique_tokens_found
- << ", multi_field_match_score: " << multi_field_match_score;*/
- continue;
- }
+ /*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"
+ << token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/
- // compute approximate match score for this field from actual query
- const std::string& field = the_fields[i].name;
- size_t words_present = 0;
+ int64_t field_num_typos = (field_id < num_typos.size()) ? num_typos[field_id] : num_typos[0];
- // FIXME: must consider phrase tokens also
- for(size_t token_index=0; token_index < field_query_tokens[i].q_include_tokens.size(); token_index++) {
- const auto& token = field_query_tokens[i].q_include_tokens[token_index].value;
- const art_leaf* leaf = (art_leaf *) art_search(search_index.at(field), (const unsigned char*) token.c_str(),
- token.length()+1);
+ auto& locale = search_schema.at(the_field.name).locale;
+ if(locale != "" && locale != "en" && !Tokenizer::is_cyrillic(locale)) {
+ // disable fuzzy trie traversal for non-english locales
+ field_num_typos = 0;
+ }
- if(!leaf) {
- continue;
- }
+ if(costs[token_index] > field_num_typos) {
+ continue;
+ }
- if(!posting_t::contains(leaf->values, seq_id)) {
- continue;
- }
+ // need less candidates for filtered searches since we already only pick tokens with results
+ art_fuzzy_search(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,
+ costs[token_index], costs[token_index], num_fuzzy_candidates, token_order, prefix_search,
+ filter_ids, filter_ids_length, leaves, unique_tokens);
- token_bits |= 1UL << token_index; // sets nth bit
- //LOG(INFO) << "token_index: " << token_index << ", pop count: " << __builtin_popcount(token_bits);
+ /*auto timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
+ std::chrono::high_resolution_clock::now() - begin).count();
+ LOG(INFO) << "Time taken for fuzzy search: " << timeMillis << "ms";*/
- words_present += 1;
+ if(leaves.empty()) {
+ // look at the next field
+ continue;
+ }
- /*if(!leaves.empty()) {
- LOG(INFO) << "tok: " << leaves[0]->key;
- }*/
- }
+ token_cost_cache.emplace(token_cost_hash, leaves);
+ for(auto leaf: leaves) {
+ std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);
+ unique_tokens.emplace(tok);
+ }
- if(words_present != 0) {
- int64_t multi_field_match_score = (int64_t(words_present) << 32) |
- (int64_t(0) << 24) |
- (int64_t(0) << 16) |
- (int64_t(0) << 8) |
- (int64_t(0) << 0);
+ if(unique_tokens.size() > num_fuzzy_candidates) {
+ break;
+ }
+ }
+ }
- if(multi_field_match_score > max_field_match_score) {
- max_field_match_score = multi_field_match_score;
- max_field_match_index = i;
+ if(!leaves.empty()) {
+ //log_leaves(costs[token_index], token, leaves);
+ std::vector<std::string> leaf_tokens;
+ for(auto leaf: leaves) {
+ std::string ltok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);
+ leaf_tokens.push_back(ltok);
+ }
+ token_candidates_vec.push_back(tok_candidates{query_tokens[token_index], costs[token_index],
+ query_tokens[token_index].is_prefix_searched, leaf_tokens});
+ } else {
+ // No result at `cost = costs[token_index]`. Remove `cost` for token and re-do combinations
+ auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
+ if(it != token_to_costs[token_index].end()) {
+ token_to_costs[token_index].erase(it);
+
+ // when no more costs are left for this token
+ if(token_to_costs[token_index].empty()) {
+ // we cannot proceed further, as this token is not found within cost limits
+ // and, dropping of tokens are done elsewhere.
+ return ;
+ }
}
- //LOG(INFO) << "!seq_id: " << seq_id << ", words_present: " << words_present;
+ // Continue outerloop on new cost combination
+ n = -1;
+ N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
+ goto resume_typo_loop;
}
+
+ token_index++;
}
- uint32_t num_tokens_found = __builtin_popcount(token_bits);
- uint64_t aggregated_score = (int64_t(num_tokens_found) << 48) |
- (int64_t(max_field_match_score) << 8) |
- (int64_t(the_fields[max_field_match_index].weight) << 0);
+ if(token_candidates_vec.size() == query_tokens.size()) {
+ std::vector<uint32_t> id_buff;
- //LOG(INFO) << "seq id: " << seq_id << ", aggregated_score: " << aggregated_score;
+ search_all_candidates(num_search_fields, the_fields, filter_ids, filter_ids_length,
+ exclude_token_ids, exclude_token_ids_size,
+ sort_fields, token_candidates_vec, searched_queries, qtoken_set, topster,
+ groups_processed, all_result_ids, all_result_ids_len,
+ typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
+ num_typos, prefixes, prioritize_exact_match, exhaustive_search,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ query_hashes, id_buff);
- /*LOG(INFO) << "seq id: " << seq_id
- << ", num_tokens_found: " << num_tokens_found
- << ", max_field_match_score: " << max_field_match_score
- << ", matched field weight: " << the_fields[max_field_match_index].weight
- << ", aggregated_score: " << aggregated_score;*/
+ if(id_buff.size() > 1) {
+ std::sort(id_buff.begin(), id_buff.end());
+ id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());
+ }
- kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;
- topster->add(kvs[0]);
- }
-}
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, &id_buff[0],
+ id_buff.size(), &new_all_result_ids);
+ delete[] all_result_ids;
+ all_result_ids = new_all_result_ids;
+ }
-void Index::search_fields(const std::vector<filter>& filters,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std,
- const size_t min_typo,
- const std::vector<uint32_t>& num_typos,
- Topster* topster, Topster* curated_topster, const token_ordering& token_order,
- const std::vector<bool>& prefixes, const size_t drop_tokens_threshold,
- spp::sparse_hash_set<uint64_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t typo_tokens_threshold,
- const size_t group_limit, const std::vector<std::string>& group_by_fields,
- bool prioritize_exact_match, const bool exhaustive_search, const size_t concurrency,
- size_t min_len_1typo, size_t min_len_2typo, const size_t max_candidates,
- const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
- const size_t max_extra_suffix, uint32_t* filter_ids, uint32_t filter_ids_length,
- const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,
- const size_t num_search_fields, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size, std::vector<Topster*>& ftopsters, bool is_wildcard_query,
- const bool split_join_tokens, std::vector<query_tokens_t>& field_query_tokens,
- const std::vector<search_field_t>& the_fields, size_t& all_result_ids_len,
- uint32_t*& all_result_ids,
- spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const {
+ resume_typo_loop:
- for(size_t i = 0; i < num_search_fields; i++) {
- uint32_t field_num_typos = (i < num_typos.size()) ? num_typos[i] : num_typos[0];
- if(min_typo > field_num_typos) {
- continue;
+ if(!exhaustive_search && all_result_ids_len >= typo_tokens_threshold) {
+ // if typo threshold is breached, we are done
+ return ;
}
- const std::string & field_name = the_fields[i].name;
- bool is_array = search_schema.at(field_name).is_array();
-
- auto actual_filter_ids = filter_ids;
- auto actual_filter_ids_length = filter_ids_length;
+ n++;
+ }
+}
- // handle phrase search by ANDing `q_phrases` to add to filters
- uint32_t* phrase_match_ids = nullptr;
- size_t phrase_match_ids_size = 0;
- do_phrase_search(filter_ids, filter_ids_length, field_name, is_array, phrase_match_ids,
- field_query_tokens[i].q_phrases, actual_filter_ids, actual_filter_ids_length,
- phrase_match_ids_size);
+void Index::search_across_fields(const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ const std::vector<search_field_t>& the_fields,
+ const size_t num_search_fields,
+ const std::vector<sort_by>& sort_fields,
+ Topster* topster,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const uint32_t* filter_ids, uint32_t filter_ids_length,
+ const uint32_t total_cost, const int syn_orig_num_tokens,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len) const {
+
+ std::vector<art_leaf*> query_suggestion;
+
+ // one iterator for each token, each underlying iterator contains results of token across multiple fields
+ std::vector<or_iterator_t> token_its;
+
+ // used to track plists that must be destructed once done
+ std::vector<posting_list_t*> expanded_plists;
+
+ result_iter_state_t istate(exclude_token_ids, exclude_token_ids_size, filter_ids, filter_ids_length);
+ std::vector<uint32_t> result_ids;
+ uint32_t* new_all_result_ids = nullptr;
- if(!field_query_tokens[i].q_phrases.empty() && phrase_match_ids_size == 0) {
- // phrase query does not produce results, so no need to do other types of searches
- continue;
- }
+ // for each token, find the posting lists across all query_by fields
+ for(size_t ti = 0; ti < query_tokens.size(); ti++) {
+ const bool prefix_search = query_tokens[ti].is_prefix_searched;
+ const uint32_t token_num_typos = query_tokens[ti].num_typos;
+ const bool token_prefix = query_tokens[ti].is_prefix_searched;
- std::vector<token_t> q_include_pos_tokens = field_query_tokens[i].q_include_tokens;
+ auto& token_str = query_tokens[ti].value;
+ auto token_c_str = (const unsigned char*) token_str.c_str();
+ const size_t token_len = token_str.size() + 1;
+ std::vector<posting_list_t::iterator_t> its;
- // these are already validated upstream, but still playing safe
- bool field_prefix = (i < prefixes.size()) ? prefixes[i] : prefixes[0];
- infix_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];
+ for(size_t i = 0; i < num_search_fields; i++) {
+ const std::string& field_name = the_fields[i].name;
+ const uint32_t field_num_typos = (i < num_typos.size()) ? num_typos[i] : num_typos[0];
+ const bool field_prefix = (i < prefixes.size()) ? prefixes[i] : prefixes[0];
- // proceed to query search only when no filters are provided or when filtering produces results
- if(filters.empty() || actual_filter_ids_length > 0) {
- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
+ if(token_num_typos > field_num_typos) {
+ // since the token can come from any field, we still have to respect per-field num_typos
+ continue;
+ }
- auto field_it = search_schema.find(field_name);
- if(field_it == search_schema.end()) {
+ if(token_prefix && !field_prefix) {
+ // even though this token is an outcome of prefix search, we can't use it for this field, since
+ // this field has prefix search disabled.
continue;
}
- std::vector<token_t> query_tokens = q_include_pos_tokens;
- size_t num_tokens_dropped = 0;
+ art_tree* tree = search_index.at(field_name);
+ art_leaf* leaf = static_cast<art_leaf*>(art_search(tree, token_c_str, token_len));
- //LOG(INFO) << "searching field_name! " << field_name;
+ if(!leaf) {
+ continue;
+ }
- // Don't waste additional cycles for single field_name searches
- Topster* actual_topster = (num_search_fields == 1) ? topster : ftopsters[i];
+ query_suggestion.push_back(leaf);
- // tracks the number of results found for the current field_name
- size_t field_num_results = 0;
- std::set<uint64> query_hashes;
+ /*LOG(INFO) << "Token: " << token_str << ", field_name: " << field_name
+ << ", num_ids: " << posting_t::num_ids(leaf->values);*/
- int last_typo = int(min_typo) - 1;
-
- if(!is_wildcard_query) {
- search_field(field_id, query_tokens, exclude_token_ids, exclude_token_ids_size,
- num_tokens_dropped, field_it->second, field_name,
- actual_filter_ids, actual_filter_ids_length, curated_ids_sorted, sort_fields_std,
- last_typo, min_typo, searched_queries, actual_topster, groups_processed,
- &all_result_ids, all_result_ids_len,
- field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
- query_hashes, token_order, field_prefix,
- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search, -1,
- min_len_1typo, min_len_2typo, max_candidates);
-
- do_infix_search(sort_fields_std, searched_queries, group_limit, group_by_fields,
- max_extra_prefix, max_extra_suffix, field_infix, field_id, field_name, query_tokens,
- actual_topster, filter_ids, filter_ids_length,
- curated_ids_sorted, field_num_results, all_result_ids, all_result_ids_len, groups_processed);
-
- if(field_num_results == 0 && split_join_tokens) {
- std::vector<std::vector<std::string>> space_resolved_queries;
- std::vector<std::string> q_include_tokens;
-
- for(auto& q_include_token: field_query_tokens[i].q_include_tokens) {
- q_include_tokens.push_back(q_include_token.value);
- }
+ if(IS_COMPACT_POSTING(leaf->values)) {
+ auto compact_posting_list = COMPACT_POSTING_PTR(leaf->values);
+ posting_list_t* full_posting_list = compact_posting_list->to_full_posting_list();
+ expanded_plists.push_back(full_posting_list);
+ its.push_back(full_posting_list->new_iterator(nullptr, nullptr, i)); // moved, not copied
+ } else {
+ posting_list_t* full_posting_list = (posting_list_t*)(leaf->values);
+ its.push_back(full_posting_list->new_iterator(nullptr, nullptr, i)); // moved, not copied
+ }
+ }
- resolve_space_as_typos(q_include_tokens, field_name,
- space_resolved_queries);
+ if(its.empty()) {
+ // this token does not have any match across *any* field: probably a typo
+ LOG(INFO) << "No matching field found for token: " << token_str;
+ continue;
+ }
- // only one query is resolved for now, so just use that
- if(!space_resolved_queries.empty()) {
- q_include_pos_tokens.clear();
- const auto& resolved_query = space_resolved_queries[0];
+ or_iterator_t token_fields(its);
+ token_its.push_back(std::move(token_fields));
+ }
- for(size_t j=0; j < resolved_query.size(); j++) {
- bool is_prefix = (j == resolved_query.size()-1);
- q_include_pos_tokens.emplace_back(j, space_resolved_queries[0][j], is_prefix);
- }
+ or_iterator_t::intersect(token_its, istate, [&](uint32_t seq_id, const std::vector<or_iterator_t>& its) {
+ // Convert [token -> fields] orientation to [field -> tokens] orientation
+ //LOG(INFO) << "seq_id: " << seq_id;
+ std::vector<std::vector<posting_list_t::iterator_t>> field_to_tokens(num_search_fields);
- query_tokens = q_include_pos_tokens;
+ for(size_t ti = 0; ti < its.size(); ti++) {
+ const or_iterator_t& token_fields_iters = its[ti];
+ const std::vector<posting_list_t::iterator_t>& field_iters = token_fields_iters.get_its();
- search_field(field_id, query_tokens, exclude_token_ids, exclude_token_ids_size,
- num_tokens_dropped, field_it->second, field_name,
- actual_filter_ids, actual_filter_ids_length, curated_ids_sorted, sort_fields_std,
- last_typo, min_typo, searched_queries, actual_topster, groups_processed,
- &all_result_ids, all_result_ids_len,
- field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
- query_hashes, token_order, field_prefix,
- 0, 0, exhaustive_search, -1, min_len_1typo, min_len_2typo, max_candidates);
- }
+ for(size_t fi = 0; fi < field_iters.size(); fi++) {
+ const posting_list_t::iterator_t& field_iter = field_iters[fi];
+ if(field_iter.id() == seq_id) {
+ // not all fields might contain a given token
+ field_to_tokens[field_iter.get_field_id()].push_back(field_iter.clone());
}
+ }
+ }
- } else if(actual_filter_ids_length != 0) {
- // indicates phrase match query
- curate_filtered_ids(filters, curated_ids, exclude_token_ids,
- exclude_token_ids_size, actual_filter_ids, actual_filter_ids_length, curated_ids_sorted);
+ int64_t max_field_match_score = 0, max_field_match_index = 0;
- search_wildcard(filters, included_ids_map, sort_fields_std, actual_topster,
- curated_topster,
- groups_processed, searched_queries, group_limit, group_by_fields,
- curated_ids, curated_ids_sorted,
- exclude_token_ids, exclude_token_ids_size, field_id, field_name,
- all_result_ids, all_result_ids_len,
- actual_filter_ids, actual_filter_ids_length, concurrency);
+ for(size_t fi = 0; fi < field_to_tokens.size(); fi++) {
+ const std::vector<posting_list_t::iterator_t>& token_postings = field_to_tokens[fi];
+ if(token_postings.empty()) {
+ continue;
}
- // do synonym based searches
+ bool field_is_array = search_schema.at(the_fields[fi].name).is_array();
+ int64_t field_match_score = 0;
- // get synonyms
- std::vector<std::string> q_include_tokens;
- for(size_t j = 0; j < field_query_tokens[i].q_include_tokens.size(); j++) {
- q_include_tokens.push_back(field_query_tokens[i].q_include_tokens[j].value);
+ bool single_exact_query_token = false;
+ if(total_cost == 0 && query_tokens.size() == 1) {
+ // does this candidate suggestion token match query token exactly?
+ single_exact_query_token = true;
}
- synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[i].q_synonyms);
- // since typos are disabled, we will use drop_tokens_threshold for typo_tokens_threshold as well
- // otherwise, we can't support dropping of tokens here.
- do_synonym_search(filters, included_ids_map, sort_fields_std, curated_topster, token_order,
- drop_tokens_threshold, drop_tokens_threshold, group_limit,
- group_by_fields, prioritize_exact_match, exhaustive_search, concurrency,
- min_len_1typo, min_len_2typo,
- max_candidates, curated_ids, curated_ids_sorted, exclude_token_ids,
- exclude_token_ids_size, i, actual_filter_ids_length, 0, field_prefix,
- field_id, field_name, field_it,
- query_tokens, num_tokens_dropped, actual_topster, field_num_results,
- field_query_tokens, all_result_ids_len, groups_processed, searched_queries,
- all_result_ids,
- actual_filter_ids, query_hashes);
+ score_results2(sort_fields, searched_queries.size(), field_is_array,
+ total_cost, field_match_score,
+ seq_id, sort_order, group_limit, group_by_fields,
+ prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, token_postings);
- // concat is done only for multi-field searches as `ftopster` will be empty for single-field search
- concat_topster_ids(ftopsters[i], topster_ids);
- collate_included_ids(field_query_tokens[i].q_include_tokens, field_name, field_id, included_ids_map, curated_topster, searched_queries);
- //LOG(INFO) << "topster_ids.size: " << topster_ids.size();
+ if(field_match_score > max_field_match_score) {
+ max_field_match_score = field_match_score;
+ max_field_match_index = fi;
+ }
}
- if(phrase_match_ids_size != 0) {
- delete [] actual_filter_ids;
+ uint64_t distinct_id = seq_id;
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ groups_processed.emplace(distinct_id);
}
- }
-}
-void
-Index::do_phrase_search(const uint32_t* filter_ids, uint32_t filter_ids_length, const string& field_name, bool is_array,
- uint32_t* phrase_match_ids, const std::vector<std::vector<std::string>>& field_phrases,
- uint32_t*& actual_filter_ids, uint32_t& actual_filter_ids_length,
- size_t& phrase_match_ids_size) const {
- for(const auto& phrase: field_phrases) {
- std::vector<void*> posting_lists;
+ int64_t scores[3] = {0};
+ int64_t match_score_index = 0;
- for(const std::string& token: phrase) {
- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
- (const unsigned char *) token.c_str(),
- token.size() + 1);
+ compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id,
+ max_field_match_score, scores, match_score_index);
- if(leaf) {
- posting_lists.push_back(leaf->values);
- }
+ size_t query_len = query_tokens.size();
+ if(syn_orig_num_tokens != -1) {
+ query_len = syn_orig_num_tokens;
}
- if(posting_lists.size() != phrase.size()) {
- // unmatched length means no matches will be found
- continue;
- }
+ // NOTE: `query_len` is total tokens matched across fields.
+ // Within a field, only a subset can match
- std::vector<uint32_t> contains_ids;
- posting_t::intersect(posting_lists, contains_ids);
+ uint64_t aggregated_score = (int64_t(query_len) << 48) |
+ (int64_t(max_field_match_score) << 8) |
+ (int64_t(the_fields[max_field_match_index].weight) << 0);
- uint32_t* phrase_ids = new uint32_t[contains_ids.size()];
- size_t phrase_ids_size = 0;
- posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),
- phrase_ids, phrase_ids_size);
+ /*LOG(INFO) << "seq_id: " << seq_id << ", query_tokens.size(): " << query_tokens.size()
+ << ", max_field_match_score: " << max_field_match_score
+ << ", aggregated_score: " << aggregated_score;*/
- if(phrase_ids_size != 0) {
- if(phrase_match_ids_size == 0) {
- phrase_match_ids_size = phrase_ids_size;
- phrase_match_ids = phrase_ids;
- } else {
- uint32_t *phrase_ids_merged = nullptr;
- phrase_match_ids_size = ArrayUtils::and_scalar(phrase_ids, phrase_ids_size, phrase_match_ids,
- phrase_match_ids_size, &phrase_ids_merged);
- delete [] phrase_match_ids;
- delete [] phrase_ids;
- phrase_match_ids = phrase_ids_merged;
- }
- } else {
- delete [] phrase_ids;
- continue;
- }
- }
+ KV kv(0, searched_queries.size(), 0, seq_id, distinct_id, match_score_index, scores);
+ kv.scores[match_score_index] = aggregated_score;
+ topster->add(&kv);
+ result_ids.push_back(seq_id);
+ });
- if(phrase_match_ids_size != 0) {
- if(filter_ids_length == 0) {
- actual_filter_ids_length = phrase_match_ids_size;
- actual_filter_ids = phrase_match_ids;
- } else {
- actual_filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, phrase_match_ids,
- phrase_match_ids_size, &actual_filter_ids);
+ all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, &result_ids[0],
+ result_ids.size(), &new_all_result_ids);
+ delete [] all_result_ids;
+ all_result_ids = new_all_result_ids;
- delete [] phrase_match_ids;
- phrase_match_ids = nullptr;
+ if(!result_ids.empty()) {
+ searched_queries.push_back(query_suggestion);
+ for(const auto& qtoken: query_tokens) {
+ qtoken_set.insert(qtoken.value, token_leaf(nullptr, qtoken.root_len, qtoken.num_typos));
}
}
-}
-void Index::do_synonym_search(const std::vector<filter>& filters,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
- const token_ordering& token_order, const size_t drop_tokens_threshold,
- const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
- const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo,
- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size, size_t i, uint32_t actual_filter_ids_length,
- int field_num_typos, bool field_prefix, const uint8_t field_id, const string& field_name,
- const std::unordered_map<string, field>::const_iterator& field_it,
- std::vector<token_t>& query_tokens,
- size_t num_tokens_dropped, Topster* actual_topster, size_t field_num_results,
- std::vector<query_tokens_t>& field_query_tokens, size_t& all_result_ids_len,
- spp::sparse_hash_set<uint64_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries, uint32_t*& all_result_ids,
- uint32_t*& actual_filter_ids, std::set<uint64>& query_hashes) const {
- std::vector<std::vector<token_t>> q_pos_synonyms;
- for(const auto& q_syn_vec: field_query_tokens[i].q_synonyms) {
- std::vector<token_t> q_pos_syn;
- for(size_t j=0; j < q_syn_vec.size(); j++) {
- bool is_prefix = (j == q_syn_vec.size()-1);
- q_pos_syn.emplace_back(j, q_syn_vec[j], is_prefix);
- }
- q_pos_synonyms.emplace_back(q_pos_syn);
+ for(posting_list_t* plist: expanded_plists) {
+ delete plist;
}
+}
- int syn_orig_num_tokens = field_query_tokens[i].q_include_tokens.size();
- bool syn_wildcard_filter_init_done = false;
+void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ uint32_t seq_id, int64_t max_field_match_score,
+ int64_t* scores, int64_t& match_score_index) const {
- for(const auto& syn_tokens: q_pos_synonyms) {
- num_tokens_dropped = 0;
- field_num_results = 0;
- query_tokens = syn_tokens;
- query_hashes.clear();
+ int64_t geopoint_distances[3];
+
+ for(auto& i: geopoint_indices) {
+ spp::sparse_hash_map<uint32_t, int64_t>* geopoints = field_values[i];
+ int64_t dist = INT32_MAX;
+
+ S2LatLng reference_lat_lng;
+ GeoPoint::unpack_lat_lng(sort_fields[i].geopoint, reference_lat_lng);
+
+ if(geopoints != nullptr) {
+ auto it = geopoints->find(seq_id);
- if(query_tokens.size() == 1 && query_tokens[0].value == "*") {
- // synonym can be a wildcard if there was an override filter used
- // we will only do filtering in that case
+ if(it != geopoints->end()) {
+ int64_t packed_latlng = it->second;
+ S2LatLng s2_lat_lng;
+ GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);
+ dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);
+ }
+ } else {
+ // indicates geo point array
+ auto field_it = geo_array_index.at(sort_fields[i].name);
+ auto it = field_it->find(seq_id);
- if(!syn_wildcard_filter_init_done) {
- curate_filtered_ids(filters, curated_ids, exclude_token_ids, exclude_token_ids_size,
- actual_filter_ids, actual_filter_ids_length, curated_ids_sorted);
- syn_wildcard_filter_init_done = true;
+ if(it != field_it->end()) {
+ int64_t* latlngs = it->second;
+ for(size_t li = 0; li < latlngs[0]; li++) {
+ S2LatLng s2_lat_lng;
+ int64_t packed_latlng = latlngs[li + 1];
+ GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);
+ int64_t this_dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);
+ if(this_dist < dist) {
+ dist = this_dist;
+ }
+ }
}
+ }
+
+ if(dist < sort_fields[i].exclude_radius) {
+ dist = 0;
+ }
+
+ if(sort_fields[i].geo_precision > 0) {
+ dist = dist + sort_fields[i].geo_precision - 1 -
+ (dist + sort_fields[i].geo_precision - 1) % sort_fields[i].geo_precision;
+ }
+
+ geopoint_distances[i] = dist;
+
+ // Swap (id -> latlong) index to (id -> distance) index
+ field_values[i] = &geo_sentinel_value;
+ }
+
+ const int64_t default_score = INT64_MIN; // to handle field that doesn't exist in document (e.g. optional)
- search_wildcard(filters, included_ids_map, sort_fields_std, actual_topster,
- curated_topster,
- groups_processed, searched_queries, group_limit, group_by_fields,
- curated_ids, curated_ids_sorted,
- exclude_token_ids, exclude_token_ids_size, field_id, field_name,
- all_result_ids, all_result_ids_len,
- actual_filter_ids, actual_filter_ids_length, concurrency);
+ // avoiding loop
+ if (sort_fields.size() > 0) {
+ if (field_values[0] == &text_match_sentinel_value) {
+ scores[0] = int64_t(max_field_match_score);
+ match_score_index = 0;
+ } else if (field_values[0] == &seq_id_sentinel_value) {
+ scores[0] = seq_id;
+ } else if(field_values[0] == &geo_sentinel_value) {
+ scores[0] = geopoint_distances[0];
+ } else if(field_values[0] == &str_sentinel_value) {
+ scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);
} else {
- search_field(field_id, query_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
- field_it->second, field_name, actual_filter_ids, actual_filter_ids_length, curated_ids_sorted, sort_fields_std,
- -1, field_num_typos, searched_queries, actual_topster, groups_processed,
- &all_result_ids, all_result_ids_len,
- field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
- query_hashes, token_order, field_prefix,
- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search, syn_orig_num_tokens,
- min_len_1typo, min_len_2typo, max_candidates);
+ auto it = field_values[0]->find(seq_id);
+ scores[0] = (it == field_values[0]->end()) ? default_score : it->second;
+ }
+
+ if (sort_order[0] == -1) {
+ scores[0] = -scores[0];
+ }
+ }
+
+ if(sort_fields.size() > 1) {
+ if (field_values[1] == &text_match_sentinel_value) {
+ scores[1] = int64_t(max_field_match_score);
+ match_score_index = 1;
+ } else if (field_values[1] == &seq_id_sentinel_value) {
+ scores[1] = seq_id;
+ } else if(field_values[1] == &geo_sentinel_value) {
+ scores[1] = geopoint_distances[1];
+ } else if(field_values[1] == &str_sentinel_value) {
+ scores[1] = str_sort_index.at(sort_fields[1].name)->rank(seq_id);
+ } else {
+ auto it = field_values[1]->find(seq_id);
+ scores[1] = (it == field_values[1]->end()) ? default_score : it->second;
+ }
+
+ if (sort_order[1] == -1) {
+ scores[1] = -scores[1];
+ }
+ }
+
+ if(sort_fields.size() > 2) {
+ if (field_values[2] == &text_match_sentinel_value) {
+ scores[2] = int64_t(max_field_match_score);
+ match_score_index = 2;
+ } else if (field_values[2] == &seq_id_sentinel_value) {
+ scores[2] = seq_id;
+ } else if(field_values[2] == &geo_sentinel_value) {
+ scores[2] = geopoint_distances[2];
+ } else if(field_values[2] == &str_sentinel_value) {
+ scores[2] = str_sort_index.at(sort_fields[2].name)->rank(seq_id);
+ } else {
+ auto it = field_values[2]->find(seq_id);
+ scores[2] = (it == field_values[2]->end()) ? default_score : it->second;
+ }
+
+ if (sort_order[2] == -1) {
+ scores[2] = -scores[2];
}
}
}
-void Index::do_infix_search(const std::vector<sort_by>& sort_fields_std,
+void Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
+ std::vector<query_tokens_t>& field_query_tokens,
+ uint32_t*& filter_ids, uint32_t& filter_ids_length) const {
+
+ uint32_t* phrase_match_ids = nullptr;
+ size_t phrase_match_ids_size = 0;
+
+ for(size_t i = 0; i < num_search_fields; i++) {
+ const std::string& field_name = search_fields[i].name;
+ bool is_array = search_schema.at(field_name).is_array();
+
+ uint32_t* field_phrase_match_ids = nullptr;
+ size_t field_phrase_match_ids_size = 0;
+
+ for(const auto& phrase: field_query_tokens[i].q_phrases) {
+ std::vector<void*> posting_lists;
+
+ for(const std::string& token: phrase) {
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
+ (const unsigned char *) token.c_str(),
+ token.size() + 1);
+ if(leaf) {
+ posting_lists.push_back(leaf->values);
+ }
+ }
+
+ if(posting_lists.size() != phrase.size()) {
+ // unmatched length means no matches will be found for this phrase, so skip to next phrase
+ continue;
+ }
+
+ std::vector<uint32_t> contains_ids;
+ posting_t::intersect(posting_lists, contains_ids);
+
+ uint32_t* this_phrase_ids = new uint32_t[contains_ids.size()];
+ size_t this_phrase_ids_size = 0;
+ posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),
+ this_phrase_ids, this_phrase_ids_size);
+
+ if(this_phrase_ids_size == 0) {
+ // no results found for this phrase, but other phrases can find results
+ delete [] this_phrase_ids;
+ continue;
+ }
+
+ // results of multiple phrases must be ANDed
+
+ if(field_phrase_match_ids_size == 0) {
+ field_phrase_match_ids_size = this_phrase_ids_size;
+ field_phrase_match_ids = this_phrase_ids;
+ } else {
+ uint32_t* phrase_ids_merged = nullptr;
+ field_phrase_match_ids_size = ArrayUtils::and_scalar(this_phrase_ids, this_phrase_ids_size, field_phrase_match_ids,
+ field_phrase_match_ids_size, &phrase_ids_merged);
+ delete [] field_phrase_match_ids;
+ delete [] this_phrase_ids;
+ field_phrase_match_ids = phrase_ids_merged;
+ }
+ }
+
+ if(field_phrase_match_ids_size == 0) {
+ continue;
+ }
+
+ // across fields, we have to OR phrase match ids
+ if(filter_ids_length == 0) {
+ phrase_match_ids = field_phrase_match_ids;
+ phrase_match_ids_size = field_phrase_match_ids_size;
+ } else {
+ uint32_t* phrase_ids_merged = nullptr;
+ phrase_match_ids_size = ArrayUtils::or_scalar(phrase_match_ids, phrase_match_ids_size, field_phrase_match_ids,
+ field_phrase_match_ids_size, &phrase_ids_merged);
+
+ delete [] phrase_match_ids;
+ delete [] field_phrase_match_ids;
+ phrase_match_ids = phrase_ids_merged;
+ }
+ }
+
+ // AND phrase id matches with filter ids
+ if(filter_ids_length == 0) {
+ filter_ids = phrase_match_ids;
+ filter_ids_length = phrase_match_ids_size;
+ } else {
+ uint32_t* filter_ids_merged = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, phrase_match_ids,
+ phrase_match_ids_size, &filter_ids_merged);
+
+ delete [] filter_ids;
+ filter_ids = filter_ids_merged;
+
+ delete [] phrase_match_ids;
+ }
+
+ if(filter_ids_length == 0) {
+ delete [] filter_ids;
+ filter_ids = nullptr;
+ }
+}
+
+void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
+ const std::vector<filter>& filters,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
+ const token_ordering& token_order,
+ const size_t typo_tokens_threshold, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo,
+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ Topster* actual_topster,
+ std::vector<query_tokens_t>& field_query_tokens,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const uint32_t* filter_ids, const uint32_t filter_ids_length,
+ std::set<uint64>& query_hashes,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ std::vector<std::vector<token_t>>& all_queries) const {
+
+ std::vector<std::string> q_include_tokens;
+ for(size_t j = 0; j < field_query_tokens[0].q_include_tokens.size(); j++) {
+ q_include_tokens.push_back(field_query_tokens[0].q_include_tokens[j].value);
+ }
+ synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);
+
+ std::vector<std::vector<token_t>> q_pos_synonyms;
+ for(const auto& q_syn_vec: field_query_tokens[0].q_synonyms) {
+ std::vector<token_t> q_pos_syn;
+ for(size_t j=0; j < q_syn_vec.size(); j++) {
+ bool is_prefix = (j == q_syn_vec.size()-1);
+ q_pos_syn.emplace_back(j, q_syn_vec[j], is_prefix, q_syn_vec[j].size(), 0);
+ }
+ q_pos_synonyms.push_back(q_pos_syn);
+ all_queries.push_back(q_pos_syn);
+ }
+
+ int syn_orig_num_tokens = field_query_tokens[0].q_include_tokens.size();
+ bool syn_wildcard_filter_init_done = false;
+
+ for(const auto& syn_tokens: q_pos_synonyms) {
+ query_hashes.clear();
+ fuzzy_search_fields(the_fields, syn_tokens, exclude_token_ids,
+ exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,
+ sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
+ query_hashes, token_order, {0}, typo_tokens_threshold,
+ exhaustive_search, min_len_1typo,
+ min_len_2typo, q_include_tokens.size(), sort_order, field_values, geopoint_indices);
+ }
+
+ collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
+}
+
+void Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
+ const std::vector<infix_t>& infixes,
+ const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
- const size_t max_extra_suffix, const infix_t& field_infix, const uint8_t field_id,
- const string& field_name, const std::vector<token_t>& query_tokens, Topster* actual_topster,
+ const size_t max_extra_suffix,
+ const std::vector<token_t>& query_tokens, Topster* actual_topster,
const uint32_t *filter_ids, size_t filter_ids_length,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
const std::vector<uint32_t>& curated_ids_sorted,
- size_t field_num_results, uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
spp::sparse_hash_set<uint64_t>& groups_processed) const {
- if(field_infix == always || (field_infix == fallback && field_num_results == 0)) {
- std::vector<uint32_t> infix_ids;
- search_infix(query_tokens[0].value, field_name, infix_ids, max_extra_prefix, max_extra_suffix);
+ for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
+ auto& field_name = the_fields[field_id].name;
+ infix_t field_infix = (field_id < infixes.size()) ? infixes[field_id] : infixes[0];
- if(!infix_ids.empty()) {
- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;
- std::vector<size_t> geopoint_indices;
- populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);
- uint32_t token_bits = 0;
+ if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {
+ std::vector<uint32_t> infix_ids;
+ search_infix(query_tokens[0].value, field_name, infix_ids, max_extra_prefix, max_extra_suffix);
- std::sort(infix_ids.begin(), infix_ids.end());
- infix_ids.erase(std::unique( infix_ids.begin(), infix_ids.end() ), infix_ids.end());
+ if(!infix_ids.empty()) {
+ std::sort(infix_ids.begin(), infix_ids.end());
+ infix_ids.erase(std::unique( infix_ids.begin(), infix_ids.end() ), infix_ids.end());
- uint32_t *raw_infix_ids = nullptr;
- size_t raw_infix_ids_length = 0;
+ uint32_t *raw_infix_ids = nullptr;
+ size_t raw_infix_ids_length = 0;
- if(curated_ids_sorted.size() != 0) {
- raw_infix_ids_length = ArrayUtils::exclude_scalar(&infix_ids[0], infix_ids.size(), &curated_ids_sorted[0],
- curated_ids_sorted.size(), &raw_infix_ids);
- infix_ids.clear();
- } else {
- raw_infix_ids = &infix_ids[0];
- raw_infix_ids_length = infix_ids.size();
- }
+ if(curated_ids_sorted.size() != 0) {
+ raw_infix_ids_length = ArrayUtils::exclude_scalar(&infix_ids[0], infix_ids.size(), &curated_ids_sorted[0],
+ curated_ids_sorted.size(), &raw_infix_ids);
+ infix_ids.clear();
+ } else {
+ raw_infix_ids = &infix_ids[0];
+ raw_infix_ids_length = infix_ids.size();
+ }
- if(filter_ids_length != 0) {
- uint32_t *filtered_raw_infix_ids = nullptr;
- raw_infix_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, raw_infix_ids,
- raw_infix_ids_length, &filtered_raw_infix_ids);
- if(raw_infix_ids != &infix_ids[0]) {
- delete [] raw_infix_ids;
+ if(filter_ids_length != 0) {
+ uint32_t *filtered_raw_infix_ids = nullptr;
+ raw_infix_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, raw_infix_ids,
+ raw_infix_ids_length, &filtered_raw_infix_ids);
+ if(raw_infix_ids != &infix_ids[0]) {
+ delete [] raw_infix_ids;
+ }
+
+ raw_infix_ids = filtered_raw_infix_ids;
}
- raw_infix_ids = filtered_raw_infix_ids;
- }
+ bool field_is_array = search_schema.at(the_fields[field_id].name).is_array();
- for(size_t i = 0; i < raw_infix_ids_length; i++) {
- auto seq_id = raw_infix_ids[i];
- score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 2,
- actual_topster, {}, groups_processed, seq_id, sort_order, field_values,
- geopoint_indices, group_limit, group_by_fields, token_bits,
- false, false, -1, {});
- }
+ for(size_t i = 0; i < raw_infix_ids_length; i++) {
+ auto seq_id = raw_infix_ids[i];
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, raw_infix_ids,
- raw_infix_ids_length, &new_all_result_ids);
- delete[] all_result_ids;
- all_result_ids = new_all_result_ids;
+ int64_t match_score = 0;
+ score_results2(sort_fields, searched_queries.size(), field_is_array,
+ 0, match_score, seq_id, sort_order, group_limit, group_by_fields,
+ false, false, -1, {});
- if(raw_infix_ids != &infix_ids[0]) {
- delete [] raw_infix_ids;
- }
+ int64_t scores[3] = {0};
+ int64_t match_score_index = 0;
+
+ compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id,
+ 100, scores, match_score_index);
+
+ uint64_t distinct_id = seq_id;
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ groups_processed.emplace(distinct_id);
+ }
- searched_queries.push_back({});
+ KV kv(field_id, searched_queries.size(), 0, seq_id, distinct_id, match_score_index, scores);
+ actual_topster->add(&kv);
+ }
+
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, raw_infix_ids,
+ raw_infix_ids_length, &new_all_result_ids);
+ delete[] all_result_ids;
+ all_result_ids = new_all_result_ids;
+
+ if(raw_infix_ids != &infix_ids[0]) {
+ delete [] raw_infix_ids;
+ }
+
+ searched_queries.push_back({});
+ }
}
}
}
@@ -2995,7 +3358,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
for (size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {
bool is_prefix = (qtoken_index == query_tokens.size()-1);
- qtokens.emplace_back(qtoken_index, query_tokens[qtoken_index], is_prefix);
+ qtokens.emplace_back(qtoken_index, query_tokens[qtoken_index], is_prefix,
+ query_tokens[qtoken_index].size(), 0);
}
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -3114,19 +3478,17 @@ void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::s
void Index::search_wildcard(const std::vector<filter>& filters,
const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,
+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
spp::sparse_hash_set<uint64_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size, const uint8_t field_id, const string& field,
uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
- uint32_t filter_ids_length, const size_t concurrency) const {
-
- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;
- std::vector<size_t> geopoint_indices;
- populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);
+ uint32_t filter_ids_length, const size_t concurrency,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const {
uint32_t token_bits = 0;
const bool check_for_circuit_break = (filter_ids_length > 1000000);
@@ -3167,9 +3529,9 @@ void Index::search_wildcard(const std::vector<filter>& filters,
topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);
thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
- thread_id, &sort_fields_std, &searched_queries, &field_id,
+ thread_id, &sort_fields, &searched_queries, &field_id,
&group_limit, &group_by_fields, &topsters, &tgroups_processed,
- &sort_order, field_values, &geopoint_indices, &token_bits, &plists,
+ &sort_order, field_values, &geopoint_indices, &plists,
check_for_circuit_break,
batch_result_ids, batch_res_len,
&num_processed, &m_process, &cv_process]() {
@@ -3180,10 +3542,26 @@ void Index::search_wildcard(const std::vector<filter>& filters,
for(size_t i = 0; i < batch_res_len; i++) {
const uint32_t seq_id = batch_result_ids[i];
- score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0,
- topsters[thread_id], {}, tgroups_processed[thread_id], seq_id, sort_order, field_values,
- geopoint_indices, group_limit, group_by_fields, token_bits,
- false, false, -1, plists);
+ int64_t match_score = 0;
+
+ score_results2(sort_fields, (uint16_t) searched_queries.size(), false, 0,
+ match_score, seq_id, sort_order, group_limit, group_by_fields, false,
+ false, -1, plists);
+
+ int64_t scores[3] = {0};
+ int64_t match_score_index = 0;
+
+ compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id,
+ 100, scores, match_score_index);
+
+ uint64_t distinct_id = seq_id;
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ tgroups_processed[thread_id].emplace(distinct_id);
+ }
+
+ KV kv(0, searched_queries.size(), 0, seq_id, distinct_id, match_score_index, scores);
+ topsters[thread_id]->add(&kv);
if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {
// check only once every 2^15 docs to reduce overhead
@@ -3215,7 +3593,7 @@ void Index::search_wildcard(const std::vector<filter>& filters,
std::chrono::high_resolution_clock::now() - beginF).count();
LOG(INFO) << "Time for raw scoring: " << timeMillisF;*/
- collate_included_ids({}, field, field_id, included_ids_map, curated_topster, searched_queries);
+ collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
uint32_t* new_all_result_ids = nullptr;
all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, filter_ids,
@@ -3320,7 +3698,8 @@ void Index::search_field(const uint8_t & field_id,
// stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"
std::vector<token_candidates> token_candidates_vec;
- std::set<art_leaf*> unique_tokens;
+
+ std::set<std::string> unique_tokens;
auto product = []( long long a, std::vector<int>& b ) { return a*b.size(); };
long long n = 0;
@@ -3361,7 +3740,7 @@ void Index::search_field(const uint8_t & field_id,
const std::string token_cost_hash = token + std::to_string(costs[token_index]);
std::vector<art_leaf*> leaves;
- const bool prefix_search = prefix && query_tokens[token_index].prefix;
+ const bool prefix_search = prefix && query_tokens[token_index].is_prefix_searched;
/*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"
<< token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/
@@ -3386,7 +3765,8 @@ void Index::search_field(const uint8_t & field_id,
if(!leaves.empty()) {
token_cost_cache.emplace(token_cost_hash, leaves);
for(auto leaf: leaves) {
- unique_tokens.emplace(leaf);
+ std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);
+ unique_tokens.emplace(tok);
}
}
}
@@ -3461,6 +3841,84 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
}
}
+int64_t Index::score_results2(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,
+ const bool field_is_array, const uint32_t total_cost,
+ int64_t& match_score,
+ const uint32_t seq_id, const int sort_order[3],
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool prioritize_exact_match,
+ const bool single_exact_query_token,
+ int syn_orig_num_tokens,
+ const std::vector<posting_list_t::iterator_t>& posting_lists) const {
+
+ //auto begin = std::chrono::high_resolution_clock::now();
+ //const std::string first_token((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1);
+
+ if (posting_lists.size() <= 1) {
+ const uint8_t is_verbatim_match = uint8_t(
+ prioritize_exact_match && single_exact_query_token &&
+ posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array)
+ );
+ size_t words_present = (syn_orig_num_tokens == -1) ? 1 : syn_orig_num_tokens;
+ size_t distance = (syn_orig_num_tokens == -1) ? 0 : syn_orig_num_tokens-1;
+ Match single_token_match = Match(words_present, distance, is_verbatim_match);
+ match_score = single_token_match.get_match_score(total_cost, words_present);
+ } else {
+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ posting_list_t::get_offsets(posting_lists, array_token_positions);
+
+ for (const auto& kv: array_token_positions) {
+ const std::vector<token_positions_t>& token_positions = kv.second;
+ if (token_positions.empty()) {
+ continue;
+ }
+
+ const Match &match = Match(seq_id, token_positions, false, prioritize_exact_match);
+ uint64_t this_match_score = match.get_match_score(total_cost, posting_lists.size());
+
+ auto unique_words = ((this_match_score >> 32) & 0xFF);
+ auto this_words_present = ((this_match_score >> 24) & 0xFF);
+ auto typo_score = ((this_match_score >> 16) & 0xFF);
+ auto proximity = ((this_match_score >> 8) & 0xFF);
+ auto verbatim = (this_match_score & 0xFF);
+
+ if(syn_orig_num_tokens != -1) {
+ unique_words = syn_orig_num_tokens;
+ this_words_present = syn_orig_num_tokens;
+ proximity = 100 - (syn_orig_num_tokens - 1);
+ }
+
+ // Within a field, only a subset of query tokens can match (unique_words), but even a smaller set
+ // might be available within the window used for proximity calculation (this_words_present)
+
+ uint64_t mod_match_score = (
+ (int64_t(unique_words) << 32) |
+ (int64_t(this_words_present) << 24) |
+ (int64_t(typo_score) << 16) |
+ (int64_t(proximity) << 8) |
+ (int64_t(verbatim) << 0)
+ );
+
+ if(mod_match_score > match_score) {
+ match_score = mod_match_score;
+ }
+
+ /*std::ostringstream os;
+ os << name << ", total_cost: " << (255 - total_cost)
+ << ", words_present: " << match.words_present
+ << ", match_score: " << match_score
+ << ", match.distance: " << match.distance
+ << ", seq_id: " << seq_id << std::endl;
+ LOG(INFO) << os.str();*/
+ }
+ }
+
+ //long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ //LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";
+
+ return 0;
+}
+
void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,
const uint8_t & field_id, const bool field_is_array, const uint32_t total_cost,
Topster* topster,
@@ -3703,6 +4161,53 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
return distinct_id;
}
+inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token_candidates_vec,
+ long long int n,
+ std::vector<token_t>& query_suggestion,
+ int syn_orig_num_tokens,
+ uint32_t& token_bits,
+ uint64& qhash) {
+ uint32_t total_cost = 0;
+ qhash = 1;
+
+ // generate the next combination from `token_leaves` and store it in `query_suggestion`
+ ldiv_t q { n, 0 };
+ for(long long i = 0 ; i < (long long) token_candidates_vec.size(); i++) {
+ size_t token_size = token_candidates_vec[i].token.value.size();
+ q = ldiv(q.quot, token_candidates_vec[i].candidates.size());
+ const auto& candidate = token_candidates_vec[i].candidates[q.rem];
+
+ bool exact_match = token_candidates_vec[i].cost == 0 && token_size == candidate.size();
+
+ // we assume that toke was found via prefix search if candidate is longer than token's typo tolerance
+ bool is_prefix_searched = token_candidates_vec[i].prefix_search &&
+ (candidate.size() > (token_size + token_candidates_vec[i].cost));
+
+ size_t actual_cost = (2 * token_candidates_vec[i].cost) + uint32_t(is_prefix_searched);
+ total_cost += actual_cost;
+
+ token_bits |= 1UL << token_candidates_vec[i].token.position; // sets n-th bit
+
+ query_suggestion[i] = token_t(i, candidate, is_prefix_searched, token_size, token_candidates_vec[i].cost);
+
+ uint64_t this_hash = StringUtils::hash_wy(query_suggestion[i].value.c_str(), query_suggestion[i].value.size());
+ qhash = StringUtils::hash_combine(qhash, this_hash);
+
+ /*LOG(INFO) << "suggestion key: " << actual_query_suggestion[i]->key << ", token: "
+ << token_candidates_vec[i].token.value << ", actual_cost: " << actual_cost;
+ LOG(INFO) << ".";*/
+ }
+
+ if(syn_orig_num_tokens != -1) {
+ token_bits = 0;
+ for(size_t i = 0; i < size_t(syn_orig_num_tokens); i++) {
+ token_bits |= 1UL << i;
+ }
+ }
+
+ return total_cost;
+}
+
inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
long long int n,
std::vector<art_leaf *>& actual_query_suggestion,
@@ -3912,6 +4417,7 @@ void Index::tokenize_string_field(const nlohmann::json& document, const field& s
}
}
+// FIXME: returning raw value is a problem!
art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {
std::shared_lock lock(mutex);
const art_tree *t = search_index.at(field_name);
diff --git a/src/or_iterator.cpp b/src/or_iterator.cpp
new file mode 100644
index 00000000..c29be99b
--- /dev/null
+++ b/src/or_iterator.cpp
@@ -0,0 +1,221 @@
+#include "or_iterator.h"
+
+
+bool or_iterator_t::at_end(const std::vector<or_iterator_t>& its) {
+ // if any iterator is empty, we stop
+ for(const auto& it : its) {
+ if(it.valid()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool or_iterator_t::at_end2(const std::vector<or_iterator_t>& its) {
+ // if any iterator is invalid, we stop
+ return !its[0].valid() || !its[1].valid();
+}
+
+bool or_iterator_t::equals(std::vector<or_iterator_t>& its) {
+ for(int i = 0; i < int(its.size()) - 1; i++) {
+ if(its[i].id() != its[i+1].id()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool or_iterator_t::equals2(std::vector<or_iterator_t>& its) {
+ return its[0].id() == its[1].id();
+}
+
+
+void or_iterator_t::advance_all(std::vector<or_iterator_t>& its) {
+ for(size_t i = 0; i < its.size(); i++) {
+ auto& it = its[i];
+ bool valid = it.next();
+ if(!valid) {
+ its.erase(its.begin() + i);
+ }
+ }
+}
+
+void or_iterator_t::advance_all2(std::vector<or_iterator_t>& its) {
+ bool valid0 = its[0].next();
+ bool valid1 = its[1].next();
+
+ if(!valid0) {
+ its.erase(its.begin() + 0);
+ if(!valid1) {
+ // 1st index will be now at 0th index
+ its.erase(its.begin() + 0);
+ }
+ } else if(!valid1) {
+ its.erase(its.begin() + 1);
+ }
+}
+
+void or_iterator_t::advance_non_largest(std::vector<or_iterator_t>& its) {
+ // we will find the iter with greatest value and then advance the rest until their value catches up
+ uint32_t greatest_value = 0;
+
+ for(size_t i = 0; i < its.size(); i++) {
+ if(its[i].id() > greatest_value) {
+ greatest_value = its[i].id();
+ }
+ }
+
+ for(size_t i = 0; i < its.size(); i++) {
+ if(its[i].id() != greatest_value) {
+ bool valid = its[i].skip_to(greatest_value);
+ if(!valid) {
+ its.erase(its.begin() + i);
+ i--;
+ }
+ }
+ }
+}
+
+void or_iterator_t::advance_non_largest2(std::vector<or_iterator_t>& its) {
+ if(its[0].id() > its[1].id()) {
+ bool valid = its[1].skip_to(its[0].id());
+ if(!valid) {
+ its.erase(its.begin() + 1);
+ }
+ } else {
+ bool valid = its[0].skip_to(its[1].id());
+ if(!valid) {
+ its.erase(its.begin() + 0);
+ }
+ }
+}
+
+bool or_iterator_t::valid() const {
+ return !its.empty();
+}
+
+bool or_iterator_t::next() {
+ size_t num_lists = its.size();
+
+ switch (num_lists) {
+ case 0:
+ break;
+ case 2:
+ if(!posting_list_t::all_ended2(its)) {
+ advance_smallest();
+ }
+ break;
+ default:
+ if(!posting_list_t::all_ended(its)) {
+ advance_smallest();
+ }
+ break;
+ }
+
+ return !its.empty();
+}
+
+void or_iterator_t::advance_smallest() {
+ // we will advance the smallest value and point current_index to next smallest value across the lists
+ auto smallest_value = its[curr_index].id();
+ curr_index = 0;
+
+ for(int i = 0; i < int(its.size()); i++) {
+ if(its[i].id() == smallest_value) {
+ its[i].next();
+ }
+
+ if(!its[i].valid()) {
+ its[i].destroy();
+ its.erase(its.cbegin() + i);
+ i--;
+ }
+ }
+
+ uint32_t new_smallest_value = UINT32_MAX;
+ for(int i = 0; i < int(its.size()); i++) {
+ if(its[i].id() < new_smallest_value) {
+ curr_index = i;
+ new_smallest_value = its[i].id();
+ }
+ }
+}
+
+bool or_iterator_t::skip_to(uint32_t id) {
+ auto current_value = UINT32_MAX;
+ curr_index = 0;
+
+ for(size_t i = 0; i < its.size(); i++) {
+ auto& it = its[i];
+ it.skip_to(id);
+
+ if(!it.valid()) {
+ its[i].destroy();
+ its.erase(its.begin() + i);
+ i--;
+ } else {
+ if(it.id() < current_value) {
+ curr_index = i;
+ current_value = it.id();
+ }
+ }
+ }
+
+ return !its.empty();
+}
+
+uint32_t or_iterator_t::id() const {
+ return its[curr_index].id();
+}
+
+bool or_iterator_t::take_id(result_iter_state_t& istate, uint32_t id) {
+ // decide if this result id should be excluded
+ if(istate.excluded_result_ids_size != 0) {
+ if (std::binary_search(istate.excluded_result_ids,
+ istate.excluded_result_ids + istate.excluded_result_ids_size, id)) {
+ return false;
+ }
+ }
+
+ // decide if this result be matched with filter results
+ if(istate.filter_ids_length != 0) {
+ return std::binary_search(istate.filter_ids, istate.filter_ids + istate.filter_ids_length, id);
+ }
+
+ return true;
+}
+
+or_iterator_t::or_iterator_t(std::vector<posting_list_t::iterator_t>& its): its(std::move(its)) {
+ curr_index = 0;
+
+ for(size_t i = 1; i < this->its.size(); i++) {
+ if(this->its[i].id() < this->its[curr_index].id()) {
+ curr_index = i;
+ }
+ }
+}
+
+or_iterator_t::or_iterator_t(or_iterator_t&& rhs) noexcept {
+ its = std::move(rhs.its);
+ curr_index = rhs.curr_index;
+}
+
+or_iterator_t& or_iterator_t::operator=(or_iterator_t&& rhs) noexcept {
+ its = std::move(rhs.its);
+ curr_index = rhs.curr_index;
+ return *this;
+}
+
+const std::vector<posting_list_t::iterator_t>& or_iterator_t::get_its() const {
+ return its;
+}
+
+or_iterator_t::~or_iterator_t() noexcept {
+ for(auto& it: its) {
+ it.destroy();
+ }
+}
+
+
diff --git a/src/posting.cpp b/src/posting.cpp
index aa38f09c..4480b2ae 100644
--- a/src/posting.cpp
+++ b/src/posting.cpp
@@ -444,9 +444,14 @@ void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>&
for(posting_list_t* pl: plists) {
its.push_back(pl->new_iterator());
its.back().skip_to(id);
+ if(!its.back().valid() || its.back().id() != id) {
+ its.pop_back();
+ }
}
- posting_list_t::get_offsets(its, array_token_positions);
+ if(!its.empty()) {
+ posting_list_t::get_offsets(its, array_token_positions);
+ }
for(posting_list_t* expanded_plist: expanded_plists) {
delete expanded_plist;
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index b5610181..66fbfd9d 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -925,7 +925,7 @@ bool posting_list_t::at_end2(const std::vector<posting_list_t::iterator_t>& its)
}
bool posting_list_t::equals(std::vector<posting_list_t::iterator_t>& its) {
- for(size_t i = 0; i < its.size() - 1; i++) {
+ for(int i = 0; i < int(its.size()) - 1; i++) {
if(its[i].id() != its[i+1].id()) {
return false;
}
@@ -938,9 +938,9 @@ bool posting_list_t::equals2(std::vector<posting_list_t::iterator_t>& its) {
return its[0].id() == its[1].id();
}
-posting_list_t::iterator_t posting_list_t::new_iterator(block_t* start_block, block_t* end_block) {
+posting_list_t::iterator_t posting_list_t::new_iterator(block_t* start_block, block_t* end_block, uint32_t field_id) {
start_block = (start_block == nullptr) ? &root_block : start_block;
- return posting_list_t::iterator_t(start_block, end_block);
+ return posting_list_t::iterator_t(start_block, end_block, true, field_id);
}
void posting_list_t::advance_all(std::vector<posting_list_t::iterator_t>& its) {
@@ -1315,10 +1315,27 @@ void posting_list_t::get_matching_array_indices(uint32_t id, std::vector<iterato
}
}
+bool posting_list_t::all_ended(const std::vector<posting_list_t::iterator_t>& its) {
+ // if all iterators are at end, we return true
+ for(const auto& it : its) {
+ if(it.valid()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool posting_list_t::all_ended2(const std::vector<posting_list_t::iterator_t>& its) {
+ // if both iterators are at end, we return true
+ return !its[0].valid() && !its[1].valid();
+}
+
/* iterator_t operations */
-posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* start, posting_list_t::block_t* end):
- curr_block(start), curr_index(0), end_block(end) {
+posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* start, posting_list_t::block_t* end,
+ bool auto_destroy, uint32_t field_id):
+ curr_block(start), curr_index(0), end_block(end), auto_destroy(auto_destroy), field_id(field_id) {
if(curr_block != end_block) {
ids = curr_block->ids.uncompress();
@@ -1393,6 +1410,12 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {
}
posting_list_t::iterator_t::~iterator_t() {
+ if(auto_destroy) {
+ destroy();
+ }
+}
+
+void posting_list_t::iterator_t::destroy() {
delete [] ids;
ids = nullptr;
@@ -1410,14 +1433,52 @@ posting_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {
ids = rhs.ids;
offset_index = rhs.offset_index;
offsets = rhs.offsets;
+ auto_destroy = rhs.auto_destroy;
+ field_id = rhs.field_id;
+
+ rhs.curr_block = nullptr;
+ rhs.end_block = nullptr;
+ rhs.ids = nullptr;
+ rhs.offset_index = nullptr;
+ rhs.offsets = nullptr;
+}
+
+posting_list_t::iterator_t& posting_list_t::iterator_t::operator=(posting_list_t::iterator_t&& rhs) noexcept {
+ curr_block = rhs.curr_block;
+ curr_index = rhs.curr_index;
+ end_block = rhs.end_block;
+ ids = rhs.ids;
+ offset_index = rhs.offset_index;
+ offsets = rhs.offsets;
+ auto_destroy = rhs.auto_destroy;
+ field_id = rhs.field_id;
rhs.curr_block = nullptr;
rhs.end_block = nullptr;
rhs.ids = nullptr;
rhs.offset_index = nullptr;
rhs.offsets = nullptr;
+
+ return *this;
}
void posting_list_t::iterator_t::set_index(uint32_t index) {
curr_index = index;
}
+
+posting_list_t::iterator_t posting_list_t::iterator_t::clone() const {
+ posting_list_t::iterator_t it(nullptr, nullptr);
+ it.curr_block = curr_block;
+ it.curr_index = curr_index;
+ it.end_block = end_block;
+ it.ids = ids;
+ it.offsets = offsets;
+ it.offset_index = offset_index;
+ it.auto_destroy = false;
+ it.field_id = field_id;
+ return it;
+}
+
+uint32_t posting_list_t::iterator_t::get_field_id() const {
+ return field_id;
+}
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 32bdcae6..3c9cb3d0 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -25,10 +25,18 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
void Tokenizer::init(const std::string& input) {
+ // init() can be called multiple times safely without leaking memory as we check for prior initialization
+ if(normalized_text) {
+ free(normalized_text);
+ normalized_text = nullptr;
+ }
+
if(locale == "zh") {
UErrorCode translit_status = U_ZERO_ERROR;
- transliterator = icu::Transliterator::createInstance("Traditional-Simplified",
- UTRANS_FORWARD, translit_status);
+ if(!transliterator) {
+ transliterator = icu::Transliterator::createInstance("Traditional-Simplified",
+ UTRANS_FORWARD, translit_status);
+ }
if(U_FAILURE(translit_status)) {
//LOG(ERROR) << "Unable to create transliteration instance for `zh` locale.";
transliterator = nullptr;
@@ -50,8 +58,10 @@ void Tokenizer::init(const std::string& input) {
} else if(is_cyrillic(locale)) {
// init transliterator but will only transliterate during tokenization
UErrorCode translit_status = U_ZERO_ERROR;
- transliterator = icu::Transliterator::createInstance("Any-Latin; Latin-ASCII",
- UTRANS_FORWARD, translit_status);
+ if(!transliterator) {
+ transliterator = icu::Transliterator::createInstance("Any-Latin; Latin-ASCII",
+ UTRANS_FORWARD, translit_status);
+ }
text = input;
} else {
text = input;
@@ -60,7 +70,9 @@ void Tokenizer::init(const std::string& input) {
if(!locale.empty() && locale != "en") {
UErrorCode status = U_ZERO_ERROR;
const icu::Locale& icu_locale = icu::Locale(locale.c_str());
- bi = icu::BreakIterator::createWordInstance(icu_locale, status);
+ if(!bi) {
+ bi = icu::BreakIterator::createWordInstance(icu_locale, status);
+ }
unicode_text = icu::UnicodeString::fromUTF8(text);
bi->setText(unicode_text);
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index f269edc9..9c9bef92 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -300,7 +300,7 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {
ASSERT_STREQ("pop", results["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());
ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("2", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("4", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());
ASSERT_EQ(2, results["grouped_hits"][1]["hits"].size());
diff --git a/test/collection_infix_search_test.cpp b/test/collection_infix_search_test.cpp
index b6df2d01..4f3fcdf2 100644
--- a/test/collection_infix_search_test.cpp
+++ b/test/collection_infix_search_test.cpp
@@ -112,6 +112,37 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionInfixSearchTest, InfixOnArray) {
+ std::vector<field> fields = {field("model_numbers", field_types::STRING_ARRAY, false, false, true, "", -1, 1),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["model_numbers"] = {"GH100037IN8900X", "GH100047IN8900X", "GH100057IN8900X"};
+ doc["points"] = 100;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("47in",
+ {"model_numbers"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "model_numbers", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_EQ(1, results["hits"][0]["highlights"].size());
+ ASSERT_EQ("model_numbers", results["hits"][0]["highlights"][0]["field"].get<std::string>());
+ ASSERT_EQ("<mark>GH100047IN8900X</mark>", results["hits"][0]["highlights"][0]["snippets"][0].get<std::string>());
+ ASSERT_EQ("<mark>GH100047IN8900X</mark>", results["hits"][0]["highlights"][0]["values"][0].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionInfixSearchTest, InfixWithFiltering) {
std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),
field("points", field_types::INT32, false),};
@@ -131,7 +162,7 @@ TEST_F(CollectionInfixSearchTest, InfixWithFiltering) {
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
- auto results = coll1->search("37IN8",
+ auto results = coll1->search("37in8",
{"title"}, "points: 200", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, "", "", {}, 0,
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index de105fbc..e76c5d2c 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -109,7 +109,7 @@ TEST_F(CollectionLocaleTest, SearchAgainstChineseText) {
// partial token should not match as prefix when prefix is set to false
results = coll1->search("并",
- {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY).get();
+ {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(0, results["found"].get<size_t>());
@@ -506,6 +506,23 @@ TEST_F(CollectionLocaleTest, SearchCyrillicText) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionLocaleTest, SearchCyrillicTextWithDefaultLocale) {
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, ""),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Test Тест";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "TEST ТЕСТ";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("тетст", {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {false}).get();
+
+ ASSERT_EQ(0, results["hits"].size());
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionLocaleTest, SearchCyrillicTextWithDropTokens) {
// this test ensures that even when tokens are dropped, the eventual text is highlighted on all query tokens
std::vector<field> fields = {field("description", field_types::STRING, false, false, true, "sr"),
@@ -554,7 +571,6 @@ TEST_F(CollectionLocaleTest, SearchAndFacetSearchForGreekText) {
ASSERT_EQ("<mark>Εμφάν</mark>ιση κάθε μέρα.", results["hits"][0]["highlights"][0]["value"].get<std::string>());
// with typo
-
results = coll1->search("Εμφάιση", {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("<mark>Εμφάνιση</mark> κάθε μέρα.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
@@ -653,3 +669,28 @@ TEST_F(CollectionLocaleTest, SearchOnCyrillicLargeText) {
ASSERT_STREQ("Петр <mark>Великий</mark>, царь России, в начале",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
}
+
+/*
+TEST_F(CollectionLocaleTest, TranslitPad) {
+ UErrorCode translit_status = U_ZERO_ERROR;
+ auto transliterator = icu::Transliterator::createInstance("Any-Latin; Latin-ASCII",
+ UTRANS_FORWARD, translit_status);
+
+ icu::UnicodeString unicode_input = icu::UnicodeString::fromUTF8("எண்ணெய்");
+ transliterator->transliterate(unicode_input);
+ std::string output;
+ unicode_input.toUTF8String(output);
+ LOG(INFO) << output;
+
+ unicode_input = icu::UnicodeString::fromUTF8("எண்");
+ transliterator->transliterate(unicode_input);
+ unicode_input.toUTF8String(output);
+ LOG(INFO) << output;
+
+ unicode_input = icu::UnicodeString::fromUTF8("என்னை");
+ transliterator->transliterate(unicode_input);
+ unicode_input.toUTF8String(output);
+ LOG(INFO) << output;
+
+ delete transliterator;
+}*/
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index ffd92ec5..44fbe967 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -1211,10 +1211,11 @@ TEST_F(CollectionOverrideTest, DynamicFilteringMultiplePlaceholders) {
auto results = coll1->search("Nike Air Jordan light yellow shoes", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
+ // not happy with this order (0,2,1 is better)
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
- ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
// query with tokens at the start that preceding the placeholders in the rule
results = coll1->search("New Nike Air Jordan yellow shoes", {"name", "category", "brand"}, "",
@@ -1366,13 +1367,13 @@ TEST_F(CollectionOverrideTest, DynamicFilteringWithNumericalFilter) {
ASSERT_TRUE(op.ok());
auto results = coll1->search("popular nike shoes", {"name", "category", "brand"}, "",
- {}, sort_fields, {2, 2, 2}, 10).get();
+ {}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(4, results["hits"].size());
coll1->add_override(override);
results = coll1->search("popular nike shoes", {"name", "category", "brand"}, "",
- {}, sort_fields, {2, 2, 2}, 10).get();
+ {}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
@@ -1381,7 +1382,7 @@ TEST_F(CollectionOverrideTest, DynamicFilteringWithNumericalFilter) {
bool enable_overrides = false;
results = coll1->search("popular nike shoes", {"name", "category", "brand"}, "",
- {}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false, false, false}, 1,
+ {}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false, false, false}, 10,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 1, 1}, 10000, true, false, enable_overrides).get();
@@ -1395,8 +1396,8 @@ TEST_F(CollectionOverrideTest, DynamicFilteringWithNumericalFilter) {
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
- ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
- ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
results = coll1->search("adidas", {"name", "category", "brand"}, "",
{}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {false}, 10).get();
@@ -1479,7 +1480,7 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatch) {
coll1->add_override(override);
auto results = coll1->search("really popular nike shoes", {"name", "category", "brand"}, "",
- {}, sort_fields, {2, 2, 2}, 10).get();
+ {}, sort_fields, {2, 2, 2}, 10, 1, FREQUENCY, {true}, 10).get();
ASSERT_EQ(4, results["hits"].size());
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index e7543d57..bf19e5e1 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -1460,10 +1460,10 @@ TEST_F(CollectionSortingTest, RepeatingTokenRanking) {
ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
- ASSERT_EQ(50291713, results["hits"][0]["text_match"].get<uint32_t>());
- ASSERT_EQ(50291712, results["hits"][1]["text_match"].get<uint32_t>());
- ASSERT_EQ(50291712, results["hits"][2]["text_match"].get<uint32_t>());
- ASSERT_EQ(50291712, results["hits"][3]["text_match"].get<uint32_t>());
+ ASSERT_EQ(4284743939, results["hits"][0]["text_match"].get<uint32_t>());
+ ASSERT_EQ(4284743683, results["hits"][1]["text_match"].get<uint32_t>());
+ ASSERT_EQ(4284743683, results["hits"][2]["text_match"].get<uint32_t>());
+ ASSERT_EQ(4284743683, results["hits"][3]["text_match"].get<uint32_t>());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index c27bf137..c62f6b10 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -876,14 +876,14 @@ TEST_F(CollectionSpecificTest, HighlightLongFieldWithDropTokens) {
ASSERT_EQ("Tripp Lite USB C to VGA Multiport Video Adapter Converter w/ USB-A Hub, "
"USB-C PD <mark>Charging</mark> <mark>Port</mark> & <mark>Gigabit</mark> Ethernet "
"<mark>Port,</mark> Thunderbolt 3 Compatible, USB Type C to VGA, USB-C, USB Type-C - for "
- "Notebook/Tablet PC - 2 x USB <mark>Ports</mark> - 2 x USB 3.0 - Network (RJ-45) - "
+ "Notebook/Tablet PC - 2 x USB <mark>Port</mark>s - 2 x USB 3.0 - Network (RJ-45) - "
"VGA - <mark>Wired</mark>",
results["hits"][0]["highlights"][0]["value"].get<std::string>());
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionSpecificTest, HighlightWithDropTokensAndPrefixSearch) {
+TEST_F(CollectionSpecificTest, HighlightWithTypoTokensAndPrefixSearch) {
std::vector<field> fields = {field("username", field_types::STRING, false),
field("name", field_types::STRING, false),
field("tags", field_types::STRING_ARRAY, false),
@@ -915,8 +915,8 @@ TEST_F(CollectionSpecificTest, HighlightWithDropTokensAndPrefixSearch) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>").get();
- ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ(2, results["hits"][0]["highlights"].size());
@@ -926,6 +926,14 @@ TEST_F(CollectionSpecificTest, HighlightWithDropTokensAndPrefixSearch) {
ASSERT_EQ("<mark>Pandaabear</mark>",
results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ(2, results["hits"][1]["highlights"].size());
+
+ ASSERT_EQ("<mark>Pandaabear</mark>",
+ results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("Panda's <mark>Bas</mark>ement",
+ results["hits"][1]["highlights"][1]["snippet"].get<std::string>());
+
results = coll1->search("pandaabear bas", {"username", "name"},
"", {}, {}, {2, 2}, 10,
1, FREQUENCY, {true, true},
@@ -957,11 +965,22 @@ TEST_F(CollectionSpecificTest, HighlightWithDropTokensAndPrefixSearch) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
"<mark>", "</mark>").get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
- // NOT asserting here because given current highlighting limitations, prefixes which are not directly matched
- // onto a token in an array is not used during highlighting
+ ASSERT_EQ(2, results["hits"][0]["highlights"].size());
+ ASSERT_EQ("<mark>Pandaabear</mark> <mark>Bas</mark>ic",
+ results["hits"][0]["highlights"][0]["snippets"][0].get<std::string>());
+ ASSERT_EQ("<mark>Pandaabear</mark>",
+ results["hits"][0]["highlights"][1]["snippet"].get<std::string>());
- // ASSERT_EQ(2, results["hits"][1]["highlights"].size());
+ ASSERT_EQ(2, results["hits"][1]["highlights"].size());
+ ASSERT_EQ("<mark>Pandaabear</mark>",
+ results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("Panda's <mark>Bas</mark>ement",
+ results["hits"][1]["highlights"][1]["snippets"][0].get<std::string>());
collectionManager.drop_collection("coll1");
}
@@ -1025,7 +1044,7 @@ TEST_F(CollectionSpecificTest, TokenStartingWithSameLetterAsPrevToken) {
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeDeemedAsVerbatimMatch) {
+TEST_F(CollectionSpecificTest, CrossFieldMatchingExactMatchOnSingleField) {
std::vector<field> fields = {field("name", field_types::STRING, false),
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
@@ -1055,8 +1074,8 @@ TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeDeemedAsVerbatimMatch) {
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
- ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
results = coll1->search("john vegatable farmer", {"name", "description"},
"", {}, {}, {1, 1}, 10,
@@ -1066,8 +1085,8 @@ TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeDeemedAsVerbatimMatch) {
"<mark>", "</mark>").get();
ASSERT_EQ(2, results["hits"].size());
- ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index 44287a6d..91426240 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -373,15 +373,14 @@ TEST_F(CollectionSynonymsTest, SynonymQueryVariantWithDropTokens) {
ASSERT_TRUE(coll1->add(doc3.dump()).ok());
auto res = coll1->search("us sneakers", {"category", "location"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 10).get();
-
ASSERT_EQ(3, res["hits"].size());
- // NOTE: "2" is ranked above "1" because synonym matches uses the root query's number of tokens for counting
- // This means that "united states" == "us" so a single token match, same as match on "sneakers" in record 2.
+ // NOTE: "1" is ranked above "0" because synonym matches uses the root query's number of tokens for counting
+ // This means that "united states" == "us" so both records have 2 tokens matched, so tie breaking happens on points.
- ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("2", res["hits"][1]["document"]["id"].get<std::string>());
- ASSERT_EQ("1", res["hits"][2]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", res["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", res["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", res["hits"][2]["document"]["id"].get<std::string>());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 14b5580f..095cfa9e 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -110,7 +110,8 @@ TEST_F(CollectionTest, RetrieveADocumentById) {
TEST_F(CollectionTest, ExactSearchShouldBeStable) {
std::vector<std::string> facets;
- nlohmann::json results = collection->search("the", query_fields, "", facets, sort_fields, {0}, 10).get();
+ nlohmann::json results = collection->search("the", query_fields, "", facets, sort_fields, {0}, 10,
+ 1, FREQUENCY, {false}).get();
ASSERT_EQ(7, results["hits"].size());
ASSERT_EQ(7, results["found"].get<int>());
@@ -131,7 +132,8 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {
// check ASC sorting
std::vector<sort_by> sort_fields_asc = { sort_by("points", "ASC") };
- results = collection->search("the", query_fields, "", facets, sort_fields_asc, {0}, 10).get();
+ results = collection->search("the", query_fields, "", facets, sort_fields_asc, {0}, 10,
+ 1, FREQUENCY, {false}).get();
ASSERT_EQ(7, results["hits"].size());
ASSERT_EQ(7, results["found"].get<int>());
@@ -145,7 +147,8 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {
}
// when a query does not return results, hits and found fields should still exist in response
- results = collection->search("zxsadqewsad", query_fields, "", facets, sort_fields_asc, {0}, 10).get();
+ results = collection->search("zxsadqewsad", query_fields, "", facets, sort_fields_asc, {0}, 10,
+ 1, FREQUENCY, {false}).get();
ASSERT_EQ(0, results["hits"].size());
ASSERT_EQ(0, results["found"].get<int>());
}
diff --git a/test/or_iterator_test.cpp b/test/or_iterator_test.cpp
new file mode 100644
index 00000000..19c949b0
--- /dev/null
+++ b/test/or_iterator_test.cpp
@@ -0,0 +1,141 @@
+#include <gtest/gtest.h>
+#include <or_iterator.h>
+#include <posting_list.h>
+#include "logger.h"
+
+TEST(OrIteratorTest, IntersectTwoListsWith3SubLists) {
+ std::vector<uint32_t> offsets = {0, 1, 3};
+
+ std::vector<std::vector<uint32_t>> plists = {
+ {0, 2, 3, 20},
+ {1, 3, 5, 10, 20},
+ {2, 3, 6, 7, 20}
+ };
+
+ std::vector<posting_list_t*> postings1(plists.size());
+
+ for(size_t i = 0; i < plists.size(); i++) {
+ postings1[i] = new posting_list_t(2);
+ for(auto n: plists[i]) {
+ postings1[i]->upsert(n, offsets);
+ }
+ }
+
+ std::vector<std::vector<uint32_t>> ulists = {
+ {0, 1, 5, 20},
+ {1, 2, 7, 11, 15},
+ {3, 5, 10, 11, 12}
+ };
+
+ std::vector<posting_list_t*> postings2(ulists.size());
+
+ for(size_t i = 0; i < ulists.size(); i++) {
+ postings2[i] = new posting_list_t(2);
+ for(auto n: ulists[i]) {
+ postings2[i]->upsert(n, offsets);
+ }
+ }
+
+ std::vector<posting_list_t::iterator_t> pits1;
+ for(auto& posting_list: postings1) {
+ pits1.push_back(posting_list->new_iterator());
+ }
+
+ std::vector<posting_list_t::iterator_t> pits2;
+ for(auto& posting_list: postings2) {
+ pits2.push_back(posting_list->new_iterator());
+ }
+
+ or_iterator_t it1(pits1);
+ or_iterator_t it2(pits2);
+
+ std::vector<or_iterator_t> or_its;
+ or_its.push_back(std::move(it1));
+ or_its.push_back(std::move(it2));
+
+ result_iter_state_t istate;
+
+ std::vector<uint32_t> results;
+
+ or_iterator_t::intersect(or_its, istate, [&results](uint32_t id, std::vector<or_iterator_t>& its) {
+ LOG(INFO) << "id: " << id;
+ results.push_back(id);
+ });
+
+
+ ASSERT_EQ(8, results.size());
+
+ std::vector<uint32_t> expected_results = {0, 1, 2, 3, 5, 7, 10, 20};
+ for(size_t i = 0; i < expected_results.size(); i++) {
+ ASSERT_EQ(expected_results[i], results[i]);
+ }
+}
+
+TEST(OrIteratorTest, IntersectTwoListsWith4SubLists) {
+ std::vector<uint32_t> offsets = {0, 1, 3};
+
+ std::vector<std::vector<uint32_t>> plists = {
+ {817, 2099, 2982, 3199, 5456, 6414, 8178, 8284, 8561, 10345, 13662, 14021, 15292},
+ {9514},
+ {5758, 13357}
+ };
+
+ std::vector<posting_list_t*> postings1(plists.size());
+
+ for(size_t i = 0; i < plists.size(); i++) {
+ postings1[i] = new posting_list_t(2);
+ for(auto n: plists[i]) {
+ postings1[i]->upsert(n, offsets);
+ }
+ }
+
+ std::vector<std::vector<uint32_t>> ulists = {
+ {15156},
+ {242, 403, 431, 449, 469, 470, 471, 474, 476, 522, 538, 616, 684, 690, 789, 797, 841, 961, 970, 981, 1012, 1016, 1073, 1106, 1115, 1153, 1256, 1282, 1291, 1306, 1313, 1317, 1454, 1530, 1555, 1558, 1583, 1594, 1596, 1650, 1652, 1669, 1686, 1718, 1805, 1809, 1811, 1816, 1840, 1854, 1879, 1887, 1939, 1983, 2041, 2049, 2091, 2138, 2152, 2220, 2350, 2409, 2459, 2491, 2507, 2545, 2687, 2740, 2754, 2789, 2804, 2907, 2933, 2935, 2964, 2970, 3024, 3084, 3126, 3149, 3177, 3199, 3227, 3271, 3300, 3314, 3403, 3547, 3575, 3587, 3601, 3692, 3804, 3833, 3834, 3928, 4019, 4022, 4045, 4086, 4135, 4145, 4232, 4444, 4451, 4460, 4467, 4578, 4588, 4632, 4709, 4721, 4757, 4777, 4833, 4880, 4927, 4996, 5117, 5133, 5156, 5158, 5288, 5311, 5361, 5558, 5649, 5654, 5658, 5666, 5794, 5818, 5829, 5852, 5857, 5859, 5893, 5909, 5959, 5970, 5983, 5986, 6009, 6016, 6020, 6189, 6192, 6202, 6308, 6326, 6365, 6402, 6414, 6416, 6433, 6448, 6454, 6460, 6583, 6589, 6702, 7006, 7010, 7273, 7335, 7340, 7677, 7678, 7722, 7775, 7807, 7861, 7903, 7950, 7975, 8123, 8201, 8288, 8359, 8373, 8392, 8497, 8502, 8566, 8613, 8635, 8720, 8827, 8847, 8873, 9079, 9374, 9394, 9404, 9486, 9587, 9796, 9859, 9958, 10054, 10101, 10105, 10120, 10135, 10180, 10234, 10246, 10299, 10400, 10777, 11213, 11361, 11776, 11888, 12054, 12133, 12506, 12957, 12959, 12985, 13046, 13054, 13189, 13299, 13316, 13324, 13377, 13657, 13734, 14563, 14651, 14666, 14681, 14688, 14700, 14729, 14849, 14983, 14985, 15003, 15046, 15049, 15052, 15056, 15077, 15156, 15249, 15558, 15583, 15725, 15761, 15770, 15810, 16278, 16588, 17123, 17223,},
+ {4, 235, 257, 261, 379, 394, 403, 449, 469, 621, 750, 758, 789, 790, 806, 820, 889, 910, 912, 921, 961, 992, 1000, 1005, 1012, 1036, 1153, 1155, 1176, 1394, 1407, 1412, 1450, 1454, 1475, 1486, 1594, 1633, 1650, 1654, 1669, 1675, 1686, 1766, 1871, 1879, 1939, 1983, 2023, 2056, 2197, 2226, 2255, 2332, 2459, 2491, 2507, 2513, 2526, 2538, 2545, 2546, 2567, 2591, 2592, 2749, 2825, 2834, 2843, 2849, 2920, 3013, 3024, 3061, 3062, 3183, 3219, 3319, 3503, 3657, 3667, 3692, 3728, 3740, 3751, 3804, 3807, 3860, 4022, 4112, 4120, 4123, 4135, 4262, 4343, 4375, 4388, 4444, 4467, 4588, 4762, 4829, 4927, 5107, 5109, 5117, 5241, 5288, 5411, 5558, 5654, 5675, 5710, 5744, 5760, 5778, 5781, 5823, 5893, 5974, 5986, 6000, 6009, 6012, 6016, 6067, 6114, 6192, 6222, 6253, 6259, 6287, 6308, 6337, 6338, 6349, 6384, 6387, 6416, 6433, 6442, 6454, 6476, 6576, 6589, 6619, 6719, 6727, 6875, 7084, 7221, 7335, 7340, 7355, 7619, 7670, 7775, 7781, 7861, 7961, 8000, 8017, 8191, 8268, 8363, 8412, 8484, 8737, 8833, 8872, 9121, 9125, 9311, 9322, 9359, 9413, 9491, 9532, 9694, 9735, 9895, 9911, 9958, 10105, 10120, 10180, 10299, 10302, 10318, 10327, 10372, 10375, 10378, 10391, 10394, 10400, 10458, 10487, 10497, 10556, 10564, 10569, 10631, 10657, 10662, 10777, 10781, 10827, 10872, 10873, 10923, 10961, 10975, 11043, 11224, 11702, 11776, 12025, 12149, 12318, 12414, 12565, 12734, 12854, 12945, 12971, 12977, 12997, 13008, 13032, 13054, 13064, 13103, 13143, 13170, 13205, 13209, 13220, 13224, 13255, 13299, 13348, 13357, 13377, 13381, 13385, 13516, 13537, 13588, 13626, 13631, 13643, 13669, 13700, 13752, 13788, 13813, 13817, 13914, 13935, 13974, 13999, 14111, 14236, 14544, 14549, 14627, 14688, 14712, 14985, 15012, 15137, 15148, 15155, 15297, 15302, 15386, 15388, 15416, 15418, 15576, 15583, 15584, 15608, 15636, 15679, 15685, 15686, 15690, 15693, 15742, 15753, 15756, 15762, 15783, 15805, 15810, 15819, 15906, 15910, 16093, 16232, 16278, 16479, 17027, 17123, 17223,}
+ };
+
+ // both lists contain common IDs: 3199, 6414, 13357
+
+ std::vector<posting_list_t*> postings2(ulists.size());
+
+ for(size_t i = 0; i < ulists.size(); i++) {
+ postings2[i] = new posting_list_t(2);
+ for(auto n: ulists[i]) {
+ postings2[i]->upsert(n, offsets);
+ }
+ }
+
+ std::vector<posting_list_t::iterator_t> pits1;
+ for(auto& posting_list: postings1) {
+ pits1.push_back(posting_list->new_iterator());
+ }
+
+ std::vector<posting_list_t::iterator_t> pits2;
+ for(auto& posting_list: postings2) {
+ pits2.push_back(posting_list->new_iterator());
+ }
+
+ or_iterator_t it1(pits1);
+ or_iterator_t it2(pits2);
+
+ std::vector<or_iterator_t> or_its;
+ or_its.push_back(std::move(it1));
+ or_its.push_back(std::move(it2));
+
+ result_iter_state_t istate;
+
+ std::vector<uint32_t> results;
+
+ or_iterator_t::intersect(or_its, istate, [&results](uint32_t id, std::vector<or_iterator_t>& its) {
+ results.push_back(id);
+ });
+
+ std::vector<uint32_t> expected_results = {3199, 6414, 13357};
+
+ ASSERT_EQ(expected_results.size(), results.size());
+
+ for(size_t i = 0; i < expected_results.size(); i++) {
+ ASSERT_EQ(expected_results[i], results[i]);
+ }
+}
\ No newline at end of file
diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cpp
index b78186e6..7c41028b 100644
--- a/test/posting_list_test.cpp
+++ b/test/posting_list_test.cpp
@@ -630,7 +630,7 @@ TEST_F(PostingListTest, SplittingOfListsSimple) {
std::vector<void*> raw_lists = {&p1, &p2, &p3};
std::vector<posting_list_t::iterator_t> its;
- posting_list_t::result_iter_state_t iter_state;
+ result_iter_state_t iter_state;
posting_t::block_intersector_t intersector(raw_lists, iter_state, pool);
std::vector<std::vector<posting_list_t::iterator_t>> partial_its_vec(4);
@@ -702,7 +702,7 @@ TEST_F(PostingListTest, IntersectionBasics) {
ASSERT_EQ(20, result_ids[1]);
std::vector<posting_list_t::iterator_t> its;
- posting_list_t::result_iter_state_t iter_state;
+ result_iter_state_t iter_state;
result_ids.clear();
posting_t::block_intersector_t(raw_lists, iter_state, pool).intersect([&](auto id, auto& its, size_t index){
@@ -727,7 +727,7 @@ TEST_F(PostingListTest, IntersectionBasics) {
ASSERT_EQ(expected_ids[i], result_ids[i]);
}
- posting_list_t::result_iter_state_t iter_state2;
+ result_iter_state_t iter_state2;
result_ids.clear();
raw_lists = {&p1};
@@ -750,7 +750,7 @@ TEST_F(PostingListTest, IntersectionBasics) {
posting_list_t::intersect(empty_list, result_ids);
ASSERT_EQ(0, result_ids.size());
- posting_list_t::result_iter_state_t iter_state3;
+ result_iter_state_t iter_state3;
result_ids.clear();
raw_lists.clear();
@@ -816,7 +816,7 @@ TEST_F(PostingListTest, ResultsAndOffsetsBasics) {
/*
std::vector<posting_list_t::iterator_t> its;
- posting_list_t::result_iter_state_t iter_state;
+ result_iter_state_t iter_state;
bool has_more = posting_list_t::block_intersect(lists, 2, its, iter_state);
ASSERT_FALSE(has_more);
@@ -1364,7 +1364,7 @@ TEST_F(PostingListTest, BlockIntersectionOnMixedLists) {
p1.upsert(20, offsets1);
std::vector<void*> raw_posting_lists = {SET_COMPACT_POSTING(list1), &p1};
- posting_list_t::result_iter_state_t iter_state;
+ result_iter_state_t iter_state;
std::vector<uint32_t> result_ids;
std::mutex vecm;