Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Remove old search_field API and code.

Kishore Nallan1b5846fe4966
diff --git a/include/index.h b/include/index.h
index 6b63456a..1c8b7460 100644
--- a/include/index.h
+++ b/include/index.h
@@ -89,13 +89,15 @@ enum enable_t {
struct search_field_t {
std::string name;
+ std::string str_name; // for lookup of non-string fields in art index
size_t weight;
size_t num_typos;
bool prefix;
enable_t infix;
- search_field_t(const std::string& name, size_t weight, size_t num_typos, bool prefix, enable_t infix):
- name(name), weight(weight), num_typos(num_typos), prefix(prefix), infix(infix) { }
+ search_field_t(const std::string& name, const std::string& str_name, size_t weight, size_t num_typos,
+ bool prefix, enable_t infix):
+ name(name), str_name(str_name), weight(weight), num_typos(num_typos), prefix(prefix), infix(infix) { }
};
enum text_match_type_t {
@@ -472,37 +474,6 @@ private:
static void aggregate_topster(Topster* agg_topster, Topster* index_topster);
- void search_field(const uint8_t & field_id,
- const std::vector<token_t>& query_tokens,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- size_t& num_tokens_dropped,
- const field& the_field, const std::string& field_name,
- const uint32_t *filter_ids, size_t filter_ids_length,
- const std::vector<uint32_t>& curated_ids,
- std::vector<sort_by> & sort_fields,
- int last_typo,
- int max_typos,
- std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t** all_result_ids, size_t & all_result_ids_len,
- size_t& field_num_results,
- size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- bool prioritize_exact_match,
- size_t concurrency,
- std::set<uint64>& query_hashes,
- token_ordering token_order, const bool prefix,
- size_t drop_tokens_threshold,
- size_t typo_tokens_threshold,
- bool exhaustive_search,
- int syn_orig_num_tokens,
- size_t min_len_1typo,
- size_t min_len_2typo,
- size_t max_candidates,
- bool enable_typos_for_numerical_tokens) const;
-
Option<bool> search_all_candidates(const size_t num_search_fields,
const text_match_type_t match_type,
const std::vector<search_field_t>& the_fields,
@@ -536,28 +507,6 @@ private:
std::set<uint64>& query_hashes,
std::vector<uint32_t>& id_buff, const std::string& collection_name = "") const;
- void search_candidates(const uint8_t & field_id,
- bool field_is_array,
- const uint32_t* filter_ids, size_t filter_ids_length,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::vector<uint32_t>& curated_ids,
- std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,
- std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t** all_result_ids,
- size_t & all_result_ids_len,
- size_t& field_num_results,
- const size_t typo_tokens_threshold,
- const size_t group_limit, const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const std::vector<token_t>& query_tokens,
- bool prioritize_exact_match,
- bool exhaustive_search,
- int syn_orig_num_tokens,
- size_t concurrency,
- std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const;
-
static void popular_fields_of_token(const spp::sparse_hash_map<std::string, art_tree*>& search_index,
const std::string& previous_token,
const std::vector<search_field_t>& the_fields,
@@ -840,13 +789,13 @@ public:
static void remove_matched_tokens(std::vector<std::string>& tokens, const std::set<std::string>& rule_token_set) ;
void compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,
- const size_t facet_query_num_typos,
- const uint32_t* all_result_ids, const size_t& all_result_ids_len,
+ const uint32_t facet_query_num_typos,
+ uint32_t* all_result_ids, const size_t& all_result_ids_len,
const std::vector<std::string>& group_by_fields,
size_t group_limit, bool is_wildcard_no_filter_query,
size_t max_candidates,
- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,
- bool enable_typos_for_numerical_tokens) const;
+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type
+ ) const;
void resolve_space_as_typos(std::vector<std::string>& qtokens, const std::string& field_name,
std::vector<std::vector<std::string>>& resolved_queries) const;
diff --git a/src/collection.cpp b/src/collection.cpp
index 61c0d127..125c355e 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1942,7 +1942,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
auto prefix = i < prefixes.size() ? prefixes[i] : prefixes[0];
auto infix = i < infixes.size() ? infixes[i] : infixes[0];
- processed_search_fields.emplace_back(expanded_search_field, query_weight, num_typo, prefix, infix);
+ processed_search_fields.emplace_back(expanded_search_field, search_field.faceted_name(),
+ query_weight, num_typo, prefix, infix);
if(!raw_query_by_weights.empty()) {
query_by_weights.push_back(query_weight);
}
diff --git a/src/index.cpp b/src/index.cpp
index daf21c1b..75a38ffd 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1673,138 +1673,6 @@ Option<bool> Index::search_all_candidates(const size_t num_search_fields,
return Option<bool>(true);
}
-void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
- const uint32_t* filter_ids, size_t filter_ids_length,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::vector<uint32_t>& curated_ids,
- std::vector<sort_by> & sort_fields,
- std::vector<token_candidates> & token_candidates_vec,
- std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t** all_result_ids, size_t & all_result_ids_len,
- size_t& field_num_results,
- const size_t typo_tokens_threshold,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const std::vector<token_t>& query_tokens,
- bool prioritize_exact_match,
- const bool exhaustive_search,
- int syn_orig_num_tokens,
- const size_t concurrency,
- std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const {
-
- auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };
- long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);
-
- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
- std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values;
- std::vector<size_t> geopoint_indices;
-
- populate_sort_mapping(sort_order, geopoint_indices, sort_fields, field_values);
-
- // escape hatch to prevent too much looping
- size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
-
- for (long long n = 0; n < N && n < combination_limit; ++n) {
- RETURN_CIRCUIT_BREAKER
-
- // every element in `query_suggestion` contains a token and its associated hits
- std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());
-
- // actual query suggestion preserves original order of tokens in query
- std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());
- uint64 qhash;
-
- uint32_t token_bits = 0;
- uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,
- query_suggestion, syn_orig_num_tokens, token_bits, qhash);
-
- if(query_hashes.find(qhash) != query_hashes.end()) {
- // skip this query since it has already been processed before
- continue;
- }
-
- query_hashes.insert(qhash);
-
- //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
- //LOG(INFO) << "n: " << n;
-
- /*std::stringstream fullq;
- for(const auto& qleaf : actual_query_suggestion) {
- std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);
- fullq << qtok << " ";
- }
- LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;*/
-
- // Prepare excluded document IDs that we can later remove from the result set
- uint32_t* excluded_result_ids = nullptr;
- size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
- &curated_ids[0], curated_ids.size(), &excluded_result_ids);
-
- std::vector<void*> posting_lists;
-
- for(auto& query_leaf : query_suggestion) {
- posting_lists.push_back(query_leaf->values);
- }
-
- result_iter_state_t iter_state(
- excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length
- );
-
- // We fetch offset positions only for multi token query
- bool single_exact_query_token = false;
-
- if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {
- // does this candidate suggestion token match query token exactly?
- single_exact_query_token = true;
- }
-
- if(topster == nullptr) {
- posting_t::block_intersector_t(posting_lists, iter_state)
- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {
- id_buff.push_back(seq_id);
- });
- } else {
- posting_t::block_intersector_t(posting_lists, iter_state)
- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {
- score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
- total_cost, topster, query_suggestion, groups_processed,
- seq_id, sort_order, field_values, geopoint_indices,
- group_limit, group_by_fields, group_missing_values, token_bits,
- prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);
-
- id_buff.push_back(seq_id);
- });
- }
-
- delete [] excluded_result_ids;
- const size_t num_result_ids = id_buff.size();
-
- if(id_buff.size() > 100000) {
- // prevents too many ORs during exhaustive searching
- std::sort(id_buff.begin(), id_buff.end());
- id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());
-
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],
- id_buff.size(), &new_all_result_ids);
- delete[] *all_result_ids;
- *all_result_ids = new_all_result_ids;
- id_buff.clear();
- }
-
- if(num_result_ids == 0) {
- continue;
- }
-
- field_num_results += num_result_ids;
- searched_queries.push_back(actual_query_suggestion);
- }
-}
-
bool Index::field_is_indexed(const std::string& field_name) const {
return search_index.count(field_name) != 0 ||
numerical_index.count(field_name) != 0 ||
@@ -2693,14 +2561,28 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&
}
std::vector<sort_by> sort_fields;
- search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it.value(), field_name,
- nullptr, 0, {}, sort_fields, -1, 0, searched_queries, topster, groups_processed,
- &result_ids, result_ids_len, field_num_results, 0, group_by_fields,
- true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4,
- enable_typos_for_numerical_tokens);
+ std::vector<search_field_t> fq_fields;
+ fq_fields.emplace_back(field_name, field_it.value().faceted_name(), 1, 0, true, enable_t::off);
+
+ uint32_t* filter_ids = nullptr;
+ filter_result_iterator_t filter_result_it(filter_ids, 0);
+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values{};
+ const std::vector<size_t> geopoint_indices;
+ tsl::htrie_map<char, token_leaf> qtoken_set;
+
+ auto fuzzy_search_fields_op = fuzzy_search_fields(
+ fq_fields, window_tokens, {}, text_match_type_t::max_score, nullptr, 0,
+ &filter_result_it, {}, {}, sort_fields, {0}, searched_queries,
+ qtoken_set, topster, groups_processed, result_ids, result_ids_len,
+ 0, group_by_fields, false, true, false, false, query_hashes, MAX_SCORE, {true}, 1,
+ false, 4, 3, 7, 0, nullptr, field_values, geopoint_indices, "", true);
+
+ if(!fuzzy_search_fields_op.ok()) {
+ continue;
+ }
if(result_ids_len != 0) {
- // we need to narraw onto the exact matches
+ // we need to narrow onto the exact matches
std::vector<void*> posting_lists;
art_tree* t = search_index.at(field_name);
@@ -3648,7 +3530,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<facet_info_t> facet_infos(facets.size());
compute_facet_infos(facets, facet_query, facet_query_num_typos, all_result_ids, all_result_ids_len,
group_by_fields, group_limit, is_wildcard_no_filter_query,
- max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);
+ max_candidates, facet_infos, facet_index_type);
std::vector<std::vector<facet>> facet_batches(num_threads);
std::vector<facet> value_facets;
@@ -3777,7 +3659,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
compute_facet_infos(facets, facet_query, facet_query_num_typos,
&included_ids_vec[0], included_ids_vec.size(), group_by_fields,
group_limit, is_wildcard_no_filter_query,
- max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);
+ max_candidates, facet_infos, facet_index_type);
do_facets(facets, facet_query, estimate_facets, facet_sample_percent,
facet_infos, group_limit, group_by_fields, group_missing_values, &included_ids_vec[0],
included_ids_vec.size(), max_facet_values, is_wildcard_no_filter_query,
@@ -4103,7 +3985,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
int64_t field_num_typos = the_fields[field_id].num_typos;
- auto& locale = search_schema.at(the_field.name).locale;
+ const auto& search_field = search_schema.at(the_field.name);
+ auto& locale = search_field.locale;
if(locale != "" && (locale == "zh" || locale == "ko" || locale == "ja")) {
// disable fuzzy trie traversal for CJK locales
field_num_typos = 0;
@@ -4117,7 +4000,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
const auto& prev_token = last_token ? token_candidates_vec.back().candidates[0] : "";
std::vector<art_leaf*> field_leaves;
- art_fuzzy_search_i(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,
+ art_fuzzy_search_i(search_index.at(search_field.faceted_name()),
+ (const unsigned char *) token.c_str(), token_len,
costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,
last_token, prev_token, filter_result_iterator, field_leaves, unique_tokens);
filter_result_iterator->reset();
@@ -4165,7 +4049,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
int64_t field_num_typos = the_fields[field_id].num_typos;
- auto& locale = search_schema.at(the_field.name).locale;
+ const auto& search_field = search_schema.at(the_field.name);
+ auto& locale = search_field.locale;
if(locale != "" && locale != "en" && locale != "th" && !Tokenizer::is_cyrillic(locale)) {
// disable fuzzy trie traversal for non-english locales
field_num_typos = 0;
@@ -4438,7 +4323,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
for(size_t i = 0; i < the_fields.size(); i++) {
const std::string& field_name = the_fields[i].name;
- art_tree* tree = search_index.at(field_name);
+ art_tree* tree = search_index.at(the_fields[i].str_name);
art_leaf* leaf = static_cast<art_leaf*>(art_search(tree, token_c_str, token.size()+1));
if(!leaf) {
@@ -4497,7 +4382,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
continue;
}
- art_tree* tree = search_index.at(field_name);
+ art_tree* tree = search_index.at(the_fields[i].str_name);
art_leaf* leaf = static_cast<art_leaf*>(art_search(tree, token_c_str, token_len));
if(!leaf) {
@@ -4539,6 +4424,12 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
or_iterator_t::intersect(token_its, istate,
[&](single_filter_result_t& filter_result, const std::vector<or_iterator_t>& its) {
auto& seq_id = filter_result.seq_id;
+
+ if(topster == nullptr) {
+ result_ids.push_back(seq_id);
+ return ;
+ }
+
auto references = std::move(filter_result.reference_filter_results);
//LOG(INFO) << "seq_id: " << seq_id;
// Convert [token -> fields] orientation to [field -> tokens] orientation
@@ -4695,6 +4586,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
}
result_ids.push_back(seq_id);
});
+
if (!status.ok()) {
for(posting_list_t* plist: expanded_plists) {
delete plist;
@@ -5745,13 +5637,12 @@ void Index::handle_exclusion(const size_t num_search_fields, std::vector<query_t
}
void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,
- const size_t facet_query_num_typos,
- const uint32_t* all_result_ids, const size_t& all_result_ids_len,
+ const uint32_t facet_query_num_typos,
+ uint32_t* all_result_ids, const size_t& all_result_ids_len,
const std::vector<std::string>& group_by_fields,
const size_t group_limit, const bool is_wildcard_no_filter_query,
const size_t max_candidates,
- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,
- bool enable_typos_for_numerical_tokens) const {
+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const {
if(all_result_ids_len == 0) {
return;
@@ -5814,12 +5705,26 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
size_t num_toks_dropped = 0;
std::vector<sort_by> sort_fields;
- search_field(0, qtokens, nullptr, 0, num_toks_dropped,
- facet_field, facet_field.faceted_name(),
- all_result_ids, all_result_ids_len, {}, sort_fields, -1, facet_query_num_typos, searched_queries, topster,
- groups_processed, &field_result_ids, field_result_ids_len, field_num_results, 0, group_by_fields,
- true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates,
- enable_typos_for_numerical_tokens);
+ std::vector<search_field_t> fq_fields;
+ fq_fields.emplace_back(facet_field.name, facet_field.faceted_name(), 1, facet_query_num_typos,
+ true, enable_t::off);
+
+ uint32_t* filter_ids = new uint32_t[all_result_ids_len];
+ std::copy(all_result_ids, all_result_ids + all_result_ids_len, filter_ids);
+ filter_result_iterator_t filter_result_it(filter_ids, all_result_ids_len);
+ tsl::htrie_map<char, token_leaf> qtoken_set;
+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values{};
+ const std::vector<size_t> geopoint_indices;
+
+ auto fuzzy_search_fields_op = fuzzy_search_fields(fq_fields, qtokens, {}, text_match_type_t::max_score, nullptr, 0,
+ &filter_result_it, {}, {}, sort_fields, {facet_query_num_typos}, searched_queries,
+ qtoken_set, topster, groups_processed, field_result_ids, field_result_ids_len,
+ group_limit, group_by_fields, false, true, false, false, query_hashes, MAX_SCORE, {true}, 1,
+ false, max_candidates, 3, 7, 0, nullptr, field_values, geopoint_indices, "", true);
+
+ if(!fuzzy_search_fields_op.ok()) {
+ continue;
+ }
//LOG(INFO) << "searched_queries.size: " << searched_queries.size();
@@ -6212,181 +6117,6 @@ void Index::populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>
populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);
}
-void Index::search_field(const uint8_t & field_id,
- const std::vector<token_t>& query_tokens,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- size_t& num_tokens_dropped,
- const field& the_field, const std::string& field_name, // to handle faceted index
- const uint32_t *filter_ids, size_t filter_ids_length,
- const std::vector<uint32_t>& curated_ids,
- std::vector<sort_by> & sort_fields,
- const int last_typo,
- const int max_typos,
- std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,
- const size_t group_limit, const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- bool prioritize_exact_match,
- const size_t concurrency,
- std::set<uint64>& query_hashes,
- const token_ordering token_order, const bool prefix,
- const size_t drop_tokens_threshold,
- const size_t typo_tokens_threshold,
- const bool exhaustive_search,
- int syn_orig_num_tokens,
- size_t min_len_1typo,
- size_t min_len_2typo,
- const size_t max_candidates,
- bool enable_typos_for_numerical_tokens) const {
-
- // NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
-
- size_t max_cost = (max_typos < 0 || max_typos > 2) ? 2 : max_typos;
-
- if(the_field.locale != "" && the_field.locale != "en" && !Tokenizer::is_cyrillic(the_field.locale)) {
- // disable fuzzy trie traversal for certain non-english locales
- max_cost = 0;
- }
-
- // To prevent us from doing ART search repeatedly as we iterate through possible corrections
- spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
-
- std::vector<std::vector<int>> token_to_costs;
-
- for(size_t stoken_index=0; stoken_index < query_tokens.size(); stoken_index++) {
- const std::string& token = query_tokens[stoken_index].value;
-
- std::vector<int> all_costs;
- // This ensures that we don't end up doing a cost of 1 for a single char etc.
- int bounded_cost = get_bounded_typo_cost(max_cost, token, token.length(), min_len_1typo, min_len_2typo,
- enable_typos_for_numerical_tokens);
-
- for(int cost = 0; cost <= bounded_cost; cost++) {
- all_costs.push_back(cost);
- }
-
- token_to_costs.push_back(all_costs);
- }
-
- // stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"
- std::vector<token_candidates> token_candidates_vec;
-
- std::set<std::string> unique_tokens;
-
- auto product = []( long long a, std::vector<int>& b ) { return a*b.size(); };
- long long n = 0;
- long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
-
- const size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
-
- while(n < N && n < combination_limit) {
- RETURN_CIRCUIT_BREAKER
-
- // Outerloop generates combinations of [cost to max_cost] for each token
- // For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.
- std::vector<uint32_t> costs(token_to_costs.size());
- ldiv_t q { n, 0 };
- bool valid_combo = false;
-
- for(long long i = (token_to_costs.size() - 1); 0 <= i ; --i ) {
- q = ldiv(q.quot, token_to_costs[i].size());
- costs[i] = token_to_costs[i][q.rem];
- if(costs[i] == uint32_t(last_typo+1)) {
- // to support progressive typo searching, there must be atleast one typo that's greater than last_typo
- valid_combo = true;
- }
- }
-
- if(last_typo != -1 && !valid_combo) {
- n++;
- continue;
- }
-
- unique_tokens.clear();
- token_candidates_vec.clear();
- size_t token_index = 0;
-
- while(token_index < query_tokens.size()) {
- // For each token, look up the generated cost for this iteration and search using that cost
- const std::string& token = query_tokens[token_index].value;
- const std::string token_cost_hash = token + std::to_string(costs[token_index]);
-
- std::vector<art_leaf*> leaves;
- const bool prefix_search = prefix && query_tokens[token_index].is_prefix_searched;
-
- /*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"
- << token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/
-
- if(token_cost_cache.count(token_cost_hash) != 0) {
- leaves = token_cost_cache[token_cost_hash];
- } else {
- const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
-
- //auto begin = std::chrono::high_resolution_clock::now();
-
- // need less candidates for filtered searches since we already only pick tokens with results
- art_fuzzy_search(search_index.at(field_name), (const unsigned char *) token.c_str(), token_len,
- costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,
- false, "", filter_ids, filter_ids_length, leaves, unique_tokens);
-
- /*auto timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - begin).count();
- LOG(INFO) << "Time taken for fuzzy search: " << timeMillis << "ms";*/
-
- if(!leaves.empty()) {
- token_cost_cache.emplace(token_cost_hash, leaves);
- for(auto leaf: leaves) {
- std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);
- unique_tokens.emplace(tok);
- }
- }
- }
-
- if(!leaves.empty()) {
- //log_leaves(costs[token_index], token, leaves);
- token_candidates_vec.push_back(
- token_candidates{query_tokens[token_index], costs[token_index], prefix_search, leaves});
- }
-
- token_index++;
- }
-
- if(token_candidates_vec.size() == query_tokens.size()) {
- std::vector<uint32_t> id_buff;
-
- // If all tokens are, go ahead and search for candidates
- search_candidates(field_id, the_field.is_array(), filter_ids, filter_ids_length,
- exclude_token_ids, exclude_token_ids_size,
- curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,
- groups_processed, all_result_ids, all_result_ids_len, field_num_results,
- typo_tokens_threshold, group_limit, group_by_fields,
- group_missing_values, query_tokens,
- prioritize_exact_match, exhaustive_search, syn_orig_num_tokens,
- concurrency, query_hashes, id_buff);
-
- if(id_buff.size() > 1) {
- std::sort(id_buff.begin(), id_buff.end());
- id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());
- }
-
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],
- id_buff.size(), &new_all_result_ids);
- delete[] *all_result_ids;
- *all_result_ids = new_all_result_ids;
- }
-
- if(!exhaustive_search && field_num_results >= typo_tokens_threshold) {
- // if typo threshold is breached, we are done
- return ;
- }
-
- n++;
- }
-}
-
int Index::get_bounded_typo_cost(const size_t max_cost, const std::string& token, const size_t token_len,
const size_t min_len_1typo, const size_t min_len_2typo,
bool enable_typos_for_numerical_tokens) {
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 615db51b..d29e6112 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -1028,9 +1028,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
std::vector<search_field_t> raw_search_fields = {
- search_field_t("title", 110, 2, true, off),
- search_field_t("brand", 25, 2, true, off),
- search_field_t("type", 55, 2, true, off),
+ search_field_t("title", "title", 110, 2, true, off),
+ search_field_t("brand", "brand", 25, 2, true, off),
+ search_field_t("type", "type", 55, 2, true, off),
};
std::vector<uint32_t> query_by_weights = {110, 25, 55};
std::vector<search_field_t> weighted_search_fields;
@@ -1051,9 +1051,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
weighted_search_fields.clear();
query_by_weights = {15, 15, 15};
raw_search_fields = {
- search_field_t{"title", 15, 2, true, off},
- search_field_t{"brand", 15, 2, true, off},
- search_field_t{"type", 15, 2, true, off},
+ search_field_t{"title", "title", 15, 2, true, off},
+ search_field_t{"brand", "brand", 15, 2, true, off},
+ search_field_t{"type", "type", 15, 2, true, off},
};
coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
@@ -1070,9 +1070,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
weighted_search_fields.clear();
query_by_weights = {800, 800, 800};
raw_search_fields = {
- search_field_t{"title", 800, 2, true, off},
- search_field_t{"brand", 800, 2, true, off},
- search_field_t{"type", 800, 2, true, off},
+ search_field_t{"title", "title", 800, 2, true, off},
+ search_field_t{"brand", "brand", 800, 2, true, off},
+ search_field_t{"type", "type", 800, 2, true, off},
};
coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
@@ -1089,9 +1089,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
weighted_search_fields.clear();
query_by_weights = {603, 602, 601};
raw_search_fields = {
- search_field_t{"title", 603, 2, true, off},
- search_field_t{"brand", 602, 2, true, off},
- search_field_t{"type", 601, 2, true, off},
+ search_field_t{"title", "title", 603, 2, true, off},
+ search_field_t{"brand", "brand", 602, 2, true, off},
+ search_field_t{"type", "type", 601, 2, true, off},
};
coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
@@ -1110,7 +1110,8 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
query_by_weights.clear();
for(size_t i = 0; i < 17; i++) {
- raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 17 - i, 2, true, off});
+ auto fname = "field" + std::to_string(17 - i);
+ raw_search_fields.push_back(search_field_t{fname, fname, 17 - i, 2, true, off});
query_by_weights.push_back(17 - i);
}
@@ -1130,7 +1131,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
query_by_weights.clear();
for(size_t i = 0; i < 17; i++) {
- raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 0, 2, true, off});
+ auto field_name = "field" + std::to_string(17 - i);
+ raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i),
+ field_name, 0, 2, true, off});
}
coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);