Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Remove old search_field API and code.
Kishore Nallan1b5846fe4966
diff --git a/include/index.h b/include/index.hindex 6b63456a..1c8b7460 100644--- a/include/index.h+++ b/include/index.h@@ -89,13 +89,15 @@ enum enable_t {struct search_field_t {std::string name;+ std::string str_name; // for lookup of non-string fields in art indexsize_t weight;size_t num_typos;bool prefix;enable_t infix;- search_field_t(const std::string& name, size_t weight, size_t num_typos, bool prefix, enable_t infix):- name(name), weight(weight), num_typos(num_typos), prefix(prefix), infix(infix) { }+ search_field_t(const std::string& name, const std::string& str_name, size_t weight, size_t num_typos,+ bool prefix, enable_t infix):+ name(name), str_name(str_name), weight(weight), num_typos(num_typos), prefix(prefix), infix(infix) { }};enum text_match_type_t {@@ -472,37 +474,6 @@ private:static void aggregate_topster(Topster* agg_topster, Topster* index_topster);- void search_field(const uint8_t & field_id,- const std::vector<token_t>& query_tokens,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- size_t& num_tokens_dropped,- const field& the_field, const std::string& field_name,- const uint32_t *filter_ids, size_t filter_ids_length,- const std::vector<uint32_t>& curated_ids,- std::vector<sort_by> & sort_fields,- int last_typo,- int max_typos,- std::vector<std::vector<art_leaf*>> & searched_queries,- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t** all_result_ids, size_t & all_result_ids_len,- size_t& field_num_results,- size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- bool prioritize_exact_match,- size_t concurrency,- std::set<uint64>& query_hashes,- token_ordering token_order, const bool prefix,- size_t drop_tokens_threshold,- size_t typo_tokens_threshold,- bool exhaustive_search,- int syn_orig_num_tokens,- size_t min_len_1typo,- size_t min_len_2typo,- size_t max_candidates,- bool enable_typos_for_numerical_tokens) const;-Option<bool> search_all_candidates(const size_t num_search_fields,const text_match_type_t match_type,const std::vector<search_field_t>& the_fields,@@ -536,28 +507,6 @@ private:std::set<uint64>& query_hashes,std::vector<uint32_t>& id_buff, const std::string& collection_name = "") const;- void search_candidates(const uint8_t & field_id,- bool field_is_array,- const uint32_t* filter_ids, size_t filter_ids_length,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::vector<uint32_t>& curated_ids,- std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,- std::vector<std::vector<art_leaf*>> & searched_queries,- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t** all_result_ids,- size_t & all_result_ids_len,- size_t& field_num_results,- const size_t typo_tokens_threshold,- const size_t group_limit, const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const std::vector<token_t>& query_tokens,- bool prioritize_exact_match,- bool exhaustive_search,- int syn_orig_num_tokens,- size_t concurrency,- std::set<uint64>& query_hashes,- std::vector<uint32_t>& id_buff) const;-static void popular_fields_of_token(const spp::sparse_hash_map<std::string, art_tree*>& search_index,const std::string& previous_token,const std::vector<search_field_t>& the_fields,@@ -840,13 +789,13 @@ public:static void remove_matched_tokens(std::vector<std::string>& tokens, const std::set<std::string>& rule_token_set) ;void compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,- const size_t facet_query_num_typos,- const uint32_t* all_result_ids, const size_t& all_result_ids_len,+ const uint32_t facet_query_num_typos,+ uint32_t* all_result_ids, const size_t& all_result_ids_len,const std::vector<std::string>& group_by_fields,size_t group_limit, bool is_wildcard_no_filter_query,size_t max_candidates,- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,- bool enable_typos_for_numerical_tokens) const;+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type+ ) const;void resolve_space_as_typos(std::vector<std::string>& qtokens, const std::string& field_name,std::vector<std::vector<std::string>>& resolved_queries) const;diff --git a/src/collection.cpp b/src/collection.cppindex 61c0d127..125c355e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1942,7 +1942,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,auto prefix = i < prefixes.size() ? prefixes[i] : prefixes[0];auto infix = i < infixes.size() ? infixes[i] : infixes[0];- processed_search_fields.emplace_back(expanded_search_field, query_weight, num_typo, prefix, infix);+ processed_search_fields.emplace_back(expanded_search_field, search_field.faceted_name(),+ query_weight, num_typo, prefix, infix);if(!raw_query_by_weights.empty()) {query_by_weights.push_back(query_weight);}diff --git a/src/index.cpp b/src/index.cppindex daf21c1b..75a38ffd 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1673,138 +1673,6 @@ Option<bool> Index::search_all_candidates(const size_t num_search_fields,return Option<bool>(true);}-void Index::search_candidates(const uint8_t & field_id, bool field_is_array,- const uint32_t* filter_ids, size_t filter_ids_length,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::vector<uint32_t>& curated_ids,- std::vector<sort_by> & sort_fields,- std::vector<token_candidates> & token_candidates_vec,- std::vector<std::vector<art_leaf*>> & searched_queries,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t** all_result_ids, size_t & all_result_ids_len,- size_t& field_num_results,- const size_t typo_tokens_threshold,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const std::vector<token_t>& query_tokens,- bool prioritize_exact_match,- const bool exhaustive_search,- int syn_orig_num_tokens,- const size_t concurrency,- std::set<uint64>& query_hashes,- std::vector<uint32_t>& id_buff) const {-- auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };- long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);-- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively- std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values;- std::vector<size_t> geopoint_indices;-- populate_sort_mapping(sort_order, geopoint_indices, sort_fields, field_values);-- // escape hatch to prevent too much looping- size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;-- for (long long n = 0; n < N && n < combination_limit; ++n) {- RETURN_CIRCUIT_BREAKER-- // every element in `query_suggestion` contains a token and its associated hits- std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());-- // actual query suggestion preserves original order of tokens in query- std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());- uint64 qhash;-- uint32_t token_bits = 0;- uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,- query_suggestion, syn_orig_num_tokens, token_bits, qhash);-- if(query_hashes.find(qhash) != query_hashes.end()) {- // skip this query since it has already been processed before- continue;- }-- query_hashes.insert(qhash);-- //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;- //LOG(INFO) << "n: " << n;-- /*std::stringstream fullq;- for(const auto& qleaf : actual_query_suggestion) {- std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);- fullq << qtok << " ";- }- LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;*/-- // Prepare excluded document IDs that we can later remove from the result set- uint32_t* excluded_result_ids = nullptr;- size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,- &curated_ids[0], curated_ids.size(), &excluded_result_ids);-- std::vector<void*> posting_lists;-- for(auto& query_leaf : query_suggestion) {- posting_lists.push_back(query_leaf->values);- }-- result_iter_state_t iter_state(- excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length- );-- // We fetch offset positions only for multi token query- bool single_exact_query_token = false;-- if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {- // does this candidate suggestion token match query token exactly?- single_exact_query_token = true;- }-- if(topster == nullptr) {- posting_t::block_intersector_t(posting_lists, iter_state)- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {- id_buff.push_back(seq_id);- });- } else {- posting_t::block_intersector_t(posting_lists, iter_state)- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {- score_results(sort_fields, searched_queries.size(), field_id, field_is_array,- total_cost, topster, query_suggestion, groups_processed,- seq_id, sort_order, field_values, geopoint_indices,- group_limit, group_by_fields, group_missing_values, token_bits,- prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);-- id_buff.push_back(seq_id);- });- }-- delete [] excluded_result_ids;- const size_t num_result_ids = id_buff.size();-- if(id_buff.size() > 100000) {- // prevents too many ORs during exhaustive searching- std::sort(id_buff.begin(), id_buff.end());- id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());-- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],- id_buff.size(), &new_all_result_ids);- delete[] *all_result_ids;- *all_result_ids = new_all_result_ids;- id_buff.clear();- }-- if(num_result_ids == 0) {- continue;- }-- field_num_results += num_result_ids;- searched_queries.push_back(actual_query_suggestion);- }-}-bool Index::field_is_indexed(const std::string& field_name) const {return search_index.count(field_name) != 0 ||numerical_index.count(field_name) != 0 ||@@ -2693,14 +2561,28 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&}std::vector<sort_by> sort_fields;- search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it.value(), field_name,- nullptr, 0, {}, sort_fields, -1, 0, searched_queries, topster, groups_processed,- &result_ids, result_ids_len, field_num_results, 0, group_by_fields,- true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4,- enable_typos_for_numerical_tokens);+ std::vector<search_field_t> fq_fields;+ fq_fields.emplace_back(field_name, field_it.value().faceted_name(), 1, 0, true, enable_t::off);++ uint32_t* filter_ids = nullptr;+ filter_result_iterator_t filter_result_it(filter_ids, 0);+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values{};+ const std::vector<size_t> geopoint_indices;+ tsl::htrie_map<char, token_leaf> qtoken_set;++ auto fuzzy_search_fields_op = fuzzy_search_fields(+ fq_fields, window_tokens, {}, text_match_type_t::max_score, nullptr, 0,+ &filter_result_it, {}, {}, sort_fields, {0}, searched_queries,+ qtoken_set, topster, groups_processed, result_ids, result_ids_len,+ 0, group_by_fields, false, true, false, false, query_hashes, MAX_SCORE, {true}, 1,+ false, 4, 3, 7, 0, nullptr, field_values, geopoint_indices, "", true);++ if(!fuzzy_search_fields_op.ok()) {+ continue;+ }if(result_ids_len != 0) {- // we need to narraw onto the exact matches+ // we need to narrow onto the exact matchesstd::vector<void*> posting_lists;art_tree* t = search_index.at(field_name);@@ -3648,7 +3530,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<facet_info_t> facet_infos(facets.size());compute_facet_infos(facets, facet_query, facet_query_num_typos, all_result_ids, all_result_ids_len,group_by_fields, group_limit, is_wildcard_no_filter_query,- max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);+ max_candidates, facet_infos, facet_index_type);std::vector<std::vector<facet>> facet_batches(num_threads);std::vector<facet> value_facets;@@ -3777,7 +3659,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscompute_facet_infos(facets, facet_query, facet_query_num_typos,&included_ids_vec[0], included_ids_vec.size(), group_by_fields,group_limit, is_wildcard_no_filter_query,- max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);+ max_candidates, facet_infos, facet_index_type);do_facets(facets, facet_query, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields, group_missing_values, &included_ids_vec[0],included_ids_vec.size(), max_facet_values, is_wildcard_no_filter_query,@@ -4103,7 +3985,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fint64_t field_num_typos = the_fields[field_id].num_typos;- auto& locale = search_schema.at(the_field.name).locale;+ const auto& search_field = search_schema.at(the_field.name);+ auto& locale = search_field.locale;if(locale != "" && (locale == "zh" || locale == "ko" || locale == "ja")) {// disable fuzzy trie traversal for CJK localesfield_num_typos = 0;@@ -4117,7 +4000,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fconst auto& prev_token = last_token ? token_candidates_vec.back().candidates[0] : "";std::vector<art_leaf*> field_leaves;- art_fuzzy_search_i(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,+ art_fuzzy_search_i(search_index.at(search_field.faceted_name()),+ (const unsigned char *) token.c_str(), token_len,costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,last_token, prev_token, filter_result_iterator, field_leaves, unique_tokens);filter_result_iterator->reset();@@ -4165,7 +4049,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fconst size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;int64_t field_num_typos = the_fields[field_id].num_typos;- auto& locale = search_schema.at(the_field.name).locale;+ const auto& search_field = search_schema.at(the_field.name);+ auto& locale = search_field.locale;if(locale != "" && locale != "en" && locale != "th" && !Tokenizer::is_cyrillic(locale)) {// disable fuzzy trie traversal for non-english localesfield_num_typos = 0;@@ -4438,7 +4323,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokenfor(size_t i = 0; i < the_fields.size(); i++) {const std::string& field_name = the_fields[i].name;- art_tree* tree = search_index.at(field_name);+ art_tree* tree = search_index.at(the_fields[i].str_name);art_leaf* leaf = static_cast<art_leaf*>(art_search(tree, token_c_str, token.size()+1));if(!leaf) {@@ -4497,7 +4382,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokencontinue;}- art_tree* tree = search_index.at(field_name);+ art_tree* tree = search_index.at(the_fields[i].str_name);art_leaf* leaf = static_cast<art_leaf*>(art_search(tree, token_c_str, token_len));if(!leaf) {@@ -4539,6 +4424,12 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokenor_iterator_t::intersect(token_its, istate,[&](single_filter_result_t& filter_result, const std::vector<or_iterator_t>& its) {auto& seq_id = filter_result.seq_id;++ if(topster == nullptr) {+ result_ids.push_back(seq_id);+ return ;+ }+auto references = std::move(filter_result.reference_filter_results);//LOG(INFO) << "seq_id: " << seq_id;// Convert [token -> fields] orientation to [field -> tokens] orientation@@ -4695,6 +4586,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token}result_ids.push_back(seq_id);});+if (!status.ok()) {for(posting_list_t* plist: expanded_plists) {delete plist;@@ -5745,13 +5637,12 @@ void Index::handle_exclusion(const size_t num_search_fields, std::vector<query_t}void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,- const size_t facet_query_num_typos,- const uint32_t* all_result_ids, const size_t& all_result_ids_len,+ const uint32_t facet_query_num_typos,+ uint32_t* all_result_ids, const size_t& all_result_ids_len,const std::vector<std::string>& group_by_fields,const size_t group_limit, const bool is_wildcard_no_filter_query,const size_t max_candidates,- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,- bool enable_typos_for_numerical_tokens) const {+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const {if(all_result_ids_len == 0) {return;@@ -5814,12 +5705,26 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&size_t num_toks_dropped = 0;std::vector<sort_by> sort_fields;- search_field(0, qtokens, nullptr, 0, num_toks_dropped,- facet_field, facet_field.faceted_name(),- all_result_ids, all_result_ids_len, {}, sort_fields, -1, facet_query_num_typos, searched_queries, topster,- groups_processed, &field_result_ids, field_result_ids_len, field_num_results, 0, group_by_fields,- true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates,- enable_typos_for_numerical_tokens);+ std::vector<search_field_t> fq_fields;+ fq_fields.emplace_back(facet_field.name, facet_field.faceted_name(), 1, facet_query_num_typos,+ true, enable_t::off);++ uint32_t* filter_ids = new uint32_t[all_result_ids_len];+ std::copy(all_result_ids, all_result_ids + all_result_ids_len, filter_ids);+ filter_result_iterator_t filter_result_it(filter_ids, all_result_ids_len);+ tsl::htrie_map<char, token_leaf> qtoken_set;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values{};+ const std::vector<size_t> geopoint_indices;++ auto fuzzy_search_fields_op = fuzzy_search_fields(fq_fields, qtokens, {}, text_match_type_t::max_score, nullptr, 0,+ &filter_result_it, {}, {}, sort_fields, {facet_query_num_typos}, searched_queries,+ qtoken_set, topster, groups_processed, field_result_ids, field_result_ids_len,+ group_limit, group_by_fields, false, true, false, false, query_hashes, MAX_SCORE, {true}, 1,+ false, max_candidates, 3, 7, 0, nullptr, field_values, geopoint_indices, "", true);++ if(!fuzzy_search_fields_op.ok()) {+ continue;+ }//LOG(INFO) << "searched_queries.size: " << searched_queries.size();@@ -6212,181 +6117,6 @@ void Index::populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);}-void Index::search_field(const uint8_t & field_id,- const std::vector<token_t>& query_tokens,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- size_t& num_tokens_dropped,- const field& the_field, const std::string& field_name, // to handle faceted index- const uint32_t *filter_ids, size_t filter_ids_length,- const std::vector<uint32_t>& curated_ids,- std::vector<sort_by> & sort_fields,- const int last_typo,- const int max_typos,- std::vector<std::vector<art_leaf*>> & searched_queries,- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,- const size_t group_limit, const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- bool prioritize_exact_match,- const size_t concurrency,- std::set<uint64>& query_hashes,- const token_ordering token_order, const bool prefix,- const size_t drop_tokens_threshold,- const size_t typo_tokens_threshold,- const bool exhaustive_search,- int syn_orig_num_tokens,- size_t min_len_1typo,- size_t min_len_2typo,- const size_t max_candidates,- bool enable_typos_for_numerical_tokens) const {-- // NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens-- size_t max_cost = (max_typos < 0 || max_typos > 2) ? 2 : max_typos;-- if(the_field.locale != "" && the_field.locale != "en" && !Tokenizer::is_cyrillic(the_field.locale)) {- // disable fuzzy trie traversal for certain non-english locales- max_cost = 0;- }-- // To prevent us from doing ART search repeatedly as we iterate through possible corrections- spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;-- std::vector<std::vector<int>> token_to_costs;-- for(size_t stoken_index=0; stoken_index < query_tokens.size(); stoken_index++) {- const std::string& token = query_tokens[stoken_index].value;-- std::vector<int> all_costs;- // This ensures that we don't end up doing a cost of 1 for a single char etc.- int bounded_cost = get_bounded_typo_cost(max_cost, token, token.length(), min_len_1typo, min_len_2typo,- enable_typos_for_numerical_tokens);-- for(int cost = 0; cost <= bounded_cost; cost++) {- all_costs.push_back(cost);- }-- token_to_costs.push_back(all_costs);- }-- // stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"- std::vector<token_candidates> token_candidates_vec;-- std::set<std::string> unique_tokens;-- auto product = []( long long a, std::vector<int>& b ) { return a*b.size(); };- long long n = 0;- long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);-- const size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;-- while(n < N && n < combination_limit) {- RETURN_CIRCUIT_BREAKER-- // Outerloop generates combinations of [cost to max_cost] for each token- // For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.- std::vector<uint32_t> costs(token_to_costs.size());- ldiv_t q { n, 0 };- bool valid_combo = false;-- for(long long i = (token_to_costs.size() - 1); 0 <= i ; --i ) {- q = ldiv(q.quot, token_to_costs[i].size());- costs[i] = token_to_costs[i][q.rem];- if(costs[i] == uint32_t(last_typo+1)) {- // to support progressive typo searching, there must be atleast one typo that's greater than last_typo- valid_combo = true;- }- }-- if(last_typo != -1 && !valid_combo) {- n++;- continue;- }-- unique_tokens.clear();- token_candidates_vec.clear();- size_t token_index = 0;-- while(token_index < query_tokens.size()) {- // For each token, look up the generated cost for this iteration and search using that cost- const std::string& token = query_tokens[token_index].value;- const std::string token_cost_hash = token + std::to_string(costs[token_index]);-- std::vector<art_leaf*> leaves;- const bool prefix_search = prefix && query_tokens[token_index].is_prefix_searched;-- /*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"- << token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/-- if(token_cost_cache.count(token_cost_hash) != 0) {- leaves = token_cost_cache[token_cost_hash];- } else {- const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;-- //auto begin = std::chrono::high_resolution_clock::now();-- // need less candidates for filtered searches since we already only pick tokens with results- art_fuzzy_search(search_index.at(field_name), (const unsigned char *) token.c_str(), token_len,- costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,- false, "", filter_ids, filter_ids_length, leaves, unique_tokens);-- /*auto timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - begin).count();- LOG(INFO) << "Time taken for fuzzy search: " << timeMillis << "ms";*/-- if(!leaves.empty()) {- token_cost_cache.emplace(token_cost_hash, leaves);- for(auto leaf: leaves) {- std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);- unique_tokens.emplace(tok);- }- }- }-- if(!leaves.empty()) {- //log_leaves(costs[token_index], token, leaves);- token_candidates_vec.push_back(- token_candidates{query_tokens[token_index], costs[token_index], prefix_search, leaves});- }-- token_index++;- }-- if(token_candidates_vec.size() == query_tokens.size()) {- std::vector<uint32_t> id_buff;-- // If all tokens are, go ahead and search for candidates- search_candidates(field_id, the_field.is_array(), filter_ids, filter_ids_length,- exclude_token_ids, exclude_token_ids_size,- curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,- groups_processed, all_result_ids, all_result_ids_len, field_num_results,- typo_tokens_threshold, group_limit, group_by_fields,- group_missing_values, query_tokens,- prioritize_exact_match, exhaustive_search, syn_orig_num_tokens,- concurrency, query_hashes, id_buff);-- if(id_buff.size() > 1) {- std::sort(id_buff.begin(), id_buff.end());- id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());- }-- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],- id_buff.size(), &new_all_result_ids);- delete[] *all_result_ids;- *all_result_ids = new_all_result_ids;- }-- if(!exhaustive_search && field_num_results >= typo_tokens_threshold) {- // if typo threshold is breached, we are done- return ;- }-- n++;- }-}-int Index::get_bounded_typo_cost(const size_t max_cost, const std::string& token, const size_t token_len,const size_t min_len_1typo, const size_t min_len_2typo,bool enable_typos_for_numerical_tokens) {diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 615db51b..d29e6112 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -1028,9 +1028,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();std::vector<search_field_t> raw_search_fields = {- search_field_t("title", 110, 2, true, off),- search_field_t("brand", 25, 2, true, off),- search_field_t("type", 55, 2, true, off),+ search_field_t("title", "title", 110, 2, true, off),+ search_field_t("brand", "brand", 25, 2, true, off),+ search_field_t("type", "type", 55, 2, true, off),};std::vector<uint32_t> query_by_weights = {110, 25, 55};std::vector<search_field_t> weighted_search_fields;@@ -1051,9 +1051,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {weighted_search_fields.clear();query_by_weights = {15, 15, 15};raw_search_fields = {- search_field_t{"title", 15, 2, true, off},- search_field_t{"brand", 15, 2, true, off},- search_field_t{"type", 15, 2, true, off},+ search_field_t{"title", "title", 15, 2, true, off},+ search_field_t{"brand", "brand", 15, 2, true, off},+ search_field_t{"type", "type", 15, 2, true, off},};coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);@@ -1070,9 +1070,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {weighted_search_fields.clear();query_by_weights = {800, 800, 800};raw_search_fields = {- search_field_t{"title", 800, 2, true, off},- search_field_t{"brand", 800, 2, true, off},- search_field_t{"type", 800, 2, true, off},+ search_field_t{"title", "title", 800, 2, true, off},+ search_field_t{"brand", "brand", 800, 2, true, off},+ search_field_t{"type", "type", 800, 2, true, off},};coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);@@ -1089,9 +1089,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {weighted_search_fields.clear();query_by_weights = {603, 602, 601};raw_search_fields = {- search_field_t{"title", 603, 2, true, off},- search_field_t{"brand", 602, 2, true, off},- search_field_t{"type", 601, 2, true, off},+ search_field_t{"title", "title", 603, 2, true, off},+ search_field_t{"brand", "brand", 602, 2, true, off},+ search_field_t{"type", "type", 601, 2, true, off},};coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);@@ -1110,7 +1110,8 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {query_by_weights.clear();for(size_t i = 0; i < 17; i++) {- raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 17 - i, 2, true, off});+ auto fname = "field" + std::to_string(17 - i);+ raw_search_fields.push_back(search_field_t{fname, fname, 17 - i, 2, true, off});query_by_weights.push_back(17 - i);}@@ -1130,7 +1131,9 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {query_by_weights.clear();for(size_t i = 0; i < 17; i++) {- raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 0, 2, true, off});+ auto field_name = "field" + std::to_string(17 - i);+ raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i),+ field_name, 0, 2, true, off});}coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);