Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor `filter_result_t::reference_filter_results`.
Harpreet Sangar31107ded93ef
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex f0d26c82..143939ed 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -4,6 +4,7 @@#include <map>#include <utility>#include <vector>+#include <memory>#include "option.h"#include "posting_list.h"@@ -14,6 +15,17 @@ struct reference_filter_result_t {uint32_t count = 0;uint32_t* docs = nullptr;+ explicit reference_filter_result_t(uint32_t count = 0, uint32_t* docs = nullptr) : count(count), docs(docs) {}++ reference_filter_result_t(const reference_filter_result_t& obj) {+ if (&obj == this)+ return;++ count = obj.count;+ docs = new uint32_t[count];+ memcpy(docs, obj.docs, count * sizeof(uint32_t));+ }+reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {if (&obj == this)return *this;@@ -25,6 +37,18 @@ struct reference_filter_result_t {return *this;}+ reference_filter_result_t& operator=(reference_filter_result_t&& obj) noexcept {+ if (&obj == this)+ return *this;++ count = obj.count;+ docs = obj.docs;++ obj.docs = nullptr;++ return *this;+ }+~reference_filter_result_t() {delete[] docs;}@@ -58,7 +82,7 @@ struct filter_result_t {uint32_t count = 0;uint32_t* docs = nullptr;// Collection name -> Reference filter result- std::map<std::string, reference_filter_result_t*> reference_filter_results;+ std::unique_ptr<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []> reference_filter_results;filter_result_t() = default;@@ -72,14 +96,7 @@ struct filter_result_t {docs = new uint32_t[count];memcpy(docs, obj.docs, count * sizeof(uint32_t));- // Copy every collection's references.- for (const auto &item: obj.reference_filter_results) {- auto& ref_coll_name = item.first;- reference_filter_results[ref_coll_name] = new reference_filter_result_t[count];- for (uint32_t i = 0; i < count; i++) {- reference_filter_results[ref_coll_name][i] = item.second[i];- }- }+ copy_references(obj, *this);}filter_result_t& operator=(const filter_result_t& obj) noexcept {@@ -90,14 +107,7 @@ struct filter_result_t {docs = new uint32_t[count];memcpy(docs, obj.docs, count * sizeof(uint32_t));- // Copy every collection's references.- for (const auto &item: obj.reference_filter_results) {- reference_filter_results[item.first] = new reference_filter_result_t[count];-- for (uint32_t i = 0; i < count; i++) {- reference_filter_results[item.first][i] = item.second[i];- }- }+ copy_references(obj, *this);return *this;}@@ -108,24 +118,22 @@ struct filter_result_t {count = obj.count;docs = obj.docs;- reference_filter_results = std::map(obj.reference_filter_results);+ reference_filter_results = std::move(obj.reference_filter_results);obj.docs = nullptr;- obj.reference_filter_results.clear();return *this;}~filter_result_t() {delete[] docs;- for (const auto &item: reference_filter_results) {- delete[] item.second;- }}static void and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);static void or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);++ static void copy_references(const filter_result_t& from, filter_result_t& to);};class filter_result_iterator_t {@@ -175,7 +183,8 @@ private:explicit filter_result_iterator_t(uint32_t approx_filter_ids_length);/// Collects n doc ids while advancing the iterator. The iterator may become invalid during this operation.- void get_n_ids(const uint32_t& n, filter_result_t& result);+ /// **The references are moved from filter_result_iterator_t.+ void get_n_ids(const uint32_t& n, filter_result_t*& result);public:uint32_t seq_id = 0;@@ -221,11 +230,11 @@ public:void next();/// Collects n doc ids while advancing the iterator. The ids present in excluded_result_ids are ignored. The- /// iterator may become invalid during this operation.+ /// iterator may become invalid during this operation. **The references are moved from filter_result_iterator_t.void get_n_ids(const uint32_t& n,uint32_t& excluded_result_index,uint32_t const* const excluded_result_ids, const size_t& excluded_result_ids_size,- filter_result_t& result);+ filter_result_t*& result);/// Advances the iterator until the doc value reaches or just overshoots id. The iterator may become invalid during/// this operation.diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex a3e0650a..5d226673 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -13,6 +13,24 @@#include "posting.h"#include "collection_manager.h"+void filter_result_t::copy_references(const filter_result_t& from, filter_result_t& to) {+ if (from.reference_filter_results == nullptr) {+ return;+ }++ auto const& count = from.count;+ to.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(count);+ for (uint32_t i = 0; i < count; i++) {+ if (from.reference_filter_results[i] == nullptr) {+ continue;+ }++ auto& ref = to.reference_filter_results[i];+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ ref->insert(from.reference_filter_results[i]->begin(), from.reference_filter_results[i]->end());+ }+}+void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {auto lenA = a.count, lenB = b.count;if (lenA == 0 || lenB == 0) {@@ -25,16 +43,8 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_const uint32_t *endA = A + lenA;const uint32_t *endB = B + lenB;- // Add an entry of references in the result for each unique collection in a and b.- for (auto const& item: a.reference_filter_results) {- if (result.reference_filter_results.count(item.first) == 0) {- result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];- }- }- for (auto const& item: b.reference_filter_results) {- if (result.reference_filter_results.count(item.first) == 0) {- result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];- }+ if (a.reference_filter_results != nullptr || b.reference_filter_results != nullptr) {+ result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(std::min(lenA, lenB));}while (true) {@@ -54,12 +64,18 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_if (*A == *B) {*out = *A;- // Copy the references of the document from every collection into result.- for (auto const& item: a.reference_filter_results) {- result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];- }- for (auto const& item: b.reference_filter_results) {- result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];+ if (result.reference_filter_results != nullptr) {+ // Copy the references of the document from every collection into result.+ auto& ref = result.reference_filter_results[out - result.docs];+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ if (a.reference_filter_results != nullptr && a.reference_filter_results[A - a.docs] != nullptr) {+ ref->insert(a.reference_filter_results[A - a.docs]->begin(),+ a.reference_filter_results[A - a.docs]->end());+ }+ if (b.reference_filter_results != nullptr && b.reference_filter_results[B - b.docs] != nullptr) {+ ref->insert(b.reference_filter_results[B - b.docs]->begin(),+ b.reference_filter_results[B - b.docs]->end());+ }}out++;@@ -92,16 +108,8 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rsize_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;result.docs = new uint32_t[lenA + lenB];- // Add an entry of references in the result for each unique collection in a and b.- for (auto const& item: a.reference_filter_results) {- if (result.reference_filter_results.count(item.first) == 0) {- result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];- }- }- for (auto const& item: b.reference_filter_results) {- if (result.reference_filter_results.count(item.first) == 0) {- result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];- }+ if (a.reference_filter_results != nullptr || b.reference_filter_results != nullptr) {+ result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(lenA + lenB);}while (indexA < lenA && indexB < lenB) {@@ -112,9 +120,13 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rres_index++;}- // Copy references of the last result document from every collection in a.- for (auto const& item: a.reference_filter_results) {- result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];+ if (a.reference_filter_results != nullptr && a.reference_filter_results[indexA] != nullptr) {+ // Copy references of the last result document from every collection in a.+ auto &ref = result.reference_filter_results[res_index - 1];+ if (ref == nullptr) {+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ }+ ref->insert(a.reference_filter_results[indexA]->begin(), a.reference_filter_results[indexA]->end());}indexA++;@@ -124,8 +136,12 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rres_index++;}- for (auto const& item: b.reference_filter_results) {- result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];+ if (b.reference_filter_results != nullptr && b.reference_filter_results[indexB] != nullptr) {+ auto &ref = result.reference_filter_results[res_index - 1];+ if (ref == nullptr) {+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ }+ ref->insert(b.reference_filter_results[indexB]->begin(), b.reference_filter_results[indexB]->end());}indexB++;@@ -138,8 +154,12 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rres_index++;}- for (auto const& item: a.reference_filter_results) {- result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];+ if (a.reference_filter_results != nullptr && a.reference_filter_results[indexA] != nullptr) {+ auto &ref = result.reference_filter_results[res_index - 1];+ if (ref == nullptr) {+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ }+ ref->insert(a.reference_filter_results[indexA]->begin(), a.reference_filter_results[indexA]->end());}indexA++;@@ -151,8 +171,12 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rres_index++;}- for (auto const& item: b.reference_filter_results) {- result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];+ if (b.reference_filter_results != nullptr && b.reference_filter_results[indexB] != nullptr) {+ auto &ref = result.reference_filter_results[res_index - 1];+ if (ref == nullptr) {+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ }+ ref->insert(b.reference_filter_results[indexB]->begin(), b.reference_filter_results[indexB]->end());}indexB++;@@ -160,21 +184,32 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rresult.count = res_index;+ if (res_index == lenA + lenB) {+ return;+ }+// shrink fitauto out = new uint32_t[res_index];memcpy(out, result.docs, res_index * sizeof(uint32_t));delete[] result.docs;result.docs = out;- for (auto &item: result.reference_filter_results) {- auto out_references = new reference_filter_result_t[res_index];+ if (result.reference_filter_results == nullptr) {+ return;+ }- for (uint32_t i = 0; i < result.count; i++) {- out_references[i] = item.second[i];+ auto out_references = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(res_index);+ for (uint32_t i = 0; i < res_index; i++) {+ if (result.reference_filter_results[i] == nullptr) {+ continue;}- delete[] item.second;- item.second = out_references;++ auto& ref = out_references[i];+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ ref->insert(result.reference_filter_results[i]->begin(), result.reference_filter_results[i]->end());}++ result.reference_filter_results = std::move(out_references);}void filter_result_iterator_t::and_filter_iterators() {@@ -410,8 +445,10 @@ void filter_result_iterator_t::next() {seq_id = filter_result.docs[result_index];reference.clear();- for (auto const& item: filter_result.reference_filter_results) {- reference[item.first] = item.second[result_index];+ if (filter_result.reference_filter_results != nullptr &&+ filter_result.reference_filter_results[result_index] != nullptr) {+ auto& ref = *filter_result.reference_filter_results[result_index];+ reference.insert(ref.begin(), ref.end());}return;@@ -666,8 +703,10 @@ void filter_result_iterator_t::init() {}seq_id = filter_result.docs[result_index];- for (auto const& item: filter_result.reference_filter_results) {- reference[item.first] = item.second[result_index];+ if (filter_result.reference_filter_results != nullptr &&+ filter_result.reference_filter_results[result_index] != nullptr) {+ auto& ref = *filter_result.reference_filter_results[result_index];+ reference.insert(ref.begin(), ref.end());}is_filter_result_initialized = true;@@ -1141,8 +1180,10 @@ void filter_result_iterator_t::skip_to(uint32_t id) {seq_id = filter_result.docs[result_index];reference.clear();- for (auto const& item: filter_result.reference_filter_results) {- reference[item.first] = item.second[result_index];+ if (filter_result.reference_filter_results != nullptr &&+ filter_result.reference_filter_results[result_index] != nullptr) {+ auto& ref = *filter_result.reference_filter_results[result_index];+ reference.insert(ref.begin(), ref.end());}return;@@ -1374,8 +1415,10 @@ void filter_result_iterator_t::reset() {seq_id = filter_result.docs[result_index];reference.clear();- for (auto const& item: filter_result.reference_filter_results) {- reference[item.first] = item.second[result_index];+ if (filter_result.reference_filter_results != nullptr &&+ filter_result.reference_filter_results[result_index] != nullptr) {+ auto& ref = *filter_result.reference_filter_results[result_index];+ reference.insert(ref.begin(), ref.end());}is_valid = true;@@ -1480,7 +1523,7 @@ void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& lenreturn;}- if (filter_result.reference_filter_results.empty()) {+ if (filter_result.reference_filter_results == nullptr) {if (is_filter_result_initialized) {result.count = ArrayUtils::and_scalar(A, lenA, filter_result.docs, filter_result.count, &result.docs);return;@@ -1528,16 +1571,20 @@ void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& lenresult.count = match_indexes.size();result.docs = new uint32_t[match_indexes.size()];- for (auto const& item: filter_result.reference_filter_results) {- result.reference_filter_results[item.first] = new reference_filter_result_t[match_indexes.size()];- }+ result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(match_indexes.size());for (uint32_t i = 0; i < match_indexes.size(); i++) {auto const& match_index = match_indexes[i];result.docs[i] = filter_result.docs[match_index];- for (auto const& item: filter_result.reference_filter_results) {- result.reference_filter_results[item.first][i] = item.second[match_index];++ if (filter_result.reference_filter_results[match_index] == nullptr) {+ continue;}++ auto& result_reference = result.reference_filter_results[i];+ result_reference = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ result_reference->insert(filter_result.reference_filter_results[match_index]->begin(),+ filter_result.reference_filter_results[match_index]->end());}}@@ -1618,22 +1665,28 @@ filter_result_iterator_t &filter_result_iterator_t::operator=(filter_result_iterreturn *this;}-void filter_result_iterator_t::get_n_ids(const uint32_t& n, filter_result_t& result) {+void filter_result_iterator_t::get_n_ids(const uint32_t& n, filter_result_t*& result) {if (!is_filter_result_initialized) {return;}- auto result_length = result.count = std::min(n, filter_result.count - result_index);- result.docs = new uint32_t[result_length];- for (const auto &item: filter_result.reference_filter_results) {- result.reference_filter_results[item.first] = new reference_filter_result_t[result_length];+ auto result_length = result->count = std::min(n, filter_result.count - result_index);+ result->docs = new uint32_t[result_length];+ if (filter_result.reference_filter_results != nullptr) {+ result->reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>>[]>(result_length);}for (uint32_t i = 0; i < result_length; i++, result_index++) {- result.docs[i] = filter_result.docs[result_index];- for (const auto &item: filter_result.reference_filter_results) {- result.reference_filter_results[item.first][i] = item.second[result_index];+ result->docs[i] = filter_result.docs[result_index];++ if (filter_result.reference_filter_results == nullptr) {+ continue;}++ auto& result_reference = result->reference_filter_results[i];+ // Moving references since get_n_ids is only called in wildcard search flow and filter_result_iterator is+ // not used afterwards.+ result_reference = std::move(filter_result.reference_filter_results[result_index]);}is_valid = result_index < filter_result.count;@@ -1642,7 +1695,7 @@ void filter_result_iterator_t::get_n_ids(const uint32_t& n, filter_result_t& resvoid filter_result_iterator_t::get_n_ids(const uint32_t& n,uint32_t& excluded_result_index,uint32_t const* const excluded_result_ids, const size_t& excluded_result_ids_size,- filter_result_t& result) {+ filter_result_t*& result) {if (excluded_result_ids == nullptr || excluded_result_ids_size == 0 ||excluded_result_index >= excluded_result_ids_size) {return get_n_ids(n, result);@@ -1663,18 +1716,24 @@ void filter_result_iterator_t::get_n_ids(const uint32_t& n,}}- result.count = match_indexes.size();- result.docs = new uint32_t[match_indexes.size()];- for (auto const& item: filter_result.reference_filter_results) {- result.reference_filter_results[item.first] = new reference_filter_result_t[match_indexes.size()];+ result->count = match_indexes.size();+ result->docs = new uint32_t[match_indexes.size()];+ if (filter_result.reference_filter_results != nullptr) {+ result->reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>>[]>(match_indexes.size());}for (uint32_t i = 0; i < match_indexes.size(); i++) {auto const& match_index = match_indexes[i];- result.docs[i] = filter_result.docs[match_index];- for (auto const& item: filter_result.reference_filter_results) {- result.reference_filter_results[item.first][i] = item.second[match_index];+ result->docs[i] = filter_result.docs[match_index];++ if (filter_result.reference_filter_results == nullptr) {+ continue;}++ auto& result_reference = result->reference_filter_results[i];+ // Moving references since get_n_ids is only called in wildcard search flow and filter_result_iterator is+ // not used afterwards.+ result_reference = std::move(filter_result.reference_filter_results[match_index]);}is_valid = result_index < filter_result.count;diff --git a/src/index.cpp b/src/index.cppindex 228c3504..f6ed5ed9 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1,5 +1,6 @@#include "index.h"+#include <memory>#include <numeric>#include <chrono>#include <set>@@ -1697,29 +1698,52 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filteruint32_t count = filter_result_iterator.to_filter_id_array(reference_docs);std::unique_ptr<uint32_t[]> docs_guard(reference_docs);- // doc id -> reference doc ids- std::map<uint32_t, std::vector<uint32_t>> reference_map;+ if (count == 0) {+ return Option(true);+ }++ // Collect all the doc ids from the reference ids.+ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;+ auto const& ref_index = *sort_index.at(reference_helper_field_name);for (uint32_t i = 0; i < count; i++) {- auto reference_doc_id = reference_docs[i];- auto doc_id = sort_index.at(reference_helper_field_name)->at(reference_doc_id);+ auto& reference_doc_id = reference_docs[i];+ auto doc_id = ref_index.at(reference_doc_id);++ id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));+ }- reference_map[doc_id].push_back(reference_doc_id);+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {+ return left.first < right.first;+ });++ // Aggregate reference ids of every doc.+ std::vector<std::pair<uint32_t, std::vector<uint32_t>>> result;+ for (uint32_t i = 0, current_doc = id_pairs[0].first + 1; i < id_pairs.size(); i++) {+ auto const& doc_id = id_pairs[i].first;+ auto const& reference_doc_id = id_pairs[i].second;++ if (doc_id != current_doc) {+ result.emplace_back(std::pair<uint32_t, std::vector<uint32_t>>(doc_id, {reference_doc_id}));+ current_doc = doc_id;+ } else {+ result.back().second.push_back(reference_doc_id);+ }}- filter_result.count = reference_map.size();- filter_result.docs = new uint32_t[reference_map.size()];- filter_result.reference_filter_results[collection_name] = new reference_filter_result_t[reference_map.size()];+ filter_result.count = result.size();+ filter_result.docs = new uint32_t[result.size()];+ filter_result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(result.size());- size_t doc_index = 0;- for (auto &item: reference_map) {- filter_result.docs[doc_index] = item.first;+ for (uint32_t result_index = 0; result_index < result.size(); result_index++) {+ filter_result.docs[result_index] = result[result_index].first;- auto& reference_result = filter_result.reference_filter_results[collection_name][doc_index];- reference_result.count = item.second.size();- reference_result.docs = new uint32_t[item.second.size()];- std::copy(item.second.begin(), item.second.end(), reference_result.docs);+ auto& reference_result = filter_result.reference_filter_results[result_index];+ reference_result = std::make_unique<std::map<std::string, reference_filter_result_t>>();- doc_index++;+ auto const& references = result[result_index].second;+ auto r = reference_filter_result_t(references.size(), new uint32_t[references.size()]);+ std::copy(references.begin(), references.end(), r.docs);+ (*reference_result)[collection_name] = std::move(r);}return Option(true);@@ -2284,6 +2308,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfilter_iterator_guard.reset(filter_result_iterator);if (!do_phrase_search_op.ok()) {+ delete [] all_result_ids;return do_phrase_search_op;}@@ -4681,9 +4706,10 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vfor(size_t i = 0; i < raw_infix_ids_length; i++) {auto seq_id = raw_infix_ids[i];- std::map<basic_string<char>, reference_filter_result_t> references;- for (const auto& item: filtered_infix_ids.reference_filter_results) {- references[item.first] = item.second[i];+ std::map<std::string, reference_filter_result_t> references;+ if (filtered_infix_ids.reference_filter_results != nullptr &&+ filtered_infix_ids.reference_filter_results[i] != nullptr) {+ references = std::move(*filtered_infix_ids.reference_filter_results[i]);}int64_t match_score = 0;@@ -5038,9 +5064,9 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootOption<bool>* compute_sort_score_statuses[num_threads];for(size_t thread_id = 0; thread_id < num_threads && filter_result_iterator->is_valid; thread_id++) {- filter_result_t batch_result;+ auto batch_result = new filter_result_t();filter_result_iterator->get_n_ids(window_size, excluded_result_index, exclude_token_ids,- exclude_token_ids_size, batch_result);+ exclude_token_ids_size, batch_result);num_queued++;@@ -5056,6 +5082,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootcheck_for_circuit_break,batch_result,&num_processed, &m_process, &cv_process, &compute_sort_score_status, collection_name]() {+ std::unique_ptr<filter_result_t> batch_result_guard(batch_result);search_begin_us = parent_search_begin;search_stop_us = parent_search_stop_ms;@@ -5063,11 +5090,12 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootsize_t filter_index = 0;- for(size_t i = 0; i < batch_result.count; i++) {- const uint32_t seq_id = batch_result.docs[i];+ for(size_t i = 0; i < batch_result->count; i++) {+ const uint32_t seq_id = batch_result->docs[i];std::map<basic_string<char>, reference_filter_result_t> references;- for (const auto& item: batch_result.reference_filter_results) {- references[item.first] = item.second[i];+ if (batch_result->reference_filter_results != nullptr &&+ batch_result->reference_filter_results[i] != nullptr) {+ references = std::move(*batch_result->reference_filter_results[i]);}int64_t match_score = 0;diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex 212e720e..fed20f59 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -651,7 +651,7 @@ TEST_F(CollectionJoinTest, AndFilterResults_NoReference) {filter_result_t::and_filter_results(a, b, result);ASSERT_EQ(2, result.count);- ASSERT_EQ(0, result.reference_filter_results.size());+ ASSERT_EQ(nullptr, result.reference_filter_results);std::vector<uint32_t> docs = {3, 6};@@ -664,30 +664,31 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {filter_result_t a;a.count = 9;a.docs = new uint32_t[a.count];- a.reference_filter_results["foo"] = new reference_filter_result_t[a.count];+ a.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(a.count);+for (size_t i = 0; i < a.count; i++) {a.docs[i] = i;+ auto& reference = a.reference_filter_results[i] = std::make_unique<std::map<std::string, reference_filter_result_t>>();// Having only one reference of each document for brevity.- auto& reference = a.reference_filter_results["foo"][i];- reference.count = 1;- reference.docs = new uint32_t[1];- reference.docs[0] = 10 - i;+ auto reference_docs = new uint32_t[1];+ reference_docs[0] = 10 - i;+ (*reference)["foo"] = reference_filter_result_t(1, reference_docs);}filter_result_t b;b.count = 0;uint32_t limit = 10;b.docs = new uint32_t[limit];- b.reference_filter_results["bar"] = new reference_filter_result_t[limit];+ b.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(limit);for (size_t i = 2; i < limit; i++) {if (i % 3 == 0) {b.docs[b.count] = i;- auto& reference = b.reference_filter_results["bar"][b.count++];- reference.count = 1;- reference.docs = new uint32_t[1];- reference.docs[0] = 2 * i;+ auto& reference = b.reference_filter_results[b.count++] = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ auto reference_docs = new uint32_t[1];+ reference_docs[0] = 2 * i;+ (*reference)["bar"] = reference_filter_result_t(1, reference_docs);}}@@ -696,9 +697,9 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {filter_result_t::and_filter_results(a, b, result);ASSERT_EQ(2, result.count);- ASSERT_EQ(2, result.reference_filter_results.size());- ASSERT_EQ(1, result.reference_filter_results.count("foo"));- ASSERT_EQ(1, result.reference_filter_results.count("bar"));+ ASSERT_EQ(2, result.reference_filter_results[0]->size());+ ASSERT_EQ(1, result.reference_filter_results[0]->count("foo"));+ ASSERT_EQ(1, result.reference_filter_results[0]->count("bar"));std::vector<uint32_t> docs = {3, 6}, foo_reference = {7, 4}, bar_reference = {6, 12};@@ -706,10 +707,10 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {ASSERT_EQ(docs[i], result.docs[i]);// result should contain correct references to the foo and bar collection.- ASSERT_EQ(1, result.reference_filter_results["foo"][i].count);- ASSERT_EQ(foo_reference[i], result.reference_filter_results["foo"][i].docs[0]);- ASSERT_EQ(1, result.reference_filter_results["bar"][i].count);- ASSERT_EQ(bar_reference[i], result.reference_filter_results["bar"][i].docs[0]);+ ASSERT_EQ(1, result.reference_filter_results[i]->at("foo").count);+ ASSERT_EQ(foo_reference[i], result.reference_filter_results[i]->at("foo").docs[0]);+ ASSERT_EQ(1, result.reference_filter_results[i]->at("bar").count);+ ASSERT_EQ(bar_reference[i], result.reference_filter_results[i]->at("bar").docs[0]);}}@@ -728,7 +729,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {filter_result_t result1;filter_result_t::or_filter_results(a, b, result1);ASSERT_EQ(3, result1.count);- ASSERT_EQ(0, result1.reference_filter_results.size());+ ASSERT_EQ(nullptr, result1.reference_filter_results);std::vector<uint32_t> expected = {3, 6, 9};for (size_t i = 0; i < result1.count; i++) {@@ -745,7 +746,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {filter_result_t result2;filter_result_t::or_filter_results(a, b, result2);ASSERT_EQ(10, result2.count);- ASSERT_EQ(0, result2.reference_filter_results.size());+ ASSERT_EQ(nullptr, result2.reference_filter_results);expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};for (size_t i = 0; i < result2.count; i++) {@@ -765,7 +766,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {// b.docs: [0..8], c.docs: [0, 4, 5]filter_result_t::or_filter_results(b, c, result3);ASSERT_EQ(9, result3.count);- ASSERT_EQ(0, result3.reference_filter_results.size());+ ASSERT_EQ(nullptr, result3.reference_filter_results);expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};for(size_t i = 0; i < result3.count; i++) {@@ -779,15 +780,15 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {a.count = 0;a.docs = new uint32_t[limit];- a.reference_filter_results["foo"] = new reference_filter_result_t[limit];+ a.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(limit);for (size_t i = 2; i < limit; i++) {if (i % 3 == 0) {a.docs[a.count] = i;- auto& reference = a.reference_filter_results["foo"][a.count++];- reference.count = 1;- reference.docs = new uint32_t[1];- reference.docs[0] = 2 * i;+ auto& reference = a.reference_filter_results[a.count++] = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ auto reference_docs = new uint32_t[1];+ reference_docs[0] = 2 * i;+ (*reference)["foo"] = reference_filter_result_t(1, reference_docs);}}@@ -796,27 +797,27 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {filter_result_t::or_filter_results(a, b, result1);ASSERT_EQ(3, result1.count);- ASSERT_EQ(1, result1.reference_filter_results.size());- ASSERT_EQ(1, result1.reference_filter_results.count("foo"));+ ASSERT_EQ(1, result1.reference_filter_results[0]->size());+ ASSERT_EQ(1, result1.reference_filter_results[0]->count("foo"));std::vector<uint32_t> expected = {3, 6, 9}, foo_reference = {6, 12, 18};for (size_t i = 0; i < result1.count; i++) {ASSERT_EQ(expected[i], result1.docs[i]);- ASSERT_EQ(1, result1.reference_filter_results["foo"][i].count);- ASSERT_EQ(foo_reference[i], result1.reference_filter_results["foo"][i].docs[0]);+ ASSERT_EQ(1, result1.reference_filter_results[i]->at("foo").count);+ ASSERT_EQ(foo_reference[i], result1.reference_filter_results[i]->at("foo").docs[0]);}b.count = 9;b.docs = new uint32_t[b.count];- b.reference_filter_results["bar"] = new reference_filter_result_t[b.count];+ b.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(b.count);for (size_t i = 0; i < b.count; i++) {b.docs[i] = i;- auto& reference = b.reference_filter_results["bar"][i];- reference.count = 1;- reference.docs = new uint32_t[1];- reference.docs[0] = 10 - i;+ auto& reference = b.reference_filter_results[i] = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ auto reference_docs = new uint32_t[1];+ reference_docs[0] = 10 - i;+ (*reference)["bar"] = reference_filter_result_t(1, reference_docs);}// a.docs: [3, 6, 9], b.docs: [0..8]@@ -833,18 +834,18 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {ASSERT_EQ(expected[i], result2.docs[i]);if (foo_map.count(i) != 0) {- ASSERT_EQ(1, result2.reference_filter_results["foo"][i].count);- ASSERT_EQ(foo_map[i], result2.reference_filter_results["foo"][i].docs[0]);+ ASSERT_EQ(1, result2.reference_filter_results[i]->at("foo").count);+ ASSERT_EQ(foo_map[i], result2.reference_filter_results[i]->at("foo").docs[0]);} else {- // Reference count should be 0 for the docs that were not present in the a result.- ASSERT_EQ(0, result2.reference_filter_results["foo"][i].count);+ // foo didn't have any reference to current doc.+ ASSERT_EQ(0, result2.reference_filter_results[i]->count("foo"));}if (bar_map.count(i) != 0) {- ASSERT_EQ(1, result2.reference_filter_results["bar"][i].count);- ASSERT_EQ(bar_map[i], result2.reference_filter_results["bar"][i].docs[0]);+ ASSERT_EQ(1, result2.reference_filter_results[i]->at("bar").count);+ ASSERT_EQ(bar_map[i], result2.reference_filter_results[i]->at("bar").docs[0]);} else {- ASSERT_EQ(0, result2.reference_filter_results["bar"][i].count);+ ASSERT_EQ(0, result2.reference_filter_results[i]->count("bar"));}}@@ -853,15 +854,15 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {std::map<uint32_t, uint32_t> baz_map = {{0, 2}, {4, 0}, {5, 8}};c.count = baz_map.size();c.docs = new uint32_t[baz_map.size()];- c.reference_filter_results["baz"] = new reference_filter_result_t[baz_map.size()];+ c.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(baz_map.size());auto j = 0;for(auto i: baz_map) {c.docs[j] = i.first;- auto& reference = c.reference_filter_results["baz"][j++];- reference.count = 1;- reference.docs = new uint32_t[1];- reference.docs[0] = i.second;+ auto& reference = c.reference_filter_results[j++] = std::make_unique<std::map<std::string, reference_filter_result_t>>();+ auto reference_docs = new uint32_t[1];+ reference_docs[0] = i.second;+ (*reference)["baz"] = reference_filter_result_t(1, reference_docs);}// b.docs: [0..8], c.docs: [0, 4, 5]@@ -873,17 +874,17 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {ASSERT_EQ(expected[i], result3.docs[i]);if (bar_map.count(i) != 0) {- ASSERT_EQ(1, result3.reference_filter_results["bar"][i].count);- ASSERT_EQ(bar_map[i], result3.reference_filter_results["bar"][i].docs[0]);+ ASSERT_EQ(1, result3.reference_filter_results[i]->at("bar").count);+ ASSERT_EQ(bar_map[i], result3.reference_filter_results[i]->at("bar").docs[0]);} else {- ASSERT_EQ(0, result3.reference_filter_results["bar"][i].count);+ ASSERT_EQ(0, result3.reference_filter_results[i]->count("bar"));}if (baz_map.count(i) != 0) {- ASSERT_EQ(1, result3.reference_filter_results["baz"][i].count);- ASSERT_EQ(baz_map[i], result3.reference_filter_results["baz"][i].docs[0]);+ ASSERT_EQ(1, result3.reference_filter_results[i]->at("baz").count);+ ASSERT_EQ(baz_map[i], result3.reference_filter_results[i]->at("baz").docs[0]);} else {- ASSERT_EQ(0, result3.reference_filter_results["baz"][i].count);+ ASSERT_EQ(0, result3.reference_filter_results[i]->count("baz"));}}}