Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor `filter_result_t::reference_filter_results`.

Harpreet Sangar31107ded93ef
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index f0d26c82..143939ed 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -4,6 +4,7 @@
#include <map>
#include <utility>
#include <vector>
+#include <memory>
#include "option.h"
#include "posting_list.h"
@@ -14,6 +15,17 @@ struct reference_filter_result_t {
uint32_t count = 0;
uint32_t* docs = nullptr;
+ explicit reference_filter_result_t(uint32_t count = 0, uint32_t* docs = nullptr) : count(count), docs(docs) {}
+
+ reference_filter_result_t(const reference_filter_result_t& obj) {
+ if (&obj == this)
+ return;
+
+ count = obj.count;
+ docs = new uint32_t[count];
+ memcpy(docs, obj.docs, count * sizeof(uint32_t));
+ }
+
reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {
if (&obj == this)
return *this;
@@ -25,6 +37,18 @@ struct reference_filter_result_t {
return *this;
}
+ reference_filter_result_t& operator=(reference_filter_result_t&& obj) noexcept {
+ if (&obj == this)
+ return *this;
+
+ count = obj.count;
+ docs = obj.docs;
+
+ obj.docs = nullptr;
+
+ return *this;
+ }
+
~reference_filter_result_t() {
delete[] docs;
}
@@ -58,7 +82,7 @@ struct filter_result_t {
uint32_t count = 0;
uint32_t* docs = nullptr;
// Collection name -> Reference filter result
- std::map<std::string, reference_filter_result_t*> reference_filter_results;
+ std::unique_ptr<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []> reference_filter_results;
filter_result_t() = default;
@@ -72,14 +96,7 @@ struct filter_result_t {
docs = new uint32_t[count];
memcpy(docs, obj.docs, count * sizeof(uint32_t));
- // Copy every collection's references.
- for (const auto &item: obj.reference_filter_results) {
- auto& ref_coll_name = item.first;
- reference_filter_results[ref_coll_name] = new reference_filter_result_t[count];
- for (uint32_t i = 0; i < count; i++) {
- reference_filter_results[ref_coll_name][i] = item.second[i];
- }
- }
+ copy_references(obj, *this);
}
filter_result_t& operator=(const filter_result_t& obj) noexcept {
@@ -90,14 +107,7 @@ struct filter_result_t {
docs = new uint32_t[count];
memcpy(docs, obj.docs, count * sizeof(uint32_t));
- // Copy every collection's references.
- for (const auto &item: obj.reference_filter_results) {
- reference_filter_results[item.first] = new reference_filter_result_t[count];
-
- for (uint32_t i = 0; i < count; i++) {
- reference_filter_results[item.first][i] = item.second[i];
- }
- }
+ copy_references(obj, *this);
return *this;
}
@@ -108,24 +118,22 @@ struct filter_result_t {
count = obj.count;
docs = obj.docs;
- reference_filter_results = std::map(obj.reference_filter_results);
+ reference_filter_results = std::move(obj.reference_filter_results);
obj.docs = nullptr;
- obj.reference_filter_results.clear();
return *this;
}
~filter_result_t() {
delete[] docs;
- for (const auto &item: reference_filter_results) {
- delete[] item.second;
- }
}
static void and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);
static void or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result);
+
+ static void copy_references(const filter_result_t& from, filter_result_t& to);
};
class filter_result_iterator_t {
@@ -175,7 +183,8 @@ private:
explicit filter_result_iterator_t(uint32_t approx_filter_ids_length);
/// Collects n doc ids while advancing the iterator. The iterator may become invalid during this operation.
- void get_n_ids(const uint32_t& n, filter_result_t& result);
+ /// **The references are moved from filter_result_iterator_t.
+ void get_n_ids(const uint32_t& n, filter_result_t*& result);
public:
uint32_t seq_id = 0;
@@ -221,11 +230,11 @@ public:
void next();
/// Collects n doc ids while advancing the iterator. The ids present in excluded_result_ids are ignored. The
- /// iterator may become invalid during this operation.
+ /// iterator may become invalid during this operation. **The references are moved from filter_result_iterator_t.
void get_n_ids(const uint32_t& n,
uint32_t& excluded_result_index,
uint32_t const* const excluded_result_ids, const size_t& excluded_result_ids_size,
- filter_result_t& result);
+ filter_result_t*& result);
/// Advances the iterator until the doc value reaches or just overshoots id. The iterator may become invalid during
/// this operation.
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index a3e0650a..5d226673 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -13,6 +13,24 @@
#include "posting.h"
#include "collection_manager.h"
+void filter_result_t::copy_references(const filter_result_t& from, filter_result_t& to) {
+ if (from.reference_filter_results == nullptr) {
+ return;
+ }
+
+ auto const& count = from.count;
+ to.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(count);
+ for (uint32_t i = 0; i < count; i++) {
+ if (from.reference_filter_results[i] == nullptr) {
+ continue;
+ }
+
+ auto& ref = to.reference_filter_results[i];
+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ ref->insert(from.reference_filter_results[i]->begin(), from.reference_filter_results[i]->end());
+ }
+}
+
void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
auto lenA = a.count, lenB = b.count;
if (lenA == 0 || lenB == 0) {
@@ -25,16 +43,8 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_
const uint32_t *endA = A + lenA;
const uint32_t *endB = B + lenB;
- // Add an entry of references in the result for each unique collection in a and b.
- for (auto const& item: a.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
- }
- }
- for (auto const& item: b.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
- }
+ if (a.reference_filter_results != nullptr || b.reference_filter_results != nullptr) {
+ result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(std::min(lenA, lenB));
}
while (true) {
@@ -54,12 +64,18 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_
if (*A == *B) {
*out = *A;
- // Copy the references of the document from every collection into result.
- for (auto const& item: a.reference_filter_results) {
- result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];
- }
- for (auto const& item: b.reference_filter_results) {
- result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];
+ if (result.reference_filter_results != nullptr) {
+ // Copy the references of the document from every collection into result.
+ auto& ref = result.reference_filter_results[out - result.docs];
+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ if (a.reference_filter_results != nullptr && a.reference_filter_results[A - a.docs] != nullptr) {
+ ref->insert(a.reference_filter_results[A - a.docs]->begin(),
+ a.reference_filter_results[A - a.docs]->end());
+ }
+ if (b.reference_filter_results != nullptr && b.reference_filter_results[B - b.docs] != nullptr) {
+ ref->insert(b.reference_filter_results[B - b.docs]->begin(),
+ b.reference_filter_results[B - b.docs]->end());
+ }
}
out++;
@@ -92,16 +108,8 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
size_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;
result.docs = new uint32_t[lenA + lenB];
- // Add an entry of references in the result for each unique collection in a and b.
- for (auto const& item: a.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
- }
- }
- for (auto const& item: b.reference_filter_results) {
- if (result.reference_filter_results.count(item.first) == 0) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
- }
+ if (a.reference_filter_results != nullptr || b.reference_filter_results != nullptr) {
+ result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(lenA + lenB);
}
while (indexA < lenA && indexB < lenB) {
@@ -112,9 +120,13 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
res_index++;
}
- // Copy references of the last result document from every collection in a.
- for (auto const& item: a.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ if (a.reference_filter_results != nullptr && a.reference_filter_results[indexA] != nullptr) {
+ // Copy references of the last result document from every collection in a.
+ auto &ref = result.reference_filter_results[res_index - 1];
+ if (ref == nullptr) {
+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ }
+ ref->insert(a.reference_filter_results[indexA]->begin(), a.reference_filter_results[indexA]->end());
}
indexA++;
@@ -124,8 +136,12 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
res_index++;
}
- for (auto const& item: b.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ if (b.reference_filter_results != nullptr && b.reference_filter_results[indexB] != nullptr) {
+ auto &ref = result.reference_filter_results[res_index - 1];
+ if (ref == nullptr) {
+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ }
+ ref->insert(b.reference_filter_results[indexB]->begin(), b.reference_filter_results[indexB]->end());
}
indexB++;
@@ -138,8 +154,12 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
res_index++;
}
- for (auto const& item: a.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ if (a.reference_filter_results != nullptr && a.reference_filter_results[indexA] != nullptr) {
+ auto &ref = result.reference_filter_results[res_index - 1];
+ if (ref == nullptr) {
+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ }
+ ref->insert(a.reference_filter_results[indexA]->begin(), a.reference_filter_results[indexA]->end());
}
indexA++;
@@ -151,8 +171,12 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
res_index++;
}
- for (auto const& item: b.reference_filter_results) {
- result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ if (b.reference_filter_results != nullptr && b.reference_filter_results[indexB] != nullptr) {
+ auto &ref = result.reference_filter_results[res_index - 1];
+ if (ref == nullptr) {
+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ }
+ ref->insert(b.reference_filter_results[indexB]->begin(), b.reference_filter_results[indexB]->end());
}
indexB++;
@@ -160,21 +184,32 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
result.count = res_index;
+ if (res_index == lenA + lenB) {
+ return;
+ }
+
// shrink fit
auto out = new uint32_t[res_index];
memcpy(out, result.docs, res_index * sizeof(uint32_t));
delete[] result.docs;
result.docs = out;
- for (auto &item: result.reference_filter_results) {
- auto out_references = new reference_filter_result_t[res_index];
+ if (result.reference_filter_results == nullptr) {
+ return;
+ }
- for (uint32_t i = 0; i < result.count; i++) {
- out_references[i] = item.second[i];
+ auto out_references = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(res_index);
+ for (uint32_t i = 0; i < res_index; i++) {
+ if (result.reference_filter_results[i] == nullptr) {
+ continue;
}
- delete[] item.second;
- item.second = out_references;
+
+ auto& ref = out_references[i];
+ ref = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ ref->insert(result.reference_filter_results[i]->begin(), result.reference_filter_results[i]->end());
}
+
+ result.reference_filter_results = std::move(out_references);
}
void filter_result_iterator_t::and_filter_iterators() {
@@ -410,8 +445,10 @@ void filter_result_iterator_t::next() {
seq_id = filter_result.docs[result_index];
reference.clear();
- for (auto const& item: filter_result.reference_filter_results) {
- reference[item.first] = item.second[result_index];
+ if (filter_result.reference_filter_results != nullptr &&
+ filter_result.reference_filter_results[result_index] != nullptr) {
+ auto& ref = *filter_result.reference_filter_results[result_index];
+ reference.insert(ref.begin(), ref.end());
}
return;
@@ -666,8 +703,10 @@ void filter_result_iterator_t::init() {
}
seq_id = filter_result.docs[result_index];
- for (auto const& item: filter_result.reference_filter_results) {
- reference[item.first] = item.second[result_index];
+ if (filter_result.reference_filter_results != nullptr &&
+ filter_result.reference_filter_results[result_index] != nullptr) {
+ auto& ref = *filter_result.reference_filter_results[result_index];
+ reference.insert(ref.begin(), ref.end());
}
is_filter_result_initialized = true;
@@ -1141,8 +1180,10 @@ void filter_result_iterator_t::skip_to(uint32_t id) {
seq_id = filter_result.docs[result_index];
reference.clear();
- for (auto const& item: filter_result.reference_filter_results) {
- reference[item.first] = item.second[result_index];
+ if (filter_result.reference_filter_results != nullptr &&
+ filter_result.reference_filter_results[result_index] != nullptr) {
+ auto& ref = *filter_result.reference_filter_results[result_index];
+ reference.insert(ref.begin(), ref.end());
}
return;
@@ -1374,8 +1415,10 @@ void filter_result_iterator_t::reset() {
seq_id = filter_result.docs[result_index];
reference.clear();
- for (auto const& item: filter_result.reference_filter_results) {
- reference[item.first] = item.second[result_index];
+ if (filter_result.reference_filter_results != nullptr &&
+ filter_result.reference_filter_results[result_index] != nullptr) {
+ auto& ref = *filter_result.reference_filter_results[result_index];
+ reference.insert(ref.begin(), ref.end());
}
is_valid = true;
@@ -1480,7 +1523,7 @@ void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& len
return;
}
- if (filter_result.reference_filter_results.empty()) {
+ if (filter_result.reference_filter_results == nullptr) {
if (is_filter_result_initialized) {
result.count = ArrayUtils::and_scalar(A, lenA, filter_result.docs, filter_result.count, &result.docs);
return;
@@ -1528,16 +1571,20 @@ void filter_result_iterator_t::and_scalar(const uint32_t* A, const uint32_t& len
result.count = match_indexes.size();
result.docs = new uint32_t[match_indexes.size()];
- for (auto const& item: filter_result.reference_filter_results) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[match_indexes.size()];
- }
+ result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(match_indexes.size());
for (uint32_t i = 0; i < match_indexes.size(); i++) {
auto const& match_index = match_indexes[i];
result.docs[i] = filter_result.docs[match_index];
- for (auto const& item: filter_result.reference_filter_results) {
- result.reference_filter_results[item.first][i] = item.second[match_index];
+
+ if (filter_result.reference_filter_results[match_index] == nullptr) {
+ continue;
}
+
+ auto& result_reference = result.reference_filter_results[i];
+ result_reference = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ result_reference->insert(filter_result.reference_filter_results[match_index]->begin(),
+ filter_result.reference_filter_results[match_index]->end());
}
}
@@ -1618,22 +1665,28 @@ filter_result_iterator_t &filter_result_iterator_t::operator=(filter_result_iter
return *this;
}
-void filter_result_iterator_t::get_n_ids(const uint32_t& n, filter_result_t& result) {
+void filter_result_iterator_t::get_n_ids(const uint32_t& n, filter_result_t*& result) {
if (!is_filter_result_initialized) {
return;
}
- auto result_length = result.count = std::min(n, filter_result.count - result_index);
- result.docs = new uint32_t[result_length];
- for (const auto &item: filter_result.reference_filter_results) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[result_length];
+ auto result_length = result->count = std::min(n, filter_result.count - result_index);
+ result->docs = new uint32_t[result_length];
+ if (filter_result.reference_filter_results != nullptr) {
+ result->reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>>[]>(result_length);
}
for (uint32_t i = 0; i < result_length; i++, result_index++) {
- result.docs[i] = filter_result.docs[result_index];
- for (const auto &item: filter_result.reference_filter_results) {
- result.reference_filter_results[item.first][i] = item.second[result_index];
+ result->docs[i] = filter_result.docs[result_index];
+
+ if (filter_result.reference_filter_results == nullptr) {
+ continue;
}
+
+ auto& result_reference = result->reference_filter_results[i];
+ // Moving references since get_n_ids is only called in wildcard search flow and filter_result_iterator is
+ // not used afterwards.
+ result_reference = std::move(filter_result.reference_filter_results[result_index]);
}
is_valid = result_index < filter_result.count;
@@ -1642,7 +1695,7 @@ void filter_result_iterator_t::get_n_ids(const uint32_t& n, filter_result_t& res
void filter_result_iterator_t::get_n_ids(const uint32_t& n,
uint32_t& excluded_result_index,
uint32_t const* const excluded_result_ids, const size_t& excluded_result_ids_size,
- filter_result_t& result) {
+ filter_result_t*& result) {
if (excluded_result_ids == nullptr || excluded_result_ids_size == 0 ||
excluded_result_index >= excluded_result_ids_size) {
return get_n_ids(n, result);
@@ -1663,18 +1716,24 @@ void filter_result_iterator_t::get_n_ids(const uint32_t& n,
}
}
- result.count = match_indexes.size();
- result.docs = new uint32_t[match_indexes.size()];
- for (auto const& item: filter_result.reference_filter_results) {
- result.reference_filter_results[item.first] = new reference_filter_result_t[match_indexes.size()];
+ result->count = match_indexes.size();
+ result->docs = new uint32_t[match_indexes.size()];
+ if (filter_result.reference_filter_results != nullptr) {
+ result->reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>>[]>(match_indexes.size());
}
for (uint32_t i = 0; i < match_indexes.size(); i++) {
auto const& match_index = match_indexes[i];
- result.docs[i] = filter_result.docs[match_index];
- for (auto const& item: filter_result.reference_filter_results) {
- result.reference_filter_results[item.first][i] = item.second[match_index];
+ result->docs[i] = filter_result.docs[match_index];
+
+ if (filter_result.reference_filter_results == nullptr) {
+ continue;
}
+
+ auto& result_reference = result->reference_filter_results[i];
+ // Moving references since get_n_ids is only called in wildcard search flow and filter_result_iterator is
+ // not used afterwards.
+ result_reference = std::move(filter_result.reference_filter_results[match_index]);
}
is_valid = result_index < filter_result.count;
diff --git a/src/index.cpp b/src/index.cpp
index 228c3504..f6ed5ed9 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1,5 +1,6 @@
#include "index.h"
+#include <memory>
#include <numeric>
#include <chrono>
#include <set>
@@ -1697,29 +1698,52 @@ Option<bool> Index::do_reference_filtering_with_lock(filter_node_t* const filter
uint32_t count = filter_result_iterator.to_filter_id_array(reference_docs);
std::unique_ptr<uint32_t[]> docs_guard(reference_docs);
- // doc id -> reference doc ids
- std::map<uint32_t, std::vector<uint32_t>> reference_map;
+ if (count == 0) {
+ return Option(true);
+ }
+
+ // Collect all the doc ids from the reference ids.
+ std::vector<std::pair<uint32_t, uint32_t>> id_pairs;
+ auto const& ref_index = *sort_index.at(reference_helper_field_name);
for (uint32_t i = 0; i < count; i++) {
- auto reference_doc_id = reference_docs[i];
- auto doc_id = sort_index.at(reference_helper_field_name)->at(reference_doc_id);
+ auto& reference_doc_id = reference_docs[i];
+ auto doc_id = ref_index.at(reference_doc_id);
+
+ id_pairs.emplace_back(std::pair(doc_id, reference_doc_id));
+ }
- reference_map[doc_id].push_back(reference_doc_id);
+ std::sort(id_pairs.begin(), id_pairs.end(), [](auto const& left, auto const& right) {
+ return left.first < right.first;
+ });
+
+ // Aggregate reference ids of every doc.
+ std::vector<std::pair<uint32_t, std::vector<uint32_t>>> result;
+ for (uint32_t i = 0, current_doc = id_pairs[0].first + 1; i < id_pairs.size(); i++) {
+ auto const& doc_id = id_pairs[i].first;
+ auto const& reference_doc_id = id_pairs[i].second;
+
+ if (doc_id != current_doc) {
+ result.emplace_back(std::pair<uint32_t, std::vector<uint32_t>>(doc_id, {reference_doc_id}));
+ current_doc = doc_id;
+ } else {
+ result.back().second.push_back(reference_doc_id);
+ }
}
- filter_result.count = reference_map.size();
- filter_result.docs = new uint32_t[reference_map.size()];
- filter_result.reference_filter_results[collection_name] = new reference_filter_result_t[reference_map.size()];
+ filter_result.count = result.size();
+ filter_result.docs = new uint32_t[result.size()];
+ filter_result.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(result.size());
- size_t doc_index = 0;
- for (auto &item: reference_map) {
- filter_result.docs[doc_index] = item.first;
+ for (uint32_t result_index = 0; result_index < result.size(); result_index++) {
+ filter_result.docs[result_index] = result[result_index].first;
- auto& reference_result = filter_result.reference_filter_results[collection_name][doc_index];
- reference_result.count = item.second.size();
- reference_result.docs = new uint32_t[item.second.size()];
- std::copy(item.second.begin(), item.second.end(), reference_result.docs);
+ auto& reference_result = filter_result.reference_filter_results[result_index];
+ reference_result = std::make_unique<std::map<std::string, reference_filter_result_t>>();
- doc_index++;
+ auto const& references = result[result_index].second;
+ auto r = reference_filter_result_t(references.size(), new uint32_t[references.size()]);
+ std::copy(references.begin(), references.end(), r.docs);
+ (*reference_result)[collection_name] = std::move(r);
}
return Option(true);
@@ -2284,6 +2308,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
filter_iterator_guard.reset(filter_result_iterator);
if (!do_phrase_search_op.ok()) {
+ delete [] all_result_ids;
return do_phrase_search_op;
}
@@ -4681,9 +4706,10 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::v
for(size_t i = 0; i < raw_infix_ids_length; i++) {
auto seq_id = raw_infix_ids[i];
- std::map<basic_string<char>, reference_filter_result_t> references;
- for (const auto& item: filtered_infix_ids.reference_filter_results) {
- references[item.first] = item.second[i];
+ std::map<std::string, reference_filter_result_t> references;
+ if (filtered_infix_ids.reference_filter_results != nullptr &&
+ filtered_infix_ids.reference_filter_results[i] != nullptr) {
+ references = std::move(*filtered_infix_ids.reference_filter_results[i]);
}
int64_t match_score = 0;
@@ -5038,9 +5064,9 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
Option<bool>* compute_sort_score_statuses[num_threads];
for(size_t thread_id = 0; thread_id < num_threads && filter_result_iterator->is_valid; thread_id++) {
- filter_result_t batch_result;
+ auto batch_result = new filter_result_t();
filter_result_iterator->get_n_ids(window_size, excluded_result_index, exclude_token_ids,
- exclude_token_ids_size, batch_result);
+ exclude_token_ids_size, batch_result);
num_queued++;
@@ -5056,6 +5082,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
check_for_circuit_break,
batch_result,
&num_processed, &m_process, &cv_process, &compute_sort_score_status, collection_name]() {
+ std::unique_ptr<filter_result_t> batch_result_guard(batch_result);
search_begin_us = parent_search_begin;
search_stop_us = parent_search_stop_ms;
@@ -5063,11 +5090,12 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
size_t filter_index = 0;
- for(size_t i = 0; i < batch_result.count; i++) {
- const uint32_t seq_id = batch_result.docs[i];
+ for(size_t i = 0; i < batch_result->count; i++) {
+ const uint32_t seq_id = batch_result->docs[i];
std::map<basic_string<char>, reference_filter_result_t> references;
- for (const auto& item: batch_result.reference_filter_results) {
- references[item.first] = item.second[i];
+ if (batch_result->reference_filter_results != nullptr &&
+ batch_result->reference_filter_results[i] != nullptr) {
+ references = std::move(*batch_result->reference_filter_results[i]);
}
int64_t match_score = 0;
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 212e720e..fed20f59 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -651,7 +651,7 @@ TEST_F(CollectionJoinTest, AndFilterResults_NoReference) {
filter_result_t::and_filter_results(a, b, result);
ASSERT_EQ(2, result.count);
- ASSERT_EQ(0, result.reference_filter_results.size());
+ ASSERT_EQ(nullptr, result.reference_filter_results);
std::vector<uint32_t> docs = {3, 6};
@@ -664,30 +664,31 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {
filter_result_t a;
a.count = 9;
a.docs = new uint32_t[a.count];
- a.reference_filter_results["foo"] = new reference_filter_result_t[a.count];
+ a.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(a.count);
+
for (size_t i = 0; i < a.count; i++) {
a.docs[i] = i;
+ auto& reference = a.reference_filter_results[i] = std::make_unique<std::map<std::string, reference_filter_result_t>>();
// Having only one reference of each document for brevity.
- auto& reference = a.reference_filter_results["foo"][i];
- reference.count = 1;
- reference.docs = new uint32_t[1];
- reference.docs[0] = 10 - i;
+ auto reference_docs = new uint32_t[1];
+ reference_docs[0] = 10 - i;
+ (*reference)["foo"] = reference_filter_result_t(1, reference_docs);
}
filter_result_t b;
b.count = 0;
uint32_t limit = 10;
b.docs = new uint32_t[limit];
- b.reference_filter_results["bar"] = new reference_filter_result_t[limit];
+ b.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(limit);
for (size_t i = 2; i < limit; i++) {
if (i % 3 == 0) {
b.docs[b.count] = i;
- auto& reference = b.reference_filter_results["bar"][b.count++];
- reference.count = 1;
- reference.docs = new uint32_t[1];
- reference.docs[0] = 2 * i;
+ auto& reference = b.reference_filter_results[b.count++] = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ auto reference_docs = new uint32_t[1];
+ reference_docs[0] = 2 * i;
+ (*reference)["bar"] = reference_filter_result_t(1, reference_docs);
}
}
@@ -696,9 +697,9 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {
filter_result_t::and_filter_results(a, b, result);
ASSERT_EQ(2, result.count);
- ASSERT_EQ(2, result.reference_filter_results.size());
- ASSERT_EQ(1, result.reference_filter_results.count("foo"));
- ASSERT_EQ(1, result.reference_filter_results.count("bar"));
+ ASSERT_EQ(2, result.reference_filter_results[0]->size());
+ ASSERT_EQ(1, result.reference_filter_results[0]->count("foo"));
+ ASSERT_EQ(1, result.reference_filter_results[0]->count("bar"));
std::vector<uint32_t> docs = {3, 6}, foo_reference = {7, 4}, bar_reference = {6, 12};
@@ -706,10 +707,10 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {
ASSERT_EQ(docs[i], result.docs[i]);
// result should contain correct references to the foo and bar collection.
- ASSERT_EQ(1, result.reference_filter_results["foo"][i].count);
- ASSERT_EQ(foo_reference[i], result.reference_filter_results["foo"][i].docs[0]);
- ASSERT_EQ(1, result.reference_filter_results["bar"][i].count);
- ASSERT_EQ(bar_reference[i], result.reference_filter_results["bar"][i].docs[0]);
+ ASSERT_EQ(1, result.reference_filter_results[i]->at("foo").count);
+ ASSERT_EQ(foo_reference[i], result.reference_filter_results[i]->at("foo").docs[0]);
+ ASSERT_EQ(1, result.reference_filter_results[i]->at("bar").count);
+ ASSERT_EQ(bar_reference[i], result.reference_filter_results[i]->at("bar").docs[0]);
}
}
@@ -728,7 +729,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {
filter_result_t result1;
filter_result_t::or_filter_results(a, b, result1);
ASSERT_EQ(3, result1.count);
- ASSERT_EQ(0, result1.reference_filter_results.size());
+ ASSERT_EQ(nullptr, result1.reference_filter_results);
std::vector<uint32_t> expected = {3, 6, 9};
for (size_t i = 0; i < result1.count; i++) {
@@ -745,7 +746,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {
filter_result_t result2;
filter_result_t::or_filter_results(a, b, result2);
ASSERT_EQ(10, result2.count);
- ASSERT_EQ(0, result2.reference_filter_results.size());
+ ASSERT_EQ(nullptr, result2.reference_filter_results);
expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
for (size_t i = 0; i < result2.count; i++) {
@@ -765,7 +766,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {
// b.docs: [0..8], c.docs: [0, 4, 5]
filter_result_t::or_filter_results(b, c, result3);
ASSERT_EQ(9, result3.count);
- ASSERT_EQ(0, result3.reference_filter_results.size());
+ ASSERT_EQ(nullptr, result3.reference_filter_results);
expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};
for(size_t i = 0; i < result3.count; i++) {
@@ -779,15 +780,15 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
a.count = 0;
a.docs = new uint32_t[limit];
- a.reference_filter_results["foo"] = new reference_filter_result_t[limit];
+ a.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(limit);
for (size_t i = 2; i < limit; i++) {
if (i % 3 == 0) {
a.docs[a.count] = i;
- auto& reference = a.reference_filter_results["foo"][a.count++];
- reference.count = 1;
- reference.docs = new uint32_t[1];
- reference.docs[0] = 2 * i;
+ auto& reference = a.reference_filter_results[a.count++] = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ auto reference_docs = new uint32_t[1];
+ reference_docs[0] = 2 * i;
+ (*reference)["foo"] = reference_filter_result_t(1, reference_docs);
}
}
@@ -796,27 +797,27 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
filter_result_t::or_filter_results(a, b, result1);
ASSERT_EQ(3, result1.count);
- ASSERT_EQ(1, result1.reference_filter_results.size());
- ASSERT_EQ(1, result1.reference_filter_results.count("foo"));
+ ASSERT_EQ(1, result1.reference_filter_results[0]->size());
+ ASSERT_EQ(1, result1.reference_filter_results[0]->count("foo"));
std::vector<uint32_t> expected = {3, 6, 9}, foo_reference = {6, 12, 18};
for (size_t i = 0; i < result1.count; i++) {
ASSERT_EQ(expected[i], result1.docs[i]);
- ASSERT_EQ(1, result1.reference_filter_results["foo"][i].count);
- ASSERT_EQ(foo_reference[i], result1.reference_filter_results["foo"][i].docs[0]);
+ ASSERT_EQ(1, result1.reference_filter_results[i]->at("foo").count);
+ ASSERT_EQ(foo_reference[i], result1.reference_filter_results[i]->at("foo").docs[0]);
}
b.count = 9;
b.docs = new uint32_t[b.count];
- b.reference_filter_results["bar"] = new reference_filter_result_t[b.count];
+ b.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(b.count);
for (size_t i = 0; i < b.count; i++) {
b.docs[i] = i;
- auto& reference = b.reference_filter_results["bar"][i];
- reference.count = 1;
- reference.docs = new uint32_t[1];
- reference.docs[0] = 10 - i;
+ auto& reference = b.reference_filter_results[i] = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ auto reference_docs = new uint32_t[1];
+ reference_docs[0] = 10 - i;
+ (*reference)["bar"] = reference_filter_result_t(1, reference_docs);
}
// a.docs: [3, 6, 9], b.docs: [0..8]
@@ -833,18 +834,18 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
ASSERT_EQ(expected[i], result2.docs[i]);
if (foo_map.count(i) != 0) {
- ASSERT_EQ(1, result2.reference_filter_results["foo"][i].count);
- ASSERT_EQ(foo_map[i], result2.reference_filter_results["foo"][i].docs[0]);
+ ASSERT_EQ(1, result2.reference_filter_results[i]->at("foo").count);
+ ASSERT_EQ(foo_map[i], result2.reference_filter_results[i]->at("foo").docs[0]);
} else {
- // Reference count should be 0 for the docs that were not present in the a result.
- ASSERT_EQ(0, result2.reference_filter_results["foo"][i].count);
+ // foo didn't have any reference to current doc.
+ ASSERT_EQ(0, result2.reference_filter_results[i]->count("foo"));
}
if (bar_map.count(i) != 0) {
- ASSERT_EQ(1, result2.reference_filter_results["bar"][i].count);
- ASSERT_EQ(bar_map[i], result2.reference_filter_results["bar"][i].docs[0]);
+ ASSERT_EQ(1, result2.reference_filter_results[i]->at("bar").count);
+ ASSERT_EQ(bar_map[i], result2.reference_filter_results[i]->at("bar").docs[0]);
} else {
- ASSERT_EQ(0, result2.reference_filter_results["bar"][i].count);
+ ASSERT_EQ(0, result2.reference_filter_results[i]->count("bar"));
}
}
@@ -853,15 +854,15 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
std::map<uint32_t, uint32_t> baz_map = {{0, 2}, {4, 0}, {5, 8}};
c.count = baz_map.size();
c.docs = new uint32_t[baz_map.size()];
- c.reference_filter_results["baz"] = new reference_filter_result_t[baz_map.size()];
+ c.reference_filter_results = std::make_unique<std::unique_ptr<std::map<std::string, reference_filter_result_t>> []>(baz_map.size());
auto j = 0;
for(auto i: baz_map) {
c.docs[j] = i.first;
- auto& reference = c.reference_filter_results["baz"][j++];
- reference.count = 1;
- reference.docs = new uint32_t[1];
- reference.docs[0] = i.second;
+ auto& reference = c.reference_filter_results[j++] = std::make_unique<std::map<std::string, reference_filter_result_t>>();
+ auto reference_docs = new uint32_t[1];
+ reference_docs[0] = i.second;
+ (*reference)["baz"] = reference_filter_result_t(1, reference_docs);
}
// b.docs: [0..8], c.docs: [0, 4, 5]
@@ -873,17 +874,17 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
ASSERT_EQ(expected[i], result3.docs[i]);
if (bar_map.count(i) != 0) {
- ASSERT_EQ(1, result3.reference_filter_results["bar"][i].count);
- ASSERT_EQ(bar_map[i], result3.reference_filter_results["bar"][i].docs[0]);
+ ASSERT_EQ(1, result3.reference_filter_results[i]->at("bar").count);
+ ASSERT_EQ(bar_map[i], result3.reference_filter_results[i]->at("bar").docs[0]);
} else {
- ASSERT_EQ(0, result3.reference_filter_results["bar"][i].count);
+ ASSERT_EQ(0, result3.reference_filter_results[i]->count("bar"));
}
if (baz_map.count(i) != 0) {
- ASSERT_EQ(1, result3.reference_filter_results["baz"][i].count);
- ASSERT_EQ(baz_map[i], result3.reference_filter_results["baz"][i].docs[0]);
+ ASSERT_EQ(1, result3.reference_filter_results[i]->at("baz").count);
+ ASSERT_EQ(baz_map[i], result3.reference_filter_results[i]->at("baz").docs[0]);
} else {
- ASSERT_EQ(0, result3.reference_filter_results["baz"][i].count);
+ ASSERT_EQ(0, result3.reference_filter_results[i]->count("baz"));
}
}
}