Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

refactoring changes

krunal1313b0a369f8ab31
diff --git a/include/facet_index.h b/include/facet_index.h
index db1e2162..c97d6097 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -11,19 +11,22 @@ private:
~count_list () = default;
- count_list(const std::string& sv, uint32_t facet_count) {
+ count_list(const std::string& sv, uint32_t facet_count, uint32_t facet_index) {
facet_value = sv;
count = facet_count;
+ index = facet_index;
}
count_list& operator=(count_list& obj) {
facet_value = obj.facet_value;
count = obj.count;
+ index = obj.index;
return *this;
}
std::string facet_value;
uint32_t count;
+ uint32_t index;
};
struct facet_index_struct {
@@ -61,8 +64,6 @@ private:
std::map<std::string, facet_index_counter> facet_field_map;
uint32_t count_index = 0;
- std::unordered_set<std::string> dropped_fields;
-
public:
facet_index_t() = default;
@@ -73,6 +74,8 @@ public:
void erase(const std::string& field);
+ void remove(const std::string& field, const uint32_t seq_id);
+
bool contains(const std::string& field);
size_t get_facet_count(const std::string& field);
@@ -81,11 +84,8 @@ public:
int max_facet_count, std::map<std::string, uint32_t>& found,
bool is_wildcard_no_filter_query);
- void get_facet_indexes(const std::string& field, std::function<void(uint32_t, uint32_t)> functor);
-
- bool get_migrated (const std::string& field) const;
-
- void set_migrated(const std::string& field, bool val);
-
- void set_dropped(const std::string& field);
+ size_t get_facet_indexes(const std::string& field,
+ std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes);
+
+ void initialize(const std::string& field);
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index 1ae09e59..78f04f76 100644
--- a/include/index.h
+++ b/include/index.h
@@ -304,13 +304,10 @@ private:
// geo_array_field => (seq_id => values) used for exact filtering of geo array records
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;
- // facet_field => (seq_id => values)
- spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;
-
- facet_index_t* facet_index_v4 = nullptr;
+ spp::sparse_hash_map<std::string, posting_list_t*> facet_hash_index;
- // facet_field => (seq_id => hash)
- spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;
+ facet_index_t* facet_index_v4 = nullptr;
+
// sort_field => (seq_id => value)
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;
diff --git a/include/num_tree.h b/include/num_tree.h
index ce0a226c..c31f0b99 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -43,7 +43,6 @@ private:
};
std::vector<count_list> counter_list;
- bool is_migrated = false;
public:
@@ -85,13 +84,5 @@ public:
size_t counter_list_size() const;
- void get_facet_indexes(std::function<void(uint32_t, uint32_t)> functor);
-
- bool get_migrated () const {
- return is_migrated;
- }
-
- void set_migrated(bool val) {
- is_migrated = true;
- }
+ size_t get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes);
};
\ No newline at end of file
diff --git a/include/posting_list.h b/include/posting_list.h
index a0df81fa..d71a8fbd 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -188,6 +188,8 @@ public:
);
static bool take_id(result_iter_state_t& istate, uint32_t id);
+
+ static void get_offsets(iterator_t& iter, std::vector<uint32_t>& positions);
static bool get_offsets(
const std::vector<iterator_t>& its,
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index e007710b..88e8ec69 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -3,22 +3,24 @@
#include "string_utils.h"
#include "array_utils.h"
+void facet_index_t::initialize(const std::string& field) {
+ const auto facet_field_map_it = facet_field_map.find(field);
+ if(facet_field_map_it == facet_field_map.end()) {
+ facet_field_map.emplace(field, facet_index_counter{});
+ }
+}
+
uint32_t facet_index_t::insert(const std::string& field, const std::string& value,
uint32_t id, bool is_string) {
- if(dropped_fields.find(field) != dropped_fields.end()) {
- //field has been dropped
- return 0;
- }
-
const auto facet_field_map_it = facet_field_map.find(field);
if(facet_field_map_it == facet_field_map.end()) {
- facet_field_map.emplace(field, facet_index_counter{});
+ return 0; //field is not initialized or dropped
}
uint32_t index = 0;
- auto& facet_index_map = facet_field_map[field].facet_index_map;
+ auto& facet_index_map = facet_field_map_it->second.facet_index_map;
const auto sv = value.substr(0, 100);
const auto it = facet_index_map.find(sv);
@@ -44,22 +46,26 @@ uint32_t facet_index_t::insert(const std::string& field, const std::string& valu
if(is_string) {
const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);
//LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;
- auto& counter_list = facet_field_map[field].counter_list;
+ auto& counter_list = facet_field_map_it->second.counter_list;
if(counter_list.empty()) {
- counter_list.emplace_back(sv, facet_count);
+ counter_list.emplace_back(sv, facet_count, index);
} else {
- count_list node(sv, facet_count);
- uint32_t ind = 0;
- for(ind; ind < counter_list.size(); ++ind) {
- if(counter_list[ind].facet_value == sv) {
+ count_list node(sv, facet_count, index);
+ auto ind = 0;
+
+ for(; ind < counter_list.size(); ++ind) {
+ if(counter_list[ind].index == index) {
counter_list[ind].count = facet_count;
- if((ind > 1) &&
- (counter_list[ind-1].count < counter_list[ind].count)) {
- count_list temp = counter_list[ind-1];
- counter_list[ind-1] = counter_list[ind];
- counter_list[ind] = temp;
+ if(ind > 1) {
+ auto curr = ind;
+ while (curr && (counter_list[curr-1].count < counter_list[curr].count)) {
+ count_list temp = counter_list[curr-1];
+ counter_list[curr-1] = counter_list[curr];
+ counter_list[curr] = temp;
+ --curr;
+ }
}
break;
}
@@ -92,6 +98,23 @@ void facet_index_t::erase(const std::string& field) {
}
}
+void facet_index_t::remove(const std::string& field, const uint32_t seq_id) {
+ const auto facet_field_it = facet_field_map.find(field);
+ if(facet_field_it != facet_field_map.end()) {
+ auto facet_index_map = facet_field_it->second.facet_index_map;
+ auto facet_index_map_it = facet_index_map.begin();
+
+ for(; facet_index_map_it != facet_index_map.end(); ++facet_index_map_it) {
+ void* ids = facet_index_map_it.value().id_list_ptr;
+ if(ids && ids_t::contains(ids, seq_id)) {
+ // ids_t::erase(ids, seq_id);
+ // facet_index_map_it.value().id_list_ptr = ids;
+ break;
+ }
+ }
+ }
+}
+
size_t facet_index_t::get_facet_count(const std::string& field) {
const auto it = facet_field_map.find(field);
@@ -157,12 +180,12 @@ facet_index_t::~facet_index_t() {
}
//used for migrating string and int64 facets
-void facet_index_t::get_facet_indexes(const std::string& field,
- std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {
+size_t facet_index_t::get_facet_indexes(const std::string& field,
+ std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes) {
const auto& facet_field_it = facet_field_map.find(field);
if(facet_field_it == facet_field_map.end()) {
- return;
+ return 0;
}
auto& facet_index_map = facet_field_it->second.facet_index_map;
@@ -177,37 +200,11 @@ void facet_index_t::get_facet_indexes(const std::string& field,
//emplacing seq_id=>count_index
for(const auto& id : id_list) {
- functor(id, facet_index_map_it->index);
+ seqid_countIndexes[id].emplace_back(facet_index_map_it->index);
}
id_list.clear();
}
- return;
-}
-
-
-bool facet_index_t::get_migrated (const std::string& field) const {
- const auto it = facet_field_map.find(field);
- if(it != facet_field_map.end()) {
- return it->second.is_migrated;
- }
-
- return false;
-}
-
-void facet_index_t::set_migrated(const std::string& field, bool val) {
- const auto it = facet_field_map.find(field);
- if(it != facet_field_map.end()) {
- it->second.is_migrated = val;
- }
-
- return;
+ return seqid_countIndexes.size();
}
-
-void facet_index_t::set_dropped(const std::string& field) {
-
- dropped_fields.emplace(field);
-
- return;
-}
\ No newline at end of file
diff --git a/src/index.cpp b/src/index.cpp
index b366c0de..460aef95 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -187,23 +187,11 @@ Index::~Index() {
str_sort_index.clear();
- for(auto& field_name_facet_map_array: facet_index_v3) {
- for(auto& facet_map: field_name_facet_map_array.second) {
- delete facet_map;
- facet_map = nullptr;
- }
+ for(const auto& facet_hash_index_it : facet_hash_index) {
+ delete facet_hash_index_it.second;
}
+ facet_hash_index.clear();
- facet_index_v3.clear();
-
- for(auto& field_name_facet_map_array: single_val_facet_index_v3) {
- for(auto& facet_map: field_name_facet_map_array.second) {
- delete facet_map;
- facet_map = nullptr;
- }
- }
-
- single_val_facet_index_v3.clear();
delete facet_index_v4;
delete seq_ids;
@@ -607,59 +595,26 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
void Index::create_facet_hash_index(const field& facet_field) {
//LOG(INFO) << "migrating facet " << field << " to old index";
- if(facet_field.is_array()) {
- auto& facet_index = facet_index_v3[facet_field.name];
-
- facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {
+ std::map<uint32_t, std::vector<uint32_t>> seqid_index_map;
+ auto facet_index = facet_hash_index[facet_field.name];
- auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
- } else {
- facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);
- }
- });
-
- auto numerical_index_it = numerical_index.find(facet_field.name);
- if(numerical_index_it != numerical_index_it) {
- auto num_tree = numerical_index_it->second;
- num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {
-
- auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
- } else {
- facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);
- }
- });
+ if(facet_index_v4->get_facet_indexes(facet_field.name, seqid_index_map)) {
+ for(const auto& kv : seqid_index_map) {
+ facet_index->upsert(kv.first, std::move(kv.second));
}
- } else {
- auto& single_facet_index = single_val_facet_index_v3[facet_field.name];
+ }
- facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {
+ seqid_index_map.clear();
- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
- } else {
- facet_dim_index->operator[](seq_id) = count_index;
+ auto numerical_index_it = numerical_index.find(facet_field.name);
+ if(numerical_index_it != numerical_index_it) {
+ auto num_tree = numerical_index_it->second;
+ if(num_tree->get_facet_indexes(seqid_index_map)) {
+ for(const auto& kv : seqid_index_map) {
+ facet_index->upsert(kv.first, std::move(kv.second));
}
- });
-
- auto numerical_index_it = numerical_index.find(facet_field.name);
- if(numerical_index_it != numerical_index_it) {
- auto num_tree = numerical_index_it->second;
- num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {
-
- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
- } else {
- facet_dim_index->operator[](seq_id) = count_index;
- }
- });
- }
- }
+ }
+ }
}
void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch) {
@@ -697,43 +652,38 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
auto facet_threshold_count = 0;
- bool is_migrated = false;
if(afield.is_string()) {
facet_threshold_count = facet_index_v4->get_facet_count(afield.name);
- is_migrated = facet_index_v4->get_migrated(afield.name);
} else {
auto numerical_index_it = numerical_index.find(afield.name);
if(numerical_index_it != numerical_index.end()) {
facet_threshold_count = numerical_index_it->second->counter_list_size();
- is_migrated = numerical_index_it->second->get_migrated();
}
}
#ifdef TEST_BUILD
facet_threshold_count = FACET_INDEX_THRESHOLD + 1;
- is_migrated = true;
#endif
const auto total_num_docs = aggragate_docs_count.load();
- if(!is_migrated && (facet_threshold_count > FACET_INDEX_THRESHOLD)
+ bool facet_hash_index_exist = facet_hash_index.find(afield.name) != facet_hash_index.end();
+
+ if(!facet_hash_index_exist && (facet_threshold_count > FACET_INDEX_THRESHOLD)
&& total_num_docs < 1000000) {
create_facet_hash_index(afield);
- if(afield.is_string()) {
- facet_index_v4->set_migrated(afield.name, true);
- } else {
- auto numerical_index_it = numerical_index.find(afield.name);
- if(numerical_index_it != numerical_index.end()) {
- numerical_index_it->second->set_migrated(true);
- }
- }
-
auto cardinality_ratio = total_num_docs / facet_threshold_count;
if(cardinality_ratio < 5) {
+ //drop the field
facet_index_v4->erase(afield.name);
- facet_index_v4->set_dropped(afield.name);
}
}
+ posting_list_t* facet_index = nullptr;
+ auto facet_index_it = facet_hash_index.find(afield.name);
+ if(facet_index_it != facet_hash_index.end()) {
+ facet_index = facet_index_it->second;
+ }
+
for(const auto& record: iter_batch) {
if(!record.indexed.ok()) {
// some records could have been invalidated upstream
@@ -757,7 +707,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
if(afield.facet) {
if(afield.is_array()) {
- facet_hash_values_t fhashvalues;
+ std::vector<uint32_t> fhashvalues;
for(size_t i = 0; i < document[afield.name].size(); ++i) {
if(afield.type == field_types::INT32_ARRAY) {
int32_t raw_val = document[afield.name][i].get<int32_t>();
@@ -778,19 +728,13 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
}
if(facet_threshold_count > FACET_INDEX_THRESHOLD) {
- fhashvalues.hashes.emplace_back(fhash);
+ fhashvalues.emplace_back(fhash);
}
}
if(facet_threshold_count > FACET_INDEX_THRESHOLD) {
- fhashvalues.length = fhashvalues.hashes.size();
-
- auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
- } else {
- facet_dim_index->emplace(seq_id, std::move(fhashvalues));
- }
+ facet_index->upsert(seq_id, std::move(fhashvalues));
+ fhashvalues.clear();
}
} else {
uint32_t fhash;
@@ -818,13 +762,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
}
if(facet_threshold_count > FACET_INDEX_THRESHOLD) {
-
- auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
- } else {
- facet_dim_index->emplace(seq_id, std::move(fhash));
- }
+ facet_index->upsert(seq_id, {fhash});
}
}
}
@@ -1195,19 +1133,13 @@ void Index::tokenize_string_array(const std::vector<std::string>& strings, bool
}
}
void Index::initialize_facet_indexes(const field& facet_field) {
- if(facet_field.is_array()) {
- array_mapped_facet_t facet_array;
- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- facet_array[i] = new facet_map_t();
- }
- facet_index_v3.emplace(facet_field.name, facet_array);
- } else {
- array_mapped_single_val_facet_t facet_array;
- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- facet_array[i] = new single_val_facet_map_t();
- }
- single_val_facet_index_v3.emplace(facet_field.name, facet_array);
+
+ if(facet_field.is_string() || facet_field.is_int64()) {
+ facet_index_v4->initialize(facet_field.name);
}
+
+ facet_hash_index.emplace(facet_field.name, new posting_list_t(256));
+
}
void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {
@@ -1337,12 +1269,10 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
use_facet_intersection = false;
use_hashes = false;
#endif
- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
-
+ const auto& facet_hash_index_it = facet_hash_index.find(a_facet.field_name);
+
bool is_wildcard_no_filter_query = is_wildcard_query && no_filters_provided;
- bool facet_hash_index_exists = ((field_facet_mapping_it != facet_index_v3.end())
- || (field_single_val_facet_mapping_it != single_val_facet_index_v3.end()));
+ bool facet_hash_index_exists = facet_hash_index_it != facet_hash_index.end();
if(results_size && num_facet_values && ((!facet_hash_index_exists || is_wildcard_no_filter_query
|| num_facet_values > 50000) && group_limit == 0) && !use_hashes || use_facet_intersection) {
@@ -1419,17 +1349,18 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
} else {
facet_map_t::iterator facet_map_it;
single_val_facet_map_t::iterator single_facet_map_it;
- uint32_t fhash = 0;
- size_t facet_hash_count = 1;
- if((field_facet_mapping_it == facet_index_v3.end())
- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
+ if(!facet_hash_index_exists) {
continue;
}
auto sort_index_it = sort_index.find(a_facet.field_name);
const auto facet_field_is_array = facet_field.is_array();
+ const auto& facet_index = facet_hash_index_it->second;
+ std::vector<uint32_t> facet_hashes;
+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();
+
for(size_t i = 0; i < results_size; i++) {
// if sampling is enabled, we will skip a portion of the results to speed up things
if(estimate_facets) {
@@ -1439,64 +1370,58 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
}
uint32_t doc_seq_id = result_ids[i];
- if(facet_field_is_array) {
- const auto& field_facet_mapping = field_facet_mapping_it->second;
- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
- facet_hash_count = facet_map_it->second.size();
- } else {
- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
- facet_hash_count = 1;
- fhash = single_facet_map_it->second;
+ if(!facet_index->contains(doc_seq_id)) {
+ continue;
}
- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
- //LOG(INFO) << "facet_hash_count " << facet_hash_count;
- if(((i + 1) % 16384) == 0) {
- RETURN_CIRCUIT_BREAKER
- }
+ facet_index_it.skip_to(doc_seq_id);
- for(size_t j = 0; j < facet_hash_count; j++) {
- if(facet_field_is_array) {
- fhash = facet_map_it->second.hashes[j];
- }
- if(should_compute_stats) {
- compute_facet_stats(a_facet, fhash, facet_field.type);
- }
+ if(facet_index_it.valid()) {
+ facet_hashes.clear();
+ posting_list_t::get_offsets(facet_index_it, facet_hashes);
- std::string fhash_str = std::to_string(fhash);
- if(a_facet.is_range_query) {
- int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);
-
- std::pair<std::string, std::string> range_pair {};
- if(a_facet.get_range(std::to_string(doc_val), range_pair)) {
- const auto& range_id = range_pair.first;
- facet_count_t& facet_count = a_facet.result_map[range_id];
- facet_count.count += 1;
- }
- } else if(!use_facet_query || fquery_hashes.find(fhash_str) != fquery_hashes.end()) {
- facet_count_t& facet_count = a_facet.result_map[fhash_str];
- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
- facet_count.doc_id = doc_seq_id;
- facet_count.array_pos = j;
- if(group_limit) {
- a_facet.hash_groups[fhash].emplace(distinct_id);
- } else {
- facet_count.count += 1;
+ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;
+ if(((i + 1) % 16384) == 0) {
+ RETURN_CIRCUIT_BREAKER
+ }
+
+ for(size_t j = 0; j < facet_hashes.size(); j++) {
+
+ const auto& fhash = facet_hashes[j];
+
+ if(should_compute_stats) {
+ compute_facet_stats(a_facet, fhash, facet_field.type);
}
- if(use_facet_query) {
- //LOG (INFO) << "adding hash tokens for hash " << fhash;
- a_facet.hash_tokens[fhash_str] = fquery_hashes.at(fhash_str);
+
+ std::string fhash_str = std::to_string(fhash);
+ if(a_facet.is_range_query) {
+ int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);
+
+ std::pair<std::string, std::string> range_pair {};
+ if(a_facet.get_range(std::to_string(doc_val), range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count += 1;
+ }
+ } else if(!use_facet_query || fquery_hashes.find(fhash_str) != fquery_hashes.end()) {
+ facet_count_t& facet_count = a_facet.result_map[fhash_str];
+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
+ facet_count.doc_id = doc_seq_id;
+ facet_count.array_pos = j;
+ if(group_limit) {
+ a_facet.hash_groups[fhash].emplace(distinct_id);
+ } else {
+ facet_count.count += 1;
+ }
+ if(use_facet_query) {
+ //LOG (INFO) << "adding hash tokens for hash " << fhash;
+ a_facet.hash_tokens[fhash_str] = fquery_hashes.at(fhash_str);
+ }
}
}
+ } else {
+ break;
}
}
}
@@ -4564,10 +4489,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
for(size_t findex=0; findex < facets.size(); findex++) {
const auto& a_facet = facets[findex];
- const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
- const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
- const auto facet_hash_index_exists = ((field_facet_mapping_it != facet_index_v3.end())
- || (field_single_val_facet_mapping_it != single_val_facet_index_v3.end()));
+ bool facet_hash_index_exist = facet_hash_index.find(a_facet.field_name) != facet_hash_index.end();
facet_infos[findex].use_facet_query = false;
@@ -4665,40 +4587,40 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
if(!use_facet_intersection) {
#else
//checking if facets is migrated to old index
- if(facet_hash_index_exists) {
-#endif
- if(facet_field.is_array()) {
- const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
+ if(facet_hash_index_exist) {
+#endif
+ std::vector<uint32_t> facet_hashes;
+
+ auto facet_index = facet_hash_index.at(a_facet.field_name);
+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();
+ facet_index_it.skip_to(seq_id);
+
+ if(facet_index_it.valid()) {
+ posting_list_t::get_offsets(facet_index_it, facet_hashes);
- std::vector<size_t> array_indices;
- posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);
+ if(facet_field.is_array()) {
+ std::vector<size_t> array_indices;
+ posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);
- for(size_t array_index: array_indices) {
- if(array_index < doc_fvalues_it->second.length) {
- std::string hash = std::to_string(doc_fvalues_it->second.hashes[array_index]);
+ for(size_t array_index: array_indices) {
+ if(array_index < facet_hashes.size()) {
+ std::string hash = std::to_string(facet_hashes[array_index]);
- /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "
- << array_index;*/
+ /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "
+ << array_index;*/
- if(facet_infos[findex].hashes.count(hash) == 0) {
- //LOG(INFO) << "adding searched_tokens for hash " << hash;
- facet_infos[findex].hashes.emplace(hash, searched_tokens);
+ if(facet_infos[findex].hashes.count(hash) == 0) {
+ //LOG(INFO) << "adding searched_tokens for hash " << hash;
+ facet_infos[findex].hashes.emplace(hash, searched_tokens);
+ }
}
}
- }
- } else {
- const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
-
- std::string hash = std::to_string(doc_single_fvalues_it->second);
- if(facet_infos[findex].hashes.count(hash) == 0) {
- //LOG(INFO) << "adding searched_tokens for hash " << hash;
- facet_infos[findex].hashes.emplace(hash, searched_tokens);
+ } else {
+ std::string hash = std::to_string(facet_hashes[0]);
+ if(facet_infos[findex].hashes.count(hash) == 0) {
+ //LOG(INFO) << "adding searched_tokens for hash " << hash;
+ facet_infos[findex].hashes.emplace(hash, searched_tokens);
+ }
}
}
} else { //for new index adding search tokens
@@ -5433,37 +5355,29 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
uint32_t distinct_id = 1; // some constant initial value
// calculate hash from group_by_fields
for(const auto& field: group_by_fields) {
- const auto& field_facet_mapping_it = facet_index_v3.find(field);
- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);
- if((field_facet_mapping_it == facet_index_v3.end())
- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
+ const auto& facet_hash_index_it = facet_hash_index.find(field);
+ if(facet_hash_index_it == facet_hash_index.end()) {
continue;
}
-
- if(search_schema.at(field).is_array()) {
- const auto& field_facet_mapping = field_facet_mapping_it->second;
- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
+ auto facet_index = facet_hash_index_it->second;
+ std::vector<uint32_t> facet_hashes;
+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();
+ facet_index_it.skip_to(seq_id);
- const auto& facet_hashes = facet_hashes_it->second;
- //LOG(INFO) << "combining hashes for facet array ";
- for(size_t i = 0; i < facet_hashes.size(); i++) {
- distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);
- }
- } else {
- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
+ if(facet_index_it.valid()) {
+ posting_list_t::get_offsets(facet_index_it, facet_hashes);
- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
+ if(search_schema.at(field).is_array()) {
+ //LOG(INFO) << "combining hashes for facet array ";
+ for(size_t i = 0; i < facet_hashes.size(); i++) {
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes[i]);
+ }
+ } else {
+ const auto& facet_hash = facet_hashes[0];
+ //LOG(INFO) << "combining hashes for facet ";
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
}
-
- const auto& facet_hash = facet_hashes_it->second;
- //LOG(INFO) << "combining hashes for facet ";
- distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
}
}
//LOG(INFO) << "distinct id " << distinct_id;
@@ -5700,23 +5614,16 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const
}
// remove facets
- const auto& field_facets_it = facet_index_v3.find(field_name);
- if(field_facets_it != facet_index_v3.end()) {
- const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
+ const auto& facet_hash_index_it = facet_hash_index.find(field_name);
+ if(facet_hash_index_it != facet_hash_index.end()) {
+ auto facet_index = facet_hash_index_it->second;
+ if(facet_index) {
+ facet_index->erase(seq_id);
}
}
- const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);
- if(field_single_val_facets_it != single_val_facet_index_v3.end()) {
- const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
- }
- }
- facet_index_v4->erase(field_name);
+ facet_index_v4->remove(field_name, seq_id);
// remove sort field
if(sort_index.count(field_name) != 0) {
@@ -5918,21 +5825,9 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
}
if(del_field.is_facet()) {
- if(del_field.is_array()) {
- auto& arr = facet_index_v3[del_field.name];
- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- delete arr[i];
- }
-
- facet_index_v3.erase(del_field.name);
- } else {
- auto& arr = single_val_facet_index_v3[del_field.name];
- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- delete arr[i];
- }
+ delete facet_hash_index.at(del_field.name);
+ facet_hash_index.erase(del_field.name);
- single_val_facet_index_v3.erase(del_field.name);
- }
facet_index_v4->erase(del_field.name);
if(!del_field.is_string()) {
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index 6a6dbe0d..9b2fc866 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -22,16 +22,21 @@ void num_tree_t::insert(int64_t value, uint32_t id, bool is_facet) {
counter_list.emplace_back(value, facet_count);
} else {
count_list node(value, facet_count);
- uint32_t ind = 0;
- for(ind; ind < counter_list.size(); ++ind) {
+
+ auto ind = 0;
+
+ for(; ind < counter_list.size(); ++ind) {
if(counter_list[ind].facet_value == value) {
counter_list[ind].count = facet_count;
- if((ind > 1) &&
- (counter_list[ind-1].count < counter_list[ind].count)) {
- count_list temp = counter_list[ind-1];
- counter_list[ind-1] = counter_list[ind];
- counter_list[ind] = temp;
+ if(ind > 1) {
+ auto curr = ind;
+ while (curr && (counter_list[curr-1].count < counter_list[curr].count)) {
+ count_list temp = counter_list[curr-1];
+ counter_list[curr-1] = counter_list[curr];
+ counter_list[curr] = temp;
+ --curr;
+ }
}
break;
}
@@ -426,11 +431,11 @@ size_t num_tree_t::intersect(const uint32_t* result_ids, int result_ids_len, int
return found.size();
}
-void num_tree_t::get_facet_indexes(std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {
+size_t num_tree_t::get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes) {
//check if facet field
if(counter_list.empty()) {
- return;
+ return 0;
}
std::vector<uint32_t> id_list;
@@ -442,11 +447,11 @@ void num_tree_t::get_facet_indexes(std::function<void(uint32_t seq_id, uint32_t
//emplacing seq_id=>count_index
for(const auto& id : id_list) {
- functor(id, int64map_it->first);
+ seqid_countIndexes[id].emplace_back(int64map_it->first);
}
id_list.clear();
}
- return;
+ return seqid_countIndexes.size();
}
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index dc82fb69..3e28cad9 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -808,6 +808,27 @@ bool posting_list_t::take_id(result_iter_state_t& istate, uint32_t id) {
return true;
}
+void posting_list_t::get_offsets(iterator_t& iter, std::vector<uint32_t>& positions) {
+ block_t* curr_block = iter.block();
+ uint32_t curr_index = iter.index();
+
+ if(curr_block == nullptr || curr_index == UINT32_MAX) {
+ return;
+ }
+
+ uint32_t* offsets = iter.offsets;
+ uint32_t start_offset = iter.offset_index[curr_index];
+ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?
+ curr_block->offsets.getLength() :
+ iter.offset_index[curr_index + 1];
+
+ while(start_offset < end_offset) {
+ int pos = offsets[start_offset];
+ positions.push_back(pos);
+ ++start_offset;
+ }
+}
+
bool posting_list_t::get_offsets(const std::vector<iterator_t>& its,
std::map<size_t, std::vector<token_positions_t>>& array_token_pos) {
@@ -1657,7 +1678,12 @@ void posting_list_t::iterator_t::next() {
}
uint32_t posting_list_t::iterator_t::last_block_id() const {
- return ids[curr_block->size() - 1];
+ auto size = curr_block->size();
+ if(size == 0) {
+ return 0;
+ }
+
+ return ids[size - 1];
}
uint32_t posting_list_t::iterator_t::id() const {