Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

picking index as per results size and improvements

krunal1313df8dda94321f
diff --git a/include/facet_index.h b/include/facet_index.h
index cf20cbcd..db1e2162 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -2,6 +2,7 @@
#include "ids_t.h"
#include "tsl/htrie_map.h"
+#include <unordered_set>
class facet_index_t {
private:
@@ -18,6 +19,7 @@ private:
count_list& operator=(count_list& obj) {
facet_value = obj.facet_value;
count = obj.count;
+ return *this;
}
std::string facet_value;
@@ -59,6 +61,8 @@ private:
std::map<std::string, facet_index_counter> facet_field_map;
uint32_t count_index = 0;
+ std::unordered_set<std::string> dropped_fields;
+
public:
facet_index_t() = default;
@@ -77,9 +81,11 @@ public:
int max_facet_count, std::map<std::string, uint32_t>& found,
bool is_wildcard_no_filter_query);
- size_t get_facet_indexes(const std::string& field, std::map<uint32_t, std::vector<uint32_t>>& seqid_index_map);
+ void get_facet_indexes(const std::string& field, std::function<void(uint32_t, uint32_t)> functor);
bool get_migrated (const std::string& field) const;
void set_migrated(const std::string& field, bool val);
+
+ void set_dropped(const std::string& field);
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index 6378490a..1ae09e59 100644
--- a/include/index.h
+++ b/include/index.h
@@ -524,7 +524,7 @@ private:
void initialize_facet_indexes(const field& facet_field);
- void create_facet_hash_index(const std::string& field);
+ void create_facet_hash_index(const field& facet_field);
static Option<bool> embed_fields(nlohmann::json& document,
diff --git a/include/num_tree.h b/include/num_tree.h
index 2397cc64..ce0a226c 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -35,6 +35,7 @@ private:
count_list& operator=(count_list& obj) {
facet_value = obj.facet_value;
count = obj.count;
+ return *this;
}
int64_t facet_value;
@@ -84,7 +85,7 @@ public:
size_t counter_list_size() const;
- size_t get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& facets);
+ void get_facet_indexes(std::function<void(uint32_t, uint32_t)> functor);
bool get_migrated () const {
return is_migrated;
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index 564394db..82e1e4f5 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -6,11 +6,18 @@
uint32_t facet_index_t::insert(const std::string& field, const std::string& value,
uint32_t id, bool is_string) {
+ if(dropped_fields.find(field) != dropped_fields.end()) {
+ //field has been dropped
+ return 0;
+ }
+
const auto facet_field_map_it = facet_field_map.find(field);
if(facet_field_map_it == facet_field_map.end()) {
facet_field_map.emplace(field, facet_index_counter{});
}
+ uint32_t index = 0;
+
auto& facet_index_map = facet_field_map[field].facet_index_map;
uint32_t index;
const auto sv = value.substr(0, 100);
@@ -151,12 +158,12 @@ facet_index_t::~facet_index_t() {
}
//used for migrating string and int64 facets
-size_t facet_index_t::get_facet_indexes(const std::string& field,
- std::map<uint32_t, std::vector<uint32_t>>& seqid_index_map) {
+void facet_index_t::get_facet_indexes(const std::string& field,
+ std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {
const auto& facet_field_it = facet_field_map.find(field);
if(facet_field_it == facet_field_map.end()) {
- return 0;
+ return;
}
auto& facet_index_map = facet_field_it->second.facet_index_map;
@@ -171,13 +178,13 @@ size_t facet_index_t::get_facet_indexes(const std::string& field,
//emplacing seq_id=>count_index
for(const auto& id : id_list) {
- seqid_index_map[id].emplace_back(facet_index_map_it->index);
+ functor(id, facet_index_map_it->index);
}
id_list.clear();
}
- return seqid_index_map.size();
+ return;
}
@@ -198,3 +205,10 @@ void facet_index_t::set_migrated(const std::string& field, bool val) {
return;
}
+
+void facet_index_t::set_dropped(const std::string& field) {
+
+ dropped_fields.emplace(field);
+
+ return;
+}
\ No newline at end of file
diff --git a/src/index.cpp b/src/index.cpp
index 5ee2ce04..b366c0de 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -604,71 +604,60 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
return num_indexed;
}
-void Index::create_facet_hash_index(const std::string& field) {
+void Index::create_facet_hash_index(const field& facet_field) {
//LOG(INFO) << "migrating facet " << field << " to old index";
+ if(facet_field.is_array()) {
+ auto& facet_index = facet_index_v3[facet_field.name];
- std::map<uint32_t, std::vector<uint32_t>> facets_indexes;
- auto& facet_index = facet_index_v3[field];
- auto& single_facet_index = single_val_facet_index_v3[field];
-
- //migrate string and int64 facets first
- if(facet_index_v4->get_facet_indexes(field, facets_indexes)) {
+ facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {
- for(const auto& kv : facets_indexes) {
+ auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
+ } else {
+ facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);
+ }
+ });
- const auto hash_count = kv.second.size();
- const auto seq_id = kv.first;
+ auto numerical_index_it = numerical_index.find(facet_field.name);
+ if(numerical_index_it != numerical_index_it) {
+ auto num_tree = numerical_index_it->second;
+ num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {
- if(hash_count > 1) {
auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];
if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << field;
+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
} else {
- facet_hash_values_t fhashvalues;
- fhashvalues.hashes = std::move(kv.second);
- facet_dim_index->emplace(seq_id, std::move(fhashvalues));
+ facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);
}
- } else {
- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << field;
- } else {
- facet_dim_index->emplace(seq_id, kv.second[0]);
- }
- }
+ });
}
- }
+ } else {
+ auto& single_facet_index = single_val_facet_index_v3[facet_field.name];
- //now extract remaining facets from numerical index
- facets_indexes.clear();
- auto numerical_index_it = numerical_index.find(field);
- if(numerical_index_it != numerical_index_it) {
- auto num_tree = numerical_index_it->second;
- if(num_tree->get_facet_indexes(facets_indexes)) {
+ facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {
- for(const auto& kv : facets_indexes) {
- const auto hash_count = kv.second.size();
- const auto seq_id = kv.first;
+ auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
+ } else {
+ facet_dim_index->operator[](seq_id) = count_index;
+ }
+ });
- if(hash_count > 1) {
- auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << field;
- } else {
- facet_hash_values_t fhashvalues;
- fhashvalues.hashes = std::move(kv.second);
- facet_dim_index->emplace(seq_id, std::move(fhashvalues));
- }
+ auto numerical_index_it = numerical_index.find(facet_field.name);
+ if(numerical_index_it != numerical_index_it) {
+ auto num_tree = numerical_index_it->second;
+ num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {
+
+ auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;
} else {
- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << field;
- } else {
- facet_dim_index->emplace(seq_id, kv.second[0]);
- }
+ facet_dim_index->operator[](seq_id) = count_index;
}
- }
+ });
}
}
}
@@ -727,7 +716,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
const auto total_num_docs = aggragate_docs_count.load();
if(!is_migrated && (facet_threshold_count > FACET_INDEX_THRESHOLD)
&& total_num_docs < 1000000) {
- create_facet_hash_index(afield.name);
+ create_facet_hash_index(afield);
if(afield.is_string()) {
facet_index_v4->set_migrated(afield.name, true);
@@ -737,6 +726,12 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
numerical_index_it->second->set_migrated(true);
}
}
+
+ auto cardinality_ratio = total_num_docs / facet_threshold_count;
+ if(cardinality_ratio < 5) {
+ facet_index_v4->erase(afield.name);
+ facet_index_v4->set_dropped(afield.name);
+ }
}
for(const auto& record: iter_batch) {
@@ -1349,8 +1344,8 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
bool facet_hash_index_exists = ((field_facet_mapping_it != facet_index_v3.end())
|| (field_single_val_facet_mapping_it != single_val_facet_index_v3.end()));
- if(results_size && num_facet_values && ((!facet_hash_index_exists || is_wildcard_no_filter_query)
- && group_limit == 0) && !use_hashes || use_facet_intersection) {
+ if(results_size && num_facet_values && ((!facet_hash_index_exists || is_wildcard_no_filter_query
+ || num_facet_values > 50000) && group_limit == 0) && !use_hashes || use_facet_intersection) {
//LOG(INFO) << "Using intersection to find facets";
a_facet.is_intersected = true;
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index 6a20c289..6a6dbe0d 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -426,11 +426,11 @@ size_t num_tree_t::intersect(const uint32_t* result_ids, int result_ids_len, int
return found.size();
}
-size_t num_tree_t::get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& facets) {
+void num_tree_t::get_facet_indexes(std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {
//check if facet field
if(counter_list.empty()) {
- return 0;
+ return;
}
std::vector<uint32_t> id_list;
@@ -442,11 +442,11 @@ size_t num_tree_t::get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>&
//emplacing seq_id=>count_index
for(const auto& id : id_list) {
- facets[id].emplace_back(int64map_it->first);
+ functor(id, int64map_it->first);
}
id_list.clear();
}
- return facets.size();
+ return;
}