Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
picking index as per results size and improvements
krunal1313df8dda94321f
diff --git a/include/facet_index.h b/include/facet_index.hindex cf20cbcd..db1e2162 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -2,6 +2,7 @@#include "ids_t.h"#include "tsl/htrie_map.h"+#include <unordered_set>class facet_index_t {private:@@ -18,6 +19,7 @@ private:count_list& operator=(count_list& obj) {facet_value = obj.facet_value;count = obj.count;+ return *this;}std::string facet_value;@@ -59,6 +61,8 @@ private:std::map<std::string, facet_index_counter> facet_field_map;uint32_t count_index = 0;+ std::unordered_set<std::string> dropped_fields;+public:facet_index_t() = default;@@ -77,9 +81,11 @@ public:int max_facet_count, std::map<std::string, uint32_t>& found,bool is_wildcard_no_filter_query);- size_t get_facet_indexes(const std::string& field, std::map<uint32_t, std::vector<uint32_t>>& seqid_index_map);+ void get_facet_indexes(const std::string& field, std::function<void(uint32_t, uint32_t)> functor);bool get_migrated (const std::string& field) const;void set_migrated(const std::string& field, bool val);++ void set_dropped(const std::string& field);};\ No newline at end of filediff --git a/include/index.h b/include/index.hindex 6378490a..1ae09e59 100644--- a/include/index.h+++ b/include/index.h@@ -524,7 +524,7 @@ private:void initialize_facet_indexes(const field& facet_field);- void create_facet_hash_index(const std::string& field);+ void create_facet_hash_index(const field& facet_field);static Option<bool> embed_fields(nlohmann::json& document,diff --git a/include/num_tree.h b/include/num_tree.hindex 2397cc64..ce0a226c 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -35,6 +35,7 @@ private:count_list& operator=(count_list& obj) {facet_value = obj.facet_value;count = obj.count;+ return *this;}int64_t facet_value;@@ -84,7 +85,7 @@ public:size_t counter_list_size() const;- size_t get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& facets);+ void get_facet_indexes(std::function<void(uint32_t, uint32_t)> functor);bool get_migrated () const {return is_migrated;diff --git a/src/facet_index.cpp b/src/facet_index.cppindex 564394db..82e1e4f5 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -6,11 +6,18 @@uint32_t facet_index_t::insert(const std::string& field, const std::string& value,uint32_t id, bool is_string) {+ if(dropped_fields.find(field) != dropped_fields.end()) {+ //field has been dropped+ return 0;+ }+const auto facet_field_map_it = facet_field_map.find(field);if(facet_field_map_it == facet_field_map.end()) {facet_field_map.emplace(field, facet_index_counter{});}+ uint32_t index = 0;+auto& facet_index_map = facet_field_map[field].facet_index_map;uint32_t index;const auto sv = value.substr(0, 100);@@ -151,12 +158,12 @@ facet_index_t::~facet_index_t() {}//used for migrating string and int64 facets-size_t facet_index_t::get_facet_indexes(const std::string& field,- std::map<uint32_t, std::vector<uint32_t>>& seqid_index_map) {+void facet_index_t::get_facet_indexes(const std::string& field,+ std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {const auto& facet_field_it = facet_field_map.find(field);if(facet_field_it == facet_field_map.end()) {- return 0;+ return;}auto& facet_index_map = facet_field_it->second.facet_index_map;@@ -171,13 +178,13 @@ size_t facet_index_t::get_facet_indexes(const std::string& field,//emplacing seq_id=>count_indexfor(const auto& id : id_list) {- seqid_index_map[id].emplace_back(facet_index_map_it->index);+ functor(id, facet_index_map_it->index);}id_list.clear();}- return seqid_index_map.size();+ return;}@@ -198,3 +205,10 @@ void facet_index_t::set_migrated(const std::string& field, bool val) {return;}++void facet_index_t::set_dropped(const std::string& field) {++ dropped_fields.emplace(field);++ return;+}\ No newline at end of filediff --git a/src/index.cpp b/src/index.cppindex 5ee2ce04..b366c0de 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -604,71 +604,60 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_breturn num_indexed;}-void Index::create_facet_hash_index(const std::string& field) {+void Index::create_facet_hash_index(const field& facet_field) {//LOG(INFO) << "migrating facet " << field << " to old index";+ if(facet_field.is_array()) {+ auto& facet_index = facet_index_v3[facet_field.name];- std::map<uint32_t, std::vector<uint32_t>> facets_indexes;- auto& facet_index = facet_index_v3[field];- auto& single_facet_index = single_val_facet_index_v3[field];-- //migrate string and int64 facets first- if(facet_index_v4->get_facet_indexes(field, facets_indexes)) {+ facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {- for(const auto& kv : facets_indexes) {+ auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;+ } else {+ facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);+ }+ });- const auto hash_count = kv.second.size();- const auto seq_id = kv.first;+ auto numerical_index_it = numerical_index.find(facet_field.name);+ if(numerical_index_it != numerical_index_it) {+ auto num_tree = numerical_index_it->second;+ num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {- if(hash_count > 1) {auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << field;+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;} else {- facet_hash_values_t fhashvalues;- fhashvalues.hashes = std::move(kv.second);- facet_dim_index->emplace(seq_id, std::move(fhashvalues));+ facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);}- } else {- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << field;- } else {- facet_dim_index->emplace(seq_id, kv.second[0]);- }- }+ });}- }+ } else {+ auto& single_facet_index = single_val_facet_index_v3[facet_field.name];- //now extract remaining facets from numerical index- facets_indexes.clear();- auto numerical_index_it = numerical_index.find(field);- if(numerical_index_it != numerical_index_it) {- auto num_tree = numerical_index_it->second;- if(num_tree->get_facet_indexes(facets_indexes)) {+ facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {- for(const auto& kv : facets_indexes) {- const auto hash_count = kv.second.size();- const auto seq_id = kv.first;+ auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;+ } else {+ facet_dim_index->operator[](seq_id) = count_index;+ }+ });- if(hash_count > 1) {- auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << field;- } else {- facet_hash_values_t fhashvalues;- fhashvalues.hashes = std::move(kv.second);- facet_dim_index->emplace(seq_id, std::move(fhashvalues));- }+ auto numerical_index_it = numerical_index.find(facet_field.name);+ if(numerical_index_it != numerical_index_it) {+ auto num_tree = numerical_index_it->second;+ num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {++ auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;} else {- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << field;- } else {- facet_dim_index->emplace(seq_id, kv.second[0]);- }+ facet_dim_index->operator[](seq_id) = count_index;}- }+ });}}}@@ -727,7 +716,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>const auto total_num_docs = aggragate_docs_count.load();if(!is_migrated && (facet_threshold_count > FACET_INDEX_THRESHOLD)&& total_num_docs < 1000000) {- create_facet_hash_index(afield.name);+ create_facet_hash_index(afield);if(afield.is_string()) {facet_index_v4->set_migrated(afield.name, true);@@ -737,6 +726,12 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>numerical_index_it->second->set_migrated(true);}}++ auto cardinality_ratio = total_num_docs / facet_threshold_count;+ if(cardinality_ratio < 5) {+ facet_index_v4->erase(afield.name);+ facet_index_v4->set_dropped(afield.name);+ }}for(const auto& record: iter_batch) {@@ -1349,8 +1344,8 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,bool facet_hash_index_exists = ((field_facet_mapping_it != facet_index_v3.end())|| (field_single_val_facet_mapping_it != single_val_facet_index_v3.end()));- if(results_size && num_facet_values && ((!facet_hash_index_exists || is_wildcard_no_filter_query)- && group_limit == 0) && !use_hashes || use_facet_intersection) {+ if(results_size && num_facet_values && ((!facet_hash_index_exists || is_wildcard_no_filter_query+ || num_facet_values > 50000) && group_limit == 0) && !use_hashes || use_facet_intersection) {//LOG(INFO) << "Using intersection to find facets";a_facet.is_intersected = true;diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 6a20c289..6a6dbe0d 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -426,11 +426,11 @@ size_t num_tree_t::intersect(const uint32_t* result_ids, int result_ids_len, intreturn found.size();}-size_t num_tree_t::get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& facets) {+void num_tree_t::get_facet_indexes(std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {//check if facet fieldif(counter_list.empty()) {- return 0;+ return;}std::vector<uint32_t> id_list;@@ -442,11 +442,11 @@ size_t num_tree_t::get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>&//emplacing seq_id=>count_indexfor(const auto& id : id_list) {- facets[id].emplace_back(int64map_it->first);+ functor(id, int64map_it->first);}id_list.clear();}- return facets.size();+ return;}