Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
using numerical index for new data structure
krunal131358e2de03ce92
diff --git a/include/facet_index.h b/include/facet_index.hindex 6cf0bb84..cafb6c74 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -43,7 +43,7 @@ public:~facet_index_t();- uint32_t insert(const std::string& field, const std::string& value, uint32_t id);+ uint32_t insert(const std::string& field, const std::string& value, uint32_t id, bool is_string = false);void erase(const std::string& field);diff --git a/include/field.h b/include/field.hindex 696c0bb0..b4832043 100644--- a/include/field.h+++ b/include/field.h@@ -864,7 +864,7 @@ struct facet {struct facet_info_t {// facet hash => resolved tokens- std::unordered_map<uint64_t, std::vector<std::string>> hashes;+ std::unordered_map<uint32_t, std::vector<std::string>> hashes;bool use_facet_query = false;bool should_compute_stats = false;field facet_field{"", "", false};diff --git a/include/index.h b/include/index.hindex 2bdc6fb6..d045d923 100644--- a/include/index.h+++ b/include/index.h@@ -188,10 +188,6 @@ struct search_args {};};-struct offsets_facet_hashes_t {- std::unordered_map<std::string, std::vector<uint32_t>> offsets;-};-struct index_record {size_t position; // position of record in the original requestuint32_t seq_id;@@ -205,7 +201,8 @@ struct index_record {bool is_update;// pre-processed data primed for indexing- std::unordered_map<std::string, offsets_facet_hashes_t> field_index;+ std::unordered_map<std::string,+ std::unordered_map<std::string, std::vector<uint32_t>>> field_index;int64_t points;Option<bool> indexed; // indicates if the indexing operation was a successdiff --git a/include/num_tree.h b/include/num_tree.hindex 5406a109..12c51e23 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -22,11 +22,23 @@ private:return ids_t::contains(ids, id);}+ struct count_list {+ count_list(const int64_t& sv, uint32_t facet_count) {+ facet_value = sv;+ count = facet_count;+ }++ int64_t facet_value;+ uint32_t count;+ };++ std::vector<count_list> counter_list;+public:~num_tree_t();- void insert(int64_t value, uint32_t id);+ void insert(int64_t value, uint32_t id, bool is_facet=false);void range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len);@@ -53,4 +65,10 @@ public:uint32_t* const& context_ids,size_t& result_ids_len,uint32_t*& result_ids) const;++ size_t intersect(const uint32_t* result_ids, int result_id_len,+ int max_facet_count, std::map<int64_t, uint32_t>& found,+ bool is_wildcard_no_filter_query);++ size_t counter_list_size() const;};\ No newline at end of filediff --git a/src/facet_index.cpp b/src/facet_index.cppindex e00a4c9b..2d49a7df 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -1,8 +1,10 @@#include "facet_index.h"#include <tokenizer.h>#include "string_utils.h"+#include "array_utils.h"-uint32_t facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {+uint32_t facet_index_t::insert(const std::string& field, const std::string& value,+ uint32_t id, bool is_string) {auto& facet_index_map = facet_field_map[field].facet_index_map;uint32_t index;const auto sv = value.substr(0, 100);@@ -12,49 +14,56 @@ uint32_t facet_index_t::insert(const std::string& field, const std::string& valuindex = ++count_index;facet_index_struct fis {};- fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));fis.index = index;+ if(is_string) {+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));+ }facet_index_map.emplace(sv, fis);} else {- auto ids = it->id_list_ptr;- if (!ids_t::contains(ids, id)) {- ids_t::upsert(ids, id);- facet_index_map[sv].id_list_ptr = ids;+ if(is_string) {+ auto ids = it->id_list_ptr;+ if (!ids_t::contains(ids, id)) {+ ids_t::upsert(ids, id);+ facet_index_map[sv].id_list_ptr = ids;+ }}index = facet_index_map[sv].index;}- const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);- //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;- auto& counter_list = facet_field_map[field].counter_list;- if(counter_list.empty()) {- counter_list.emplace_back(sv, facet_count);- } else {- auto counter_it = counter_list.begin();- //remove node from list- for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {- if(counter_it->facet_value == sv) {- //found facet in first node- counter_list.erase(counter_it);- break;+ if(is_string) {+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);+ //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;+ auto& counter_list = facet_field_map[field].counter_list;++ if(counter_list.empty()) {+ counter_list.emplace_back(sv, facet_count);+ } else {+ auto counter_it = counter_list.begin();+ //remove node from list+ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {+ if(counter_it->facet_value == sv) {+ //found facet in first node+ counter_list.erase(counter_it);+ break;+ }}- }- //find position in list and add node with updated count- count_list node(sv, facet_count);+ //find position in list and add node with updated count+ count_list node(sv, facet_count);- for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {- // LOG (INFO) << "inserting in middle or front facet " << node.facet_value- // << " with count " << node.count;- if(counter_it->count <= facet_count) {- counter_list.emplace(counter_it, node);- break;+ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {+ // LOG (INFO) << "inserting in middle or front facet " << node.facet_value+ // << " with count " << node.count;+ if(counter_it->count <= facet_count) {+ counter_list.emplace(counter_it, node);+ break;+ }+ }+ if(counter_it == counter_list.end()) {+ // LOG (INFO) << "inserting at last facet " << node.facet_value+ // << " with count " << node.count;+ counter_list.emplace_back(node);}- }- if(counter_it == counter_list.end()) {- // LOG (INFO) << "inserting at last facet " << node.facet_value- // << " with count " << node.count;- counter_list.emplace_back(node);}}@@ -118,9 +127,12 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* resultids_t::uncompress(ids, id_list);const auto ids_len = id_list.size();for(int i = 0; i < result_ids_len; ++i) {- if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {- ++count;- }+ // if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {+ // ++count;+ // }+ uint32_t* out = nullptr;+ count = ArrayUtils::and_scalar(id_list.data(), id_list.size(),+ result_ids, result_ids_len, &out);}id_list.clear();}diff --git a/src/index.cpp b/src/index.cppindex 638f7837..26d2fb49 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -656,37 +656,28 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>if(afield.facet) {if(afield.is_array()) {- facet_hash_values_t fhashvalues;-- if(afield.type == field_types::INT32_ARRAY) {- for(size_t i = 0; i < document[afield.name].size(); ++i) {+ facet_hash_values_t fhashvalues;+ for(size_t i = 0; i < document[afield.name].size(); ++i) {+ if(afield.type == field_types::INT32_ARRAY) {int32_t raw_val = document[afield.name][i].get<int32_t>();value = std::to_string(raw_val);auto index = facet_index_v4->insert(afield.name, value, seq_id);fhashvalues.hashes.emplace_back(index);- }- } else if(afield.type == field_types::INT64_ARRAY) {- for(size_t i = 0; i < document[afield.name].size(); ++i) {+ } else if(afield.type == field_types::INT64_ARRAY) {int64_t raw_val = document[afield.name][i].get<int64_t>();value = std::to_string(raw_val);auto index = facet_index_v4->insert(afield.name, value, seq_id);fhashvalues.hashes.emplace_back(index);- }- } else if(afield.type == field_types::STRING_ARRAY) {- for(size_t i = 0; i < document[afield.name].size(); ++i) {+ } else if(afield.type == field_types::STRING_ARRAY) {value = document[afield.name][i];- auto index = facet_index_v4->insert(afield.name, value, seq_id);+ auto index = facet_index_v4->insert(afield.name, value, seq_id, true);fhashvalues.hashes.emplace_back(index);- }- } else if(afield.type == field_types::FLOAT_ARRAY) {- for(size_t i = 0; i < document[afield.name].size(); ++i) {+ } else if(afield.type == field_types::FLOAT_ARRAY) {float raw_val = document[afield.name][i].get<float>();value = StringUtils::float_to_str(raw_val);auto index = facet_index_v4->insert(afield.name, value, seq_id);fhashvalues.hashes.emplace_back(index);- }- } else if(afield.type == field_types::BOOL_ARRAY) {- for(size_t i = 0; i < document[afield.name].size(); ++i) {+ } else if(afield.type == field_types::BOOL_ARRAY) {value = std::to_string(document[afield.name][i].get<bool>());auto index = facet_index_v4->insert(afield.name, value, seq_id);fhashvalues.hashes.emplace_back(index);@@ -694,12 +685,12 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}fhashvalues.length = fhashvalues.hashes.size();- auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- } else {- facet_dim_index->emplace(seq_id, std::move(fhashvalues));- }+ auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;+ } else {+ facet_dim_index->emplace(seq_id, std::move(fhashvalues));+ }} else {uint32_t fhash;@@ -715,7 +706,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}else if(afield.type == field_types::STRING) {value = document[afield.name];- fhash = facet_index_v4->insert(afield.name, value, seq_id);+ fhash = facet_index_v4->insert(afield.name, value, seq_id, true);}else if(afield.type == field_types::FLOAT) {float raw_val = document[afield.name].get<float>();@@ -740,7 +731,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>max_score = record.points;}- for(auto &token_offsets: field_index_it->second.offsets) {+ for(auto &token_offsets: field_index_it->second) {token_to_doc_offsets[token_offsets.first].emplace_back(seq_id, record.points, token_offsets.second);if(afield.infix) {@@ -776,7 +767,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree](const index_record& record, uint32_t seq_id) {int32_t value = record.doc[afield.name].get<int32_t>();- num_tree->insert(value, seq_id);+ num_tree->insert(value, seq_id, afield.is_facet());});}@@ -785,7 +776,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree](const index_record& record, uint32_t seq_id) {int64_t value = record.doc[afield.name].get<int64_t>();- num_tree->insert(value, seq_id);+ num_tree->insert(value, seq_id, afield.is_facet());});}@@ -795,14 +786,14 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>(const index_record& record, uint32_t seq_id) {float fvalue = record.doc[afield.name].get<float>();int64_t value = float_to_int64_t(fvalue);- num_tree->insert(value, seq_id);+ num_tree->insert(value, seq_id, afield.is_facet());});} else if(afield.type == field_types::BOOL) {auto num_tree = numerical_index.at(afield.name);iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree](const index_record& record, uint32_t seq_id) {bool value = record.doc[afield.name].get<bool>();- num_tree->insert(value, seq_id);+ num_tree->insert(value, seq_id, afield.is_facet());});} else if(afield.type == field_types::GEOPOINT || afield.type == field_types::GEOPOINT_ARRAY) {auto geo_index = geopoint_index.at(afield.name);@@ -941,23 +932,23 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>if(afield.type == field_types::INT32_ARRAY) {const int32_t value = arr_value;- num_tree->insert(value, seq_id);+ num_tree->insert(value, seq_id, afield.is_facet());}else if(afield.type == field_types::INT64_ARRAY) {const int64_t value = arr_value;- num_tree->insert(value, seq_id);+ num_tree->insert(value, seq_id, afield.is_facet());}else if(afield.type == field_types::FLOAT_ARRAY) {const float fvalue = arr_value;int64_t value = float_to_int64_t(fvalue);- num_tree->insert(value, seq_id);+ num_tree->insert(value, seq_id, afield.is_facet());}else if(afield.type == field_types::BOOL_ARRAY) {const bool value = record.doc[afield.name][arr_i];- num_tree->insert(int64_t(value), seq_id);+ num_tree->insert(int64_t(value), seq_id, afield.is_facet());}}});@@ -1164,7 +1155,20 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,size_t mod_value = 100 / facet_sample_percent;- const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);+ auto facet_records = 0;++ if(!facet_field.name.empty()) {+ if(facet_field.is_string()) {+ facet_records = facet_index_v4->get_facet_count(a_facet.field_name);+ } else {+ auto numerical_index_it = numerical_index.find(a_facet.field_name);+ if(numerical_index_it != numerical_index.end()) {+ facet_records = numerical_index_it->second->counter_list_size();+ } else {+ LOG(ERROR) << "facet " << a_facet.field_name << " not found in numerical index";+ }+ }+ }if(results_size && facet_records && (facet_records <= 10 || is_wildcard_query) &&!use_facet_query && group_limit == 0 && no_filters_provided) {@@ -1172,9 +1176,29 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,a_facet.is_intersected = true;std::map<std::string, uint32_t> facet_results;- facet_index_v4->intersect(a_facet.field_name, result_ids,- results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);-++ if(facet_field.is_string()) {+ facet_index_v4->intersect(a_facet.field_name, result_ids,+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);+ } else {+ std::map<int64_t, uint32_t> facet_counts;+ numerical_index.at(a_facet.field_name)->intersect(result_ids,+ results_size, max_facet_count, facet_counts, is_wildcard_query & no_filters_provided);++ for(const auto& kv : facet_counts) {+ std::string val;+ if(facet_field.is_float()) {+ val = std::to_string(int64_t_to_float(kv.first));+ } else if(facet_field.is_bool()) {+ val = kv.first == 1 ? "true" : "false";+ } else {+ val = std::to_string(kv.first);+ }++ facet_results[val] = kv.second;+ }+ }+for(const auto& kv : facet_results) {//range facet processingif(a_facet.is_range_query) {diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 1bcdbc9f..6e433eb1 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -2,7 +2,7 @@#include "parasort.h"#include "timsort.hpp"-void num_tree_t::insert(int64_t value, uint32_t id) {+void num_tree_t::insert(int64_t value, uint32_t id, bool is_facet) {if (int64map.count(value) == 0) {int64map.emplace(value, SET_COMPACT_IDS(compact_id_list_t::create(1, {id})));} else {@@ -12,6 +12,43 @@ void num_tree_t::insert(int64_t value, uint32_t id) {int64map[value] = ids;}}++ if(is_facet) {++ const auto facet_count = ids_t::num_ids(int64map.at(value));+ //LOG(INFO) << "Facet count in facet " << value << " : " << facet_count;++ if(counter_list.empty()) {+ counter_list.emplace_back(value, facet_count);+ } else {+ auto counter_it = counter_list.begin();+ //remove node from list+ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {+ if(counter_it->facet_value == value) {+ //found facet in first node+ counter_list.erase(counter_it);+ break;+ }+ }++ //find position in list and add node with updated count+ count_list node(value, facet_count);++ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {+ // LOG (INFO) << "inserting in middle or front facet " << node.facet_value+ // << " with count " << node.count;+ if(counter_it->count <= facet_count) {+ counter_list.emplace(counter_it, node);+ break;+ }+ }+ if(counter_it == counter_list.end()) {+ // LOG (INFO) << "inserting at last facet " << node.facet_value+ // << " with count " << node.count;+ counter_list.emplace_back(node);+ }+ }+ }}void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len) {@@ -322,8 +359,53 @@ size_t num_tree_t::size() {return int64map.size();}+size_t num_tree_t::counter_list_size() const {+ return counter_list.size();+}+num_tree_t::~num_tree_t() {for(auto& kv: int64map) {ids_t::destroy_list(kv.second);}}++size_t num_tree_t::intersect(const uint32_t* result_ids, int result_ids_len, int max_facet_count,+ std::map<int64_t, uint32_t>& found, bool is_wildcard_no_filter_query) {+ //LOG (INFO) << "intersecting field " << field;++ // LOG (INFO) << "int64map size " << int64map.size()+ // << " , counter_list size " << counter_list.size();++ std::vector<uint32_t> id_list;+ for(const auto& counter_list_it : counter_list) {+ // LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value+ // << " having total count " << counter_list_it.count;+ uint32_t count = 0;++ if(is_wildcard_no_filter_query) {+ count = counter_list_it.count;+ } else {+ auto ids = int64map.at(counter_list_it.facet_value);+ ids_t::uncompress(ids, id_list);+ const auto ids_len = id_list.size();+ for(int i = 0; i < result_ids_len; ++i) {+ // if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {+ // ++count;+ // }+ uint32_t* out = nullptr;+ count = ArrayUtils::and_scalar(id_list.data(), id_list.size(),+ result_ids, result_ids_len, &out);+ }+ id_list.clear();+ }++ if(count) {+ found[counter_list_it.facet_value] = count;+ if(found.size() == max_facet_count) {+ break;+ }+ }+ }++ return found.size();+}diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 8872759f..649a91d0 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -252,7 +252,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_FLOAT_EQ(5, results["facet_counts"][0]["stats"]["total_values"].get<size_t>());// check for "0" case- ASSERT_STREQ("0", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("0.000000", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());// facet query on a float field@@ -506,8 +506,8 @@ TEST_F(CollectionFacetingTest, FacetCountsFloatPrecision) {ASSERT_STREQ("points", results["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("113.4", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("113.4",results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("113.400002", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("113.400002",results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}