Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `geo_range_index`.
Harpreet Sangar334ea25b2194
diff --git a/include/index.h b/include/index.hindex ef23dc54..11d6155d 100644--- a/include/index.h+++ b/include/index.h@@ -308,7 +308,9 @@ private:spp::sparse_hash_map<std::string, NumericTrie*> range_index;- spp::sparse_hash_map<std::string, spp::sparse_hash_map<std::string, std::vector<uint32_t>>*> geopoint_index;+ spp::sparse_hash_map<std::string, NumericTrie*> geo_range_index;++// spp::sparse_hash_map<std::string, spp::sparse_hash_map<std::string, std::vector<uint32_t>>*> geopoint_index;// geo_array_field => (seq_id => values) used for exact filtering of geo array recordsspp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;diff --git a/include/numeric_range_trie_test.h b/include/numeric_range_trie_test.hindex a8422524..ed695a70 100644--- a/include/numeric_range_trie_test.h+++ b/include/numeric_range_trie_test.h@@ -14,6 +14,8 @@ class NumericTrie {void insert_helper(const int64_t& value, const uint32_t& seq_id, char& level, const char& max_level);+ void insert_geopoint_helper(const uint64_t& cell_id, const uint32_t& seq_id, char& level, const char& max_level);+void search_range_helper(const int64_t& low,const int64_t& high, const char& max_level,std::vector<Node*>& matches);@@ -35,7 +37,13 @@ class NumericTrie {delete [] children;}- void insert(const int64_t& value, const uint32_t& seq_id, const char& max_level);+ void insert(const int64_t& cell_id, const uint32_t& seq_id, const char& max_level);++ void insert_geopoint(const uint64_t& cell_id, const uint32_t& seq_id, const char& max_level);++ void search_geopoint(const uint64_t& cell_id, const char& max_index_level, uint32_t*& ids, uint32_t& ids_length);++ void delete_geopoint(const uint64_t& cell_id, uint32_t id, const char& max_level);void get_all_ids(uint32_t*& ids, uint32_t& ids_length);@@ -110,6 +118,12 @@ public:void insert(const int64_t& value, const uint32_t& seq_id);+ void insert_geopoint(const uint64_t& cell_id, const uint32_t& seq_id);++ void search_geopoint(const uint64_t& cell_id, uint32_t*& ids, uint32_t& ids_length);++ void delete_geopoint(const uint64_t& cell_id, uint32_t id);+void search_range(const int64_t& low, const bool& low_inclusive,const int64_t& high, const bool& high_inclusive,uint32_t*& ids, uint32_t& ids_length);diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex ee326b32..5fb432f2 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -892,13 +892,21 @@ void filter_result_iterator_t::init() {S2RegionTermIndexer::Options options;options.set_index_contains_points_only(true);S2RegionTermIndexer indexer(options);+ auto const& geo_range_index = index->geo_range_index.at(a_filter.field_name);for (const auto& term : indexer.GetQueryTerms(*query_region, "")) {- auto geo_index = index->geopoint_index.at(a_filter.field_name);- const auto& ids_it = geo_index->find(term);- if(ids_it != geo_index->end()) {- geo_result_ids.insert(geo_result_ids.end(), ids_it->second.begin(), ids_it->second.end());+ auto cell = S2CellId::FromToken(term);+ uint32_t* geo_ids = nullptr;+ uint32_t geo_ids_length = 0;++ geo_range_index->search_geopoint(cell.id(), geo_ids, geo_ids_length);++ geo_result_ids.reserve(geo_result_ids.size() + geo_ids_length);+ for (uint32_t i = 0; i < geo_ids_length; i++) {+ geo_result_ids.push_back(geo_ids[i]);}++ delete [] geo_ids;}gfx::timsort(geo_result_ids.begin(), geo_result_ids.end());diff --git a/src/index.cpp b/src/index.cppindex 949d3740..48b9b6bc 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -78,8 +78,7 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*art_tree_init(t);search_index.emplace(a_field.name, t);} else if(a_field.is_geopoint()) {- auto field_geo_index = new spp::sparse_hash_map<std::string, std::vector<uint32_t>>();- geopoint_index.emplace(a_field.name, field_geo_index);+ geo_range_index.emplace(a_field.name, new NumericTrie(64));if(!a_field.is_single_geopoint()) {spp::sparse_hash_map<uint32_t, int64_t*> * doc_to_geos = new spp::sparse_hash_map<uint32_t, int64_t*>();@@ -141,12 +140,12 @@ Index::~Index() {search_index.clear();- for(auto & name_index: geopoint_index) {+ for(auto & name_index: geo_range_index) {delete name_index.second;name_index.second = nullptr;}- geopoint_index.clear();+ geo_range_index.clear();for(auto& name_index: geo_array_index) {for(auto& kv: *name_index.second) {@@ -810,10 +809,10 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>num_tree->insert(value, seq_id);});} else if(afield.type == field_types::GEOPOINT || afield.type == field_types::GEOPOINT_ARRAY) {- auto geo_index = geopoint_index.at(afield.name);+ auto geopoint_range_index = geo_range_index.at(afield.name);iterate_and_index_numerical_field(iter_batch, afield,- [&afield, &geo_array_index=geo_array_index, geo_index](const index_record& record, uint32_t seq_id) {+ [&afield, &geo_array_index=geo_array_index, geopoint_range_index](const index_record& record, uint32_t seq_id) {// nested geopoint value inside an array of object will be a simple array so must be treated as geopointbool nested_obj_arr_geopoint = (afield.nested && afield.type == field_types::GEOPOINT_ARRAY &&!record.doc[afield.name].empty() && record.doc[afield.name][0].is_number());@@ -827,9 +826,8 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>S2RegionTermIndexer indexer(options);S2Point point = S2LatLng::FromDegrees(latlongs[li], latlongs[li+1]).ToPoint();- for(const auto& term: indexer.GetIndexTerms(point, "")) {- (*geo_index)[term].push_back(seq_id);- }+ auto cell = S2CellId(point);+ geopoint_range_index->insert_geopoint(cell.id(), seq_id);}if(nested_obj_arr_geopoint) {@@ -857,9 +855,9 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>for(size_t li = 0; li < latlongs.size(); li++) {auto& latlong = latlongs[li];S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();- for(const auto& term: indexer.GetIndexTerms(point, "")) {- (*geo_index)[term].push_back(seq_id);- }++ auto cell = S2CellId(point);+ geopoint_range_index->insert_geopoint(cell.id(), seq_id);int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);packed_latlongs[li + 1] = packed_latlong;@@ -1589,7 +1587,7 @@ void Index::numeric_not_equals_filter(num_tree_t* const num_tree,bool Index::field_is_indexed(const std::string& field_name) const {return search_index.count(field_name) != 0 ||numerical_index.count(field_name) != 0 ||- geopoint_index.count(field_name) != 0;+ geo_range_index.count(field_name) != 0;}void Index::aproximate_numerical_match(num_tree_t* const num_tree,@@ -5471,7 +5469,7 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const}}} else if(search_field.is_geopoint()) {- auto geo_index = geopoint_index[field_name];+ auto geopoint_range_index = geo_range_index[field_name];S2RegionTermIndexer::Options options;options.set_index_contains_points_only(true);S2RegionTermIndexer indexer(options);@@ -5482,17 +5480,8 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, constfor(const std::vector<double>& latlong: latlongs) {S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();- for(const auto& term: indexer.GetIndexTerms(point, "")) {- auto term_it = geo_index->find(term);- if(term_it == geo_index->end()) {- continue;- }- std::vector<uint32_t>& ids = term_it->second;- ids.erase(std::remove(ids.begin(), ids.end(), seq_id), ids.end());- if(ids.empty()) {- geo_index->erase(term);- }- }+ auto cell = S2CellId(point);+ geopoint_range_index->delete_geopoint(cell.id(), seq_id);}if(!search_field.is_single_geopoint()) {@@ -5644,8 +5633,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecart_tree_init(t);search_index.emplace(new_field.name, t);} else if(new_field.is_geopoint()) {- auto field_geo_index = new spp::sparse_hash_map<std::string, std::vector<uint32_t>>();- geopoint_index.emplace(new_field.name, field_geo_index);+ geo_range_index.emplace(new_field.name, new NumericTrie(64));if(!new_field.is_single_geopoint()) {auto geo_array_map = new spp::sparse_hash_map<uint32_t, int64_t*>();geo_array_index.emplace(new_field.name, geo_array_map);@@ -5695,8 +5683,8 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecdelete search_index[del_field.name];search_index.erase(del_field.name);} else if(del_field.is_geopoint()) {- delete geopoint_index[del_field.name];- geopoint_index.erase(del_field.name);+ delete geo_range_index[del_field.name];+ geo_range_index.erase(del_field.name);if(!del_field.is_single_geopoint()) {spp::sparse_hash_map<uint32_t, int64_t*>* geo_array_map = geo_array_index[del_field.name];diff --git a/src/numeric_range_trie.cpp b/src/numeric_range_trie.cppindex 9d9f4aa0..86090304 100644--- a/src/numeric_range_trie.cpp+++ b/src/numeric_range_trie.cpp@@ -18,6 +18,30 @@ void NumericTrie::insert(const int64_t& value, const uint32_t& seq_id) {}}+void NumericTrie::insert_geopoint(const uint64_t& cell_id, const uint32_t& seq_id) {+ if (positive_trie == nullptr) {+ positive_trie = new NumericTrie::Node();+ }++ positive_trie->insert_geopoint(cell_id, seq_id, max_level);+}++void NumericTrie::search_geopoint(const uint64_t& cell_id, uint32_t*& ids, uint32_t& ids_length) {+ if (positive_trie == nullptr) {+ return;+ }++ positive_trie->search_geopoint(cell_id, max_level, ids, ids_length);+}++void NumericTrie::delete_geopoint(const uint64_t& cell_id, uint32_t id) {+ if (positive_trie == nullptr) {+ return;+ }++ positive_trie->delete_geopoint(cell_id, id, max_level);+}+void NumericTrie::search_range(const int64_t& low, const bool& low_inclusive,const int64_t& high, const bool& high_inclusive,uint32_t*& ids, uint32_t& ids_length) {@@ -369,9 +393,14 @@ NumericTrie::iterator_t NumericTrie::search_equal_to(const int64_t& value) {return NumericTrie::iterator_t(matches);}-void NumericTrie::Node::insert(const int64_t& value, const uint32_t& seq_id, const char& max_level) {+void NumericTrie::Node::insert(const int64_t& cell_id, const uint32_t& seq_id, const char& max_level) {char level = 0;- return insert_helper(value, seq_id, level, max_level);+ return insert_helper(cell_id, seq_id, level, max_level);+}++void NumericTrie::Node::insert_geopoint(const uint64_t& cell_id, const uint32_t& seq_id, const char& max_level) {+ char level = 0;+ return insert_geopoint_helper(cell_id, seq_id, level, max_level);}inline int get_index(const int64_t& value, const char& level, const char& max_level) {@@ -385,6 +414,10 @@ inline int get_index(const int64_t& value, const char& level, const char& max_lereturn (value >> (8 * (max_level - level))) & 0xFF;}+inline int get_geopoint_index(const uint64_t& cell_id, const char& level, const char& max_level) {+ return (cell_id >> (8 * (max_level - level))) & 0xFF;+}+void NumericTrie::Node::insert_helper(const int64_t& value, const uint32_t& seq_id, char& level, const char& max_level) {if (level > max_level) {return;@@ -409,6 +442,85 @@ void NumericTrie::Node::insert_helper(const int64_t& value, const uint32_t& seq_}}+void NumericTrie::Node::insert_geopoint_helper(const uint64_t& cell_id, const uint32_t& seq_id, char& level,+ const char& max_level) {+ if (level > max_level) {+ return;+ }++ // Root node contains all the sequence ids present in the tree.+ if (!seq_ids.contains(seq_id)) {+ seq_ids.append(seq_id);+ }++ if (++level <= max_level) {+ if (children == nullptr) {+ children = new NumericTrie::Node* [EXPANSE]{nullptr};+ }++ auto index = get_geopoint_index(cell_id, level, max_level);+ if (children[index] == nullptr) {+ children[index] = new NumericTrie::Node();+ }++ return children[index]->insert_geopoint_helper(cell_id, seq_id, level, max_level);+ }+}++char get_max_search_level(const uint64_t& cell_id, const char& max_level) {+ // For cell id 0x47E66C3000000000, we only have to prefix match the top four bytes since rest of the bytes are 0.+ // So the max search level would be 4 in this case.++ uint64_t mask = 0xff;+ char i = max_level;+ while (((cell_id & mask) == 0) && --i > 0) {+ mask <<= 8;+ }++ return i;+}++void NumericTrie::Node::search_geopoint(const uint64_t& cell_id, const char& max_index_level,+ uint32_t*& ids, uint32_t& ids_length) {+ char level = 1;+ Node* root = this;+ auto index = get_geopoint_index(cell_id, level, max_index_level);+ auto max_search_level = get_max_search_level(cell_id, max_index_level);++ while (level < max_search_level) {+ if (root->children == nullptr || root->children[index] == nullptr) {+ return;+ }++ root = root->children[index];+ index = get_geopoint_index(cell_id, ++level, max_index_level);+ }++ root->get_all_ids(ids, ids_length);+}++void NumericTrie::Node::delete_geopoint(const uint64_t& cell_id, uint32_t id, const char& max_level) {+ char level = 1;+ Node* root = this;+ auto index = get_geopoint_index(cell_id, level, max_level);++ while (level < max_level) {+ root->seq_ids.remove_value(id);++ if (root->children == nullptr || root->children[index] == nullptr) {+ return;+ }++ root = root->children[index];+ index = get_geopoint_index(cell_id, ++level, max_level);+ }++ if (root->children != nullptr || root->children[index] != nullptr) {+ delete root->children[index];+ root->children[index] = nullptr;+ }+}+void NumericTrie::Node::get_all_ids(uint32_t*& ids, uint32_t& ids_length) {ids = seq_ids.uncompress();ids_length = seq_ids.getLength();