Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support array of geo points.
Kishore Nallan043535ee6b9b
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 972e6e95..f4e656f4 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -58,7 +58,6 @@ include(cmake/GoogleTest.cmake)include(cmake/TestResources.cmake)include(cmake/Iconv.cmake)include(cmake/Jemalloc.cmake)-include(cmake/h3.cmake)include(cmake/s2.cmake)include(cmake/lrucache.cmake)include(cmake/kakasi.cmake)@@ -92,7 +91,6 @@ include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)-include_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/src/h3lib/include)include_directories(${DEP_ROOT_DIR}/${S2_NAME}/src)include_directories(${DEP_ROOT_DIR}/${LRUCACHE_NAME}/include)include_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/build/include)@@ -105,7 +103,6 @@ link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)-link_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/lib)link_directories(${DEP_ROOT_DIR}/${S2_NAME}/build)link_directories(${DEP_ROOT_DIR}/${KAKASI_NAME}/build/lib)@@ -156,7 +153,7 @@ else()endif()set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})-set(CORE_LIBS kakasi h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for s2 h3+set(CORE_LIBS kakasi h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for s2${LevelDB_LIBRARIES} ${ROCKSDB_LIBS}glog ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${STACKTRACE_LIBS}${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${JEMALLOC_LIBRARIES}diff --git a/cmake/h3.cmake b/cmake/h3.cmakedeleted file mode 100644index 0dc5bf6e..00000000--- a/cmake/h3.cmake+++ /dev/null@@ -1,44 +0,0 @@-# Download and build H3--set(H3_VERSION 3.7.1)-set(H3_NAME h3-${H3_VERSION})-set(H3_TAR_PATH ${DEP_ROOT_DIR}/${H3_NAME}.tar.gz)--if(NOT EXISTS ${H3_TAR_PATH})- message(STATUS "Downloading https://github.com/uber/h3/archive/v${H3_VERSION}.tar.gz")- file(DOWNLOAD https://github.com/uber/h3/archive/v${H3_VERSION}.tar.gz ${H3_TAR_PATH})-endif()--if(NOT EXISTS ${DEP_ROOT_DIR}/${H3_NAME})- message(STATUS "Extracting ${H3_NAME}...")- execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${H3_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}/)-endif()--if(NOT EXISTS ${DEP_ROOT_DIR}/${H3_NAME}/build/lib/libh3.a)- message("Configuring ${H3_NAME}...")-- file(REMOVE_RECURSE ${DEP_ROOT_DIR}/${H3_NAME}/build)- file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${H3_NAME}/build)-- execute_process(COMMAND ${CMAKE_COMMAND}- "-DCMAKE_FIND_LIBRARY_SUFFIXES=.a"- "-H${DEP_ROOT_DIR}/${H3_NAME}"- "-B${DEP_ROOT_DIR}/${H3_NAME}/build"- RESULT_VARIABLE- H3_CONFIGURE)- if(NOT H3_CONFIGURE EQUAL 0)- message(FATAL_ERROR "${H3_NAME} configure failed!")- endif()-- if(BUILD_DEPS STREQUAL "yes")- message("Building ${H3_NAME} locally...")- execute_process(COMMAND ${CMAKE_COMMAND} --build- "${DEP_ROOT_DIR}/${H3_NAME}/build"- --target h3- RESULT_VARIABLE- H3_BUILD)- if(NOT H3_BUILD EQUAL 0)- message(FATAL_ERROR "${H3_NAME} build failed!")- endif()- endif()-endif()diff --git a/include/field.h b/include/field.hindex 756b6692..67c66168 100644--- a/include/field.h+++ b/include/field.h@@ -24,6 +24,7 @@ namespace field_types {static const std::string INT64_ARRAY = "int64[]";static const std::string FLOAT_ARRAY = "float[]";static const std::string BOOL_ARRAY = "bool[]";+ static const std::string GEOPOINT_ARRAY = "geopoint[]";static bool is_string_or_array(const std::string& type_def) {return type_def == "string*";@@ -70,6 +71,10 @@ struct field {return (type == field_types::BOOL);}+ bool is_single_geopoint() const {+ return (type == field_types::GEOPOINT);+ }+bool is_integer() const {return (type == field_types::INT32 || type == field_types::INT32_ARRAY ||type == field_types::INT64 || type == field_types::INT64_ARRAY);@@ -92,7 +97,7 @@ struct field {}bool is_geopoint() const {- return (type == field_types::GEOPOINT);+ return (type == field_types::GEOPOINT || type == field_types::GEOPOINT_ARRAY);}bool is_string() const {@@ -106,7 +111,8 @@ struct field {bool is_array() const {return (type == field_types::STRING_ARRAY || type == field_types::INT32_ARRAY ||type == field_types::FLOAT_ARRAY ||- type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY);+ type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY ||+ type == field_types::GEOPOINT_ARRAY);}bool is_singular() const {diff --git a/include/index.h b/include/index.hindex e8edf62a..8d1fc1e1 100644--- a/include/index.h+++ b/include/index.h@@ -15,7 +15,6 @@#include <field.h>#include <option.h>#include <set>-#include <h3api.h>#include "string_utils.h"#include "num_tree.h"#include "magic_enum.hpp"@@ -385,6 +384,9 @@ private:// sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;+ // geo_array_field => (seq_id => values) used for exact filtering of geo array records+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;+// this is used for wildcard queriessorted_array seq_ids;@@ -632,11 +634,11 @@ public:const std::map<std::string, field> & facet_schema,const std::string& fallback_field_type);- static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);+ //static bool is_point_in_polygon(const Geofence& poly, const GeoCoord& point);- static double transform_for_180th_meridian(Geofence& poly);+ //static double transform_for_180th_meridian(Geofence& poly);- static void transform_for_180th_meridian(GeoCoord& point, double offset);+ //static void transform_for_180th_meridian(GeoCoord& point, double offset);art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);@@ -644,8 +646,6 @@ public:void refresh_schemas(const std::vector<field>& new_fields);- bool field_contains_string(const std::string& field_name, const std::string& value);-// the following methods are not synchronized because their parent calls are synchronized or they are const/staticstatic Option<uint32_t> validate_index_in_memory(nlohmann::json &document, uint32_t seq_id,@@ -670,5 +670,9 @@ public:void curate_filtered_ids(const std::vector<filter>& filters, const std::set<uint32_t>& curated_ids,const uint32_t* exclude_token_ids, size_t exclude_token_ids_size, uint32_t*& filter_ids,uint32_t& filter_ids_length, const std::vector<uint32_t>& curated_ids_sorted) const;++ void populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,+ const std::vector<sort_by>& sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;};diff --git a/src/collection.cpp b/src/collection.cppindex 19bc6638..697b2c97 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -12,7 +12,6 @@#include <system_metrics.h>#include <tokenizer.h>#include <collection_manager.h>-#include <h3api.h>#include <regex>#include <list>#include <posting.h>diff --git a/src/index.cpp b/src/index.cppindex 9c81aca4..f7e8f837 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -9,7 +9,6 @@#include <string_utils.h>#include <art.h>#include <tokenizer.h>-#include <h3api.h>#include <s2/s2point.h>#include <s2/s2latlng.h>#include <s2/s2region_term_indexer.h>@@ -42,6 +41,11 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*} else if(fname_field.second.is_geopoint()) {auto field_geo_index = new spp::sparse_hash_map<std::string, std::vector<uint32_t>>();geopoint_index.emplace(fname_field.first, field_geo_index);++ if(!fname_field.second.is_single_geopoint()) {+ spp::sparse_hash_map<uint32_t, int64_t*> * doc_to_geos = new spp::sparse_hash_map<uint32_t, int64_t*>();+ geo_array_index.emplace(fname_field.first, doc_to_geos);+ }} else {num_tree_t* num_tree = new num_tree_t;numerical_index.emplace(fname_field.first, num_tree);@@ -56,8 +60,10 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*}for(const auto & pair: sort_schema) {- spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();- sort_index.emplace(pair.first, doc_to_score);+ if(pair.second.type != field_types::GEOPOINT_ARRAY) {+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();+ sort_index.emplace(pair.first, doc_to_score);+ }}for(const auto& pair: facet_schema) {@@ -86,6 +92,17 @@ Index::~Index() {geopoint_index.clear();+ for(auto& name_index: geo_array_index) {+ for(auto& kv: *name_index.second) {+ delete [] kv.second;+ }++ delete name_index.second;+ name_index.second = nullptr;+ }++ geo_array_index.clear();+for(auto & name_tree: numerical_index) {delete name_tree.second;name_tree.second = nullptr;@@ -168,7 +185,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tbool is_facet = (facet_schema.count(field_name) != 0);// non-string, non-geo faceted field should be indexed as faceted string field as well- if(field_pair.second.facet && !field_pair.second.is_string() && field_pair.second.type != field_types::GEOPOINT) {+ if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {art_tree *t = search_index.at(field_pair.second.faceted_name());if(field_pair.second.is_array()) {@@ -233,18 +250,47 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tbool value = document[field_name];num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::GEOPOINT) {+ const std::vector<double>& latlong = document[field_name];+ auto geo_index = geopoint_index.at(field_name);+S2RegionTermIndexer::Options options;options.set_index_contains_points_only(true);S2RegionTermIndexer indexer(options);- const std::vector<double>& latlong = document[field_name];S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();+for(const auto& term: indexer.GetIndexTerms(point, "")) {- auto geo_index = geopoint_index.at(field_name);(*geo_index)[term].push_back(seq_id);}} else if(field_pair.second.type == field_types::STRING_ARRAY) {art_tree *t = search_index.at(field_name);index_string_array_field(document[field_name], points, t, seq_id, is_facet, field_pair.second);+ } else if(field_pair.second.type == field_types::GEOPOINT_ARRAY) {+ const std::vector<std::vector<double>>& latlongs = document[field_name];+ auto geo_index = geopoint_index.at(field_name);+ S2RegionTermIndexer::Options options;+ options.set_index_contains_points_only(true);+ S2RegionTermIndexer indexer(options);++ int64_t* packed_latlongs = new int64_t[latlongs.size() + 1];+ packed_latlongs[0] = latlongs.size();++ for(size_t li = 0; li < latlongs.size(); li++) {+ auto& latlong = latlongs[li];+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();+ std::set<std::string> terms;+ for(const auto& term: indexer.GetIndexTerms(point, "")) {+ terms.insert(term);+ }++ for(const auto& term: terms) {+ (*geo_index)[term].push_back(seq_id);+ }++ int64_t packed_latlong = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);+ packed_latlongs[li + 1] = packed_latlong;+ }++ geo_array_index.at(field_name)->emplace(seq_id, packed_latlongs);}else if(field_pair.second.is_array()) {@@ -433,6 +479,18 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3if (!coerce_op.ok()) {return coerce_op;}+ } else if (a_field.type == field_types::GEOPOINT_ARRAY) {+ if(!item.is_array() || item.size() != 2) {+ return Option<>(400, "Field `" + field_name + "` must contain 2 element arrays: [ [lat, lng],... ].");+ }++ if(!(item[0].is_number() && item[1].is_number())) {+ // one or more elements is not an number, try to coerce+ Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name, it, true, array_ele_erased);+ if(!coerce_op.ok()) {+ return coerce_op;+ }+ }}if(!array_ele_erased) {@@ -938,24 +996,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;std::vector<size_t> geopoint_indices;- for (size_t i = 0; i < sort_fields.size(); i++) {- sort_order[i] = 1;- if (sort_fields[i].order == sort_field_const::asc) {- sort_order[i] = -1;- }-- if (sort_fields[i].name == sort_field_const::text_match) {- field_values[i] = &text_match_sentinel_value;- } else if (sort_fields[i].name == sort_field_const::seq_id) {- field_values[i] = &seq_id_sentinel_value;- } else if (sort_index.count(sort_fields[i].name) != 0) {- field_values[i] = sort_index.at(sort_fields[i].name);-- if (sort_schema.at(sort_fields[i].name).is_geopoint()) {- geopoint_indices.push_back(i);- }- }- }+ populate_sort_mapping(sort_order, geopoint_indices, sort_fields, field_values);size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;@@ -1241,14 +1282,37 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,std::vector<uint32_t> exact_geo_result_ids;- for(auto result_id: geo_result_ids) {- int64_t lat_lng = sort_index.at(f.name)->at(result_id);- S2LatLng s2_lat_lng;- GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);- if (!query_region->Contains(s2_lat_lng.ToPoint())) {- continue;+ if(f.is_single_geopoint()) {+ for(auto result_id: geo_result_ids) {+ // no need to check for existence of `result_id` because of indexer based pre-filtering above+ int64_t lat_lng = sort_index.at(f.name)->at(result_id);+ S2LatLng s2_lat_lng;+ GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);+ if (query_region->Contains(s2_lat_lng.ToPoint())) {+ exact_geo_result_ids.push_back(result_id);+ }+ }+ } else {+ for(auto result_id: geo_result_ids) {+ int64_t* lat_lngs = geo_array_index.at(f.name)->at(result_id);++ bool point_found = false;++ // any one point should exist+ for(size_t li = 0; li < lat_lngs[0]; li++) {+ int64_t lat_lng = lat_lngs[li + 1];+ S2LatLng s2_lat_lng;+ GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);+ if (query_region->Contains(s2_lat_lng.ToPoint())) {+ point_found = true;+ break;+ }+ }++ if(point_found) {+ exact_geo_result_ids.push_back(result_id);+ }}- exact_geo_result_ids.push_back(result_id);}std::sort(exact_geo_result_ids.begin(), exact_geo_result_ids.end());@@ -2335,29 +2399,11 @@ void Index::search_wildcard(const std::vector<std::string>& qtokens, const std::const string& field, uint32_t*& all_result_ids, size_t& all_result_ids_len,const uint32_t* filter_ids, uint32_t filter_ids_length) const {- // FIXME: duplicatedint sort_order[3]; // 1 or -1 based on DESC or ASC respectivelystd::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;std::vector<size_t> geopoint_indices;- for (size_t i = 0; i < sort_fields_std.size(); i++) {- sort_order[i] = 1;- if (sort_fields_std[i].order == sort_field_const::asc) {- sort_order[i] = -1;- }-- if (sort_fields_std[i].name == sort_field_const::text_match) {- field_values[i] = &text_match_sentinel_value;- } else if (sort_fields_std[i].name == sort_field_const::seq_id) {- field_values[i] = &seq_id_sentinel_value;- } else if (sort_index.count(sort_fields_std[i].name) != 0) {- field_values[i] = sort_index.at(sort_fields_std[i].name);-- if (sort_schema.at(sort_fields_std[i].name).is_geopoint()) {- geopoint_indices.push_back(i);- }- }- }+ populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);uint32_t token_bits = 255;std::vector<posting_list_t::iterator_t> plists;@@ -2379,6 +2425,34 @@ void Index::search_wildcard(const std::vector<std::string>& qtokens, const std::all_result_ids = new_all_result_ids;}+void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,+ const std::vector<sort_by>& sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const {+ for (size_t i = 0; i < sort_fields_std.size(); i++) {+ sort_order[i] = 1;+ if (sort_fields_std[i].order == sort_field_const::asc) {+ sort_order[i] = -1;+ }++ if (sort_fields_std[i].name == sort_field_const::text_match) {+ field_values[i] = &text_match_sentinel_value;+ } else if (sort_fields_std[i].name == sort_field_const::seq_id) {+ field_values[i] = &seq_id_sentinel_value;+ } else if (sort_schema.count(sort_fields_std[i].name) != 0) {+ if (sort_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {+ geopoint_indices.push_back(i);+ field_values[i] = nullptr; // GEOPOINT_ARRAY uses a multi-valued index+ } else {+ field_values[i] = sort_index.at(sort_fields_std[i].name);++ if (sort_schema.at(sort_fields_std[i].name).is_geopoint()) {+ geopoint_indices.push_back(i);+ }+ }+ }+ }+}+/*1. Split the query into tokens2. Outer loop will generate bounded cartesian product with costs for each token@@ -2633,18 +2707,37 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16for(auto& i: geopoint_indices) {spp::sparse_hash_map<uint32_t, int64_t>* geopoints = field_values[i];+ int64_t dist = INT32_MAX;S2LatLng reference_lat_lng;GeoPoint::unpack_lat_lng(sort_fields[i].geopoint, reference_lat_lng);- auto it = geopoints->find(seq_id);- int64_t dist = INT32_MAX;+ if(geopoints != nullptr) {+ auto it = geopoints->find(seq_id);- if(it != geopoints->end()) {- int64_t packed_latlng = it->second;- S2LatLng s2_lat_lng;- GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);- dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);+ if(it != geopoints->end()) {+ int64_t packed_latlng = it->second;+ S2LatLng s2_lat_lng;+ GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);+ dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);+ }+ } else {+ // indicates geo point array+ auto field_it = geo_array_index.at(sort_fields[i].name);+ auto it = field_it->find(seq_id);++ if(it != field_it->end()) {+ int64_t* latlngs = it->second;+ for(size_t li = 0; li < latlngs[0]; li++) {+ S2LatLng s2_lat_lng;+ int64_t packed_latlng = latlngs[li + 1];+ GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);+ int64_t this_dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);+ if(this_dist < dist) {+ dist = this_dist;+ }+ }+ }}if(dist < sort_fields[i].exclude_radius) {@@ -2939,13 +3032,31 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docoptions.set_index_contains_points_only(true);S2RegionTermIndexer indexer(options);- const std::vector<double>& latlong = document[field_name];- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();- for(const auto& term: indexer.GetIndexTerms(point, "")) {- std::vector<uint32_t>& ids = (*geo_index)[term];- ids.erase(std::remove(ids.begin(), ids.end(), seq_id), ids.end());- if(ids.empty()) {- geo_index->erase(term);+ const std::vector<std::vector<double>>& latlongs = search_field.is_single_geopoint() ?+ std::vector<std::vector<double>>{document[field_name].get<std::vector<double>>()} :+ document[field_name].get<std::vector<std::vector<double>>>();++ for(const std::vector<double>& latlong: latlongs) {+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();+ for(const auto& term: indexer.GetIndexTerms(point, "")) {+ auto term_it = geo_index->find(term);+ if(term_it == geo_index->end()) {+ continue;+ }+ std::vector<uint32_t>& ids = term_it->second;+ ids.erase(std::remove(ids.begin(), ids.end(), seq_id), ids.end());+ if(ids.empty()) {+ geo_index->erase(term);+ }+ }+ }++ if(!search_field.is_single_geopoint()) {+ spp::sparse_hash_map<uint32_t, int64_t*>*& field_geo_array_map = geo_array_index.at(field_name);+ auto geo_array_it = field_geo_array_map->find(seq_id);+ if(geo_array_it != field_geo_array_map->end()) {+ delete [] geo_array_it->second;+ field_geo_array_map->erase(seq_id);}}}@@ -3017,6 +3128,10 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {} else if(new_field.is_geopoint()) {auto field_geo_index = new spp::sparse_hash_map<std::string, std::vector<uint32_t>>();geopoint_index.emplace(new_field.name, field_geo_index);+ if(!new_field.is_single_geopoint()) {+ auto geo_array_map = new spp::sparse_hash_map<uint32_t, int64_t*>();+ geo_array_index.emplace(new_field.name, geo_array_map);+ }} else {num_tree_t* num_tree = new num_tree_t;numerical_index.emplace(new_field.name, num_tree);@@ -3290,7 +3405,7 @@ Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const fielOption<uint32_t> Index::coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name,nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {- std::string suffix = is_array ? "a array of" : "a";+ std::string suffix = is_array ? "an array of" : "a";auto& item = is_array ? array_iter.value() : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {@@ -3313,19 +3428,19 @@ Option<uint32_t> Index::coerce_geopoint(const DIRTY_VALUES& dirty_values, const// try to value coerce into a geopoint- if(!document[field_name][0].is_number() && document[field_name][0].is_string()) {- if(StringUtils::is_float(document[field_name][0])) {- document[field_name][0] = std::stof(document[field_name][0].get<std::string>());+ if(!item[0].is_number() && item[0].is_string()) {+ if(StringUtils::is_float(item[0])) {+ item[0] = std::stof(item[0].get<std::string>());}}- if(!document[field_name][1].is_number() && document[field_name][1].is_string()) {- if(StringUtils::is_float(document[field_name][1])) {- document[field_name][1] = std::stof(document[field_name][1].get<std::string>());+ if(!item[1].is_number() && item[1].is_string()) {+ if(StringUtils::is_float(item[1])) {+ item[1] = std::stof(item[1].get<std::string>());}}- if(!document[field_name][0].is_number() || !document[field_name][1].is_number()) {+ if(!item[0].is_number() || !item[1].is_number()) {if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {if(!a_field.optional) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");@@ -3457,6 +3572,7 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do}}+/*// https://stackoverflow.com/questions/924171/geo-fencing-point-inside-outside-polygon// NOTE: polygon and point should have been transformed with `transform_for_180th_meridian`bool Index::is_point_in_polygon(const Geofence& poly, const GeoCoord &point) {@@ -3511,16 +3627,4 @@ double Index::transform_for_180th_meridian(Geofence &poly) {void Index::transform_for_180th_meridian(GeoCoord &point, double offset) {point.lon = point.lon < 0.0 ? point.lon + offset : point.lon;}--bool Index::field_contains_string(const std::string& field_name, const std::string& value) {- std::shared_lock lock(mutex);-- auto field_it = search_index.find(field_name);- if(field_it != search_index.end()) {- art_tree* t = field_it->second;- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char *)value.c_str(), value.size()+1);- return (leaf != nullptr);- }-- return false;-}+*/diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 51b31796..3b8dabd6 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -4,7 +4,6 @@#include <fstream>#include <algorithm>#include <collection_manager.h>-#include <h3api.h>#include "collection.h"class CollectionFilteringTest : public ::testing::Test {@@ -1069,10 +1068,151 @@ TEST_F(CollectionFilteringTest, GeoPointFiltering) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionFilteringTest, GeoPointArrayFiltering) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("loc", field_types::GEOPOINT_ARRAY, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::vector<std::string>>> records = {+ { {"Alpha Inc", "Ennore", "13.22112, 80.30511"},+ {"Alpha Inc", "Velachery", "12.98973, 80.23095"}+ },++ {+ {"Veera Inc", "Thiruvallur", "13.12752, 79.90136"},+ },++ {+ {"B1 Inc", "Bengaluru", "12.98246, 77.5847"},+ {"B1 Inc", "Hosur", "12.74147, 77.82915"},+ {"B1 Inc", "Vellore", "12.91866, 79.13075"},+ },++ {+ {"M Inc", "Nashik", "20.11282, 73.79458"},+ {"M Inc", "Pune", "18.56309, 73.855"},+ }+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0][0];+ doc["points"] = i;++ std::vector<std::vector<double>> lat_lngs;+ for(size_t k = 0; k < records[i].size(); k++) {+ std::vector<std::string> lat_lng_str;+ StringUtils::split(records[i][k][2], lat_lng_str, ", ");++ std::vector<double> lat_lng = {+ std::stod(lat_lng_str[0]),+ std::stod(lat_lng_str[1])+ };++ lat_lngs.push_back(lat_lng);+ }++ doc["loc"] = lat_lngs;+ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());+ }++ // pick a location close to Chennai+ auto results = coll1->search("*",+ {}, "loc: (13.12631, 80.20252, 100 km)",+ {}, {}, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ // pick location close to none of the spots+ results = coll1->search("*",+ {}, "loc: (13.62601, 79.39559, 10 km)",+ {}, {}, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(0, results["found"].get<size_t>());++ // pick a large radius covering all points++ results = coll1->search("*",+ {}, "loc: (21.20714729927276, 78.99153966917213, 1000 km)",+ {}, {}, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(4, results["found"].get<size_t>());++ // 1 mile radius++ results = coll1->search("*",+ {}, "loc: (12.98941, 80.23073, 1 mi)",+ {}, {}, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());++ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ // when geo field is formatted badly, show meaningful error+ nlohmann::json bad_doc;+ bad_doc["id"] = "1000";+ bad_doc["title"] = "Test record";+ bad_doc["loc"] = {"48.91", "2.33"};+ bad_doc["points"] = 1000;++ auto add_op = coll1->add(bad_doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `loc` must contain 2 element arrays: [ [lat, lng],... ].", add_op.error());++ bad_doc["loc"] = "foobar";+ add_op = coll1->add(bad_doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `loc` must be an array.", add_op.error());++ bad_doc["loc"] = nlohmann::json::array();+ nlohmann::json points = nlohmann::json::array();+ points.push_back("foo");+ points.push_back("bar");+ bad_doc["loc"].push_back(points);++ add_op = coll1->add(bad_doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `loc` must be an array of geopoint.", add_op.error());++ bad_doc["loc"][0][0] = "2.33";+ bad_doc["loc"][0][1] = "bar";+ add_op = coll1->add(bad_doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `loc` must be an array of geopoint.", add_op.error());++ bad_doc["loc"][0][0] = "foo";+ bad_doc["loc"][0][1] = "2.33";+ add_op = coll1->add(bad_doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `loc` must be an array of geopoint.", add_op.error());++ // under coercion mode, it should work+ bad_doc["loc"][0][0] = "48.91";+ bad_doc["loc"][0][1] = "2.33";+ add_op = coll1->add(bad_doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_TRUE(add_op.ok());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionFilteringTest, GeoPointRemoval) {std::vector<field> fields = {field("title", field_types::STRING, false),field("loc1", field_types::GEOPOINT, false),- field("loc2", field_types::GEOPOINT, false),+ field("loc2", field_types::GEOPOINT_ARRAY, false),field("points", field_types::INT32, false),};Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();@@ -1081,7 +1221,8 @@ TEST_F(CollectionFilteringTest, GeoPointRemoval) {doc["id"] = "0";doc["title"] = "Palais Garnier";doc["loc1"] = {48.872576479306765, 2.332291112241466};- doc["loc2"] = {48.84620987789056, 2.345152755563131};+ doc["loc2"] = nlohmann::json::array();+ doc["loc2"][0] = {48.84620987789056, 2.345152755563131};doc["points"] = 100;ASSERT_TRUE(coll1->add(doc.dump()).ok());@@ -1093,6 +1234,13 @@ TEST_F(CollectionFilteringTest, GeoPointRemoval) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());+ results = coll1->search("*",+ {}, "loc2: (48.87491151802846, 2.343945883701618, 10 km)",+ {}, {}, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+// remove the document, index another document and try querying againcoll1->remove("0");doc["id"] = "1";@@ -1105,6 +1253,13 @@ TEST_F(CollectionFilteringTest, GeoPointRemoval) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());++ results = coll1->search("*",+ {}, "loc2: (48.87491151802846, 2.343945883701618, 10 km)",+ {}, {}, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());}TEST_F(CollectionFilteringTest, GeoPolygonFiltering) {diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 540748de..8d3c0d1b 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -4,7 +4,6 @@#include <fstream>#include <algorithm>#include <collection_manager.h>-#include <h3api.h>#include "collection.h"class CollectionSortingTest : public ::testing::Test {@@ -860,3 +859,157 @@ TEST_F(CollectionSortingTest, GeoPointSortingWithPrecision) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionSortingTest, GeoPointAsOptionalField) {+ Collection* coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("loc", field_types::GEOPOINT, false, true),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Tibetan Colony", "32.24678, 77.19239"},+ {"Civil Hospital", "32.23959, 77.18763"},+ {"Johnson Lodge", "32.24751, 77.18814"},++ {"Lion King Rock", "32.24493, 77.17038"},+ {"Jai Durga Handloom", "32.25749, 77.17583"},+ {"Panduropa", "32.26059, 77.21798"},++ {"Police Station", "32.23743, 77.18639"},+ {"Panduropa Post", "32.26263, 77.2196"},+ };++ for (size_t i = 0; i < records.size(); i++) {+ nlohmann::json doc;++ std::vector<std::string> lat_lng;+ StringUtils::split(records[i][1], lat_lng, ", ");++ double lat = std::stod(lat_lng[0]);+ double lng = std::stod(lat_lng[1]);++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];++ if(i != 2) {+ doc["loc"] = {lat, lng};+ }++ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ std::vector<sort_by> geo_sort_fields = {+ sort_by("loc(32.24348, 77.1893, precision: 0.9 km)", "ASC"),+ sort_by("points", "DESC"),+ };++ auto results = coll1->search("*",+ {}, "loc: (32.24348, 77.1893, 20 km)",+ {}, geo_sort_fields, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(7, results["found"].get<size_t>());+ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSortingTest, GeoPointArraySorting) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("loc", field_types::GEOPOINT_ARRAY, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::vector<std::string>>> records = {+ { {"Alpha Inc", "Ennore", "13.22112, 80.30511"},+ {"Alpha Inc", "Velachery", "12.98973, 80.23095"}+ },++ {+ {"Veera Inc", "Thiruvallur", "13.12752, 79.90136"},+ },++ {+ {"B1 Inc", "Bengaluru", "12.98246, 77.5847"},+ {"B1 Inc", "Hosur", "12.74147, 77.82915"},+ {"B1 Inc", "Vellore", "12.91866, 79.13075"},+ },++ {+ {"M Inc", "Nashik", "20.11282, 73.79458"},+ {"M Inc", "Pune", "18.56309, 73.855"},+ }+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0][0];+ doc["points"] = i;++ std::vector<std::vector<double>> lat_lngs;+ for(size_t k = 0; k < records[i].size(); k++) {+ std::vector<std::string> lat_lng_str;+ StringUtils::split(records[i][k][2], lat_lng_str, ", ");++ std::vector<double> lat_lng = {+ std::stod(lat_lng_str[0]),+ std::stod(lat_lng_str[1])+ };++ lat_lngs.push_back(lat_lng);+ }++ doc["loc"] = lat_lngs;+ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());+ }++ std::vector<sort_by> geo_sort_fields = {+ sort_by("loc(13.12631, 80.20252)", "ASC"),+ sort_by("points", "DESC"),+ };++ // pick a location close to Chennai+ auto results = coll1->search("*",+ {}, "loc: (13.12631, 80.20252, 100 km)",+ {}, geo_sort_fields, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ // pick a large radius covering all points++ geo_sort_fields = {+ sort_by("loc(13.03388, 79.25868)", "ASC"),+ sort_by("points", "DESC"),+ };++ results = coll1->search("*",+ {}, "loc: (13.03388, 79.25868, 1000 km)",+ {}, geo_sort_fields, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(4, results["found"].get<size_t>());++ ASSERT_STREQ("2", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][2]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", results["hits"][3]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}diff --git a/test/index_test.cpp b/test/index_test.cppindex b06e8ae8..934fe1c4 100644--- a/test/index_test.cpp+++ b/test/index_test.cpp@@ -63,7 +63,7 @@ TEST(IndexTest, ScrubReindexDoc) {pool.shutdown();}-TEST(IndexTest, PointInPolygon180thMeridian) {+/*TEST(IndexTest, PointInPolygon180thMeridian) {// somewhere in far eastern russiaGeoCoord verts[3] = {{67.63378886620751, 179.87924212491276},@@ -72,13 +72,13 @@ TEST(IndexTest, PointInPolygon180thMeridian) {};- /*std::vector<S2Point> vertices;+ *//*std::vector<S2Point> vertices;for(size_t point_index = 0; point_index < 4; point_index++) {S2Point vertex = S2LatLng::FromDegrees(verts[point_index].lat, verts[point_index].lon).ToPoint();vertices.emplace_back(vertex);}- S2Loop region(vertices);*/+ S2Loop region(vertices);*//*Geofence poly1{3, verts};double offset = Index::transform_for_180th_meridian(poly1);@@ -96,19 +96,19 @@ TEST(IndexTest, PointInPolygon180thMeridian) {Index::transform_for_180th_meridian(point4, offset);Index::transform_for_180th_meridian(point5, offset);- /*ASSERT_TRUE(region.Contains(S2LatLng::FromDegrees(point1.lat, point1.lon).ToPoint()));+ *//*ASSERT_TRUE(region.Contains(S2LatLng::FromDegrees(point1.lat, point1.lon).ToPoint()));ASSERT_TRUE(region.Contains(S2LatLng::FromDegrees(point2.lat, point2.lon).ToPoint()));ASSERT_TRUE(region.Contains(S2LatLng::FromDegrees(point3.lat, point3.lon).ToPoint()));ASSERT_FALSE(region.Contains(S2LatLng::FromDegrees(point4.lat, point4.lon).ToPoint()));ASSERT_FALSE(region.Contains(S2LatLng::FromDegrees(point5.lat, point5.lon).ToPoint()));-*/+*//*ASSERT_TRUE(Index::is_point_in_polygon(poly1, point1));ASSERT_TRUE(Index::is_point_in_polygon(poly1, point2));ASSERT_TRUE(Index::is_point_in_polygon(poly1, point3));ASSERT_FALSE(Index::is_point_in_polygon(poly1, point4));ASSERT_FALSE(Index::is_point_in_polygon(poly1, point5));-}+}*/TEST(IndexTest, GeoPointPackUnpack) {std::vector<std::pair<double, double>> latlngs = {