Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Point datatype for basic geo queries.
kishorenc4eb7058568ce
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 3c0ea9fa..bceefe8d 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -3,11 +3,16 @@ project(typesense)cmake_policy(SET CMP0074 NEW)-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -O2 -g -DNDEBUG")-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -std=c++17 -O0 -g")+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++17 -O0 -g")set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess")+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-class-memaccess")+endif()+# will be overridden from command line during buildset(TYPESENSE_VERSION "nightly" CACHE STRING "")set(BUILD_DEPS "yes" CACHE STRING "")@@ -50,6 +55,7 @@ include(cmake/GoogleTest.cmake)include(cmake/TestResources.cmake)include(cmake/Iconv.cmake)include(cmake/Jemalloc.cmake)+include(cmake/h3.cmake)FIND_PACKAGE(OpenSSL 1.1.1 REQUIRED)FIND_PACKAGE(Snappy REQUIRED)@@ -79,6 +85,7 @@ include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)+include_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/src/h3lib/include)link_directories(/usr/local/lib)link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)@@ -87,6 +94,7 @@ link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)+link_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/lib)set(JEMALLOC_ROOT_DIR "${DEP_ROOT_DIR}/${JEMALLOC_NAME}")FIND_PACKAGE(Jemalloc REQUIRED)@@ -135,7 +143,7 @@ else()endif()set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})-set(CORE_LIBS h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for+set(CORE_LIBS h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for h3${LevelDB_LIBRARIES} ${ROCKSDB_LIBS}glog ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${STACKTRACE_LIBS}${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${JEMALLOC_LIBRARIES}diff --git a/cmake/h3.cmake b/cmake/h3.cmakenew file mode 100644index 00000000..6490a6a9--- /dev/null+++ b/cmake/h3.cmake@@ -0,0 +1,41 @@+# Download and build H3++set(H3_VERSION 3.7.1)+set(H3_NAME h3-${H3_VERSION})+set(H3_TAR_PATH ${DEP_ROOT_DIR}/${H3_NAME}.tar.gz)++if(NOT EXISTS ${H3_TAR_PATH})+ message(STATUS "Downloading https://github.com/uber/h3/archive/v${H3_VERSION}.tar.gz")+ file(DOWNLOAD https://github.com/uber/h3/archive/v${H3_VERSION}.tar.gz ${H3_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${H3_NAME})+ message(STATUS "Extracting ${H3_NAME}...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${H3_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}/)+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${H3_NAME}/build/lib/libh3.a)+ message("Configuring ${H3_NAME}...")+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${H3_NAME}/build)++ execute_process(COMMAND ${CMAKE_COMMAND}+ "-DCMAKE_FIND_LIBRARY_SUFFIXES=.a"+ "-H${DEP_ROOT_DIR}/${H3_NAME}"+ "-B${DEP_ROOT_DIR}/${H3_NAME}/build"+ RESULT_VARIABLE+ H3_CONFIGURE)+ if(NOT H3_CONFIGURE EQUAL 0)+ message(FATAL_ERROR "${H3_NAME} configure failed!")+ endif()++ if(BUILD_DEPS STREQUAL "yes")+ message("Building ${H3_NAME} locally...")+ execute_process(COMMAND ${CMAKE_COMMAND} --build+ "${DEP_ROOT_DIR}/${H3_NAME}/build"+ RESULT_VARIABLE+ H3_BUILD)+ if(NOT H3_BUILD EQUAL 0)+ message(FATAL_ERROR "${H3_NAME} build failed!")+ endif()+ endif()+endif()diff --git a/include/collection.h b/include/collection.hindex 2f60b51d..ec9c79ae 100644--- a/include/collection.h+++ b/include/collection.h@@ -361,6 +361,11 @@ private:Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters) const;+ static Option<bool> parse_geopoint_filter_value(std::string& raw_value,+ const std::string& format_err_msg,+ std::string& processed_filter_val,+ NUM_COMPARATOR& num_comparator);+static Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,std::map<size_t, std::vector<std::string>>& pinned_hits);diff --git a/include/field.h b/include/field.hindex 30562a15..a04e5840 100644--- a/include/field.h+++ b/include/field.h@@ -11,6 +11,7 @@ namespace field_types {static const std::string INT64 = "int64";static const std::string FLOAT = "float";static const std::string BOOL = "bool";+ static const std::string GEOPOINT = "geopoint";static const std::string STRING_ARRAY = "string[]";static const std::string INT32_ARRAY = "int32[]";static const std::string INT64_ARRAY = "int64[]";@@ -25,19 +26,30 @@ namespace fields {static const std::string optional = "optional";}+static const uint8_t DEFAULT_GEO_RESOLUTION = 7;+static const uint8_t FINEST_GEO_RESOLUTION = 15;+struct field {std::string name;std::string type;bool facet;bool optional;+ uint8_t geo_resolution;+field(const std::string & name, const std::string & type, const bool facet):- name(name), type(type), facet(facet), optional(false) {+ name(name), type(type), facet(facet), optional(false), geo_resolution(DEFAULT_GEO_RESOLUTION) {}field(const std::string & name, const std::string & type, const bool facet, const bool optional):- name(name), type(type), facet(facet), optional(optional) {+ name(name), type(type), facet(facet), optional(optional), geo_resolution(DEFAULT_GEO_RESOLUTION) {++ }++ field(const std::string & name, const std::string & type, const bool facet, const bool optional,+ const uint8_t geo_resolution):+ name(name), type(type), facet(facet), optional(optional), geo_resolution(geo_resolution) {}@@ -74,6 +86,10 @@ struct field {return (type == field_types::BOOL || type == field_types::BOOL_ARRAY);}+ bool is_geopoint() const {+ return (type == field_types::GEOPOINT);+ }+bool is_string() const {return (type == field_types::STRING || type == field_types::STRING_ARRAY);}@@ -89,7 +105,7 @@ struct field {}bool has_valid_type() const {- return is_string() || is_integer() || is_float() || is_bool();+ return is_string() || is_integer() || is_float() || is_bool() || is_geopoint();}std::string faceted_name() const {@@ -177,14 +193,21 @@ namespace sort_field_const {struct sort_by {std::string name;std::string order;+ int64_t geopoint;++ sort_by(const std::string & name, const std::string & order): name(name), order(order), geopoint(0) {++ }- sort_by(const std::string & name, const std::string & order): name(name), order(order) {+ sort_by(const std::string &name, const std::string &order, int64_t geopoint) :+ name(name), order(order), geopoint(geopoint) {}sort_by& operator=(sort_by other) {name = other.name;order = other.order;+ geopoint = other.geopoint;return *this;}};diff --git a/src/collection.cpp b/src/collection.cppindex 914cfc28..b83d24fe 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -12,6 +12,7 @@#include <system_metrics.h>#include <tokenizer.h>#include <collection_manager.h>+#include <h3api.h>#include "topster.h"#include "logger.h"@@ -677,6 +678,43 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::for(const sort_by& _sort_field: sort_fields) {sort_by sort_field_std(_sort_field.name, _sort_field.order);+ if(sort_field_std.name.back() == ')') {+ // check if this is a geo field+ size_t paran_start = 0;+ while(paran_start < sort_field_std.name.size() && sort_field_std.name[paran_start] != '(') {+ paran_start++;+ }++ const std::string& actual_field_name = sort_field_std.name.substr(0, paran_start);++ if(sort_schema.count(actual_field_name) == 0) {+ std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";+ return Option<nlohmann::json>(404, error);+ }++ const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);+ std::vector<std::string> geo_coords;+ StringUtils::split(geo_coordstr, geo_coords, ",");++ if(geo_coords.size() != 2) {+ std::string error = "Geopoint sorting field `" + actual_field_name ++ "` must be in the `field(24.56,10.45):ASC` format.";+ return Option<nlohmann::json>(404, error);+ }++ if(!StringUtils::is_float(geo_coords[0]) || !StringUtils::is_float(geo_coords[1])) {+ std::string error = "Geopoint sorting field `" + actual_field_name ++ "` must be in the `field(24.56,10.45):ASC` format.";+ return Option<nlohmann::json>(404, error);+ }++ GeoCoord x {degsToRads(std::stod(geo_coords[0])), degsToRads(std::stod(geo_coords[1]))};+ H3Index geoHash = geoToH3(&x, FINEST_GEO_RESOLUTION);++ sort_field_std.name = actual_field_name;+ sort_field_std.geopoint = geoHash;+ }+if(sort_field_std.name != sort_field_const::text_match && sort_schema.count(sort_field_std.name) == 0) {std::string error = "Could not find a field named `" + sort_field_std.name + "` in the schema for sorting.";return Option<nlohmann::json>(404, error);@@ -1729,6 +1767,47 @@ const std::vector<Index *> &Collection::_get_indexes() const {return indices;}+Option<bool> Collection::parse_geopoint_filter_value(std::string& raw_value,+ const std::string& format_err_msg,+ std::string& processed_filter_val,+ NUM_COMPARATOR& num_comparator) {++ num_comparator = LESS_THAN_EQUALS;++ if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {+ return Option<bool>(400, format_err_msg);+ }++ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");++ // we will end up with: "10.45 34.56 2 km"++ if(filter_values.size() != 3) {+ return Option<bool>(400, format_err_msg);+ }++ if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {+ return Option<bool>(400, format_err_msg);+ }++ std::vector<std::string> dist_values;+ StringUtils::split(filter_values[2], dist_values, " ");++ if(dist_values.size() != 2) {+ return Option<bool>(400, format_err_msg);+ }++ if(dist_values[1] != "km" && dist_values[1] != "mi") {+ return Option<bool>(400, "Unit must be either `km` or `mi`.");+ }++ processed_filter_val = filter_values[0] + " " + filter_values[1] + " " + // co-ords+ dist_values[0] + " " + dist_values[1]; // X km++ return Option<bool>(true);+}+Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,std::vector<filter>& filters) const {std::vector<std::string> filter_blocks;@@ -1852,6 +1931,44 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_quef = {field_name, {bool_value}, {EQUALS}};}+ } else if(_field.is_geopoint()) {+ f = {field_name, {}, {}};++ const std::string& format_err_msg = "Value of filter field `" + _field.name ++ "`: must be in the `(-44.50, 170.29, 0.75 km)` format.";++ NUM_COMPARATOR num_comparator;++ // could be a single value or a list+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");++ for(std::string& filter_value: filter_values) {+ filter_value += ")";+ std::string processed_filter_val;+ auto parse_op = parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val, num_comparator);++ if(!parse_op.ok()) {+ return parse_op;+ }++ f.values.push_back(processed_filter_val);+ f.comparators.push_back(num_comparator);+ }+ } else {+ // single value, e.g. (10.45, 34.56, 2 km)+ std::string processed_filter_val;+ auto parse_op = parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val, num_comparator);++ if(!parse_op.ok()) {+ return parse_op;+ }++ f.values.push_back(processed_filter_val);+ f.comparators.push_back(num_comparator);+ }+} else if(_field.is_string()) {size_t filter_value_index = 0;NUM_COMPARATOR str_comparator = CONTAINS;@@ -2108,7 +2225,7 @@ std::vector<Index *> Collection::init_indices() {facet_schema.emplace(field.name, field);}- if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {+ if(field.is_single_integer() || field.is_single_float() || field.is_single_bool() || field.is_geopoint()) {sort_schema.emplace(field.name, field);}}diff --git a/src/index.cpp b/src/index.cppindex 9975a91a..d98ce80f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -9,6 +9,7 @@#include <string_utils.h>#include <art.h>#include <tokenizer.h>+#include <h3api.h>#include "logger.h"Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,@@ -130,8 +131,8 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tfacet_id = facet_to_id[field_name];}- // non-string faceted field should be indexed as faceted string field as well- if(field_pair.second.facet && !field_pair.second.is_string()) {+ // non-string, non-geo faceted field should be indexed as faceted string field as well+ if(field_pair.second.facet && !field_pair.second.is_string() && field_pair.second.type != field_types::GEOPOINT) {art_tree *t = search_index.at(field_pair.second.faceted_name());if(field_pair.second.is_array()) {@@ -193,6 +194,13 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tauto num_tree = numerical_index.at(field_name);bool value = document[field_name];num_tree->insert(value, seq_id);+ } else if(field_pair.second.type == field_types::GEOPOINT) {+ auto num_tree = numerical_index.at(field_name);+ const std::vector<double>& latlong = document[field_name];+ GeoCoord x {degsToRads(latlong[0]), degsToRads(latlong[1])};+ H3Index geoHash = geoToH3(&x, field_pair.second.geo_resolution);+ //LOG(INFO) << "Indexing h3 index " << geoHash << " for seq_id " << seq_id << " at res: " << size_t(field_pair.second.geo_resolution);+ num_tree->insert(geoHash, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {art_tree *t = search_index.at(field_name);const std::vector<std::string>& strings = document[field_name];@@ -226,7 +234,8 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t// add numerical values automatically into sort indexif(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||- field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL) {+ field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL ||+ field_pair.second.type == field_types::GEOPOINT) {spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(field_pair.first);if(field_pair.second.is_integer() ) {@@ -236,6 +245,11 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tdoc_to_score->emplace(seq_id, ifloat);} else if(field_pair.second.is_bool()) {doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());+ } else if(field_pair.second.is_geopoint()) {+ const std::vector<double>& latlong = document[field_pair.first];+ GeoCoord x {degsToRads(latlong[0]), degsToRads(latlong[1])};+ H3Index geoHash = geoToH3(&x, FINEST_GEO_RESOLUTION);+ doc_to_score->emplace(seq_id, (int64_t)(geoHash));}}}@@ -303,6 +317,16 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,if(!document[field_name].is_boolean()) {return Option<>(400, "Field `" + field_name + "` must be a bool.");}+ } else if(field_pair.second.type == field_types::GEOPOINT) {+ if(!document[field_name].is_array()) {+ return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");+ }+ if(document[field_name].size() != 2) {+ return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");+ }+ if(!document[field_name][0].is_number() || !document[field_name][1].is_number()) {+ return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");+ }} else if(field_pair.second.type == field_types::STRING_ARRAY) {if(!document[field_name].is_array()) {return Option<>(400, "Field `" + field_name + "` must be a string array.");@@ -1024,6 +1048,80 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filtervalue_index++;}+ } else if(f.is_geopoint()) {+ spp::sparse_hash_map<size_t, size_t> result_to_distance;+ auto num_tree = numerical_index.at(a_filter.field_name);+ uint32_t* temp_ids = nullptr;+ size_t temp_ids_len = 0;++ auto record_to_geo = sort_index.at(a_filter.field_name);++ double indexed_edge_len = edgeLengthM(f.geo_resolution);++ for(const std::string& filter_value: a_filter.values) {+ std::vector<std::string> filter_value_parts;+ StringUtils::split(filter_value, filter_value_parts, " "); // x y 2 km+ double radius = std::stof(filter_value_parts[2]);+ const auto& unit = filter_value_parts[3];++ if(unit == "km") {+ radius *= 1000;+ } else {+ // assume "mi" (validated upstream)+ radius *= 1609.34;+ }++ GeoCoord location;+ location.lat = degsToRads(std::stod(filter_value_parts[0]));+ location.lon = degsToRads(std::stod(filter_value_parts[1]));+ H3Index query_index = geoToH3(&location, f.geo_resolution);++ //LOG(INFO) << "query latlon: " << std::stod(filter_value_parts[0]) << ", " << std::stod(filter_value_parts[1]);+ //LOG(INFO) << "query h3 index: " << query_index << " at res: " << size_t(f.geo_resolution);++ size_t k_rings = size_t(std::ceil(radius / indexed_edge_len));+ size_t max_neighboring = maxKringSize(k_rings);+ H3Index* neighboring_indices = static_cast<H3Index *>(calloc(max_neighboring, sizeof(H3Index)));+ kRing(query_index, k_rings, neighboring_indices);++ for (size_t hex_index = 0; hex_index < max_neighboring; hex_index++) {+ // Some indexes may be 0 to indicate fewer than the maximum number of indexes.+ if (neighboring_indices[hex_index] != 0) {+ //LOG(INFO) << "Neighbour index: " << neighboring_indices[hex_index];+ num_tree->search(EQUALS, neighboring_indices[hex_index], &temp_ids, temp_ids_len);+ }+ }++ free(neighboring_indices);++ // `result_ids` will contain all IDs that are within K-ring hexagons+ // we still need to do another round of exact filtering on them++ H3Index query_point_index = geoToH3(&location, FINEST_GEO_RESOLUTION);+ for(size_t result_index = 0; result_index < temp_ids_len; result_index++) {+ uint32_t result_id = temp_ids[result_index];+ size_t actual_dist_meters = h3Distance(query_point_index, record_to_geo->at(result_id));+ if(actual_dist_meters <= radius) {+ auto it = result_to_distance.find(result_id);+ if(it == result_to_distance.end()) {+ result_to_distance.emplace(result_id, actual_dist_meters);+ } else {+ result_to_distance.emplace(result_id, std::min(actual_dist_meters, it->second));+ }+ }+ }+ }++ result_ids = new uint32_t[result_to_distance.size()];++ // add IDs from `result_to_distance` to `result_ids`+ size_t kv_index = 0;+ for(const auto& kv: result_to_distance) {+ result_ids[kv_index++] = kv.first;+ }++ result_ids_len = result_to_distance.size();+} else if(f.is_string()) {art_tree* t = search_index.at(a_filter.field_name);@@ -1827,15 +1925,31 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16int sort_order[3]; // 1 or -1 based on DESC or ASC respectivelyspp::sparse_hash_map<uint32_t, int64_t>* field_values[3];+ spp::sparse_hash_map<uint32_t, int64_t> geopoint_distances[3];+for(size_t i = 0; i < sort_fields.size(); i++) {sort_order[i] = 1;if(sort_fields[i].order == sort_field_const::asc) {sort_order[i] = -1;}- field_values[i] = (sort_fields[i].name != sort_field_const::text_match) ?- sort_index.at(sort_fields[i].name) :- nullptr;+ if(sort_fields[i].name == sort_field_const::text_match) {+ field_values[i] = nullptr;+ } else if(sort_schema.at(sort_fields[i].name).is_geopoint()) {+ // we have to populate distances that will be used for match scoring+ spp::sparse_hash_map<uint32_t, int64_t>* geopoints = sort_index.at(sort_fields[i].name);++ for(size_t rindex=0; rindex<result_size; rindex++) {+ const uint32_t seq_id = result_ids[rindex];+ auto it = geopoints->find(seq_id);+ int64_t dist = (it == geopoints->end()) ? INT32_MAX : h3Distance(sort_fields[i].geopoint, it->second);+ geopoint_distances[i].emplace(seq_id, dist);+ }++ field_values[i] = &geopoint_distances[i];+ } else {+ field_values[i] = sort_index.at(sort_fields[i].name);+ }}//auto begin = std::chrono::high_resolution_clock::now();diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 81dcd370..0ab29065 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -718,6 +718,87 @@ TEST_F(CollectionFilteringTest, ComparatorsOnMultiValuedNumericalField) {collectionManager.drop_collection("coll_array_fields");}+TEST_F(CollectionFilteringTest, GeoPointFiltering) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("loc", field_types::GEOPOINT, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Palais Garnier", "48.872576479306765, 2.332291112241466"},+ {"Sacre Coeur", "48.888286721920934, 2.342340862419206"},+ {"Arc de Triomphe", "48.87538726829884, 2.296113163780903"},+ {"Place de la Concorde", "48.86536119187326, 2.321850747347093"},+ {"Louvre Musuem", "48.86065813197502, 2.3381285349616725"},+ {"Les Invalides", "48.856648379569904, 2.3118555692631357"},+ {"Eiffel Tower", "48.85821022164442, 2.294239067890161"},+ {"Notre-Dame de Paris", "48.852455825574495, 2.35071182406452"},+ {"Musee Grevin", "48.872370541246816, 2.3431536410008906"},+ {"Pantheon", "48.84620987789056, 2.345152755563131"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ std::vector<std::string> lat_lng;+ StringUtils::split(records[i][1], lat_lng, ", ");++ double lat = std::stod(lat_lng[0]);+ double lng = std::stod(lat_lng[1]);++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["loc"] = {lat, lng};+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ // pick a location close to only the Sacre Coeur+ auto results = coll1->search("*",+ {}, "loc: (48.90615915923891, 2.3435897727061175, 3 km)",+ {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ // pick location close to none of the spots+ results = coll1->search("*",+ {}, "loc: (48.910544830985785, 2.337218333651177, 2 km)",+ {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(0, results["found"].get<size_t>());++ // pick a large radius covering all points++ results = coll1->search("*",+ {}, "loc: (48.910544830985785, 2.337218333651177, 20 km)",+ {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(10, results["found"].get<size_t>());++ // 1 mile radius++ results = coll1->search("*",+ {}, "loc: (48.85825332869331, 2.303816427653377, 1 mi)",+ {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(2, results["found"].get<size_t>());++ ASSERT_STREQ("6", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("5", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionFilteringTest, FilteringWithPrefixSearch) {Collection *coll1;diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 3ad37630..abd904cc 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -4,6 +4,7 @@#include <fstream>#include <algorithm>#include <collection_manager.h>+#include <h3api.h>#include "collection.h"class CollectionSortingTest : public ::testing::Test {@@ -339,5 +340,140 @@ TEST_F(CollectionSortingTest, NegativeInt64Value) {ASSERT_EQ(0, res["found"].get<size_t>());+ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSortingTest, GeoPointFiltering) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("loc", field_types::GEOPOINT, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Palais Garnier", "48.872576479306765, 2.332291112241466"},+ {"Sacre Coeur", "48.888286721920934, 2.342340862419206"},+ {"Arc de Triomphe", "48.87538726829884, 2.296113163780903"},+ {"Place de la Concorde", "48.86536119187326, 2.321850747347093"},+ {"Louvre Musuem", "48.86065813197502, 2.3381285349616725"},+ {"Les Invalides", "48.856648379569904, 2.3118555692631357"},+ {"Eiffel Tower", "48.85821022164442, 2.294239067890161"},+ {"Notre-Dame de Paris", "48.852455825574495, 2.35071182406452"},+ {"Musee Grevin", "48.872370541246816, 2.3431536410008906"},+ {"Pantheon", "48.84620987789056, 2.345152755563131"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ std::vector<std::string> lat_lng;+ StringUtils::split(records[i][1], lat_lng, ", ");++ double lat = std::stod(lat_lng[0]);+ double lng = std::stod(lat_lng[1]);++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["loc"] = {lat, lng};+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ // pick a large radius covering all points, with a point close to Pantheon+ std::vector<sort_by> geo_sort_fields = {+ sort_by("loc(48.84442912268208, 2.3490714964332353)", "ASC")+ };++ auto results = coll1->search("*",+ {}, "loc: (48.84442912268208, 2.3490714964332353, 20 km)",+ {}, geo_sort_fields, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(10, results["found"].get<size_t>());++ std::vector<std::string> expected_ids = {+ "9", "7", "4", "3", "8", "5", "0", "6", "1", "2"+ };++ for(size_t i=0; i < expected_ids.size(); i++) {+ ASSERT_STREQ(expected_ids[i].c_str(), results["hits"][i]["document"]["id"].get<std::string>().c_str());+ }++ // desc, without filter+ geo_sort_fields = {+ sort_by("loc(48.84442912268208, 2.3490714964332353)", "DESC")+ };++ results = coll1->search("*",+ {}, "",+ {}, geo_sort_fields, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(10, results["found"].get<size_t>());++ for(size_t i=0; i < expected_ids.size(); i++) {+ ASSERT_STREQ(expected_ids[expected_ids.size() - 1 - i].c_str(), results["hits"][i]["document"]["id"].get<std::string>().c_str());+ }++ // with bad sort field formats+ std::vector<sort_by> bad_geo_sort_fields = {+ sort_by("loc(,2.3490714964332353)", "ASC")+ };++ auto res_op = coll1->search("*",+ {}, "",+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);++ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("Geopoint sorting field `loc` must be in the `field(24.56,10.45):ASC` format.", res_op.error().c_str());++ bad_geo_sort_fields = {+ sort_by("loc(x, y)", "ASC")+ };++ res_op = coll1->search("*",+ {}, "",+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);++ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("Geopoint sorting field `loc` must be in the `field(24.56,10.45):ASC` format.", res_op.error().c_str());++ bad_geo_sort_fields = {+ sort_by("loc(", "ASC")+ };++ res_op = coll1->search("*",+ {}, "",+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);++ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("Could not find a field named `loc(` in the schema for sorting.", res_op.error().c_str());++ bad_geo_sort_fields = {+ sort_by("loc)", "ASC")+ };++ res_op = coll1->search("*",+ {}, "",+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);++ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("Could not find a field named `loc)` in the schema for sorting.", res_op.error().c_str());++ bad_geo_sort_fields = {+ sort_by("l()", "ASC")+ };++ res_op = coll1->search("*",+ {}, "",+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);++ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("Could not find a field named `l` in the schema for sorting.", res_op.error().c_str());+collectionManager.drop_collection("coll1");}\ No newline at end of file