Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Point datatype for basic geo queries.

kishorenc4eb7058568ce
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3c0ea9fa..bceefe8d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,11 +3,16 @@ project(typesense)
cmake_policy(SET CMP0074 NEW)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -O2 -g -DNDEBUG")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -std=c++17 -O0 -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++17 -O0 -g")
set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-class-memaccess")
+endif()
+
# will be overridden from command line during build
set(TYPESENSE_VERSION "nightly" CACHE STRING "")
set(BUILD_DEPS "yes" CACHE STRING "")
@@ -50,6 +55,7 @@ include(cmake/GoogleTest.cmake)
include(cmake/TestResources.cmake)
include(cmake/Iconv.cmake)
include(cmake/Jemalloc.cmake)
+include(cmake/h3.cmake)
FIND_PACKAGE(OpenSSL 1.1.1 REQUIRED)
FIND_PACKAGE(Snappy REQUIRED)
@@ -79,6 +85,7 @@ include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)
+include_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/src/h3lib/include)
link_directories(/usr/local/lib)
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
@@ -87,6 +94,7 @@ link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)
link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)
+link_directories(${DEP_ROOT_DIR}/${H3_NAME}/build/lib)
set(JEMALLOC_ROOT_DIR "${DEP_ROOT_DIR}/${JEMALLOC_NAME}")
FIND_PACKAGE(Jemalloc REQUIRED)
@@ -135,7 +143,7 @@ else()
endif()
set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})
-set(CORE_LIBS h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for
+set(CORE_LIBS h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for h3
${LevelDB_LIBRARIES} ${ROCKSDB_LIBS}
glog ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${STACKTRACE_LIBS}
${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${JEMALLOC_LIBRARIES}
diff --git a/cmake/h3.cmake b/cmake/h3.cmake
new file mode 100644
index 00000000..6490a6a9
--- /dev/null
+++ b/cmake/h3.cmake
@@ -0,0 +1,41 @@
+# Download and build H3
+
+set(H3_VERSION 3.7.1)
+set(H3_NAME h3-${H3_VERSION})
+set(H3_TAR_PATH ${DEP_ROOT_DIR}/${H3_NAME}.tar.gz)
+
+if(NOT EXISTS ${H3_TAR_PATH})
+ message(STATUS "Downloading https://github.com/uber/h3/archive/v${H3_VERSION}.tar.gz")
+ file(DOWNLOAD https://github.com/uber/h3/archive/v${H3_VERSION}.tar.gz ${H3_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${H3_NAME})
+ message(STATUS "Extracting ${H3_NAME}...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${H3_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}/)
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${H3_NAME}/build/lib/libh3.a)
+ message("Configuring ${H3_NAME}...")
+ file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${H3_NAME}/build)
+
+ execute_process(COMMAND ${CMAKE_COMMAND}
+ "-DCMAKE_FIND_LIBRARY_SUFFIXES=.a"
+ "-H${DEP_ROOT_DIR}/${H3_NAME}"
+ "-B${DEP_ROOT_DIR}/${H3_NAME}/build"
+ RESULT_VARIABLE
+ H3_CONFIGURE)
+ if(NOT H3_CONFIGURE EQUAL 0)
+ message(FATAL_ERROR "${H3_NAME} configure failed!")
+ endif()
+
+ if(BUILD_DEPS STREQUAL "yes")
+ message("Building ${H3_NAME} locally...")
+ execute_process(COMMAND ${CMAKE_COMMAND} --build
+ "${DEP_ROOT_DIR}/${H3_NAME}/build"
+ RESULT_VARIABLE
+ H3_BUILD)
+ if(NOT H3_BUILD EQUAL 0)
+ message(FATAL_ERROR "${H3_NAME} build failed!")
+ endif()
+ endif()
+endif()
diff --git a/include/collection.h b/include/collection.h
index 2f60b51d..ec9c79ae 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -361,6 +361,11 @@ private:
Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters) const;
+ static Option<bool> parse_geopoint_filter_value(std::string& raw_value,
+ const std::string& format_err_msg,
+ std::string& processed_filter_val,
+ NUM_COMPARATOR& num_comparator);
+
static Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,
std::map<size_t, std::vector<std::string>>& pinned_hits);
diff --git a/include/field.h b/include/field.h
index 30562a15..a04e5840 100644
--- a/include/field.h
+++ b/include/field.h
@@ -11,6 +11,7 @@ namespace field_types {
static const std::string INT64 = "int64";
static const std::string FLOAT = "float";
static const std::string BOOL = "bool";
+ static const std::string GEOPOINT = "geopoint";
static const std::string STRING_ARRAY = "string[]";
static const std::string INT32_ARRAY = "int32[]";
static const std::string INT64_ARRAY = "int64[]";
@@ -25,19 +26,30 @@ namespace fields {
static const std::string optional = "optional";
}
+static const uint8_t DEFAULT_GEO_RESOLUTION = 7;
+static const uint8_t FINEST_GEO_RESOLUTION = 15;
+
struct field {
std::string name;
std::string type;
bool facet;
bool optional;
+ uint8_t geo_resolution;
+
field(const std::string & name, const std::string & type, const bool facet):
- name(name), type(type), facet(facet), optional(false) {
+ name(name), type(type), facet(facet), optional(false), geo_resolution(DEFAULT_GEO_RESOLUTION) {
}
field(const std::string & name, const std::string & type, const bool facet, const bool optional):
- name(name), type(type), facet(facet), optional(optional) {
+ name(name), type(type), facet(facet), optional(optional), geo_resolution(DEFAULT_GEO_RESOLUTION) {
+
+ }
+
+ field(const std::string & name, const std::string & type, const bool facet, const bool optional,
+ const uint8_t geo_resolution):
+ name(name), type(type), facet(facet), optional(optional), geo_resolution(geo_resolution) {
}
@@ -74,6 +86,10 @@ struct field {
return (type == field_types::BOOL || type == field_types::BOOL_ARRAY);
}
+ bool is_geopoint() const {
+ return (type == field_types::GEOPOINT);
+ }
+
bool is_string() const {
return (type == field_types::STRING || type == field_types::STRING_ARRAY);
}
@@ -89,7 +105,7 @@ struct field {
}
bool has_valid_type() const {
- return is_string() || is_integer() || is_float() || is_bool();
+ return is_string() || is_integer() || is_float() || is_bool() || is_geopoint();
}
std::string faceted_name() const {
@@ -177,14 +193,21 @@ namespace sort_field_const {
struct sort_by {
std::string name;
std::string order;
+ int64_t geopoint;
+
+ sort_by(const std::string & name, const std::string & order): name(name), order(order), geopoint(0) {
+
+ }
- sort_by(const std::string & name, const std::string & order): name(name), order(order) {
+ sort_by(const std::string &name, const std::string &order, int64_t geopoint) :
+ name(name), order(order), geopoint(geopoint) {
}
sort_by& operator=(sort_by other) {
name = other.name;
order = other.order;
+ geopoint = other.geopoint;
return *this;
}
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 914cfc28..b83d24fe 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -12,6 +12,7 @@
#include <system_metrics.h>
#include <tokenizer.h>
#include <collection_manager.h>
+#include <h3api.h>
#include "topster.h"
#include "logger.h"
@@ -677,6 +678,43 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
for(const sort_by& _sort_field: sort_fields) {
sort_by sort_field_std(_sort_field.name, _sort_field.order);
+ if(sort_field_std.name.back() == ')') {
+ // check if this is a geo field
+ size_t paran_start = 0;
+ while(paran_start < sort_field_std.name.size() && sort_field_std.name[paran_start] != '(') {
+ paran_start++;
+ }
+
+ const std::string& actual_field_name = sort_field_std.name.substr(0, paran_start);
+
+ if(sort_schema.count(actual_field_name) == 0) {
+ std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";
+ return Option<nlohmann::json>(404, error);
+ }
+
+ const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);
+ std::vector<std::string> geo_coords;
+ StringUtils::split(geo_coordstr, geo_coords, ",");
+
+ if(geo_coords.size() != 2) {
+ std::string error = "Geopoint sorting field `" + actual_field_name +
+ "` must be in the `field(24.56,10.45):ASC` format.";
+ return Option<nlohmann::json>(404, error);
+ }
+
+ if(!StringUtils::is_float(geo_coords[0]) || !StringUtils::is_float(geo_coords[1])) {
+ std::string error = "Geopoint sorting field `" + actual_field_name +
+ "` must be in the `field(24.56,10.45):ASC` format.";
+ return Option<nlohmann::json>(404, error);
+ }
+
+ GeoCoord x {degsToRads(std::stod(geo_coords[0])), degsToRads(std::stod(geo_coords[1]))};
+ H3Index geoHash = geoToH3(&x, FINEST_GEO_RESOLUTION);
+
+ sort_field_std.name = actual_field_name;
+ sort_field_std.geopoint = geoHash;
+ }
+
if(sort_field_std.name != sort_field_const::text_match && sort_schema.count(sort_field_std.name) == 0) {
std::string error = "Could not find a field named `" + sort_field_std.name + "` in the schema for sorting.";
return Option<nlohmann::json>(404, error);
@@ -1729,6 +1767,47 @@ const std::vector<Index *> &Collection::_get_indexes() const {
return indices;
}
+Option<bool> Collection::parse_geopoint_filter_value(std::string& raw_value,
+ const std::string& format_err_msg,
+ std::string& processed_filter_val,
+ NUM_COMPARATOR& num_comparator) {
+
+ num_comparator = LESS_THAN_EQUALS;
+
+ if(!(raw_value[0] == '(' && raw_value[raw_value.size() - 1] == ')')) {
+ return Option<bool>(400, format_err_msg);
+ }
+
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+
+ // we will end up with: "10.45 34.56 2 km"
+
+ if(filter_values.size() != 3) {
+ return Option<bool>(400, format_err_msg);
+ }
+
+ if(!StringUtils::is_float(filter_values[0]) || !StringUtils::is_float(filter_values[1])) {
+ return Option<bool>(400, format_err_msg);
+ }
+
+ std::vector<std::string> dist_values;
+ StringUtils::split(filter_values[2], dist_values, " ");
+
+ if(dist_values.size() != 2) {
+ return Option<bool>(400, format_err_msg);
+ }
+
+ if(dist_values[1] != "km" && dist_values[1] != "mi") {
+ return Option<bool>(400, "Unit must be either `km` or `mi`.");
+ }
+
+ processed_filter_val = filter_values[0] + " " + filter_values[1] + " " + // co-ords
+ dist_values[0] + " " + dist_values[1]; // X km
+
+ return Option<bool>(true);
+}
+
Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,
std::vector<filter>& filters) const {
std::vector<std::string> filter_blocks;
@@ -1852,6 +1931,44 @@ Option<bool> Collection::parse_filter_query(const std::string& simple_filter_que
f = {field_name, {bool_value}, {EQUALS}};
}
+ } else if(_field.is_geopoint()) {
+ f = {field_name, {}, {}};
+
+ const std::string& format_err_msg = "Value of filter field `" + _field.name +
+ "`: must be in the `(-44.50, 170.29, 0.75 km)` format.";
+
+ NUM_COMPARATOR num_comparator;
+
+ // could be a single value or a list
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, "),");
+
+ for(std::string& filter_value: filter_values) {
+ filter_value += ")";
+ std::string processed_filter_val;
+ auto parse_op = parse_geopoint_filter_value(filter_value, format_err_msg, processed_filter_val, num_comparator);
+
+ if(!parse_op.ok()) {
+ return parse_op;
+ }
+
+ f.values.push_back(processed_filter_val);
+ f.comparators.push_back(num_comparator);
+ }
+ } else {
+ // single value, e.g. (10.45, 34.56, 2 km)
+ std::string processed_filter_val;
+ auto parse_op = parse_geopoint_filter_value(raw_value, format_err_msg, processed_filter_val, num_comparator);
+
+ if(!parse_op.ok()) {
+ return parse_op;
+ }
+
+ f.values.push_back(processed_filter_val);
+ f.comparators.push_back(num_comparator);
+ }
+
} else if(_field.is_string()) {
size_t filter_value_index = 0;
NUM_COMPARATOR str_comparator = CONTAINS;
@@ -2108,7 +2225,7 @@ std::vector<Index *> Collection::init_indices() {
facet_schema.emplace(field.name, field);
}
- if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {
+ if(field.is_single_integer() || field.is_single_float() || field.is_single_bool() || field.is_geopoint()) {
sort_schema.emplace(field.name, field);
}
}
diff --git a/src/index.cpp b/src/index.cpp
index 9975a91a..d98ce80f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -9,6 +9,7 @@
#include <string_utils.h>
#include <art.h>
#include <tokenizer.h>
+#include <h3api.h>
#include "logger.h"
Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
@@ -130,8 +131,8 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
facet_id = facet_to_id[field_name];
}
- // non-string faceted field should be indexed as faceted string field as well
- if(field_pair.second.facet && !field_pair.second.is_string()) {
+ // non-string, non-geo faceted field should be indexed as faceted string field as well
+ if(field_pair.second.facet && !field_pair.second.is_string() && field_pair.second.type != field_types::GEOPOINT) {
art_tree *t = search_index.at(field_pair.second.faceted_name());
if(field_pair.second.is_array()) {
@@ -193,6 +194,13 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
auto num_tree = numerical_index.at(field_name);
bool value = document[field_name];
num_tree->insert(value, seq_id);
+ } else if(field_pair.second.type == field_types::GEOPOINT) {
+ auto num_tree = numerical_index.at(field_name);
+ const std::vector<double>& latlong = document[field_name];
+ GeoCoord x {degsToRads(latlong[0]), degsToRads(latlong[1])};
+ H3Index geoHash = geoToH3(&x, field_pair.second.geo_resolution);
+ //LOG(INFO) << "Indexing h3 index " << geoHash << " for seq_id " << seq_id << " at res: " << size_t(field_pair.second.geo_resolution);
+ num_tree->insert(geoHash, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
art_tree *t = search_index.at(field_name);
const std::vector<std::string>& strings = document[field_name];
@@ -226,7 +234,8 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
// add numerical values automatically into sort index
if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||
- field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL) {
+ field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL ||
+ field_pair.second.type == field_types::GEOPOINT) {
spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(field_pair.first);
if(field_pair.second.is_integer() ) {
@@ -236,6 +245,11 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
doc_to_score->emplace(seq_id, ifloat);
} else if(field_pair.second.is_bool()) {
doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());
+ } else if(field_pair.second.is_geopoint()) {
+ const std::vector<double>& latlong = document[field_pair.first];
+ GeoCoord x {degsToRads(latlong[0]), degsToRads(latlong[1])};
+ H3Index geoHash = geoToH3(&x, FINEST_GEO_RESOLUTION);
+ doc_to_score->emplace(seq_id, (int64_t)(geoHash));
}
}
}
@@ -303,6 +317,16 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
if(!document[field_name].is_boolean()) {
return Option<>(400, "Field `" + field_name + "` must be a bool.");
}
+ } else if(field_pair.second.type == field_types::GEOPOINT) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");
+ }
+ if(document[field_name].size() != 2) {
+ return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");
+ }
+ if(!document[field_name][0].is_number() || !document[field_name][1].is_number()) {
+ return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");
+ }
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
if(!document[field_name].is_array()) {
return Option<>(400, "Field `" + field_name + "` must be a string array.");
@@ -1024,6 +1048,80 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
value_index++;
}
+ } else if(f.is_geopoint()) {
+ spp::sparse_hash_map<size_t, size_t> result_to_distance;
+ auto num_tree = numerical_index.at(a_filter.field_name);
+ uint32_t* temp_ids = nullptr;
+ size_t temp_ids_len = 0;
+
+ auto record_to_geo = sort_index.at(a_filter.field_name);
+
+ double indexed_edge_len = edgeLengthM(f.geo_resolution);
+
+ for(const std::string& filter_value: a_filter.values) {
+ std::vector<std::string> filter_value_parts;
+ StringUtils::split(filter_value, filter_value_parts, " "); // x y 2 km
+ double radius = std::stof(filter_value_parts[2]);
+ const auto& unit = filter_value_parts[3];
+
+ if(unit == "km") {
+ radius *= 1000;
+ } else {
+ // assume "mi" (validated upstream)
+ radius *= 1609.34;
+ }
+
+ GeoCoord location;
+ location.lat = degsToRads(std::stod(filter_value_parts[0]));
+ location.lon = degsToRads(std::stod(filter_value_parts[1]));
+ H3Index query_index = geoToH3(&location, f.geo_resolution);
+
+ //LOG(INFO) << "query latlon: " << std::stod(filter_value_parts[0]) << ", " << std::stod(filter_value_parts[1]);
+ //LOG(INFO) << "query h3 index: " << query_index << " at res: " << size_t(f.geo_resolution);
+
+ size_t k_rings = size_t(std::ceil(radius / indexed_edge_len));
+ size_t max_neighboring = maxKringSize(k_rings);
+ H3Index* neighboring_indices = static_cast<H3Index *>(calloc(max_neighboring, sizeof(H3Index)));
+ kRing(query_index, k_rings, neighboring_indices);
+
+ for (size_t hex_index = 0; hex_index < max_neighboring; hex_index++) {
+ // Some indexes may be 0 to indicate fewer than the maximum number of indexes.
+ if (neighboring_indices[hex_index] != 0) {
+ //LOG(INFO) << "Neighbour index: " << neighboring_indices[hex_index];
+ num_tree->search(EQUALS, neighboring_indices[hex_index], &temp_ids, temp_ids_len);
+ }
+ }
+
+ free(neighboring_indices);
+
+ // `result_ids` will contain all IDs that are within K-ring hexagons
+ // we still need to do another round of exact filtering on them
+
+ H3Index query_point_index = geoToH3(&location, FINEST_GEO_RESOLUTION);
+ for(size_t result_index = 0; result_index < temp_ids_len; result_index++) {
+ uint32_t result_id = temp_ids[result_index];
+ size_t actual_dist_meters = h3Distance(query_point_index, record_to_geo->at(result_id));
+ if(actual_dist_meters <= radius) {
+ auto it = result_to_distance.find(result_id);
+ if(it == result_to_distance.end()) {
+ result_to_distance.emplace(result_id, actual_dist_meters);
+ } else {
+ result_to_distance.emplace(result_id, std::min(actual_dist_meters, it->second));
+ }
+ }
+ }
+ }
+
+ result_ids = new uint32_t[result_to_distance.size()];
+
+ // add IDs from `result_to_distance` to `result_ids`
+ size_t kv_index = 0;
+ for(const auto& kv: result_to_distance) {
+ result_ids[kv_index++] = kv.first;
+ }
+
+ result_ids_len = result_to_distance.size();
+
} else if(f.is_string()) {
art_tree* t = search_index.at(a_filter.field_name);
@@ -1827,15 +1925,31 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
spp::sparse_hash_map<uint32_t, int64_t>* field_values[3];
+ spp::sparse_hash_map<uint32_t, int64_t> geopoint_distances[3];
+
for(size_t i = 0; i < sort_fields.size(); i++) {
sort_order[i] = 1;
if(sort_fields[i].order == sort_field_const::asc) {
sort_order[i] = -1;
}
- field_values[i] = (sort_fields[i].name != sort_field_const::text_match) ?
- sort_index.at(sort_fields[i].name) :
- nullptr;
+ if(sort_fields[i].name == sort_field_const::text_match) {
+ field_values[i] = nullptr;
+ } else if(sort_schema.at(sort_fields[i].name).is_geopoint()) {
+ // we have to populate distances that will be used for match scoring
+ spp::sparse_hash_map<uint32_t, int64_t>* geopoints = sort_index.at(sort_fields[i].name);
+
+ for(size_t rindex=0; rindex<result_size; rindex++) {
+ const uint32_t seq_id = result_ids[rindex];
+ auto it = geopoints->find(seq_id);
+ int64_t dist = (it == geopoints->end()) ? INT32_MAX : h3Distance(sort_fields[i].geopoint, it->second);
+ geopoint_distances[i].emplace(seq_id, dist);
+ }
+
+ field_values[i] = &geopoint_distances[i];
+ } else {
+ field_values[i] = sort_index.at(sort_fields[i].name);
+ }
}
//auto begin = std::chrono::high_resolution_clock::now();
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 81dcd370..0ab29065 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -718,6 +718,87 @@ TEST_F(CollectionFilteringTest, ComparatorsOnMultiValuedNumericalField) {
collectionManager.drop_collection("coll_array_fields");
}
+TEST_F(CollectionFilteringTest, GeoPointFiltering) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("loc", field_types::GEOPOINT, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Palais Garnier", "48.872576479306765, 2.332291112241466"},
+ {"Sacre Coeur", "48.888286721920934, 2.342340862419206"},
+ {"Arc de Triomphe", "48.87538726829884, 2.296113163780903"},
+ {"Place de la Concorde", "48.86536119187326, 2.321850747347093"},
+ {"Louvre Musuem", "48.86065813197502, 2.3381285349616725"},
+ {"Les Invalides", "48.856648379569904, 2.3118555692631357"},
+ {"Eiffel Tower", "48.85821022164442, 2.294239067890161"},
+ {"Notre-Dame de Paris", "48.852455825574495, 2.35071182406452"},
+ {"Musee Grevin", "48.872370541246816, 2.3431536410008906"},
+ {"Pantheon", "48.84620987789056, 2.345152755563131"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ std::vector<std::string> lat_lng;
+ StringUtils::split(records[i][1], lat_lng, ", ");
+
+ double lat = std::stod(lat_lng[0]);
+ double lng = std::stod(lat_lng[1]);
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["loc"] = {lat, lng};
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ // pick a location close to only the Sacre Coeur
+ auto results = coll1->search("*",
+ {}, "loc: (48.90615915923891, 2.3435897727061175, 3 km)",
+ {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ // pick location close to none of the spots
+ results = coll1->search("*",
+ {}, "loc: (48.910544830985785, 2.337218333651177, 2 km)",
+ {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(0, results["found"].get<size_t>());
+
+ // pick a large radius covering all points
+
+ results = coll1->search("*",
+ {}, "loc: (48.910544830985785, 2.337218333651177, 20 km)",
+ {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(10, results["found"].get<size_t>());
+
+ // 1 mile radius
+
+ results = coll1->search("*",
+ {}, "loc: (48.85825332869331, 2.303816427653377, 1 mi)",
+ {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+
+ ASSERT_STREQ("6", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("5", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionFilteringTest, FilteringWithPrefixSearch) {
Collection *coll1;
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 3ad37630..abd904cc 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -4,6 +4,7 @@
#include <fstream>
#include <algorithm>
#include <collection_manager.h>
+#include <h3api.h>
#include "collection.h"
class CollectionSortingTest : public ::testing::Test {
@@ -339,5 +340,140 @@ TEST_F(CollectionSortingTest, NegativeInt64Value) {
ASSERT_EQ(0, res["found"].get<size_t>());
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSortingTest, GeoPointFiltering) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("loc", field_types::GEOPOINT, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Palais Garnier", "48.872576479306765, 2.332291112241466"},
+ {"Sacre Coeur", "48.888286721920934, 2.342340862419206"},
+ {"Arc de Triomphe", "48.87538726829884, 2.296113163780903"},
+ {"Place de la Concorde", "48.86536119187326, 2.321850747347093"},
+ {"Louvre Musuem", "48.86065813197502, 2.3381285349616725"},
+ {"Les Invalides", "48.856648379569904, 2.3118555692631357"},
+ {"Eiffel Tower", "48.85821022164442, 2.294239067890161"},
+ {"Notre-Dame de Paris", "48.852455825574495, 2.35071182406452"},
+ {"Musee Grevin", "48.872370541246816, 2.3431536410008906"},
+ {"Pantheon", "48.84620987789056, 2.345152755563131"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ std::vector<std::string> lat_lng;
+ StringUtils::split(records[i][1], lat_lng, ", ");
+
+ double lat = std::stod(lat_lng[0]);
+ double lng = std::stod(lat_lng[1]);
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["loc"] = {lat, lng};
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ // pick a large radius covering all points, with a point close to Pantheon
+ std::vector<sort_by> geo_sort_fields = {
+ sort_by("loc(48.84442912268208, 2.3490714964332353)", "ASC")
+ };
+
+ auto results = coll1->search("*",
+ {}, "loc: (48.84442912268208, 2.3490714964332353, 20 km)",
+ {}, geo_sort_fields, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(10, results["found"].get<size_t>());
+
+ std::vector<std::string> expected_ids = {
+ "9", "7", "4", "3", "8", "5", "0", "6", "1", "2"
+ };
+
+ for(size_t i=0; i < expected_ids.size(); i++) {
+ ASSERT_STREQ(expected_ids[i].c_str(), results["hits"][i]["document"]["id"].get<std::string>().c_str());
+ }
+
+ // desc, without filter
+ geo_sort_fields = {
+ sort_by("loc(48.84442912268208, 2.3490714964332353)", "DESC")
+ };
+
+ results = coll1->search("*",
+ {}, "",
+ {}, geo_sort_fields, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(10, results["found"].get<size_t>());
+
+ for(size_t i=0; i < expected_ids.size(); i++) {
+ ASSERT_STREQ(expected_ids[expected_ids.size() - 1 - i].c_str(), results["hits"][i]["document"]["id"].get<std::string>().c_str());
+ }
+
+ // with bad sort field formats
+ std::vector<sort_by> bad_geo_sort_fields = {
+ sort_by("loc(,2.3490714964332353)", "ASC")
+ };
+
+ auto res_op = coll1->search("*",
+ {}, "",
+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);
+
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("Geopoint sorting field `loc` must be in the `field(24.56,10.45):ASC` format.", res_op.error().c_str());
+
+ bad_geo_sort_fields = {
+ sort_by("loc(x, y)", "ASC")
+ };
+
+ res_op = coll1->search("*",
+ {}, "",
+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);
+
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("Geopoint sorting field `loc` must be in the `field(24.56,10.45):ASC` format.", res_op.error().c_str());
+
+ bad_geo_sort_fields = {
+ sort_by("loc(", "ASC")
+ };
+
+ res_op = coll1->search("*",
+ {}, "",
+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);
+
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("Could not find a field named `loc(` in the schema for sorting.", res_op.error().c_str());
+
+ bad_geo_sort_fields = {
+ sort_by("loc)", "ASC")
+ };
+
+ res_op = coll1->search("*",
+ {}, "",
+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);
+
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("Could not find a field named `loc)` in the schema for sorting.", res_op.error().c_str());
+
+ bad_geo_sort_fields = {
+ sort_by("l()", "ASC")
+ };
+
+ res_op = coll1->search("*",
+ {}, "",
+ {}, bad_geo_sort_fields, 0, 10, 1, FREQUENCY);
+
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("Could not find a field named `l` in the schema for sorting.", res_op.error().c_str());
+
collectionManager.drop_collection("coll1");
}
\ No newline at end of file