Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Swap out underlying geo library.

Kishore Nalland827ddbc7a2c
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ae7312bc..7c568b64 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,6 +23,7 @@ endif()
# Needed to enable Snappy on RocksDB
add_definitions(-DSNAPPY)
+add_definitions(-DS2_USE_GLOG)
if (APPLE)
add_definitions(-DGTEST_USE_OWN_TR1_TUPLE)
@@ -154,7 +155,7 @@ else()
endif()
set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIES})
-set(CORE_LIBS kakasi h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for h3
+set(CORE_LIBS kakasi h2o-evloop braft brpc iconv ${ICU_ALL_LIBRARIES} ${CURL_LIBRARIES} for s2 h3
${LevelDB_LIBRARIES} ${ROCKSDB_LIBS}
glog ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${STACKTRACE_LIBS}
${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${JEMALLOC_LIBRARIES}
diff --git a/include/field.h b/include/field.h
index 4a1a5e59..dd797d94 100644
--- a/include/field.h
+++ b/include/field.h
@@ -1,6 +1,7 @@
#pragma once
#include <string>
+#include <s2/s2latlng.h>
#include "art.h"
#include "option.h"
#include "string_utils.h"
@@ -501,6 +502,32 @@ struct sort_by {
}
};
+class GeoPoint {
+ constexpr static const double EARTH_RADIUS = 3958.75;
+ constexpr static const double METER_CONVERT = 1609.00;
+public:
+ static int64_t pack_lat_lng(double lat, double lng) {
+ // https://stackoverflow.com/a/1220393/131050
+ int32_t ilat = lat * 1000000;
+ int32_t ilng = lng * 1000000;
+ int64_t lat_lng = (int64_t(ilat) << 32) | ilng;
+ return lat_lng;
+ }
+
+ static void unpack_lat_lng(int64_t packed_lat_lng, S2LatLng& latlng) {
+ double lat = double(packed_lat_lng >> 32) / 1000000;
+ double lng = double(packed_lat_lng & 0xffffffffUL) / 1000000;
+ latlng = S2LatLng::FromDegrees(lat, lng);
+ }
+
+ // distance in meters
+ static int64_t distance(const S2LatLng& a, const S2LatLng& b) {
+ double rdist = a.GetDistance(b).radians();
+ double dist = EARTH_RADIUS * rdist;
+ return dist * METER_CONVERT;
+ }
+};
+
struct token_pos_cost_t {
size_t pos;
uint32_t cost;
diff --git a/include/index.h b/include/index.h
index d29abf5f..300414a4 100644
--- a/include/index.h
+++ b/include/index.h
@@ -169,6 +169,8 @@ private:
spp::sparse_hash_map<std::string, num_tree_t*> numerical_index;
+ spp::sparse_hash_map<std::string, std::vector<uint32_t>> geopoint_index;
+
// facet_field => (seq_id => values)
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, facet_hash_values_t>*> facet_index_v3;
diff --git a/include/match_score.h b/include/match_score.h
index 3a5d0d97..b89523cc 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -7,14 +7,6 @@
#include <limits>
#include "logger.h"
-#ifdef DEBUG
-#define D(x) x
-#else
-#define D(x)
-#endif
-
-#define TokenOffsetHeap std::priority_queue<TokenOffset, std::vector<TokenOffset>, TokenOffset>
-
const size_t WINDOW_SIZE = 10;
const uint16_t MAX_DISPLACEMENT = std::numeric_limits<uint16_t>::max();
diff --git a/src/collection.cpp b/src/collection.cpp
index b3d8c3c1..f48f5d26 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -726,11 +726,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return Option<nlohmann::json>(404, error);
}
- GeoCoord x {degsToRads(std::stod(geo_coords[0])), degsToRads(std::stod(geo_coords[1]))};
- H3Index geoHash = geoToH3(&x, FINEST_GEO_RESOLUTION);
-
+ int64_t lat_lng = GeoPoint::pack_lat_lng(std::stod(geo_coords[0]), std::stod(geo_coords[1]));
sort_field_std.name = actual_field_name;
- sort_field_std.geopoint = geoHash;
+ sort_field_std.geopoint = lat_lng;
}
if(sort_field_std.name != sort_field_const::text_match && sort_schema.count(sort_field_std.name) == 0) {
diff --git a/src/index.cpp b/src/index.cpp
index e57769fa..fa6db017 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -10,6 +10,13 @@
#include <art.h>
#include <tokenizer.h>
#include <h3api.h>
+#include <s2/s2point.h>
+#include <s2/s2latlng.h>
+#include <s2/s2region_term_indexer.h>
+#include <s2/s2cap.h>
+#include <s2/s2earth.h>
+#include <s2/s2loop.h>
+#include <s2/s2builder.h>
#include "logger.h"
Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
@@ -205,12 +212,14 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
bool value = document[field_name];
num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::GEOPOINT) {
- auto num_tree = numerical_index.at(field_name);
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
const std::vector<double>& latlong = document[field_name];
- GeoCoord x {degsToRads(latlong[0]), degsToRads(latlong[1])};
- H3Index geoHash = geoToH3(&x, field_pair.second.geo_resolution);
- //LOG(INFO) << "Indexing h3 index " << geoHash << " for seq_id " << seq_id << " at res: " << size_t(field_pair.second.geo_resolution);
- num_tree->insert(geoHash, seq_id);
+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
+ for(const auto& term: indexer.GetIndexTerms(point, "")) {
+ geopoint_index[term].push_back(seq_id);
+ }
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
art_tree *t = search_index.at(field_name);
index_string_array_field(document[field_name], points, t, seq_id, is_facet, field_pair.second);
@@ -260,9 +269,8 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());
} else if(field_pair.second.is_geopoint()) {
const std::vector<double>& latlong = document[field_pair.first];
- GeoCoord x {degsToRads(latlong[0]), degsToRads(latlong[1])};
- H3Index geoHash = geoToH3(&x, FINEST_GEO_RESOLUTION);
- doc_to_score->emplace(seq_id, (int64_t)(geoHash));
+ int64_t lat_lng = GeoPoint::pack_lat_lng(latlong[0], latlong[1]);
+ doc_to_score->emplace(seq_id, lat_lng);
}
}
}
@@ -1063,90 +1071,26 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
}
} else if(f.is_geopoint()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
- auto record_to_geo = sort_index.at(a_filter.field_name);
-
- double indexed_edge_len = edgeLengthM(f.geo_resolution);
+ std::set<uint32_t> geo_result_ids;
for(const std::string& filter_value: a_filter.values) {
std::vector<std::string> filter_value_parts;
StringUtils::split(filter_value, filter_value_parts, ","); // x, y, 2 km (or) list of points
- std::vector<uint32_t> geo_result_ids;
-
bool is_polygon = StringUtils::is_float(filter_value_parts.back());
+ S2Region* query_region;
if(is_polygon) {
const int num_verts = int(filter_value_parts.size()) / 2;
- GeoCoord* verts = new GeoCoord[num_verts];
-
+ std::vector<S2Point> vertices;
for(size_t point_index = 0; point_index < size_t(num_verts); point_index++) {
- double lat = degsToRads(std::stod(filter_value_parts[point_index * 2]));
- double lon = degsToRads(std::stod(filter_value_parts[point_index * 2 + 1]));
- verts[point_index] = {lat, lon};
- }
-
- Geofence geo_fence = {num_verts, verts};
- GeoPolygon geo_polygon = {geo_fence, 0, nullptr};
- double lon_offset = transform_for_180th_meridian(geo_fence);
-
- size_t num_hexagons = maxPolyfillSize(&geo_polygon, f.geo_resolution);
-
- H3Index* hexagons = static_cast<H3Index *>(calloc(num_hexagons, sizeof(H3Index)));
-
- polyfill(&geo_polygon, f.geo_resolution, hexagons);
-
- // we will have to expand by kring=1 to ensure that hexagons completely cover the polygon
- // see: https://github.com/uber/h3/issues/332
- std::set<uint64_t> expanded_hexagons;
-
- for (size_t hex_index = 0; hex_index < num_hexagons; hex_index++) {
- // Some indexes may be 0 to indicate fewer than the maximum number of indexes.
- if (hexagons[hex_index] != 0) {
- expanded_hexagons.emplace(hexagons[hex_index]);
-
- size_t k_rings = 1;
- size_t max_neighboring = maxKringSize(k_rings);
- H3Index* neighboring_indices = static_cast<H3Index *>(calloc(max_neighboring, sizeof(H3Index)));
- kRing(hexagons[hex_index], k_rings, neighboring_indices);
-
- for (size_t neighbour_index = 0; neighbour_index < max_neighboring; neighbour_index++) {
- if (neighboring_indices[neighbour_index] != 0) {
- expanded_hexagons.emplace(neighboring_indices[neighbour_index]);
- }
- }
-
- free(neighboring_indices);
- }
- }
-
- for(auto hex_id: expanded_hexagons) {
- num_tree->get(hex_id, geo_result_ids);
+ double lat = std::stod(filter_value_parts[point_index * 2]);
+ double lon = std::stod(filter_value_parts[point_index * 2 + 1]);
+ S2Point vertex = S2LatLng::FromDegrees(lat, lon).ToPoint();
+ vertices.emplace_back(vertex);
}
- // we will do an exact filtering again with point-in-poly checks
- std::vector<uint32_t> exact_geo_result_ids;
- for(auto result_id: geo_result_ids) {
- GeoCoord point;
- h3ToGeo(record_to_geo->at(result_id), &point);
- point.lon = point.lon < 0.0 ? point.lon + lon_offset : point.lon;
-
- if(is_point_in_polygon(geo_fence, point)) {
- exact_geo_result_ids.push_back(result_id);
- }
- }
-
- std::sort(exact_geo_result_ids.begin(), exact_geo_result_ids.end());
-
- uint32_t *out = nullptr;
- result_ids_len = ArrayUtils::or_scalar(&exact_geo_result_ids[0], exact_geo_result_ids.size(),
- result_ids, result_ids_len, &out);
-
- delete [] result_ids;
- result_ids = out;
-
- free(hexagons);
- delete [] verts;
+ query_region = new S2Loop(vertices);
} else {
double radius = std::stof(filter_value_parts[2]);
const auto& unit = filter_value_parts[3];
@@ -1158,51 +1102,49 @@ uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter
radius *= 1609.34;
}
- GeoCoord location;
- location.lat = degsToRads(std::stod(filter_value_parts[0]));
- location.lon = degsToRads(std::stod(filter_value_parts[1]));
- H3Index query_index = geoToH3(&location, f.geo_resolution);
-
- //LOG(INFO) << "query latlon: " << std::stod(filter_value_parts[0]) << ", " << std::stod(filter_value_parts[1]);
- //LOG(INFO) << "query h3 index: " << query_index << " at res: " << size_t(f.geo_resolution);
+ S1Angle query_radius = S1Angle::Radians(S2Earth::MetersToRadians(radius));
+ double query_lat = std::stod(filter_value_parts[0]);
+ double query_lng = std::stod(filter_value_parts[1]);
+ S2Point center = S2LatLng::FromDegrees(query_lat, query_lng).ToPoint();
+ query_region = new S2Cap(center, query_radius);
+ }
- size_t k_rings = size_t(std::ceil(radius / indexed_edge_len));
- size_t max_neighboring = maxKringSize(k_rings);
- H3Index* neighboring_indices = static_cast<H3Index *>(calloc(max_neighboring, sizeof(H3Index)));
- kRing(query_index, k_rings, neighboring_indices);
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
- for (size_t hex_index = 0; hex_index < max_neighboring; hex_index++) {
- // Some indexes may be 0 to indicate fewer than the maximum number of indexes.
- if (neighboring_indices[hex_index] != 0) {
- //LOG(INFO) << "Neighbour index: " << neighboring_indices[hex_index];
- num_tree->get(neighboring_indices[hex_index], geo_result_ids);
- }
+ for (const auto& term : indexer.GetQueryTerms(*query_region, "")) {
+ const auto& ids_it = geopoint_index.find(term);
+ if(ids_it != geopoint_index.end()) {
+ geo_result_ids.insert(ids_it->second.begin(), ids_it->second.end());
}
+ }
- free(neighboring_indices);
-
- // `geo_result_ids` will contain all IDs that are within K-ring hexagons
- // we still need to do another round of exact filtering on them
+ // `geo_result_ids` will contain all IDs that are within approximately within query radius
+ // we still need to do another round of exact filtering on them
- std::vector<uint32_t> exact_geo_result_ids;
+ std::vector<uint32_t> exact_geo_result_ids;
- H3Index query_point_index = geoToH3(&location, FINEST_GEO_RESOLUTION);
- for(auto result_id: geo_result_ids) {
- size_t actual_dist_meters = h3Distance(query_point_index, record_to_geo->at(result_id));
- if(actual_dist_meters <= radius) {
- exact_geo_result_ids.push_back(result_id);
- }
+ for(auto result_id: geo_result_ids) {
+ int64_t lat_lng = sort_index.at(f.name)->at(result_id);
+ S2LatLng s2_lat_lng;
+ GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);
+ if (!query_region->Contains(s2_lat_lng.ToPoint())) {
+ continue;
}
+ exact_geo_result_ids.push_back(result_id);
+ }
- std::sort(exact_geo_result_ids.begin(), exact_geo_result_ids.end());
+ std::sort(exact_geo_result_ids.begin(), exact_geo_result_ids.end());
- uint32_t *out = nullptr;
- result_ids_len = ArrayUtils::or_scalar(&exact_geo_result_ids[0], exact_geo_result_ids.size(),
- result_ids, result_ids_len, &out);
+ uint32_t *out = nullptr;
+ result_ids_len = ArrayUtils::or_scalar(&exact_geo_result_ids[0], exact_geo_result_ids.size(),
+ result_ids, result_ids_len, &out);
- delete [] result_ids;
- result_ids = out;
- }
+ delete [] result_ids;
+ result_ids = out;
+
+ delete query_region;
}
} else if(f.is_string()) {
@@ -2072,10 +2014,21 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
// we have to populate distances that will be used for match scoring
spp::sparse_hash_map<uint32_t, int64_t> *geopoints = sort_index.at(sort_fields[i].name);
+ S2LatLng reference_lat_lng;
+ GeoPoint::unpack_lat_lng(sort_fields[i].geopoint, reference_lat_lng);
+
for (size_t rindex = 0; rindex < result_size; rindex++) {
const uint32_t seq_id = result_ids[rindex];
auto it = geopoints->find(seq_id);
- int64_t dist = (it == geopoints->end()) ? INT32_MAX : h3Distance(sort_fields[i].geopoint, it->second);
+ int64_t dist = INT32_MAX;
+
+ if(it != geopoints->end()) {
+ int64_t packed_latlng = it->second;
+ S2LatLng s2_lat_lng;
+ GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);
+ dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);
+ }
+
geopoint_distances[i].emplace(seq_id, dist);
}
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index ca360d79..9d732e4e 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -1005,10 +1005,11 @@ TEST_F(CollectionFilteringTest, GeoPointFiltering) {
{}, "loc: (48.85825332869331, 2.303816427653377, 1 mi)",
{}, {}, 0, 10, 1, FREQUENCY).get();
- ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["found"].get<size_t>());
ASSERT_STREQ("6", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("5", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", results["hits"][2]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index b44b3771..788c3208 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -492,7 +492,7 @@ TEST_F(CollectionSortingTest, GeoPointFiltering) {
ASSERT_EQ(10, results["found"].get<size_t>());
std::vector<std::string> expected_ids = {
- "9", "7", "4", "3", "8", "5", "0", "6", "1", "2"
+ "9", "7", "4", "5", "3", "8", "0", "6", "1", "2"
};
for(size_t i=0; i < expected_ids.size(); i++) {
diff --git a/test/index_test.cpp b/test/index_test.cpp
index c49358b0..0d0c6d84 100644
--- a/test/index_test.cpp
+++ b/test/index_test.cpp
@@ -1,6 +1,7 @@
#include <gtest/gtest.h>
#include "index.h"
#include <vector>
+#include <s2/s2loop.h>
TEST(IndexTest, ScrubReindexDoc) {
std::unordered_map<std::string, field> search_schema;
@@ -66,6 +67,15 @@ TEST(IndexTest, PointInPolygon180thMeridian) {
{67.5749950728145, 179.94421673458666}
};
+
+ /*std::vector<S2Point> vertices;
+ for(size_t point_index = 0; point_index < 4; point_index++) {
+ S2Point vertex = S2LatLng::FromDegrees(verts[point_index].lat, verts[point_index].lon).ToPoint();
+ vertices.emplace_back(vertex);
+ }
+
+ S2Loop region(vertices);*/
+
Geofence poly1{3, verts};
double offset = Index::transform_for_180th_meridian(poly1);
@@ -82,6 +92,12 @@ TEST(IndexTest, PointInPolygon180thMeridian) {
Index::transform_for_180th_meridian(point4, offset);
Index::transform_for_180th_meridian(point5, offset);
+ /*ASSERT_TRUE(region.Contains(S2LatLng::FromDegrees(point1.lat, point1.lon).ToPoint()));
+ ASSERT_TRUE(region.Contains(S2LatLng::FromDegrees(point2.lat, point2.lon).ToPoint()));
+ ASSERT_TRUE(region.Contains(S2LatLng::FromDegrees(point3.lat, point3.lon).ToPoint()));
+ ASSERT_FALSE(region.Contains(S2LatLng::FromDegrees(point4.lat, point4.lon).ToPoint()));
+ ASSERT_FALSE(region.Contains(S2LatLng::FromDegrees(point5.lat, point5.lon).ToPoint()));
+*/
ASSERT_TRUE(Index::is_point_in_polygon(poly1, point1));
ASSERT_TRUE(Index::is_point_in_polygon(poly1, point2));
ASSERT_TRUE(Index::is_point_in_polygon(poly1, point3));