Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix nested geopoint array issues.

Kishore Nallana610501e32d2
diff --git a/include/validator.h b/include/validator.h
index c9533cb9..a6d51af4 100644
--- a/include/validator.h
+++ b/include/validator.h
@@ -60,8 +60,10 @@ public:
const std::string &field_name,
nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);
- static Option<uint32_t> coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name,
- nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);
+ static Option<uint32_t> coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field,
+ nlohmann::json &document, const std::string &field_name,
+ nlohmann::json& lat, nlohmann::json& lng,
+ nlohmann::json::iterator& array_iter,
+ bool is_array, bool& array_ele_erased);
};
\ No newline at end of file
diff --git a/src/index.cpp b/src/index.cpp
index 5ea1f9c3..90ae73a0 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -746,18 +746,34 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
[&afield, &geo_array_index=geo_array_index, geo_index](const index_record& record, uint32_t seq_id) {
// nested geopoint value inside an array of object will be a simple array so must be treated as geopoint
bool nested_obj_arr_geopoint = (afield.nested && afield.type == field_types::GEOPOINT_ARRAY &&
- record.doc[afield.name].size() == 2 && record.doc[afield.name][0].is_number());
+ !record.doc[afield.name].empty() && record.doc[afield.name][0].is_number());
if(afield.type == field_types::GEOPOINT || nested_obj_arr_geopoint) {
- const std::vector<double>& latlong = record.doc[afield.name];
+ // this could be a nested gepoint array so can have more than 2 array values
+ const std::vector<double>& latlongs = record.doc[afield.name];
+ for(size_t li = 0; li < latlongs.size(); li+=2) {
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
+ S2Point point = S2LatLng::FromDegrees(latlongs[li], latlongs[li+1]).ToPoint();
- S2RegionTermIndexer::Options options;
- options.set_index_contains_points_only(true);
- S2RegionTermIndexer indexer(options);
- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
+ for(const auto& term: indexer.GetIndexTerms(point, "")) {
+ (*geo_index)[term].push_back(seq_id);
+ }
+ }
+
+ if(nested_obj_arr_geopoint) {
+ int64_t* packed_latlongs = new int64_t[(latlongs.size()/2) + 1];
+ packed_latlongs[0] = latlongs.size()/2;
+ size_t j_packed_latlongs = 0;
+
+ for(size_t li = 0; li < latlongs.size(); li+=2) {
+ int64_t packed_latlong = GeoPoint::pack_lat_lng(latlongs[li], latlongs[li+1]);
+ packed_latlongs[j_packed_latlongs + 1] = packed_latlong;
+ j_packed_latlongs++;
+ }
- for(const auto& term: indexer.GetIndexTerms(point, "")) {
- (*geo_index)[term].push_back(seq_id);
+ geo_array_index.at(afield.name)->emplace(seq_id, packed_latlongs);
}
} else {
const std::vector<std::vector<double>>& latlongs = record.doc[afield.name];
@@ -771,12 +787,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
for(size_t li = 0; li < latlongs.size(); li++) {
auto& latlong = latlongs[li];
S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
- std::set<std::string> terms;
for(const auto& term: indexer.GetIndexTerms(point, "")) {
- terms.insert(term);
- }
-
- for(const auto& term: terms) {
(*geo_index)[term].push_back(seq_id);
}
diff --git a/src/validator.cpp b/src/validator.cpp
index 25a77884..d3c828b7 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -43,8 +43,10 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso
}
if(!(doc_ele[0].is_number() && doc_ele[1].is_number())) {
- // one or more elements is not an number, try to coerce
- Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
+ // one or more elements is not a number, try to coerce
+ Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,
+ doc_ele[0], doc_ele[1],
+ dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
@@ -62,16 +64,27 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso
nlohmann::json::iterator it = doc_ele.begin();
- // Handle a geopoint[] type inside an array of object: it won't be an array of array, so cannot iterate
- if(a_field.nested && a_field.type == field_types::GEOPOINT_ARRAY &&
- it->is_number() && doc_ele.size() == 2) {
+ // have to differentiate the geopoint[] type of a nested array object's geopoint[] vs a simple nested field
+ // geopoint[] type of an array of objects field won't be an array of array
+ if(a_field.nested && a_field.type == field_types::GEOPOINT_ARRAY && it != doc_ele.end() && it->is_number()) {
+ if(!doc_ele.empty() && doc_ele.size() % 2 != 0) {
+ return Option<>(400, "Nested field `" + field_name + "` does not contain valid geopoint values.");
+ }
+
const auto& item = doc_ele;
- if(!(item[0].is_number() && item[1].is_number())) {
- // one or more elements is not an number, try to coerce
- Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
- if(!coerce_op.ok()) {
- return coerce_op;
+
+ for(size_t ai = 0; ai < doc_ele.size(); ai+=2) {
+ if(!(doc_ele[ai].is_number() && doc_ele[ai+1].is_number())) {
+ // one or more elements is not an number, try to coerce
+ Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,
+ doc_ele[ai], doc_ele[ai+1],
+ it, true, array_ele_erased);
+ if(!coerce_op.ok()) {
+ return coerce_op;
+ }
}
+
+ it++;
}
return Option<uint32_t>(200);
@@ -83,7 +96,7 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso
}
for(; it != doc_ele.end(); ) {
- const auto& item = it.value();
+ nlohmann::json& item = it.value();
array_ele_erased = false;
if (a_field.type == field_types::STRING_ARRAY && !item.is_string()) {
@@ -118,8 +131,10 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso
}
if(!(item[0].is_number() && item[1].is_number())) {
- // one or more elements is not an number, try to coerce
- Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
+ // one or more elements is not a number, try to coerce
+ Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,
+ item[0], item[1],
+ it, true, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
@@ -396,11 +411,12 @@ Option<uint32_t> validator_t::coerce_bool(const DIRTY_VALUES& dirty_values, cons
return Option<uint32_t>(200);
}
-Option<uint32_t> validator_t::coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name,
- nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
+Option<uint32_t> validator_t::coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field,
+ nlohmann::json &document, const std::string &field_name,
+ nlohmann::json& lat, nlohmann::json& lng,
+ nlohmann::json::iterator& array_iter,
+ bool is_array, bool& array_ele_erased) {
std::string suffix = is_array ? "an array of" : "a";
- auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");
@@ -422,19 +438,19 @@ Option<uint32_t> validator_t::coerce_geopoint(const DIRTY_VALUES& dirty_values,
// try to value coerce into a geopoint
- if(!item[0].is_number() && item[0].is_string()) {
- if(StringUtils::is_float(item[0])) {
- item[0] = std::stof(item[0].get<std::string>());
+ if(!lat.is_number() && lat.is_string()) {
+ if(StringUtils::is_float(lat)) {
+ lat = std::stof(lat.get<std::string>());
}
}
- if(!item[1].is_number() && item[1].is_string()) {
- if(StringUtils::is_float(item[1])) {
- item[1] = std::stof(item[1].get<std::string>());
+ if(!lng.is_number() && lng.is_string()) {
+ if(StringUtils::is_float(lng)) {
+ lng = std::stof(lng.get<std::string>());
}
}
- if(!item[0].is_number() || !item[1].is_number()) {
+ if(!lat.is_number() || !lng.is_number()) {
if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(!a_field.optional) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 39c59030..5baa6a15 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -1788,7 +1788,7 @@ TEST_F(CollectionNestedFieldsTest, NestedFieldWithGeopointArray) {
Collection* coll1 = op.get();
auto doc1 = R"({
- "addresses": [{"geoPoint": [1.91, 23.5]}]
+ "addresses": [{"geoPoint": [1.91, 23.5]}, {"geoPoint": [12.91, 23.5]}]
})"_json;
ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
@@ -1796,16 +1796,39 @@ TEST_F(CollectionNestedFieldsTest, NestedFieldWithGeopointArray) {
auto results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(1, results["found"].get<size_t>());
+ results = coll1->search("*", {}, "addresses.geoPoint: (12.911, 23.5, 1 mi)",
+ {}, {}, {0}, 10, 1, FREQUENCY).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
// with nested geopoint array
auto doc2 = R"({
- "addresses": [{"geoPoint": [[1.91, 23.5]]}]
+ "addresses": [{"geoPoint": [[1.91, 23.5]]}, {"geoPoint": [[1.91, 23.5], [1.95, 24.5]]}]
})"_json;
ASSERT_TRUE(coll1->add(doc2.dump(), CREATE).ok());
results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
ASSERT_EQ(2, results["found"].get<size_t>());
+ // simply nested geopoint array
+
+ auto doc3 = R"({
+ "addresses": {"geoPoint": [[1.91, 23.5]]}
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc3.dump(), CREATE).ok());
+ results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
+ ASSERT_EQ(3, results["found"].get<size_t>());
+
+ // simply nested geopoint
+ // this technically cannot be allowed but it's really tricky to detect so we allow
+ auto doc4 = R"({
+ "addresses": {"geoPoint": [1.91, 23.5]}
+ })"_json;
+
+ auto simple_geopoint_op = coll1->add(doc4.dump(), CREATE);
+ ASSERT_TRUE(simple_geopoint_op.ok());
+
// data validation
auto bad_doc = R"({
"addresses": [{"geoPoint": [1.91, "x"]}]