Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix nested geopoint array issues.
Kishore Nallana610501e32d2
diff --git a/include/validator.h b/include/validator.hindex c9533cb9..a6d51af4 100644--- a/include/validator.h+++ b/include/validator.h@@ -60,8 +60,10 @@ public:const std::string &field_name,nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);- static Option<uint32_t> coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name,- nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);+ static Option<uint32_t> coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field,+ nlohmann::json &document, const std::string &field_name,+ nlohmann::json& lat, nlohmann::json& lng,+ nlohmann::json::iterator& array_iter,+ bool is_array, bool& array_ele_erased);};\ No newline at end of filediff --git a/src/index.cpp b/src/index.cppindex 5ea1f9c3..90ae73a0 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -746,18 +746,34 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>[&afield, &geo_array_index=geo_array_index, geo_index](const index_record& record, uint32_t seq_id) {// nested geopoint value inside an array of object will be a simple array so must be treated as geopointbool nested_obj_arr_geopoint = (afield.nested && afield.type == field_types::GEOPOINT_ARRAY &&- record.doc[afield.name].size() == 2 && record.doc[afield.name][0].is_number());+ !record.doc[afield.name].empty() && record.doc[afield.name][0].is_number());if(afield.type == field_types::GEOPOINT || nested_obj_arr_geopoint) {- const std::vector<double>& latlong = record.doc[afield.name];+ // this could be a nested gepoint array so can have more than 2 array values+ const std::vector<double>& latlongs = record.doc[afield.name];+ for(size_t li = 0; li < latlongs.size(); li+=2) {+ S2RegionTermIndexer::Options options;+ options.set_index_contains_points_only(true);+ S2RegionTermIndexer indexer(options);+ S2Point point = S2LatLng::FromDegrees(latlongs[li], latlongs[li+1]).ToPoint();- S2RegionTermIndexer::Options options;- options.set_index_contains_points_only(true);- S2RegionTermIndexer indexer(options);- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();+ for(const auto& term: indexer.GetIndexTerms(point, "")) {+ (*geo_index)[term].push_back(seq_id);+ }+ }++ if(nested_obj_arr_geopoint) {+ int64_t* packed_latlongs = new int64_t[(latlongs.size()/2) + 1];+ packed_latlongs[0] = latlongs.size()/2;+ size_t j_packed_latlongs = 0;++ for(size_t li = 0; li < latlongs.size(); li+=2) {+ int64_t packed_latlong = GeoPoint::pack_lat_lng(latlongs[li], latlongs[li+1]);+ packed_latlongs[j_packed_latlongs + 1] = packed_latlong;+ j_packed_latlongs++;+ }- for(const auto& term: indexer.GetIndexTerms(point, "")) {- (*geo_index)[term].push_back(seq_id);+ geo_array_index.at(afield.name)->emplace(seq_id, packed_latlongs);}} else {const std::vector<std::vector<double>>& latlongs = record.doc[afield.name];@@ -771,12 +787,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>for(size_t li = 0; li < latlongs.size(); li++) {auto& latlong = latlongs[li];S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();- std::set<std::string> terms;for(const auto& term: indexer.GetIndexTerms(point, "")) {- terms.insert(term);- }-- for(const auto& term: terms) {(*geo_index)[term].push_back(seq_id);}diff --git a/src/validator.cpp b/src/validator.cppindex 25a77884..d3c828b7 100644--- a/src/validator.cpp+++ b/src/validator.cpp@@ -43,8 +43,10 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso}if(!(doc_ele[0].is_number() && doc_ele[1].is_number())) {- // one or more elements is not an number, try to coerce- Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);+ // one or more elements is not a number, try to coerce+ Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,+ doc_ele[0], doc_ele[1],+ dummy_iter, false, array_ele_erased);if(!coerce_op.ok()) {return coerce_op;}@@ -62,16 +64,27 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jsonlohmann::json::iterator it = doc_ele.begin();- // Handle a geopoint[] type inside an array of object: it won't be an array of array, so cannot iterate- if(a_field.nested && a_field.type == field_types::GEOPOINT_ARRAY &&- it->is_number() && doc_ele.size() == 2) {+ // have to differentiate the geopoint[] type of a nested array object's geopoint[] vs a simple nested field+ // geopoint[] type of an array of objects field won't be an array of array+ if(a_field.nested && a_field.type == field_types::GEOPOINT_ARRAY && it != doc_ele.end() && it->is_number()) {+ if(!doc_ele.empty() && doc_ele.size() % 2 != 0) {+ return Option<>(400, "Nested field `" + field_name + "` does not contain valid geopoint values.");+ }+const auto& item = doc_ele;- if(!(item[0].is_number() && item[1].is_number())) {- // one or more elements is not an number, try to coerce- Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name, it, true, array_ele_erased);- if(!coerce_op.ok()) {- return coerce_op;++ for(size_t ai = 0; ai < doc_ele.size(); ai+=2) {+ if(!(doc_ele[ai].is_number() && doc_ele[ai+1].is_number())) {+ // one or more elements is not an number, try to coerce+ Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,+ doc_ele[ai], doc_ele[ai+1],+ it, true, array_ele_erased);+ if(!coerce_op.ok()) {+ return coerce_op;+ }}++ it++;}return Option<uint32_t>(200);@@ -83,7 +96,7 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso}for(; it != doc_ele.end(); ) {- const auto& item = it.value();+ nlohmann::json& item = it.value();array_ele_erased = false;if (a_field.type == field_types::STRING_ARRAY && !item.is_string()) {@@ -118,8 +131,10 @@ Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::jso}if(!(item[0].is_number() && item[1].is_number())) {- // one or more elements is not an number, try to coerce- Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name, it, true, array_ele_erased);+ // one or more elements is not a number, try to coerce+ Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,+ item[0], item[1],+ it, true, array_ele_erased);if(!coerce_op.ok()) {return coerce_op;}@@ -396,11 +411,12 @@ Option<uint32_t> validator_t::coerce_bool(const DIRTY_VALUES& dirty_values, consreturn Option<uint32_t>(200);}-Option<uint32_t> validator_t::coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name,- nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {+Option<uint32_t> validator_t::coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field,+ nlohmann::json &document, const std::string &field_name,+ nlohmann::json& lat, nlohmann::json& lng,+ nlohmann::json::iterator& array_iter,+ bool is_array, bool& array_ele_erased) {std::string suffix = is_array ? "an array of" : "a";- auto& item = is_array ? array_iter.value() : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");@@ -422,19 +438,19 @@ Option<uint32_t> validator_t::coerce_geopoint(const DIRTY_VALUES& dirty_values,// try to value coerce into a geopoint- if(!item[0].is_number() && item[0].is_string()) {- if(StringUtils::is_float(item[0])) {- item[0] = std::stof(item[0].get<std::string>());+ if(!lat.is_number() && lat.is_string()) {+ if(StringUtils::is_float(lat)) {+ lat = std::stof(lat.get<std::string>());}}- if(!item[1].is_number() && item[1].is_string()) {- if(StringUtils::is_float(item[1])) {- item[1] = std::stof(item[1].get<std::string>());+ if(!lng.is_number() && lng.is_string()) {+ if(StringUtils::is_float(lng)) {+ lng = std::stof(lng.get<std::string>());}}- if(!item[0].is_number() || !item[1].is_number()) {+ if(!lat.is_number() || !lng.is_number()) {if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {if(!a_field.optional) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 39c59030..5baa6a15 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -1788,7 +1788,7 @@ TEST_F(CollectionNestedFieldsTest, NestedFieldWithGeopointArray) {Collection* coll1 = op.get();auto doc1 = R"({- "addresses": [{"geoPoint": [1.91, 23.5]}]+ "addresses": [{"geoPoint": [1.91, 23.5]}, {"geoPoint": [12.91, 23.5]}]})"_json;ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());@@ -1796,16 +1796,39 @@ TEST_F(CollectionNestedFieldsTest, NestedFieldWithGeopointArray) {auto results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();ASSERT_EQ(1, results["found"].get<size_t>());+ results = coll1->search("*", {}, "addresses.geoPoint: (12.911, 23.5, 1 mi)",+ {}, {}, {0}, 10, 1, FREQUENCY).get();+ ASSERT_EQ(1, results["found"].get<size_t>());+// with nested geopoint arrayauto doc2 = R"({- "addresses": [{"geoPoint": [[1.91, 23.5]]}]+ "addresses": [{"geoPoint": [[1.91, 23.5]]}, {"geoPoint": [[1.91, 23.5], [1.95, 24.5]]}]})"_json;ASSERT_TRUE(coll1->add(doc2.dump(), CREATE).ok());results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();ASSERT_EQ(2, results["found"].get<size_t>());+ // simply nested geopoint array++ auto doc3 = R"({+ "addresses": {"geoPoint": [[1.91, 23.5]]}+ })"_json;++ ASSERT_TRUE(coll1->add(doc3.dump(), CREATE).ok());+ results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(3, results["found"].get<size_t>());++ // simply nested geopoint+ // this technically cannot be allowed but it's really tricky to detect so we allow+ auto doc4 = R"({+ "addresses": {"geoPoint": [1.91, 23.5]}+ })"_json;++ auto simple_geopoint_op = coll1->add(doc4.dump(), CREATE);+ ASSERT_TRUE(simple_geopoint_op.ok());+// data validationauto bad_doc = R"({"addresses": [{"geoPoint": [1.91, "x"]}]