Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix edge cases in schema detection.
kishorenc3a4d21992cd5
diff --git a/include/collection.h b/include/collection.hindex 7fda6a31..5a742de6 100644--- a/include/collection.h+++ b/include/collection.h@@ -349,7 +349,7 @@ private:std::map<size_t, std::vector<uint32_t>>& include_ids,std::vector<uint32_t> & excluded_ids) const;- Option<bool> check_and_update_schema(nlohmann::json& document);+ Option<bool> check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values);static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,const std::pair<uint64_t, facet_count_t>& b) {diff --git a/include/index.h b/include/index.hindex c91f98d6..800c4c62 100644--- a/include/index.h+++ b/include/index.h@@ -246,20 +246,28 @@ private:static void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,nlohmann::json &new_doc, nlohmann::json &del_doc);- static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index);+ static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, const std::string& auto_detect_schema,+ const field& a_field, nlohmann::json &document,+ const std::string &field_name,+ nlohmann::json::iterator& array_iter,+ bool is_array,+ bool& array_ele_erased);static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index);+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index);+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index);+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index);+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);public:// for limiting number of results on multiple candidates / query rewritesdiff --git a/src/collection.cpp b/src/collection.cppindex 767a26d0..40554c77 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -180,6 +180,10 @@ Option<nlohmann::json> Collection::add(const std::string & json_str,std::vector<std::string> json_lines = {json_str};const nlohmann::json& res = add_many(json_lines, document, operation, id, dirty_values);+ if(document.count(Collection::DOC_META_KEY) != 0) {+ document.erase(Collection::DOC_META_KEY);+ }+if(!res["success"].get<bool>()) {nlohmann::json res_doc;@@ -217,10 +221,6 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaconst uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get().seq_id : 0;index_record record(i, seq_id, document, operation, dirty_values);- if(document.count(Collection::DOC_META_KEY) != 0) {- document.erase(Collection::DOC_META_KEY);- }-// NOTE: we overwrite the input json_lines with result to avoid memory pressurerecord.is_update = false;@@ -235,7 +235,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma// if `auto_detect_schema` is enabled, we will have to update schema first before indexingif(auto_detect_schema != schema_detect_types::OFF) {- Option<bool> schema_change_op = check_and_update_schema(document);+ Option<bool> schema_change_op = check_and_update_schema(record.doc, dirty_values);if(!schema_change_op.ok()) {record.index_failure(schema_change_op.code(), schema_change_op.error());}@@ -263,6 +263,11 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaif((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {batch_index(iter_batch, json_lines, num_indexed);for(size_t i_index = 0; i_index < get_num_memory_shards(); i_index++) {+ // to return the document for the single doc add cases+ if(iter_batch[i_index].size() == 1) {+ const auto& rec = iter_batch[i_index][0];+ document = rec.is_update ? rec.new_doc : rec.doc;+ }iter_batch[i_index].clear();}}@@ -292,8 +297,6 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batchif(index_record.indexed.ok()) {if(index_record.is_update) {- //get_doc_changes(index_record.doc, index_record.old_doc, index_record.new_doc, index_record.del_doc);-const std::string& serialized_json = index_record.new_doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);@@ -2247,36 +2250,45 @@ spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {return synonym_definitions;}-Option<bool> Collection::check_and_update_schema(nlohmann::json& document) {+Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values) {std::unique_lock lock(mutex);std::vector<field> new_fields;- for(const auto& kv: document.items()) {- // we will not index the special "id" key- if (search_schema.count(kv.key()) == 0 && kv.key() != "id") {++ auto kv = document.begin();+ for(; kv != document.end(); ) {+ // we will not index the special "id" or doc meta keys+ if (search_schema.count(kv.key()) == 0 && kv.key() != "id" && kv.key() != Collection::DOC_META_KEY) {std::string field_type;bool parseable = field::get_type(kv.value(), field_type);- if (parseable) {- const std::string& fname = kv.key();- field new_field(fname, field_type, false, true);-- if(auto_detect_schema == schema_detect_types::STRINGIFY) {- if(new_field.is_array()) {- new_field.type = field_types::STRING_ARRAY;- } else {- new_field.type = field_types::STRING;- }+ if(!parseable) {+ if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {+ return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");+ } else {+ // DROP or COERCE_OR_DROP+ kv = document.erase(kv);+ continue;}+ }- search_schema.emplace(fname, new_field);- fields.emplace_back(new_field);- new_fields.emplace_back(new_field);-- if(new_field.is_sortable()) {- sort_schema.emplace(new_field.name, new_field);+ const std::string &fname = kv.key();+ field new_field(fname, field_type, false, true);+ if (auto_detect_schema == schema_detect_types::STRINGIFY) {+ if (new_field.is_array()) {+ new_field.type = field_types::STRING_ARRAY;+ } else {+ new_field.type = field_types::STRING;}}+ search_schema.emplace(fname, new_field);+ fields.emplace_back(new_field);+ new_fields.emplace_back(new_field);+ if (new_field.is_sortable()) {+ sort_schema.emplace(new_field.name, new_field);+ }}++ kv++;}if(!new_fields.empty()) {diff --git a/src/core_api.cpp b/src/core_api.cppindex a13f2a3b..f47cc9c3 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -543,7 +543,7 @@ bool post_import_documents(http_req& req, http_res& res) {bool post_add_document(http_req & req, http_res & res) {const char *ACTION = "action";- const char *DIRTY_VALUES = "dirty_values";+ const char *DIRTY_VALUES_PARAM = "dirty_values";if(req.params.count(ACTION) == 0) {req.params[ACTION] = "create";@@ -554,8 +554,8 @@ bool post_add_document(http_req & req, http_res & res) {return false;}- if(req.params.count(DIRTY_VALUES) == 0) {- req.params[DIRTY_VALUES] = ""; // set it empty as default will depend on `index_all_fields`+ if(req.params.count(DIRTY_VALUES_PARAM) == 0) {+ req.params[DIRTY_VALUES_PARAM] = ""; // set it empty as default will depend on whether schema is enabled}CollectionManager & collectionManager = CollectionManager::get_instance();@@ -567,7 +567,7 @@ bool post_add_document(http_req & req, http_res & res) {}const index_operation_t operation = get_index_operation(req.params[ACTION]);- const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES]);+ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES_PARAM]);Option<nlohmann::json> inserted_doc_op = collection->add(req.body, operation, "", dirty_values);@@ -591,7 +591,14 @@ bool patch_update_document(http_req & req, http_res & res) {return false;}- Option<nlohmann::json> upserted_doc_op = collection->add(req.body, index_operation_t::UPDATE, doc_id);+ const char* DIRTY_VALUES_PARAM = "dirty_values";++ if(req.params.count(DIRTY_VALUES_PARAM) == 0) {+ req.params[DIRTY_VALUES_PARAM] = ""; // set it empty as default will depend on whether schema is enabled+ }++ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES_PARAM]);+ Option<nlohmann::json> upserted_doc_op = collection->add(req.body, index_operation_t::UPDATE, doc_id, dirty_values);if(!upserted_doc_op.ok()) {res.set(upserted_doc_op.code(), upserted_doc_op.error());diff --git a/src/index.cpp b/src/index.cppindex 7f06dfd4..525f5de1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -296,14 +296,17 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3"but is not found in the document.");}+ nlohmann::json::iterator dummy_iter;+ bool array_ele_erased = false;+if(a_field.type == field_types::STRING && !document[field_name].is_string()) {- Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, -1);+ Option<uint32_t> coerce_op = coerce_string(dirty_values, auto_detect_schema, a_field, document, field_name, dummy_iter, false, array_ele_erased);if(!coerce_op.ok()) {return coerce_op;}} else if(a_field.type == field_types::INT32) {if(!document[field_name].is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, -1);+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);if(!coerce_op.ok()) {return coerce_op;}@@ -318,18 +321,18 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3}}} else if(a_field.type == field_types::INT64 && !document[field_name].is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, -1);+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);if(!coerce_op.ok()) {return coerce_op;}} else if(a_field.type == field_types::FLOAT && !document[field_name].is_number()) {// using `is_number` allows integer to be passed to a float field- Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, -1);+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);if(!coerce_op.ok()) {return coerce_op;}} else if(a_field.type == field_types::BOOL && !document[field_name].is_boolean()) {- Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, -1);+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);if(!coerce_op.ok()) {return coerce_op;}@@ -344,36 +347,43 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3}}- for(size_t arr_index=0; arr_index < document[field_name].size(); arr_index++) {- const auto& item = document[field_name][arr_index];+ nlohmann::json::iterator it = document[field_name].begin();+ for(; it != document[field_name].end(); ) {+ const auto& item = it.value();+ array_ele_erased = false;if (a_field.type == field_types::STRING_ARRAY && !item.is_string()) {- Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, arr_index);+ Option<uint32_t> coerce_op = coerce_string(dirty_values, auto_detect_schema, a_field, document, field_name, it, true, array_ele_erased);if (!coerce_op.ok()) {return coerce_op;}} else if (a_field.type == field_types::INT32_ARRAY && !item.is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, arr_index);+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, it, true, array_ele_erased);if (!coerce_op.ok()) {return coerce_op;}} else if (a_field.type == field_types::INT64_ARRAY && !item.is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, arr_index);+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, it, true, array_ele_erased);if (!coerce_op.ok()) {return coerce_op;}} else if (a_field.type == field_types::FLOAT_ARRAY && !item.is_number()) {// we check for `is_number` to allow whole numbers to be passed into float fields- Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, arr_index);+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, it, true, array_ele_erased);if (!coerce_op.ok()) {return coerce_op;}} else if (a_field.type == field_types::BOOL_ARRAY && !item.is_boolean()) {- Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, arr_index);+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, it, true, array_ele_erased);if (!coerce_op.ok()) {return coerce_op;}}++ if(!array_ele_erased) {+ // if it is erased, the iterator will be reassigned+ it++;+ }}}}@@ -2389,20 +2399,27 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {}}-Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index) {- std::string suffix = array_index != -1 ? "an array of" : "a";- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];+Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const std::string& auto_detect_schema,+ const field& a_field, nlohmann::json &document,+ const std::string &field_name, nlohmann::json::iterator& array_iter,+ bool is_array, bool& array_ele_erased) {+ std::string suffix = is_array ? "an array of" : "a";+ auto& item = is_array ? array_iter.value() : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");}- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}return Option<uint32_t>(200);}@@ -2422,11 +2439,16 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const fi}else {- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}} else {// COERCE_OR_REJECT / non-optional + DROP@@ -2438,19 +2460,25 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const fi}Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index) {- std::string suffix = array_index != -1 ? "an array of" : "an";- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {+ std::string suffix = is_array ? "an array of" : "an";+ auto& item = is_array ? array_iter.value() : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");}- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}return Option<uint32_t>(200);}@@ -2470,11 +2498,16 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const f}else {- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");+ }++ if(!is_array) {document.erase(field_name);} else {- item.erase(item.begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}} else {// COERCE_OR_REJECT / non-optional + DROP@@ -2486,19 +2519,25 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const f}Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index) {- std::string suffix = array_index != -1 ? "an array of" : "an";- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {+ std::string suffix = is_array ? "an array of" : "an";+ auto& item = is_array ? array_iter.value() : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");}- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}return Option<uint32_t>(200);}@@ -2518,11 +2557,16 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const f}else {- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}} else {// COERCE_OR_REJECT / non-optional + DROP@@ -2534,19 +2578,25 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const f}Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index) {- std::string suffix = array_index != -1 ? "a array of" : "a";- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {+ std::string suffix = is_array ? "a array of" : "a";+ auto& item = is_array ? array_iter.value() : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");}- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}return Option<uint32_t>(200);}@@ -2570,11 +2620,16 @@ Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const fiel}else {- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}} else {// COERCE_OR_REJECT / non-optional + DROP@@ -2586,19 +2641,25 @@ Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const fiel}Option<uint32_t> Index::coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,- const std::string &field_name, const int array_index) {- std::string suffix = array_index != -1 ? "a array of" : "a";- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];+ const std::string &field_name,+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {+ std::string suffix = is_array ? "a array of" : "a";+ auto& item = is_array ? array_iter.value() : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");}- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}return Option<uint32_t>(200);}@@ -2614,11 +2675,16 @@ Option<uint32_t> Index::coerce_float(const DIRTY_VALUES& dirty_values, const fie}else {- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {- if(array_index == -1) {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {+ if(!a_field.optional) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");+ }++ if(!is_array) {document.erase(field_name);} else {- document[field_name].erase(document[field_name].begin() + array_index);+ array_iter = document[field_name].erase(array_iter);+ array_ele_erased = true;}} else {// COERCE_OR_REJECT / non-optional + DROPdiff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 2888d12a..1f592e6c 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -346,9 +346,13 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0");ASSERT_TRUE(add_op.ok());+ auto added_doc = add_op.get();- auto results = coll1->search("first", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ("1", added_doc["int_values"][0].get<std::string>());+ ASSERT_EQ("2", added_doc["int_values"][1].get<std::string>());+ auto results = coll1->search("first", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("FIRST", results["hits"][0]["document"]["title"].get<std::string>());ASSERT_EQ(1, results["hits"][0]["document"].count("int_values"));@@ -356,6 +360,88 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {ASSERT_EQ("1", results["hits"][0]["document"]["int_values"][0].get<std::string>());ASSERT_EQ("2", results["hits"][0]["document"]["int_values"][1].get<std::string>());+ // try with DROP+ doc["title"] = "SECOND";+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("second", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("SECOND", results["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(1, results["hits"][0]["document"].count("int_values"));+ ASSERT_EQ(0, results["hits"][0]["document"]["int_values"].size()); // since both array values are dropped++ // try with REJECT+ doc["title"] = "THIRD";+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());++ // singular field coercion+ doc["single_int"] = 100;+ doc["title"] = "FOURTH";++ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());++ // uncoercable field, e.g. nested dict+ doc["dict"] = nlohmann::json::object();+ doc["dict"]["one"] = 1;+ doc["dict"]["two"] = 2;++ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());++ // try with coerce_or_reject+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());++ // try with drop+ doc["title"] = "FIFTH";+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("fifth", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("FIFTH", results["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));++ // try with coerce_or_drop+ doc["title"] = "SIXTH";+ add_op = coll1->add(doc.dump(), CREATE, "66", DIRTY_VALUES::COERCE_OR_DROP);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("sixth", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("SIXTH", results["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionAllFieldsTest, UpdateOfDocumentsInAutoMode) {+ Collection *coll1;++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, {}, "", 0, schema_detect_types::AUTO).get();+ }++ nlohmann::json doc;+ doc["title"] = "FIRST";+ doc["single_float"] = 50.50;++ auto add_op = coll1->add(doc.dump(), CREATE, "0", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_TRUE(add_op.ok());++ // try updating a value+ nlohmann::json update_doc;+ update_doc["single_float"] = "123";++ add_op = coll1->add(update_doc.dump(), UPDATE, "0", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_TRUE(add_op.ok());+collectionManager.drop_collection("coll1");}