Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix edge cases in schema detection.

kishorenc3a4d21992cd5
diff --git a/include/collection.h b/include/collection.h
index 7fda6a31..5a742de6 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -349,7 +349,7 @@ private:
std::map<size_t, std::vector<uint32_t>>& include_ids,
std::vector<uint32_t> & excluded_ids) const;
- Option<bool> check_and_update_schema(nlohmann::json& document);
+ Option<bool> check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values);
static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,
const std::pair<uint64_t, facet_count_t>& b) {
diff --git a/include/index.h b/include/index.h
index c91f98d6..800c4c62 100644
--- a/include/index.h
+++ b/include/index.h
@@ -246,20 +246,28 @@ private:
static void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,
nlohmann::json &new_doc, nlohmann::json &del_doc);
- static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index);
+ static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, const std::string& auto_detect_schema,
+ const field& a_field, nlohmann::json &document,
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter,
+ bool is_array,
+ bool& array_ele_erased);
static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index);
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);
static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index);
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);
static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index);
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);
static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index);
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);
public:
// for limiting number of results on multiple candidates / query rewrites
diff --git a/src/collection.cpp b/src/collection.cpp
index 767a26d0..40554c77 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -180,6 +180,10 @@ Option<nlohmann::json> Collection::add(const std::string & json_str,
std::vector<std::string> json_lines = {json_str};
const nlohmann::json& res = add_many(json_lines, document, operation, id, dirty_values);
+ if(document.count(Collection::DOC_META_KEY) != 0) {
+ document.erase(Collection::DOC_META_KEY);
+ }
+
if(!res["success"].get<bool>()) {
nlohmann::json res_doc;
@@ -217,10 +221,6 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get().seq_id : 0;
index_record record(i, seq_id, document, operation, dirty_values);
- if(document.count(Collection::DOC_META_KEY) != 0) {
- document.erase(Collection::DOC_META_KEY);
- }
-
// NOTE: we overwrite the input json_lines with result to avoid memory pressure
record.is_update = false;
@@ -235,7 +235,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
// if `auto_detect_schema` is enabled, we will have to update schema first before indexing
if(auto_detect_schema != schema_detect_types::OFF) {
- Option<bool> schema_change_op = check_and_update_schema(document);
+ Option<bool> schema_change_op = check_and_update_schema(record.doc, dirty_values);
if(!schema_change_op.ok()) {
record.index_failure(schema_change_op.code(), schema_change_op.error());
}
@@ -263,6 +263,11 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {
batch_index(iter_batch, json_lines, num_indexed);
for(size_t i_index = 0; i_index < get_num_memory_shards(); i_index++) {
+ // to return the document for the single doc add cases
+ if(iter_batch[i_index].size() == 1) {
+ const auto& rec = iter_batch[i_index][0];
+ document = rec.is_update ? rec.new_doc : rec.doc;
+ }
iter_batch[i_index].clear();
}
}
@@ -292,8 +297,6 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch
if(index_record.indexed.ok()) {
if(index_record.is_update) {
- //get_doc_changes(index_record.doc, index_record.old_doc, index_record.new_doc, index_record.del_doc);
-
const std::string& serialized_json = index_record.new_doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);
bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);
@@ -2247,36 +2250,45 @@ spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {
return synonym_definitions;
}
-Option<bool> Collection::check_and_update_schema(nlohmann::json& document) {
+Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values) {
std::unique_lock lock(mutex);
std::vector<field> new_fields;
- for(const auto& kv: document.items()) {
- // we will not index the special "id" key
- if (search_schema.count(kv.key()) == 0 && kv.key() != "id") {
+
+ auto kv = document.begin();
+ for(; kv != document.end(); ) {
+ // we will not index the special "id" or doc meta keys
+ if (search_schema.count(kv.key()) == 0 && kv.key() != "id" && kv.key() != Collection::DOC_META_KEY) {
std::string field_type;
bool parseable = field::get_type(kv.value(), field_type);
- if (parseable) {
- const std::string& fname = kv.key();
- field new_field(fname, field_type, false, true);
-
- if(auto_detect_schema == schema_detect_types::STRINGIFY) {
- if(new_field.is_array()) {
- new_field.type = field_types::STRING_ARRAY;
- } else {
- new_field.type = field_types::STRING;
- }
+ if(!parseable) {
+ if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {
+ return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");
+ } else {
+ // DROP or COERCE_OR_DROP
+ kv = document.erase(kv);
+ continue;
}
+ }
- search_schema.emplace(fname, new_field);
- fields.emplace_back(new_field);
- new_fields.emplace_back(new_field);
-
- if(new_field.is_sortable()) {
- sort_schema.emplace(new_field.name, new_field);
+ const std::string &fname = kv.key();
+ field new_field(fname, field_type, false, true);
+ if (auto_detect_schema == schema_detect_types::STRINGIFY) {
+ if (new_field.is_array()) {
+ new_field.type = field_types::STRING_ARRAY;
+ } else {
+ new_field.type = field_types::STRING;
}
}
+ search_schema.emplace(fname, new_field);
+ fields.emplace_back(new_field);
+ new_fields.emplace_back(new_field);
+ if (new_field.is_sortable()) {
+ sort_schema.emplace(new_field.name, new_field);
+ }
}
+
+ kv++;
}
if(!new_fields.empty()) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index a13f2a3b..f47cc9c3 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -543,7 +543,7 @@ bool post_import_documents(http_req& req, http_res& res) {
bool post_add_document(http_req & req, http_res & res) {
const char *ACTION = "action";
- const char *DIRTY_VALUES = "dirty_values";
+ const char *DIRTY_VALUES_PARAM = "dirty_values";
if(req.params.count(ACTION) == 0) {
req.params[ACTION] = "create";
@@ -554,8 +554,8 @@ bool post_add_document(http_req & req, http_res & res) {
return false;
}
- if(req.params.count(DIRTY_VALUES) == 0) {
- req.params[DIRTY_VALUES] = ""; // set it empty as default will depend on `index_all_fields`
+ if(req.params.count(DIRTY_VALUES_PARAM) == 0) {
+ req.params[DIRTY_VALUES_PARAM] = ""; // set it empty as default will depend on whether schema is enabled
}
CollectionManager & collectionManager = CollectionManager::get_instance();
@@ -567,7 +567,7 @@ bool post_add_document(http_req & req, http_res & res) {
}
const index_operation_t operation = get_index_operation(req.params[ACTION]);
- const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES]);
+ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES_PARAM]);
Option<nlohmann::json> inserted_doc_op = collection->add(req.body, operation, "", dirty_values);
@@ -591,7 +591,14 @@ bool patch_update_document(http_req & req, http_res & res) {
return false;
}
- Option<nlohmann::json> upserted_doc_op = collection->add(req.body, index_operation_t::UPDATE, doc_id);
+ const char* DIRTY_VALUES_PARAM = "dirty_values";
+
+ if(req.params.count(DIRTY_VALUES_PARAM) == 0) {
+ req.params[DIRTY_VALUES_PARAM] = ""; // set it empty as default will depend on whether schema is enabled
+ }
+
+ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES_PARAM]);
+ Option<nlohmann::json> upserted_doc_op = collection->add(req.body, index_operation_t::UPDATE, doc_id, dirty_values);
if(!upserted_doc_op.ok()) {
res.set(upserted_doc_op.code(), upserted_doc_op.error());
diff --git a/src/index.cpp b/src/index.cpp
index 7f06dfd4..525f5de1 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -296,14 +296,17 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
"but is not found in the document.");
}
+ nlohmann::json::iterator dummy_iter;
+ bool array_ele_erased = false;
+
if(a_field.type == field_types::STRING && !document[field_name].is_string()) {
- Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, -1);
+ Option<uint32_t> coerce_op = coerce_string(dirty_values, auto_detect_schema, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
} else if(a_field.type == field_types::INT32) {
if(!document[field_name].is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, -1);
+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
@@ -318,18 +321,18 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
}
}
} else if(a_field.type == field_types::INT64 && !document[field_name].is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, -1);
+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
} else if(a_field.type == field_types::FLOAT && !document[field_name].is_number()) {
// using `is_number` allows integer to be passed to a float field
- Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, -1);
+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
} else if(a_field.type == field_types::BOOL && !document[field_name].is_boolean()) {
- Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, -1);
+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
@@ -344,36 +347,43 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
}
}
- for(size_t arr_index=0; arr_index < document[field_name].size(); arr_index++) {
- const auto& item = document[field_name][arr_index];
+ nlohmann::json::iterator it = document[field_name].begin();
+ for(; it != document[field_name].end(); ) {
+ const auto& item = it.value();
+ array_ele_erased = false;
if (a_field.type == field_types::STRING_ARRAY && !item.is_string()) {
- Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, arr_index);
+ Option<uint32_t> coerce_op = coerce_string(dirty_values, auto_detect_schema, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::INT32_ARRAY && !item.is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, arr_index);
+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::INT64_ARRAY && !item.is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, arr_index);
+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::FLOAT_ARRAY && !item.is_number()) {
// we check for `is_number` to allow whole numbers to be passed into float fields
- Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, arr_index);
+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::BOOL_ARRAY && !item.is_boolean()) {
- Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, arr_index);
+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
}
+
+ if(!array_ele_erased) {
+ // if it is erased, the iterator will be reassigned
+ it++;
+ }
}
}
}
@@ -2389,20 +2399,27 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {
}
}
-Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
- std::string suffix = array_index != -1 ? "an array of" : "a";
- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
+Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const std::string& auto_detect_schema,
+ const field& a_field, nlohmann::json &document,
+ const std::string &field_name, nlohmann::json::iterator& array_iter,
+ bool is_array, bool& array_ele_erased) {
+ std::string suffix = is_array ? "an array of" : "a";
+ auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
return Option<uint32_t>(200);
}
@@ -2422,11 +2439,16 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const fi
}
else {
- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
@@ -2438,19 +2460,25 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const fi
}
Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
- std::string suffix = array_index != -1 ? "an array of" : "an";
- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
+ std::string suffix = is_array ? "an array of" : "an";
+ auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
}
- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
return Option<uint32_t>(200);
}
@@ -2470,11 +2498,16 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const f
}
else {
- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- item.erase(item.begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
@@ -2486,19 +2519,25 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const f
}
Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
- std::string suffix = array_index != -1 ? "an array of" : "an";
- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
+ std::string suffix = is_array ? "an array of" : "an";
+ auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
}
- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
return Option<uint32_t>(200);
}
@@ -2518,11 +2557,16 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const f
}
else {
- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
@@ -2534,19 +2578,25 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const f
}
Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
- std::string suffix = array_index != -1 ? "a array of" : "a";
- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
+ std::string suffix = is_array ? "a array of" : "a";
+ auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
return Option<uint32_t>(200);
}
@@ -2570,11 +2620,16 @@ Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const fiel
}
else {
- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
@@ -2586,19 +2641,25 @@ Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const fiel
}
Option<uint32_t> Index::coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
- std::string suffix = array_index != -1 ? "a array of" : "a";
- auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
+ const std::string &field_name,
+ nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
+ std::string suffix = is_array ? "a array of" : "a";
+ auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
}
- if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
return Option<uint32_t>(200);
}
@@ -2614,11 +2675,16 @@ Option<uint32_t> Index::coerce_float(const DIRTY_VALUES& dirty_values, const fie
}
else {
- if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
- if(array_index == -1) {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
+ if(!a_field.optional) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
+ }
+
+ if(!is_array) {
document.erase(field_name);
} else {
- document[field_name].erase(document[field_name].begin() + array_index);
+ array_iter = document[field_name].erase(array_iter);
+ array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 2888d12a..1f592e6c 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -346,9 +346,13 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {
Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0");
ASSERT_TRUE(add_op.ok());
+ auto added_doc = add_op.get();
- auto results = coll1->search("first", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ("1", added_doc["int_values"][0].get<std::string>());
+ ASSERT_EQ("2", added_doc["int_values"][1].get<std::string>());
+ auto results = coll1->search("first", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("FIRST", results["hits"][0]["document"]["title"].get<std::string>());
ASSERT_EQ(1, results["hits"][0]["document"].count("int_values"));
@@ -356,6 +360,88 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {
ASSERT_EQ("1", results["hits"][0]["document"]["int_values"][0].get<std::string>());
ASSERT_EQ("2", results["hits"][0]["document"]["int_values"][1].get<std::string>());
+ // try with DROP
+ doc["title"] = "SECOND";
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("second", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("SECOND", results["hits"][0]["document"]["title"].get<std::string>());
+ ASSERT_EQ(1, results["hits"][0]["document"].count("int_values"));
+ ASSERT_EQ(0, results["hits"][0]["document"]["int_values"].size()); // since both array values are dropped
+
+ // try with REJECT
+ doc["title"] = "THIRD";
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
+ ASSERT_FALSE(add_op.ok());
+
+ // singular field coercion
+ doc["single_int"] = 100;
+ doc["title"] = "FOURTH";
+
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
+ ASSERT_FALSE(add_op.ok());
+
+ // uncoercable field, e.g. nested dict
+ doc["dict"] = nlohmann::json::object();
+ doc["dict"]["one"] = 1;
+ doc["dict"]["two"] = 2;
+
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());
+
+ // try with coerce_or_reject
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());
+
+ // try with drop
+ doc["title"] = "FIFTH";
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("fifth", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("FIFTH", results["hits"][0]["document"]["title"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));
+
+ // try with coerce_or_drop
+ doc["title"] = "SIXTH";
+ add_op = coll1->add(doc.dump(), CREATE, "66", DIRTY_VALUES::COERCE_OR_DROP);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("sixth", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("SIXTH", results["hits"][0]["document"]["title"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionAllFieldsTest, UpdateOfDocumentsInAutoMode) {
+ Collection *coll1;
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, {}, "", 0, schema_detect_types::AUTO).get();
+ }
+
+ nlohmann::json doc;
+ doc["title"] = "FIRST";
+ doc["single_float"] = 50.50;
+
+ auto add_op = coll1->add(doc.dump(), CREATE, "0", DIRTY_VALUES::COERCE_OR_REJECT);
+ ASSERT_TRUE(add_op.ok());
+
+ // try updating a value
+ nlohmann::json update_doc;
+ update_doc["single_float"] = "123";
+
+ add_op = coll1->add(update_doc.dump(), UPDATE, "0", DIRTY_VALUES::COERCE_OR_REJECT);
+ ASSERT_TRUE(add_op.ok());
+
collectionManager.drop_collection("coll1");
}