Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Persist per-doc coerce setting + allow dropping of bad values.
kishorencc24fc02d4d21
diff --git a/include/collection.h b/include/collection.hindex 738db45d..1a62ab52 100644--- a/include/collection.h+++ b/include/collection.h@@ -411,6 +411,11 @@ public:static constexpr const char* COLLECTION_NUM_MEMORY_SHARDS = "num_memory_shards";static constexpr const char* COLLECTION_INDEX_ALL_FIELDS = "index_all_fields";+ // DON'T CHANGE THESE VALUES!+ // this key is used as namespace key to store metadata about the document+ static constexpr const char* DOC_META_KEY = "$TSM$_";+ static constexpr const char* DOC_META_DIRTY_VALUES_KEY = "dirty_values";+// methodsCollection() = delete;@@ -453,7 +458,9 @@ public:std::string get_default_sorting_field();Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,- const index_operation_t& operation, const std::string& id="");+ const index_operation_t& operation,+ const DIRTY_VALUES dirty_values,+ const std::string& id="");static uint32_t get_seq_id_from_key(const std::string & key);diff --git a/include/field.h b/include/field.hindex 811f58f1..c2c1206b 100644--- a/include/field.h+++ b/include/field.h@@ -105,6 +105,15 @@ struct field {type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY);}+ bool is_singular() const {+ return !is_array();+ }++ bool has_numerical_index() const {+ return (type == field_types::INT32 || type == field_types::INT64 ||+ type == field_types::FLOAT || type == field_types::BOOL);+ }+bool is_sortable() const {return is_single_integer() || is_single_float() || is_single_bool() || is_geopoint();}diff --git a/include/index.h b/include/index.hindex f0b71333..667f8086 100644--- a/include/index.h+++ b/include/index.h@@ -107,10 +107,10 @@ enum index_operation_t {};enum class DIRTY_VALUES {- COERCE_OR_IGNORE = 1,- COERCE_OR_REJECT = 2,- REJECT = 3,- IGNORE = 4,+ REJECT = 1,+ DROP = 2,+ COERCE_OR_REJECT = 3,+ COERCE_OR_DROP = 4,};struct index_record {@@ -243,19 +243,19 @@ private:static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);- static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name, const int array_index);- static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name, const int array_index);- static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name, const int array_index);- static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name, const int array_index);- static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name, const int array_index);public:@@ -359,7 +359,6 @@ public:Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);-Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,const std::string & default_sorting_field, bool is_update);diff --git a/src/collection.cpp b/src/collection.cppindex 9c3c8a1a..cfad8171 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -63,7 +63,9 @@ uint32_t Collection::get_next_seq_id() {}Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::json& document,- const index_operation_t& operation, const std::string& id) {+ const index_operation_t& operation,+ const DIRTY_VALUES dirty_values,+ const std::string& id) {try {document = nlohmann::json::parse(json_str);} catch(const std::exception& e) {@@ -88,6 +90,17 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::return Option<doc_seq_id_t>(400, "The `id` should not be empty.");}+ bool meta_key_found = document.count(DOC_META_KEY) != 0;+ if(operation == CREATE && meta_key_found) {+ return Option<doc_seq_id_t>(400, "Document cannot contain a `" + std::string(DOC_META_KEY) + "` key.");+ }++ int dirty_values_integer = magic_enum::enum_integer(dirty_values);+ if(!meta_key_found) {+ document[DOC_META_KEY] = nlohmann::json::object();+ }+ document[DOC_META_KEY][DOC_META_DIRTY_VALUES_KEY] = dirty_values_integer;+if(document.count("id") == 0) {if(operation == UPDATE) {return Option<doc_seq_id_t>(400, "For update, the `id` key must be provided.");@@ -219,7 +232,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmafor(size_t i=0; i < json_lines.size(); i++) {const std::string & json_line = json_lines[i];- Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, id);+ Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, dirty_values, id);const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get().seq_id : 0;index_record record(i, seq_id, document, operation, dirty_values);@@ -351,15 +364,12 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_tbool is_update, const DIRTY_VALUES& dirty_values) {std::unique_lock lock(mutex);- if(!is_update) {- // for non-update, validation should be done prior- Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,- search_schema, facet_schema, is_update,- index_all_fields, dirty_values);+ Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,+ search_schema, facet_schema, is_update,+ index_all_fields, dirty_values);- if(!validation_op.ok()) {- return validation_op;- }+ if(!validation_op.ok()) {+ return validation_op;}Index* index = indices[seq_id % num_memory_shards];@@ -732,11 +742,6 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::return Option<nlohmann::json>(404, error);}- if(sort_schema.count(sort_field_std.name) != 0 && sort_schema.at(sort_field_std.name).optional) {- std::string error = "Cannot sort by `" + sort_field_std.name + "` as it is defined as an optional field.";- return Option<nlohmann::json>(400, error);- }-StringUtils::toupper(sort_field_std.order);if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {@@ -2251,7 +2256,7 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document) {bool parseable = field::get_type(kv.value(), field_type);if (parseable) {const std::string& fname = kv.key();- field new_field(fname, field_type, false);+ field new_field(fname, field_type, false, true);search_schema.emplace(fname, new_field);fields.emplace_back(new_field);new_fields.emplace_back(new_field);@@ -2327,6 +2332,7 @@ std::vector<Index *> Collection::init_indices() {}DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) const {+ // no need for a shared lock here since `index_all_fields` is atomicStringUtils::toupper(dirty_values);auto dirty_values_op = magic_enum::enum_cast<DIRTY_VALUES>(dirty_values);DIRTY_VALUES dirty_values_action;@@ -2334,7 +2340,7 @@ DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) coif(dirty_values_op.has_value()) {dirty_values_action = dirty_values_op.value();} else {- dirty_values_action = index_all_fields ? DIRTY_VALUES::COERCE_OR_IGNORE : DIRTY_VALUES::REJECT;+ dirty_values_action = index_all_fields ? DIRTY_VALUES::COERCE_OR_REJECT : DIRTY_VALUES::REJECT;}return dirty_values_action;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 411a59e7..fb7be8dc 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -189,9 +189,6 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {auto begin = std::chrono::high_resolution_clock::now();- std::string default_dirty_values = "";- auto dirty_values = collection->parse_dirty_values_option(default_dirty_values);-while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {num_found_docs++;const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());@@ -205,6 +202,18 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {return Option<bool>(false, "Bad JSON.");}+ auto dirty_values = DIRTY_VALUES::REJECT;++ if (document.count(Collection::DOC_META_KEY) != 0 &&+ document[Collection::DOC_META_KEY].count(Collection::DOC_META_DIRTY_VALUES_KEY) != 0) {++ auto dirty_values_op = magic_enum::enum_cast<DIRTY_VALUES>(+ document[Collection::DOC_META_KEY][Collection::DOC_META_DIRTY_VALUES_KEY]);+ if(dirty_values_op.has_value()) {+ dirty_values = dirty_values_op.value();+ }+ }+num_valid_docs++;iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));diff --git a/src/index.cpp b/src/index.cppindex ea78907f..9845a88b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -91,7 +91,8 @@ int64_t Index::float_to_in64_t(float f) {}Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,- const std::string & default_sorting_field, bool is_update) {+ const std::string & default_sorting_field,+ bool is_update) {std::unique_lock lock(mutex);@@ -181,7 +182,9 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tart_tree *t = search_index.at(field_name);const std::string & text = document[field_name];index_string_field(text, points, t, seq_id, facet_id, field_pair.second);- } else if(field_pair.second.type == field_types::INT32) {+ }++ else if(field_pair.second.type == field_types::INT32) {auto num_tree = numerical_index.at(field_name);uint32_t value = document[field_name];num_tree->insert(value, seq_id);@@ -207,32 +210,35 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tnum_tree->insert(geoHash, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {art_tree *t = search_index.at(field_name);- const std::vector<std::string>& strings = document[field_name];- index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);- } else if(field_pair.second.type == field_types::INT32_ARRAY) {+ index_string_array_field(document[field_name], points, t, seq_id, facet_id, field_pair.second);+ }++ else if(field_pair.second.is_array()) {auto num_tree = numerical_index.at(field_name);- const std::vector<int32_t>& arr_values = document[field_name];- for(const int32_t value: arr_values) {- num_tree->insert(value, seq_id);- }- } else if(field_pair.second.type == field_types::INT64_ARRAY) {- auto& num_tree = numerical_index.at(field_name);- const std::vector<int64_t>& arr_values = document[field_name];- for(const int64_t value: arr_values) {- num_tree->insert(value, seq_id);- }- } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {- const std::vector<float>& arr_values = document[field_name];- for(const float fvalue: arr_values) {- auto& num_tree = numerical_index.at(field_name);- int64_t value = float_to_in64_t(fvalue);- num_tree->insert(value, seq_id);- }- } else if(field_pair.second.type == field_types::BOOL_ARRAY) {- auto& num_tree = numerical_index.at(field_name);- const std::vector<bool>& arr_values = document[field_name];- for(const int64_t value: arr_values) {- num_tree->insert(value, seq_id);++ for(size_t arr_i = 0; arr_i < document[field_name].size(); arr_i++) {+ const auto& arr_value = document[field_name][arr_i];++ if(field_pair.second.type == field_types::INT32_ARRAY) {+ const int32_t value = arr_value;+ num_tree->insert(value, seq_id);+ }++ else if(field_pair.second.type == field_types::INT64_ARRAY) {+ const int64_t value = arr_value;+ num_tree->insert(value, seq_id);+ }++ else if(field_pair.second.type == field_types::FLOAT_ARRAY) {+ const float fvalue = arr_value;+ int64_t value = float_to_in64_t(fvalue);+ num_tree->insert(value, seq_id);+ }++ else if(field_pair.second.type == field_types::BOOL_ARRAY) {+ const bool value = document[field_name][arr_i];+ num_tree->insert(int64_t(value), seq_id);+ }}}@@ -279,8 +285,9 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3for(const auto& field_pair: search_schema) {const std::string& field_name = field_pair.first;+ const field& a_field = field_pair.second;- if((field_pair.second.optional || is_update || index_all_fields) && document.count(field_name) == 0) {+ if((a_field.optional || is_update) && document.count(field_name) == 0) {continue;}@@ -289,46 +296,47 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3"but is not found in the document.");}- if(field_pair.second.type == field_types::STRING && !document[field_name].is_string()) {- Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, -1);+ if(a_field.type == field_types::STRING && !document[field_name].is_string()) {+ Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, -1);if(!coerce_op.ok()) {return coerce_op;}- } else if(field_pair.second.type == field_types::INT32) {+ } else if(a_field.type == field_types::INT32) {if(!document[field_name].is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, -1);+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, -1);if(!coerce_op.ok()) {return coerce_op;}}if(document[field_name].get<int64_t>() > INT32_MAX) {- if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT)) {document.erase(field_name);continue;} else {return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");}}- } else if(field_pair.second.type == field_types::INT64 && !document[field_name].is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, -1);+ } else if(a_field.type == field_types::INT64 && !document[field_name].is_number_integer()) {+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, -1);if(!coerce_op.ok()) {return coerce_op;}- } else if(field_pair.second.type == field_types::FLOAT && !document[field_name].is_number()) {+ } else if(a_field.type == field_types::FLOAT && !document[field_name].is_number()) {// using `is_number` allows integer to be passed to a float field- Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, -1);+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, -1);if(!coerce_op.ok()) {return coerce_op;}- } else if(field_pair.second.type == field_types::BOOL && !document[field_name].is_boolean()) {- Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, -1);+ } else if(a_field.type == field_types::BOOL && !document[field_name].is_boolean()) {+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, -1);if(!coerce_op.ok()) {return coerce_op;}- } else if(field_pair.second.is_array()) {+ } else if(a_field.is_array()) {if(!document[field_name].is_array()) {- if(dirty_values == DIRTY_VALUES::IGNORE || dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||+ dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) {document.erase(field_name);continue;} else {@@ -339,29 +347,29 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3for(size_t arr_index=0; arr_index < document[field_name].size(); arr_index++) {const auto& item = document[field_name][arr_index];- if (field_pair.second.type == field_types::STRING_ARRAY && !item.is_string()) {- Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, arr_index);+ if (a_field.type == field_types::STRING_ARRAY && !item.is_string()) {+ Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, arr_index);if (!coerce_op.ok()) {return coerce_op;}- } else if (field_pair.second.type == field_types::INT32_ARRAY && !item.is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, arr_index);+ } else if (a_field.type == field_types::INT32_ARRAY && !item.is_number_integer()) {+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, arr_index);if (!coerce_op.ok()) {return coerce_op;}- } else if (field_pair.second.type == field_types::INT64_ARRAY && !item.is_number_integer()) {- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, arr_index);+ } else if (a_field.type == field_types::INT64_ARRAY && !item.is_number_integer()) {+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, arr_index);if (!coerce_op.ok()) {return coerce_op;}- } else if (field_pair.second.type == field_types::FLOAT_ARRAY && !item.is_number()) {+ } else if (a_field.type == field_types::FLOAT_ARRAY && !item.is_number()) {// we check for `is_number` to allow whole numbers to be passed into float fields- Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, arr_index);+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, arr_index);if (!coerce_op.ok()) {return coerce_op;}- } else if (field_pair.second.type == field_types::BOOL_ARRAY && !item.is_boolean()) {- Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, arr_index);+ } else if (a_field.type == field_types::BOOL_ARRAY && !item.is_boolean()) {+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, arr_index);if (!coerce_op.ok()) {return coerce_op;}@@ -1992,7 +2000,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}}- const int64_t default_score = INT64_MIN;+ const int64_t default_score = INT64_MIN; // to handle field that doesn't exist in document (e.g. optional)int64_t scores[3] = {0};size_t match_score_index = 0;@@ -2380,15 +2388,16 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {}}-Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name, const int array_index) {std::string suffix = array_index != -1 ? "an array of" : "a";+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");}- if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {if(array_index == -1) {document.erase(field_name);} else {@@ -2399,27 +2408,27 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmann// we will try to coerce the value to a string- if (document[field_name].is_number_integer()) {- document[field_name] = std::to_string((int64_t)document[field_name]);+ if (item.is_number_integer()) {+ item = std::to_string((int64_t)item);}- else if(document[field_name].is_number_float()) {- document[field_name] = std::to_string((float)document[field_name]);+ else if(item.is_number_float()) {+ item = std::to_string((float)item);}- else if(document[field_name].is_boolean()) {- document[field_name] = document[field_name] == true ? "true" : "false";+ else if(item.is_boolean()) {+ item = item == true ? "true" : "false";}else {- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {if(array_index == -1) {document.erase(field_name);} else {document[field_name].erase(document[field_name].begin() + array_index);}} else {- // DIRTY_FIELD_COERCE_REJECT+ // COERCE_OR_REJECT / non-optional + DROPreturn Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");}}@@ -2427,15 +2436,16 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmannreturn Option<>(200);}-Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,- const std::string &field_name, const int array_index) {+Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,+ const std::string &field_name, const int array_index) {std::string suffix = array_index != -1 ? "an array of" : "an";+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");}- if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {if(array_index == -1) {document.erase(field_name);} else {@@ -2446,23 +2456,27 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohman// try to value coerce into an integer- if(document[field_name].is_number_float()) {- document[field_name] = static_cast<int32_t>(document[field_name].get<float>());+ if(item.is_number_float()) {+ item = static_cast<int32_t>(item.get<float>());}- else if(document[field_name].is_boolean()) {- document[field_name] = document[field_name] == true ? 1 : 0;+ else if(item.is_boolean()) {+ item = item == true ? 1 : 0;}- else if(document[field_name].is_string() && StringUtils::is_int32_t(document[field_name])) {- document[field_name] = std::atol(document[field_name].get<std::string>().c_str());+ else if(item.is_string() && StringUtils::is_int32_t(item)) {+ item = std::atol(item.get<std::string>().c_str());}else {- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {- document.erase(field_name);+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ item.erase(item.begin() + array_index);+ }} else {- // DIRTY_FIELD_COERCE_REJECT+ // COERCE_OR_REJECT / non-optional + DROPreturn Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");}}@@ -2470,15 +2484,16 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmanreturn Option<uint32_t>(200);}-Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,const std::string &field_name, const int array_index) {std::string suffix = array_index != -1 ? "an array of" : "an";+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");}- if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {if(array_index == -1) {document.erase(field_name);} else {@@ -2489,27 +2504,27 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohman// try to value coerce into an integer- if(document[field_name].is_number_float()) {- document[field_name] = static_cast<int64_t>(document[field_name].get<float>());+ if(item.is_number_float()) {+ item = static_cast<int64_t>(item.get<float>());}- else if(document[field_name].is_boolean()) {- document[field_name] = document[field_name] == true ? 1 : 0;+ else if(item.is_boolean()) {+ item = item == true ? 1 : 0;}- else if(document[field_name].is_string() && StringUtils::is_int64_t(document[field_name])) {- document[field_name] = std::atoll(document[field_name].get<std::string>().c_str());+ else if(item.is_string() && StringUtils::is_int64_t(item)) {+ item = std::atoll(item.get<std::string>().c_str());}else {- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {if(array_index == -1) {document.erase(field_name);} else {document[field_name].erase(document[field_name].begin() + array_index);}} else {- // DIRTY_FIELD_COERCE_REJECT+ // COERCE_OR_REJECT / non-optional + DROPreturn Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");}}@@ -2517,15 +2532,16 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmanreturn Option<uint32_t>(200);}-Option<uint32_t>-Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,- const std::string &field_name, const int array_index) {+Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,+ const std::string &field_name, const int array_index) {std::string suffix = array_index != -1 ? "a array of" : "a";+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];+if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");}- if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {if(array_index == -1) {document.erase(field_name);} else {@@ -2535,32 +2551,32 @@ Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,}// try to value coerce into a bool- if (document[field_name].is_number_integer() &&- (document[field_name].get<int64_t>() == 1 || document[field_name].get<int64_t>() == 0)) {- document[field_name] = document[field_name].get<int64_t>() == 1;+ if (item.is_number_integer() &&+ (item.get<int64_t>() == 1 || item.get<int64_t>() == 0)) {+ item = item.get<int64_t>() == 1;}- else if(document[field_name].is_string()) {- std::string str_val = document[field_name].get<std::string>();+ else if(item.is_string()) {+ std::string str_val = item.get<std::string>();StringUtils::tolowercase(str_val);if(str_val == "true") {- document[field_name] = true;+ item = true;return Option<uint32_t>(200);} else if(str_val == "false") {- document[field_name] = false;+ item = false;return Option<uint32_t>(200);}}else {- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {if(array_index == -1) {document.erase(field_name);} else {document[field_name].erase(document[field_name].begin() + array_index);}} else {- // DIRTY_FIELD_COERCE_REJECT+ // COERCE_OR_REJECT / non-optional + DROPreturn Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");}}@@ -2568,16 +2584,16 @@ Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,return Option<uint32_t>(200);}-Option<uint32_t>-Index::coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,- const std::string &field_name, const int array_index) {+Option<uint32_t> Index::coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,+ const std::string &field_name, const int array_index) {std::string suffix = array_index != -1 ? "a array of" : "a";+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];if(dirty_values == DIRTY_VALUES::REJECT) {return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");}- if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {if(array_index == -1) {document.erase(field_name);} else {@@ -2588,23 +2604,23 @@ Index::coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,// try to value coerce into a float- if(document[field_name].is_string() && StringUtils::is_float(document[field_name])) {- document[field_name] = std::atof(document[field_name].get<std::string>().c_str());+ if(item.is_string() && StringUtils::is_float(item)) {+ item = std::atof(item.get<std::string>().c_str());}- else if(document[field_name].is_boolean()) {- document[field_name] = document[field_name] == true ? 1.0 : 0.0;+ else if(item.is_boolean()) {+ item = item == true ? 1.0 : 0.0;}else {- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {if(array_index == -1) {document.erase(field_name);} else {document[field_name].erase(document[field_name].begin() + array_index);}} else {- // DIRTY_FIELD_COERCE_REJECT+ // COERCE_OR_REJECT / non-optional + DROPreturn Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");}}diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 664ef652..a7514077 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -60,6 +60,22 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {query_fields = {"starring"};std::vector<std::string> facets;+ // check default no specific dirty values option is sent for a collection that has schema detection enabled+ std::string dirty_values;+ ASSERT_EQ(DIRTY_VALUES::COERCE_OR_REJECT, coll1->parse_dirty_values_option(dirty_values));++ dirty_values = "coerce_or_reject";+ ASSERT_EQ(DIRTY_VALUES::COERCE_OR_REJECT, coll1->parse_dirty_values_option(dirty_values));++ dirty_values = "COERCE_OR_DROP";+ ASSERT_EQ(DIRTY_VALUES::COERCE_OR_DROP, coll1->parse_dirty_values_option(dirty_values));++ dirty_values = "reject";+ ASSERT_EQ(DIRTY_VALUES::REJECT, coll1->parse_dirty_values_option(dirty_values));++ dirty_values = "DROP";+ ASSERT_EQ(DIRTY_VALUES::DROP, coll1->parse_dirty_values_option(dirty_values));+// same should succeed when verbatim filter is madeauto results = coll1->search("will", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();@@ -100,12 +116,12 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {ASSERT_EQ(1, results["hits"].size());ASSERT_STREQ("300", results["hits"][0]["document"]["title"].get<std::string>().c_str());- // with dirty values set to `DIRTY_FIELD_COERCE_IGNORE`+ // with dirty values set to `COERCE_OR_DROP`// `cast` field should not be indexed into storedoc_json = R"({"cast":"William Barnes","points":63,"starring":"Will Ferrell","starring_facet":"Will Ferrell","title":"With bad cast field."})";- add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_IGNORE);+ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);ASSERT_TRUE(add_op.ok());results = coll1->search("With bad cast field", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();@@ -113,12 +129,12 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {ASSERT_STREQ("With bad cast field.", results["hits"][0]["document"]["title"].get<std::string>().c_str());ASSERT_EQ(0, results["hits"][0]["document"].count("cast"));- // with dirty values set to `DIRTY_FIELD_IGNORE`- // no coercion should happen, `title` field will just be ignored, but record indexed+ // with dirty values set to `DROP`+ // no coercion should happen, `title` field will just be dropped, but record indexeddoc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell","starring_facet":"Will Ferrell","title": 1200 })";- add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::IGNORE);+ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::DROP);ASSERT_TRUE(add_op.ok());results = coll1->search("1200", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();@@ -128,7 +144,7 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ(0, results["hits"][0]["document"].count("title"));- // with dirty values set to `DIRTY_FIELD_REJECT`+ // with dirty values set to `REJECT`doc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell","starring_facet":"Will Ferrell","title": 1200 })";@@ -144,3 +160,85 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionAllFieldsTest, HandleArrayTypes) {+ Collection *coll1;++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, {}, "", 0, true).get();+ }++ nlohmann::json doc;+ doc["title"] = "FIRST";+ doc["int_values"] = {1, 2};++ Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0");+ ASSERT_TRUE(add_op.ok());++ // coercion of string -> int++ doc["int_values"] = {"3"};++ add_op = coll1->add(doc.dump(), UPDATE, "0");+ ASSERT_TRUE(add_op.ok());++ // bad array type value should be dropped when stored++ doc["title"] = "SECOND";+ doc["int_values"] = {{3}};+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES:: DROP);+ ASSERT_TRUE(add_op.ok());++ auto results = coll1->search("second", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());++ // check that the "bad" value does not exists in the stored document+ ASSERT_EQ(1, results["hits"][0]["document"].count("int_values"));+ ASSERT_EQ(0, results["hits"][0]["document"]["int_values"].size());++ // bad array type should follow coercion rules+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `int_values` must be an array of int64.", add_op.error());++ // non array field should be handled as per coercion rule+ doc["title"] = "THIRD";+ doc["int_values"] = 3;+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `int_values` must be an array.", add_op.error());++ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);+ ASSERT_TRUE(add_op.ok());+ results = coll1->search("third", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(0, results["hits"][0]["document"].count("int_values"));++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionAllFieldsTest, NonOptionalFieldShouldNotBeDropped) {+ Collection *coll1;++ std::vector<field> fields = {+ field("points", field_types::INT32, false, false)+ };++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, true).get();+ }++ nlohmann::json doc;+ doc["title"] = "FIRST";+ doc["points"] = {100};++ Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0", DIRTY_VALUES::DROP);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `points` must be an int32.", add_op.error());++ add_op = coll1->add(doc.dump(), CREATE, "0", DIRTY_VALUES::COERCE_OR_DROP);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `points` must be an int32.", add_op.error());+}\ No newline at end of filediff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex d083c3b7..4e07e882 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -304,11 +304,11 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {ASSERT_EQ(1, coll1->get_collection_id());ASSERT_EQ(3, coll1->get_sort_fields().size());- // index a document with a bad field value with COERCE_OR_IGNORE setting+ // index a document with a bad field value with COERCE_OR_DROP settingauto doc_json = R"({"title": "Unique record.", "max": 25, "scores": [22, "how", 44],"average": "bad data", "is_valid": true})";- Option<nlohmann::json> add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_IGNORE);+ Option<nlohmann::json> add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);ASSERT_TRUE(add_op.ok());std::unordered_map<std::string, field> schema = collection1->get_schema();@@ -351,8 +351,14 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {ASSERT_EQ(1, restored_schema.count("average"));ASSERT_EQ(1, restored_schema.count("is_valid"));+ // all detected schema are optional fields, while defined schema is not+for(const auto& kv: restored_schema) {- ASSERT_FALSE(kv.second.optional);+ if(kv.first == "max") {+ ASSERT_FALSE(kv.second.optional);+ } else {+ ASSERT_TRUE(kv.second.optional);+ }}// try searching for record with bad datadiff --git a/test/collection_test.cpp b/test/collection_test.cppindex 7a506a99..033779f9 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -85,6 +85,10 @@ protected:TEST_F(CollectionTest, VerifyCountOfDocuments) {// we have 1 dummy record to match the line numbers on the fixtures file with sequence numbersASSERT_EQ(24+1, collection->get_num_documents());++ // check default no specific dirty values option is sent for a collection that has explicit schema+ std::string empty_dirty_values;+ ASSERT_EQ(DIRTY_VALUES::REJECT, collection->parse_dirty_values_option(empty_dirty_values));}TEST_F(CollectionTest, RetrieveADocumentById) {@@ -1622,7 +1626,8 @@ TEST_F(CollectionTest, IndexingWithBadData) {ASSERT_STREQ("Field `tags` must be an array.", bad_facet_field_op.error().c_str());doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";- const Option<nlohmann::json> & bad_array_field_op = sample_collection->add(doc_str);+ const Option<nlohmann::json> & bad_array_field_op = sample_collection->add(doc_str, CREATE, "",+ DIRTY_VALUES::REJECT);ASSERT_FALSE(bad_array_field_op.ok());ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_op.error().c_str());@@ -1630,8 +1635,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";const Option<nlohmann::json> &bad_array_field_coercion_op = sample_collection->add(doc_str, CREATE, "",DIRTY_VALUES::COERCE_OR_REJECT);- ASSERT_FALSE(bad_array_field_coercion_op.ok());- ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_coercion_op.error().c_str());+ ASSERT_TRUE(bad_array_field_coercion_op.ok());doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";const Option<nlohmann::json> & empty_facet_field_op = sample_collection->add(doc_str);@@ -2495,12 +2499,15 @@ TEST_F(CollectionTest, OptionalFields) {ASSERT_EQ(5, res["facet_counts"][0]["counts"][0]["count"].get<size_t>());ASSERT_STREQ("description", res["facet_counts"][0]["field_name"].get<std::string>().c_str());- // sort_by optional `average` field should be rejected+ // sort_by optional `average` field should be allowed (default used for missing values)std::vector<sort_by> sort_fields = { sort_by("average", "DESC") };auto res_op = coll1->search("*", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);- ASSERT_FALSE(res_op.ok());- ASSERT_STREQ("Cannot sort by `average` as it is defined as an optional field.", res_op.error().c_str());-+ ASSERT_TRUE(res_op.ok());+ res = res_op.get();++ ASSERT_EQ(6, res["found"].get<size_t>());+ ASSERT_EQ(0, res["hits"][5]["document"].count("average")); // record with missing average is last+// try deleting a record having optional fieldOption<std::string> remove_op = coll1->remove("1");ASSERT_TRUE(remove_op.ok());