Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Persist per-doc coerce setting + allow dropping of bad values.

kishorencc24fc02d4d21
diff --git a/include/collection.h b/include/collection.h
index 738db45d..1a62ab52 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -411,6 +411,11 @@ public:
static constexpr const char* COLLECTION_NUM_MEMORY_SHARDS = "num_memory_shards";
static constexpr const char* COLLECTION_INDEX_ALL_FIELDS = "index_all_fields";
+ // DON'T CHANGE THESE VALUES!
+ // this key is used as namespace key to store metadata about the document
+ static constexpr const char* DOC_META_KEY = "$TSM$_";
+ static constexpr const char* DOC_META_DIRTY_VALUES_KEY = "dirty_values";
+
// methods
Collection() = delete;
@@ -453,7 +458,9 @@ public:
std::string get_default_sorting_field();
Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,
- const index_operation_t& operation, const std::string& id="");
+ const index_operation_t& operation,
+ const DIRTY_VALUES dirty_values,
+ const std::string& id="");
static uint32_t get_seq_id_from_key(const std::string & key);
diff --git a/include/field.h b/include/field.h
index 811f58f1..c2c1206b 100644
--- a/include/field.h
+++ b/include/field.h
@@ -105,6 +105,15 @@ struct field {
type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY);
}
+ bool is_singular() const {
+ return !is_array();
+ }
+
+ bool has_numerical_index() const {
+ return (type == field_types::INT32 || type == field_types::INT64 ||
+ type == field_types::FLOAT || type == field_types::BOOL);
+ }
+
bool is_sortable() const {
return is_single_integer() || is_single_float() || is_single_bool() || is_geopoint();
}
diff --git a/include/index.h b/include/index.h
index f0b71333..667f8086 100644
--- a/include/index.h
+++ b/include/index.h
@@ -107,10 +107,10 @@ enum index_operation_t {
};
enum class DIRTY_VALUES {
- COERCE_OR_IGNORE = 1,
- COERCE_OR_REJECT = 2,
- REJECT = 3,
- IGNORE = 4,
+ REJECT = 1,
+ DROP = 2,
+ COERCE_OR_REJECT = 3,
+ COERCE_OR_DROP = 4,
};
struct index_record {
@@ -243,19 +243,19 @@ private:
static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
- static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name, const int array_index);
- static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name, const int array_index);
- static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name, const int array_index);
- static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name, const int array_index);
- static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name, const int array_index);
public:
@@ -359,7 +359,6 @@ public:
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);
-
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,
const std::string & default_sorting_field, bool is_update);
diff --git a/src/collection.cpp b/src/collection.cpp
index 9c3c8a1a..cfad8171 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -63,7 +63,9 @@ uint32_t Collection::get_next_seq_id() {
}
Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::json& document,
- const index_operation_t& operation, const std::string& id) {
+ const index_operation_t& operation,
+ const DIRTY_VALUES dirty_values,
+ const std::string& id) {
try {
document = nlohmann::json::parse(json_str);
} catch(const std::exception& e) {
@@ -88,6 +90,17 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::
return Option<doc_seq_id_t>(400, "The `id` should not be empty.");
}
+ bool meta_key_found = document.count(DOC_META_KEY) != 0;
+ if(operation == CREATE && meta_key_found) {
+ return Option<doc_seq_id_t>(400, "Document cannot contain a `" + std::string(DOC_META_KEY) + "` key.");
+ }
+
+ int dirty_values_integer = magic_enum::enum_integer(dirty_values);
+ if(!meta_key_found) {
+ document[DOC_META_KEY] = nlohmann::json::object();
+ }
+ document[DOC_META_KEY][DOC_META_DIRTY_VALUES_KEY] = dirty_values_integer;
+
if(document.count("id") == 0) {
if(operation == UPDATE) {
return Option<doc_seq_id_t>(400, "For update, the `id` key must be provided.");
@@ -219,7 +232,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
for(size_t i=0; i < json_lines.size(); i++) {
const std::string & json_line = json_lines[i];
- Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, id);
+ Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, dirty_values, id);
const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get().seq_id : 0;
index_record record(i, seq_id, document, operation, dirty_values);
@@ -351,15 +364,12 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
bool is_update, const DIRTY_VALUES& dirty_values) {
std::unique_lock lock(mutex);
- if(!is_update) {
- // for non-update, validation should be done prior
- Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,
- search_schema, facet_schema, is_update,
- index_all_fields, dirty_values);
+ Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,
+ search_schema, facet_schema, is_update,
+ index_all_fields, dirty_values);
- if(!validation_op.ok()) {
- return validation_op;
- }
+ if(!validation_op.ok()) {
+ return validation_op;
}
Index* index = indices[seq_id % num_memory_shards];
@@ -732,11 +742,6 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return Option<nlohmann::json>(404, error);
}
- if(sort_schema.count(sort_field_std.name) != 0 && sort_schema.at(sort_field_std.name).optional) {
- std::string error = "Cannot sort by `" + sort_field_std.name + "` as it is defined as an optional field.";
- return Option<nlohmann::json>(400, error);
- }
-
StringUtils::toupper(sort_field_std.order);
if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {
@@ -2251,7 +2256,7 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document) {
bool parseable = field::get_type(kv.value(), field_type);
if (parseable) {
const std::string& fname = kv.key();
- field new_field(fname, field_type, false);
+ field new_field(fname, field_type, false, true);
search_schema.emplace(fname, new_field);
fields.emplace_back(new_field);
new_fields.emplace_back(new_field);
@@ -2327,6 +2332,7 @@ std::vector<Index *> Collection::init_indices() {
}
DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) const {
+ // no need for a shared lock here since `index_all_fields` is atomic
StringUtils::toupper(dirty_values);
auto dirty_values_op = magic_enum::enum_cast<DIRTY_VALUES>(dirty_values);
DIRTY_VALUES dirty_values_action;
@@ -2334,7 +2340,7 @@ DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) co
if(dirty_values_op.has_value()) {
dirty_values_action = dirty_values_op.value();
} else {
- dirty_values_action = index_all_fields ? DIRTY_VALUES::COERCE_OR_IGNORE : DIRTY_VALUES::REJECT;
+ dirty_values_action = index_all_fields ? DIRTY_VALUES::COERCE_OR_REJECT : DIRTY_VALUES::REJECT;
}
return dirty_values_action;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 411a59e7..fb7be8dc 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -189,9 +189,6 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
auto begin = std::chrono::high_resolution_clock::now();
- std::string default_dirty_values = "";
- auto dirty_values = collection->parse_dirty_values_option(default_dirty_values);
-
while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
num_found_docs++;
const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());
@@ -205,6 +202,18 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
return Option<bool>(false, "Bad JSON.");
}
+ auto dirty_values = DIRTY_VALUES::REJECT;
+
+ if (document.count(Collection::DOC_META_KEY) != 0 &&
+ document[Collection::DOC_META_KEY].count(Collection::DOC_META_DIRTY_VALUES_KEY) != 0) {
+
+ auto dirty_values_op = magic_enum::enum_cast<DIRTY_VALUES>(
+ document[Collection::DOC_META_KEY][Collection::DOC_META_DIRTY_VALUES_KEY]);
+ if(dirty_values_op.has_value()) {
+ dirty_values = dirty_values_op.value();
+ }
+ }
+
num_valid_docs++;
iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));
diff --git a/src/index.cpp b/src/index.cpp
index ea78907f..9845a88b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -91,7 +91,8 @@ int64_t Index::float_to_in64_t(float f) {
}
Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,
- const std::string & default_sorting_field, bool is_update) {
+ const std::string & default_sorting_field,
+ bool is_update) {
std::unique_lock lock(mutex);
@@ -181,7 +182,9 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
art_tree *t = search_index.at(field_name);
const std::string & text = document[field_name];
index_string_field(text, points, t, seq_id, facet_id, field_pair.second);
- } else if(field_pair.second.type == field_types::INT32) {
+ }
+
+ else if(field_pair.second.type == field_types::INT32) {
auto num_tree = numerical_index.at(field_name);
uint32_t value = document[field_name];
num_tree->insert(value, seq_id);
@@ -207,32 +210,35 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
num_tree->insert(geoHash, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
art_tree *t = search_index.at(field_name);
- const std::vector<std::string>& strings = document[field_name];
- index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);
- } else if(field_pair.second.type == field_types::INT32_ARRAY) {
+ index_string_array_field(document[field_name], points, t, seq_id, facet_id, field_pair.second);
+ }
+
+ else if(field_pair.second.is_array()) {
auto num_tree = numerical_index.at(field_name);
- const std::vector<int32_t>& arr_values = document[field_name];
- for(const int32_t value: arr_values) {
- num_tree->insert(value, seq_id);
- }
- } else if(field_pair.second.type == field_types::INT64_ARRAY) {
- auto& num_tree = numerical_index.at(field_name);
- const std::vector<int64_t>& arr_values = document[field_name];
- for(const int64_t value: arr_values) {
- num_tree->insert(value, seq_id);
- }
- } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
- const std::vector<float>& arr_values = document[field_name];
- for(const float fvalue: arr_values) {
- auto& num_tree = numerical_index.at(field_name);
- int64_t value = float_to_in64_t(fvalue);
- num_tree->insert(value, seq_id);
- }
- } else if(field_pair.second.type == field_types::BOOL_ARRAY) {
- auto& num_tree = numerical_index.at(field_name);
- const std::vector<bool>& arr_values = document[field_name];
- for(const int64_t value: arr_values) {
- num_tree->insert(value, seq_id);
+
+ for(size_t arr_i = 0; arr_i < document[field_name].size(); arr_i++) {
+ const auto& arr_value = document[field_name][arr_i];
+
+ if(field_pair.second.type == field_types::INT32_ARRAY) {
+ const int32_t value = arr_value;
+ num_tree->insert(value, seq_id);
+ }
+
+ else if(field_pair.second.type == field_types::INT64_ARRAY) {
+ const int64_t value = arr_value;
+ num_tree->insert(value, seq_id);
+ }
+
+ else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
+ const float fvalue = arr_value;
+ int64_t value = float_to_in64_t(fvalue);
+ num_tree->insert(value, seq_id);
+ }
+
+ else if(field_pair.second.type == field_types::BOOL_ARRAY) {
+ const bool value = document[field_name][arr_i];
+ num_tree->insert(int64_t(value), seq_id);
+ }
}
}
@@ -279,8 +285,9 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
for(const auto& field_pair: search_schema) {
const std::string& field_name = field_pair.first;
+ const field& a_field = field_pair.second;
- if((field_pair.second.optional || is_update || index_all_fields) && document.count(field_name) == 0) {
+ if((a_field.optional || is_update) && document.count(field_name) == 0) {
continue;
}
@@ -289,46 +296,47 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
"but is not found in the document.");
}
- if(field_pair.second.type == field_types::STRING && !document[field_name].is_string()) {
- Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, -1);
+ if(a_field.type == field_types::STRING && !document[field_name].is_string()) {
+ Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, -1);
if(!coerce_op.ok()) {
return coerce_op;
}
- } else if(field_pair.second.type == field_types::INT32) {
+ } else if(a_field.type == field_types::INT32) {
if(!document[field_name].is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, -1);
+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, -1);
if(!coerce_op.ok()) {
return coerce_op;
}
}
if(document[field_name].get<int64_t>() > INT32_MAX) {
- if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT)) {
document.erase(field_name);
continue;
} else {
return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
}
}
- } else if(field_pair.second.type == field_types::INT64 && !document[field_name].is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, -1);
+ } else if(a_field.type == field_types::INT64 && !document[field_name].is_number_integer()) {
+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, -1);
if(!coerce_op.ok()) {
return coerce_op;
}
- } else if(field_pair.second.type == field_types::FLOAT && !document[field_name].is_number()) {
+ } else if(a_field.type == field_types::FLOAT && !document[field_name].is_number()) {
// using `is_number` allows integer to be passed to a float field
- Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, -1);
+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, -1);
if(!coerce_op.ok()) {
return coerce_op;
}
- } else if(field_pair.second.type == field_types::BOOL && !document[field_name].is_boolean()) {
- Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, -1);
+ } else if(a_field.type == field_types::BOOL && !document[field_name].is_boolean()) {
+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, -1);
if(!coerce_op.ok()) {
return coerce_op;
}
- } else if(field_pair.second.is_array()) {
+ } else if(a_field.is_array()) {
if(!document[field_name].is_array()) {
- if(dirty_values == DIRTY_VALUES::IGNORE || dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||
+ dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) {
document.erase(field_name);
continue;
} else {
@@ -339,29 +347,29 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
for(size_t arr_index=0; arr_index < document[field_name].size(); arr_index++) {
const auto& item = document[field_name][arr_index];
- if (field_pair.second.type == field_types::STRING_ARRAY && !item.is_string()) {
- Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, arr_index);
+ if (a_field.type == field_types::STRING_ARRAY && !item.is_string()) {
+ Option<uint32_t> coerce_op = coerce_string(dirty_values, a_field, document, field_name, arr_index);
if (!coerce_op.ok()) {
return coerce_op;
}
- } else if (field_pair.second.type == field_types::INT32_ARRAY && !item.is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, arr_index);
+ } else if (a_field.type == field_types::INT32_ARRAY && !item.is_number_integer()) {
+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, arr_index);
if (!coerce_op.ok()) {
return coerce_op;
}
- } else if (field_pair.second.type == field_types::INT64_ARRAY && !item.is_number_integer()) {
- Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, arr_index);
+ } else if (a_field.type == field_types::INT64_ARRAY && !item.is_number_integer()) {
+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, arr_index);
if (!coerce_op.ok()) {
return coerce_op;
}
- } else if (field_pair.second.type == field_types::FLOAT_ARRAY && !item.is_number()) {
+ } else if (a_field.type == field_types::FLOAT_ARRAY && !item.is_number()) {
// we check for `is_number` to allow whole numbers to be passed into float fields
- Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, arr_index);
+ Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, arr_index);
if (!coerce_op.ok()) {
return coerce_op;
}
- } else if (field_pair.second.type == field_types::BOOL_ARRAY && !item.is_boolean()) {
- Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, arr_index);
+ } else if (a_field.type == field_types::BOOL_ARRAY && !item.is_boolean()) {
+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, arr_index);
if (!coerce_op.ok()) {
return coerce_op;
}
@@ -1992,7 +2000,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
}
- const int64_t default_score = INT64_MIN;
+ const int64_t default_score = INT64_MIN; // to handle field that doesn't exist in document (e.g. optional)
int64_t scores[3] = {0};
size_t match_score_index = 0;
@@ -2380,15 +2388,16 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {
}
}
-Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name, const int array_index) {
std::string suffix = array_index != -1 ? "an array of" : "a";
+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
- if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
@@ -2399,27 +2408,27 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmann
// we will try to coerce the value to a string
- if (document[field_name].is_number_integer()) {
- document[field_name] = std::to_string((int64_t)document[field_name]);
+ if (item.is_number_integer()) {
+ item = std::to_string((int64_t)item);
}
- else if(document[field_name].is_number_float()) {
- document[field_name] = std::to_string((float)document[field_name]);
+ else if(item.is_number_float()) {
+ item = std::to_string((float)item);
}
- else if(document[field_name].is_boolean()) {
- document[field_name] = document[field_name] == true ? "true" : "false";
+ else if(item.is_boolean()) {
+ item = item == true ? "true" : "false";
}
else {
- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
document[field_name].erase(document[field_name].begin() + array_index);
}
} else {
- // DIRTY_FIELD_COERCE_REJECT
+ // COERCE_OR_REJECT / non-optional + DROP
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
}
@@ -2427,15 +2436,16 @@ Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmann
return Option<>(200);
}
-Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
+Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
std::string suffix = array_index != -1 ? "an array of" : "an";
+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
}
- if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
@@ -2446,23 +2456,27 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohman
// try to value coerce into an integer
- if(document[field_name].is_number_float()) {
- document[field_name] = static_cast<int32_t>(document[field_name].get<float>());
+ if(item.is_number_float()) {
+ item = static_cast<int32_t>(item.get<float>());
}
- else if(document[field_name].is_boolean()) {
- document[field_name] = document[field_name] == true ? 1 : 0;
+ else if(item.is_boolean()) {
+ item = item == true ? 1 : 0;
}
- else if(document[field_name].is_string() && StringUtils::is_int32_t(document[field_name])) {
- document[field_name] = std::atol(document[field_name].get<std::string>().c_str());
+ else if(item.is_string() && StringUtils::is_int32_t(item)) {
+ item = std::atol(item.get<std::string>().c_str());
}
else {
- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
- document.erase(field_name);
+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ item.erase(item.begin() + array_index);
+ }
} else {
- // DIRTY_FIELD_COERCE_REJECT
+ // COERCE_OR_REJECT / non-optional + DROP
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
}
}
@@ -2470,15 +2484,16 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohman
return Option<uint32_t>(200);
}
-Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name, const int array_index) {
std::string suffix = array_index != -1 ? "an array of" : "an";
+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
}
- if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
@@ -2489,27 +2504,27 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohman
// try to value coerce into an integer
- if(document[field_name].is_number_float()) {
- document[field_name] = static_cast<int64_t>(document[field_name].get<float>());
+ if(item.is_number_float()) {
+ item = static_cast<int64_t>(item.get<float>());
}
- else if(document[field_name].is_boolean()) {
- document[field_name] = document[field_name] == true ? 1 : 0;
+ else if(item.is_boolean()) {
+ item = item == true ? 1 : 0;
}
- else if(document[field_name].is_string() && StringUtils::is_int64_t(document[field_name])) {
- document[field_name] = std::atoll(document[field_name].get<std::string>().c_str());
+ else if(item.is_string() && StringUtils::is_int64_t(item)) {
+ item = std::atoll(item.get<std::string>().c_str());
}
else {
- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
document[field_name].erase(document[field_name].begin() + array_index);
}
} else {
- // DIRTY_FIELD_COERCE_REJECT
+ // COERCE_OR_REJECT / non-optional + DROP
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
}
}
@@ -2517,15 +2532,16 @@ Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohman
return Option<uint32_t>(200);
}
-Option<uint32_t>
-Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
+Option<uint32_t> Index::coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
std::string suffix = array_index != -1 ? "a array of" : "a";
+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
+
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
- if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
@@ -2535,32 +2551,32 @@ Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
}
// try to value coerce into a bool
- if (document[field_name].is_number_integer() &&
- (document[field_name].get<int64_t>() == 1 || document[field_name].get<int64_t>() == 0)) {
- document[field_name] = document[field_name].get<int64_t>() == 1;
+ if (item.is_number_integer() &&
+ (item.get<int64_t>() == 1 || item.get<int64_t>() == 0)) {
+ item = item.get<int64_t>() == 1;
}
- else if(document[field_name].is_string()) {
- std::string str_val = document[field_name].get<std::string>();
+ else if(item.is_string()) {
+ std::string str_val = item.get<std::string>();
StringUtils::tolowercase(str_val);
if(str_val == "true") {
- document[field_name] = true;
+ item = true;
return Option<uint32_t>(200);
} else if(str_val == "false") {
- document[field_name] = false;
+ item = false;
return Option<uint32_t>(200);
}
}
else {
- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
document[field_name].erase(document[field_name].begin() + array_index);
}
} else {
- // DIRTY_FIELD_COERCE_REJECT
+ // COERCE_OR_REJECT / non-optional + DROP
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
}
@@ -2568,16 +2584,16 @@ Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
return Option<uint32_t>(200);
}
-Option<uint32_t>
-Index::coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
- const std::string &field_name, const int array_index) {
+Option<uint32_t> Index::coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
std::string suffix = array_index != -1 ? "a array of" : "a";
+ auto& item = array_index != -1 ? document[field_name][array_index] : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
}
- if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
@@ -2588,23 +2604,23 @@ Index::coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
// try to value coerce into a float
- if(document[field_name].is_string() && StringUtils::is_float(document[field_name])) {
- document[field_name] = std::atof(document[field_name].get<std::string>().c_str());
+ if(item.is_string() && StringUtils::is_float(item)) {
+ item = std::atof(item.get<std::string>().c_str());
}
- else if(document[field_name].is_boolean()) {
- document[field_name] = document[field_name] == true ? 1.0 : 0.0;
+ else if(item.is_boolean()) {
+ item = item == true ? 1.0 : 0.0;
}
else {
- if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(a_field.optional && dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(array_index == -1) {
document.erase(field_name);
} else {
document[field_name].erase(document[field_name].begin() + array_index);
}
} else {
- // DIRTY_FIELD_COERCE_REJECT
+ // COERCE_OR_REJECT / non-optional + DROP
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
}
}
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 664ef652..a7514077 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -60,6 +60,22 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
query_fields = {"starring"};
std::vector<std::string> facets;
+ // check default no specific dirty values option is sent for a collection that has schema detection enabled
+ std::string dirty_values;
+ ASSERT_EQ(DIRTY_VALUES::COERCE_OR_REJECT, coll1->parse_dirty_values_option(dirty_values));
+
+ dirty_values = "coerce_or_reject";
+ ASSERT_EQ(DIRTY_VALUES::COERCE_OR_REJECT, coll1->parse_dirty_values_option(dirty_values));
+
+ dirty_values = "COERCE_OR_DROP";
+ ASSERT_EQ(DIRTY_VALUES::COERCE_OR_DROP, coll1->parse_dirty_values_option(dirty_values));
+
+ dirty_values = "reject";
+ ASSERT_EQ(DIRTY_VALUES::REJECT, coll1->parse_dirty_values_option(dirty_values));
+
+ dirty_values = "DROP";
+ ASSERT_EQ(DIRTY_VALUES::DROP, coll1->parse_dirty_values_option(dirty_values));
+
// same should succeed when verbatim filter is made
auto results = coll1->search("will", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
@@ -100,12 +116,12 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_STREQ("300", results["hits"][0]["document"]["title"].get<std::string>().c_str());
- // with dirty values set to `DIRTY_FIELD_COERCE_IGNORE`
+ // with dirty values set to `COERCE_OR_DROP`
// `cast` field should not be indexed into store
doc_json = R"({"cast":"William Barnes","points":63,"starring":"Will Ferrell",
"starring_facet":"Will Ferrell","title":"With bad cast field."})";
- add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_IGNORE);
+ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);
ASSERT_TRUE(add_op.ok());
results = coll1->search("With bad cast field", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
@@ -113,12 +129,12 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
ASSERT_STREQ("With bad cast field.", results["hits"][0]["document"]["title"].get<std::string>().c_str());
ASSERT_EQ(0, results["hits"][0]["document"].count("cast"));
- // with dirty values set to `DIRTY_FIELD_IGNORE`
- // no coercion should happen, `title` field will just be ignored, but record indexed
+ // with dirty values set to `DROP`
+ // no coercion should happen, `title` field will just be dropped, but record indexed
doc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell",
"starring_facet":"Will Ferrell","title": 1200 })";
- add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::IGNORE);
+ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::DROP);
ASSERT_TRUE(add_op.ok());
results = coll1->search("1200", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
@@ -128,7 +144,7 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(0, results["hits"][0]["document"].count("title"));
- // with dirty values set to `DIRTY_FIELD_REJECT`
+ // with dirty values set to `REJECT`
doc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell",
"starring_facet":"Will Ferrell","title": 1200 })";
@@ -144,3 +160,85 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionAllFieldsTest, HandleArrayTypes) {
+ Collection *coll1;
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, {}, "", 0, true).get();
+ }
+
+ nlohmann::json doc;
+ doc["title"] = "FIRST";
+ doc["int_values"] = {1, 2};
+
+ Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0");
+ ASSERT_TRUE(add_op.ok());
+
+ // coercion of string -> int
+
+ doc["int_values"] = {"3"};
+
+ add_op = coll1->add(doc.dump(), UPDATE, "0");
+ ASSERT_TRUE(add_op.ok());
+
+ // bad array type value should be dropped when stored
+
+ doc["title"] = "SECOND";
+ doc["int_values"] = {{3}};
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES:: DROP);
+ ASSERT_TRUE(add_op.ok());
+
+ auto results = coll1->search("second", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+
+ // check that the "bad" value does not exists in the stored document
+ ASSERT_EQ(1, results["hits"][0]["document"].count("int_values"));
+ ASSERT_EQ(0, results["hits"][0]["document"]["int_values"].size());
+
+ // bad array type should follow coercion rules
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `int_values` must be an array of int64.", add_op.error());
+
+ // non array field should be handled as per coercion rule
+ doc["title"] = "THIRD";
+ doc["int_values"] = 3;
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `int_values` must be an array.", add_op.error());
+
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);
+ ASSERT_TRUE(add_op.ok());
+ results = coll1->search("third", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(0, results["hits"][0]["document"].count("int_values"));
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionAllFieldsTest, NonOptionalFieldShouldNotBeDropped) {
+ Collection *coll1;
+
+ std::vector<field> fields = {
+ field("points", field_types::INT32, false, false)
+ };
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, true).get();
+ }
+
+ nlohmann::json doc;
+ doc["title"] = "FIRST";
+ doc["points"] = {100};
+
+ Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0", DIRTY_VALUES::DROP);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `points` must be an int32.", add_op.error());
+
+ add_op = coll1->add(doc.dump(), CREATE, "0", DIRTY_VALUES::COERCE_OR_DROP);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `points` must be an int32.", add_op.error());
+}
\ No newline at end of file
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index d083c3b7..4e07e882 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -304,11 +304,11 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {
ASSERT_EQ(1, coll1->get_collection_id());
ASSERT_EQ(3, coll1->get_sort_fields().size());
- // index a document with a bad field value with COERCE_OR_IGNORE setting
+ // index a document with a bad field value with COERCE_OR_DROP setting
auto doc_json = R"({"title": "Unique record.", "max": 25, "scores": [22, "how", 44],
"average": "bad data", "is_valid": true})";
- Option<nlohmann::json> add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_IGNORE);
+ Option<nlohmann::json> add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);
ASSERT_TRUE(add_op.ok());
std::unordered_map<std::string, field> schema = collection1->get_schema();
@@ -351,8 +351,14 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {
ASSERT_EQ(1, restored_schema.count("average"));
ASSERT_EQ(1, restored_schema.count("is_valid"));
+ // all detected schema are optional fields, while defined schema is not
+
for(const auto& kv: restored_schema) {
- ASSERT_FALSE(kv.second.optional);
+ if(kv.first == "max") {
+ ASSERT_FALSE(kv.second.optional);
+ } else {
+ ASSERT_TRUE(kv.second.optional);
+ }
}
// try searching for record with bad data
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 7a506a99..033779f9 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -85,6 +85,10 @@ protected:
TEST_F(CollectionTest, VerifyCountOfDocuments) {
// we have 1 dummy record to match the line numbers on the fixtures file with sequence numbers
ASSERT_EQ(24+1, collection->get_num_documents());
+
+ // check default no specific dirty values option is sent for a collection that has explicit schema
+ std::string empty_dirty_values;
+ ASSERT_EQ(DIRTY_VALUES::REJECT, collection->parse_dirty_values_option(empty_dirty_values));
}
TEST_F(CollectionTest, RetrieveADocumentById) {
@@ -1622,7 +1626,8 @@ TEST_F(CollectionTest, IndexingWithBadData) {
ASSERT_STREQ("Field `tags` must be an array.", bad_facet_field_op.error().c_str());
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";
- const Option<nlohmann::json> & bad_array_field_op = sample_collection->add(doc_str);
+ const Option<nlohmann::json> & bad_array_field_op = sample_collection->add(doc_str, CREATE, "",
+ DIRTY_VALUES::REJECT);
ASSERT_FALSE(bad_array_field_op.ok());
ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_op.error().c_str());
@@ -1630,8 +1635,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";
const Option<nlohmann::json> &bad_array_field_coercion_op = sample_collection->add(doc_str, CREATE, "",
DIRTY_VALUES::COERCE_OR_REJECT);
- ASSERT_FALSE(bad_array_field_coercion_op.ok());
- ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_coercion_op.error().c_str());
+ ASSERT_TRUE(bad_array_field_coercion_op.ok());
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";
const Option<nlohmann::json> & empty_facet_field_op = sample_collection->add(doc_str);
@@ -2495,12 +2499,15 @@ TEST_F(CollectionTest, OptionalFields) {
ASSERT_EQ(5, res["facet_counts"][0]["counts"][0]["count"].get<size_t>());
ASSERT_STREQ("description", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
- // sort_by optional `average` field should be rejected
+ // sort_by optional `average` field should be allowed (default used for missing values)
std::vector<sort_by> sort_fields = { sort_by("average", "DESC") };
auto res_op = coll1->search("*", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);
- ASSERT_FALSE(res_op.ok());
- ASSERT_STREQ("Cannot sort by `average` as it is defined as an optional field.", res_op.error().c_str());
-
+ ASSERT_TRUE(res_op.ok());
+ res = res_op.get();
+
+ ASSERT_EQ(6, res["found"].get<size_t>());
+ ASSERT_EQ(0, res["hits"][5]["document"].count("average")); // record with missing average is last
+
// try deleting a record having optional field
Option<std::string> remove_op = coll1->remove("1");
ASSERT_TRUE(remove_op.ok());