Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle bad data gracefully.

kishorenc11c41804e5ca
diff --git a/include/collection.h b/include/collection.h
index 184925ed..738db45d 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -323,7 +323,7 @@ private:
const std::vector<Index*> indices;
- const bool index_all_fields;
+ const std::atomic<bool> index_all_fields;
// methods
@@ -349,6 +349,8 @@ private:
std::map<size_t, std::vector<uint32_t>>& include_ids,
std::vector<uint32_t> & excluded_ids) const;
+ Option<bool> check_and_update_schema(nlohmann::json& document);
+
static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,
const std::pair<uint64_t, facet_count_t>& b) {
return std::tie(a.second.count, a.first) > std::tie(b.second.count, b.first);
@@ -457,7 +459,8 @@ public:
Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document) const;
- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, bool is_update);
+ Option<uint32_t> index_in_memory(nlohmann::json & document, uint32_t seq_id,
+ bool is_update, const DIRTY_VALUES& dirty_values);
static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,
const spp::sparse_hash_set<std::string> & exclude_fields);
@@ -490,10 +493,12 @@ public:
size_t par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch, std::vector<size_t>& indexed_counts);
Option<nlohmann::json> add(const std::string & json_str,
- const index_operation_t& operation=CREATE, const std::string& id="");
+ const index_operation_t& operation=CREATE, const std::string& id="",
+ const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT);
nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,
- const index_operation_t& operation=CREATE, const std::string& id="");
+ const index_operation_t& operation=CREATE, const std::string& id="",
+ const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT);
Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,
const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
@@ -534,6 +539,8 @@ public:
size_t get_num_documents() const;
+ DIRTY_VALUES parse_dirty_values_option(std::string& dirty_values) const;
+
// Override operations
Option<uint32_t> add_override(const override_t & override);
diff --git a/include/field.h b/include/field.h
index cf48da5a..121f5e51 100644
--- a/include/field.h
+++ b/include/field.h
@@ -105,6 +105,10 @@ struct field {
type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY);
}
+ bool is_sortable() const {
+ return is_single_integer() || is_single_float() || is_single_bool() || is_geopoint();
+ }
+
bool has_valid_type() const {
return is_string() || is_integer() || is_float() || is_bool() || is_geopoint();
}
diff --git a/include/index.h b/include/index.h
index 36811d9b..c2076b36 100644
--- a/include/index.h
+++ b/include/index.h
@@ -17,6 +17,7 @@
#include <set>
#include "string_utils.h"
#include "num_tree.h"
+#include "magic_enum.hpp"
struct token_t {
size_t position;
@@ -103,6 +104,12 @@ enum index_operation_t {
DELETE
};
+enum class DIRTY_VALUES {
+ COERCE_OR_IGNORE = 1,
+ COERCE_OR_REJECT = 2,
+ REJECT = 3,
+ IGNORE = 4,
+};
struct index_record {
size_t position; // position of record in the original request
@@ -118,8 +125,12 @@ struct index_record {
Option<bool> indexed; // indicates if the indexing operation was a success
- index_record(size_t record_pos, uint32_t seq_id, const nlohmann::json& doc, index_operation_t operation):
- position(record_pos), seq_id(seq_id), doc(doc), operation(operation), is_update(false), indexed(false) {
+ const DIRTY_VALUES dirty_values;
+
+ index_record(size_t record_pos, uint32_t seq_id, const nlohmann::json& doc, index_operation_t operation,
+ const DIRTY_VALUES& dirty_values):
+ position(record_pos), seq_id(seq_id), doc(doc), operation(operation), is_update(false),
+ indexed(false), dirty_values(dirty_values) {
}
@@ -227,6 +238,21 @@ private:
static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
+ static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index);
+
+ static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index);
+
+ static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index);
+
+ static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index);
+
+ static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index);
+
public:
// for limiting number of results on multiple candidates / query rewrites
enum {TYPO_TOKENS_THRESHOLD = 100};
@@ -335,7 +361,8 @@ public:
std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema);
+ const std::map<std::string, field> & facet_schema,
+ bool index_all_fields);
static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
const std::vector<uint32_t*>& leaf_to_indices,
@@ -348,13 +375,15 @@ public:
uint32_t do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const;
- static Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,
+ static Option<uint32_t> validate_index_in_memory(nlohmann::json &document, uint32_t seq_id,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
const std::map<std::string, field> & facet_schema,
- bool is_update);
+ bool is_update,
+ bool index_all_fields,
+ const DIRTY_VALUES& dirty_values);
- void refresh_search_schema(const std::unordered_map<std::string, field>& src_search_schema);
+ void refresh_schemas(const std::vector<field>& new_fields);
};
diff --git a/include/magic_enum.hpp b/include/magic_enum.hpp
new file mode 100644
index 00000000..aa17bc6e
--- /dev/null
+++ b/include/magic_enum.hpp
@@ -0,0 +1,1131 @@
+// __ __ _ ______ _____
+// | \/ | (_) | ____| / ____|_ _
+// | \ / | __ _ __ _ _ ___ | |__ _ __ _ _ _ __ ___ | | _| |_ _| |_
+// | |\/| |/ _` |/ _` | |/ __| | __| | '_ \| | | | '_ ` _ \ | | |_ _|_ _|
+// | | | | (_| | (_| | | (__ | |____| | | | |_| | | | | | | | |____|_| |_|
+// |_| |_|\__,_|\__, |_|\___| |______|_| |_|\__,_|_| |_| |_| \_____|
+// __/ | https://github.com/Neargye/magic_enum
+// |___/ version 0.7.2
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2019 - 2021 Daniil Goncharov <[email protected]>.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef NEARGYE_MAGIC_ENUM_HPP
+#define NEARGYE_MAGIC_ENUM_HPP
+
+#define MAGIC_ENUM_VERSION_MAJOR 0
+#define MAGIC_ENUM_VERSION_MINOR 7
+#define MAGIC_ENUM_VERSION_PATCH 2
+
+#include <array>
+#include <cassert>
+#include <cstdint>
+#include <cstddef>
+#include <iosfwd>
+#include <limits>
+#include <type_traits>
+#include <utility>
+
+#if !defined(MAGIC_ENUM_USING_ALIAS_OPTIONAL)
+#include <optional>
+#endif
+#if !defined(MAGIC_ENUM_USING_ALIAS_STRING)
+#include <string>
+#endif
+#if !defined(MAGIC_ENUM_USING_ALIAS_STRING_VIEW)
+#include <string_view>
+#endif
+
+#if defined(__clang__)
+# pragma clang diagnostic push
+#elif defined(__GNUC__)
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" // May be used uninitialized 'return {};'.
+#elif defined(_MSC_VER)
+# pragma warning(push)
+# pragma warning(disable : 26495) // Variable 'static_string<N>::chars_' is uninitialized.
+# pragma warning(disable : 28020) // Arithmetic overflow: Using operator '-' on a 4 byte value and then casting the result to a 8 byte value.
+# pragma warning(disable : 26451) // The expression '0<=_Param_(1)&&_Param_(1)<=1-1' is not true at this call.
+#endif
+
+// Checks magic_enum compiler compatibility.
+#if defined(__clang__) && __clang_major__ >= 5 || defined(__GNUC__) && __GNUC__ >= 9 || defined(_MSC_VER) && _MSC_VER >= 1910
+# undef MAGIC_ENUM_SUPPORTED
+# define MAGIC_ENUM_SUPPORTED 1
+#endif
+
+// Checks magic_enum compiler aliases compatibility.
+#if defined(__clang__) && __clang_major__ >= 5 || defined(__GNUC__) && __GNUC__ >= 9 || defined(_MSC_VER) && _MSC_VER >= 1920
+# undef MAGIC_ENUM_SUPPORTED_ALIASES
+# define MAGIC_ENUM_SUPPORTED_ALIASES 1
+#endif
+
+// Enum value must be greater or equals than MAGIC_ENUM_RANGE_MIN. By default MAGIC_ENUM_RANGE_MIN = -128.
+// If need another min range for all enum types by default, redefine the macro MAGIC_ENUM_RANGE_MIN.
+#if !defined(MAGIC_ENUM_RANGE_MIN)
+# define MAGIC_ENUM_RANGE_MIN -128
+#endif
+
+// Enum value must be less or equals than MAGIC_ENUM_RANGE_MAX. By default MAGIC_ENUM_RANGE_MAX = 128.
+// If need another max range for all enum types by default, redefine the macro MAGIC_ENUM_RANGE_MAX.
+#if !defined(MAGIC_ENUM_RANGE_MAX)
+# define MAGIC_ENUM_RANGE_MAX 128
+#endif
+
+namespace magic_enum {
+
+// If need another optional type, define the macro MAGIC_ENUM_USING_ALIAS_OPTIONAL.
+#if defined(MAGIC_ENUM_USING_ALIAS_OPTIONAL)
+ MAGIC_ENUM_USING_ALIAS_OPTIONAL
+#else
+ template <typename T>
+ using optional = std::optional<T>;
+#endif
+
+// If need another string_view type, define the macro MAGIC_ENUM_USING_ALIAS_STRING_VIEW.
+#if defined(MAGIC_ENUM_USING_ALIAS_STRING_VIEW)
+ MAGIC_ENUM_USING_ALIAS_STRING_VIEW
+#else
+ using string_view = std::string_view;
+#endif
+
+// If need another string type, define the macro MAGIC_ENUM_USING_ALIAS_STRING.
+#if defined(MAGIC_ENUM_USING_ALIAS_STRING)
+ MAGIC_ENUM_USING_ALIAS_STRING
+#else
+ using string = std::string;
+#endif
+
+ namespace customize {
+
+// Enum value must be in range [MAGIC_ENUM_RANGE_MIN, MAGIC_ENUM_RANGE_MAX]. By default MAGIC_ENUM_RANGE_MIN = -128, MAGIC_ENUM_RANGE_MAX = 128.
+// If need another range for all enum types by default, redefine the macro MAGIC_ENUM_RANGE_MIN and MAGIC_ENUM_RANGE_MAX.
+// If need another range for specific enum type, add specialization enum_range for necessary enum type.
+ template <typename E>
+ struct enum_range {
+ static_assert(std::is_enum_v<E>, "magic_enum::customize::enum_range requires enum type.");
+ inline static constexpr int min = MAGIC_ENUM_RANGE_MIN;
+ inline static constexpr int max = MAGIC_ENUM_RANGE_MAX;
+ static_assert(max > min, "magic_enum::customize::enum_range requires max > min.");
+ };
+
+ static_assert(MAGIC_ENUM_RANGE_MIN <= 0, "MAGIC_ENUM_RANGE_MIN must be less or equals than 0.");
+ static_assert(MAGIC_ENUM_RANGE_MIN > (std::numeric_limits<std::int16_t>::min)(), "MAGIC_ENUM_RANGE_MIN must be greater than INT16_MIN.");
+
+ static_assert(MAGIC_ENUM_RANGE_MAX > 0, "MAGIC_ENUM_RANGE_MAX must be greater than 0.");
+ static_assert(MAGIC_ENUM_RANGE_MAX < (std::numeric_limits<std::int16_t>::max)(), "MAGIC_ENUM_RANGE_MAX must be less than INT16_MAX.");
+
+ static_assert(MAGIC_ENUM_RANGE_MAX > MAGIC_ENUM_RANGE_MIN, "MAGIC_ENUM_RANGE_MAX must be greater than MAGIC_ENUM_RANGE_MIN.");
+
+// If need cunstom names for enum, add specialization enum_name for necessary enum type.
+ template <typename E>
+ constexpr string_view enum_name(E) noexcept {
+ static_assert(std::is_enum_v<E>, "magic_enum::customize::enum_name requires enum type.");
+
+ return {};
+ }
+
+ } // namespace magic_enum::customize
+
+ namespace detail {
+
+ template <typename T>
+ struct supported
+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED || defined(MAGIC_ENUM_NO_CHECK_SUPPORT)
+ : std::true_type {};
+#else
+ : std::false_type {};
+#endif
+
+ struct char_equal_to {
+ constexpr bool operator()(char lhs, char rhs) const noexcept {
+ return lhs == rhs;
+ }
+ };
+
+ template <std::size_t N>
+ class static_string {
+ public:
+ constexpr explicit static_string(string_view str) noexcept : static_string{str, std::make_index_sequence<N>{}} {
+ assert(str.size() == N);
+ }
+
+ constexpr const char* data() const noexcept { return chars_.data(); }
+
+ constexpr std::size_t size() const noexcept { return N; }
+
+ constexpr operator string_view() const noexcept { return {data(), size()}; }
+
+ private:
+ template <std::size_t... I>
+ constexpr static_string(string_view str, std::index_sequence<I...>) noexcept : chars_{{str[I]..., '\0'}} {}
+
+ const std::array<char, N + 1> chars_;
+ };
+
+ template <>
+ class static_string<0> {
+ public:
+ constexpr explicit static_string(string_view) noexcept {}
+
+ constexpr const char* data() const noexcept { return nullptr; }
+
+ constexpr std::size_t size() const noexcept { return 0; }
+
+ constexpr operator string_view() const noexcept { return {}; }
+ };
+
+ constexpr string_view pretty_name(string_view name) noexcept {
+ for (std::size_t i = name.size(); i > 0; --i) {
+ if (!((name[i - 1] >= '0' && name[i - 1] <= '9') ||
+ (name[i - 1] >= 'a' && name[i - 1] <= 'z') ||
+ (name[i - 1] >= 'A' && name[i - 1] <= 'Z') ||
+ (name[i - 1] == '_'))) {
+ name.remove_prefix(i);
+ break;
+ }
+ }
+
+ if (name.size() > 0 && ((name.front() >= 'a' && name.front() <= 'z') ||
+ (name.front() >= 'A' && name.front() <= 'Z') ||
+ (name.front() == '_'))) {
+ return name;
+ }
+
+ return {}; // Invalid name.
+ }
+
+ constexpr std::size_t find(string_view str, char c) noexcept {
+#if defined(__clang__) && __clang_major__ < 9 && defined(__GLIBCXX__) || defined(_MSC_VER) && _MSC_VER < 1920 && !defined(__clang__)
+ // https://stackoverflow.com/questions/56484834/constexpr-stdstring-viewfind-last-of-doesnt-work-on-clang-8-with-libstdc
+// https://developercommunity.visualstudio.com/content/problem/360432/vs20178-regression-c-failed-in-test.html
+ constexpr bool workaround = true;
+#else
+ constexpr bool workaround = false;
+#endif
+ if constexpr (workaround) {
+ for (std::size_t i = 0; i < str.size(); ++i) {
+ if (str[i] == c) {
+ return i;
+ }
+ }
+
+ return string_view::npos;
+ } else {
+ return str.find_first_of(c);
+ }
+ }
+
+ template <typename BinaryPredicate>
+ constexpr bool cmp_equal(string_view lhs, string_view rhs, BinaryPredicate&& p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) {
+#if defined(_MSC_VER) && _MSC_VER < 1920 && !defined(__clang__)
+ // https://developercommunity.visualstudio.com/content/problem/360432/vs20178-regression-c-failed-in-test.html
+ // https://developercommunity.visualstudio.com/content/problem/232218/c-constexpr-string-view.html
+ constexpr bool workaround = true;
+#else
+ constexpr bool workaround = false;
+#endif
+ constexpr bool default_predicate = std::is_same_v<std::decay_t<BinaryPredicate>, char_equal_to>;
+
+ if constexpr (default_predicate && !workaround) {
+ static_cast<void>(p);
+ return lhs == rhs;
+ } else {
+ if (lhs.size() != rhs.size()) {
+ return false;
+ }
+
+ const auto size = lhs.size();
+ for (std::size_t i = 0; i < size; ++i) {
+ if (!p(lhs[i], rhs[i])) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }
+
+ template <typename L, typename R>
+ constexpr bool cmp_less(L lhs, R rhs) noexcept {
+ static_assert(std::is_integral_v<L> && std::is_integral_v<R>, "magic_enum::detail::cmp_less requires integral type.");
+
+ if constexpr (std::is_signed_v<L> == std::is_signed_v<R>) {
+ // If same signedness (both signed or both unsigned).
+ return lhs < rhs;
+ } else if constexpr (std::is_signed_v<R>) {
+ // If 'right' is negative, then result is 'false', otherwise cast & compare.
+ return rhs > 0 && lhs < static_cast<std::make_unsigned_t<R>>(rhs);
+ } else {
+ // If 'left' is negative, then result is 'true', otherwise cast & compare.
+ return lhs < 0 || static_cast<std::make_unsigned_t<L>>(lhs) < rhs;
+ }
+ }
+
+ template <typename I>
+ constexpr I log2(I value) noexcept {
+ static_assert(std::is_integral_v<I>, "magic_enum::detail::log2 requires integral type.");
+
+ auto ret = I{0};
+ for (; value > I{1}; value >>= I{1}, ++ret) {}
+
+ return ret;
+ }
+
+ template <typename I>
+ constexpr bool is_pow2(I x) noexcept {
+ static_assert(std::is_integral_v<I>, "magic_enum::detail::is_pow2 requires integral type.");
+
+ return x != 0 && (x & (x - 1)) == 0;
+ }
+
+ template <typename T>
+ inline constexpr bool is_enum_v = std::is_enum_v<T> && std::is_same_v<T, std::decay_t<T>>;
+
+ template <typename E>
+ constexpr auto n() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::n requires enum type.");
+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED
+# if defined(__clang__)
+ constexpr string_view name{__PRETTY_FUNCTION__ + 34, sizeof(__PRETTY_FUNCTION__) - 36};
+# elif defined(__GNUC__)
+ constexpr string_view name{__PRETTY_FUNCTION__ + 49, sizeof(__PRETTY_FUNCTION__) - 51};
+# elif defined(_MSC_VER)
+ constexpr string_view name{__FUNCSIG__ + 40, sizeof(__FUNCSIG__) - 57};
+# endif
+ return static_string<name.size()>{name};
+#else
+ return string_view{}; // Unsupported compiler.
+#endif
+ }
+
+ template <typename E>
+ inline constexpr auto type_name_v = n<E>();
+
+ template <typename E, E V>
+ constexpr auto n() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::n requires enum type.");
+ constexpr auto custom_name = customize::enum_name<E>(V);
+
+ if constexpr (custom_name.empty()) {
+ static_cast<void>(custom_name);
+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED
+# if defined(__clang__) || defined(__GNUC__)
+ constexpr auto name = pretty_name({__PRETTY_FUNCTION__, sizeof(__PRETTY_FUNCTION__) - 2});
+# elif defined(_MSC_VER)
+ constexpr auto name = pretty_name({__FUNCSIG__, sizeof(__FUNCSIG__) - 17});
+# endif
+ return static_string<name.size()>{name};
+#else
+ return string_view{}; // Unsupported compiler.
+#endif
+ } else {
+ return static_string<custom_name.size()>{custom_name};
+ }
+ }
+
+ template <typename E, E V>
+ inline constexpr auto enum_name_v = n<E, V>();
+
+ template <typename E, auto V>
+ constexpr bool is_valid() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::is_valid requires enum type.");
+
+ return n<E, static_cast<E>(V)>().size() != 0;
+ }
+
+ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>
+ constexpr int reflected_min() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::reflected_min requires enum type.");
+
+ if constexpr (IsFlags) {
+ return 0;
+ } else {
+ constexpr auto lhs = customize::enum_range<E>::min;
+ static_assert(lhs > (std::numeric_limits<std::int16_t>::min)(), "magic_enum::enum_range requires min must be greater than INT16_MIN.");
+ constexpr auto rhs = (std::numeric_limits<U>::min)();
+
+ if constexpr (cmp_less(lhs, rhs)) {
+ return rhs;
+ } else {
+ return lhs;
+ }
+ }
+ }
+
+ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>
+ constexpr int reflected_max() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::reflected_max requires enum type.");
+
+ if constexpr (IsFlags) {
+ return std::numeric_limits<U>::digits - 1;
+ } else {
+ constexpr auto lhs = customize::enum_range<E>::max;
+ static_assert(lhs < (std::numeric_limits<std::int16_t>::max)(), "magic_enum::enum_range requires max must be less than INT16_MAX.");
+ constexpr auto rhs = (std::numeric_limits<U>::max)();
+
+ if constexpr (cmp_less(lhs, rhs)) {
+ return lhs;
+ } else {
+ return rhs;
+ }
+ }
+ }
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto reflected_min_v = reflected_min<E, IsFlags>();
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto reflected_max_v = reflected_max<E, IsFlags>();
+
+ template <typename E, int O, bool IsFlags = false, typename U = std::underlying_type_t<E>>
+ constexpr E value(std::size_t i) noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::value requires enum type.");
+
+ if constexpr (IsFlags) {
+ return static_cast<E>(U{1} << static_cast<U>(static_cast<int>(i) + O));
+ } else {
+ return static_cast<E>(static_cast<int>(i) + O);
+ }
+ }
+
+ template <std::size_t N>
+ constexpr std::size_t values_count(const std::array<bool, N>& valid) noexcept {
+ auto count = std::size_t{0};
+ for (std::size_t i = 0; i < valid.size(); ++i) {
+ if (valid[i]) {
+ ++count;
+ }
+ }
+
+ return count;
+ }
+
+ template <typename E, bool IsFlags, int Min, std::size_t... I>
+ constexpr auto values(std::index_sequence<I...>) noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::values requires enum type.");
+ constexpr std::array<bool, sizeof...(I)> valid{{is_valid<E, value<E, Min, IsFlags>(I)>()...}};
+ constexpr std::size_t count = values_count(valid);
+
+ std::array<E, count> values{};
+ for (std::size_t i = 0, v = 0; v < count; ++i) {
+ if (valid[i]) {
+ values[v++] = value<E, Min, IsFlags>(i);
+ }
+ }
+
+ return values;
+ }
+
+ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>
+ constexpr auto values() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::values requires enum type.");
+ constexpr auto min = reflected_min_v<E, IsFlags>;
+ constexpr auto max = reflected_max_v<E, IsFlags>;
+ constexpr auto range_size = max - min + 1;
+ static_assert(range_size > 0, "magic_enum::enum_range requires valid size.");
+ static_assert(range_size < (std::numeric_limits<std::uint16_t>::max)(), "magic_enum::enum_range requires valid size.");
+ if constexpr (cmp_less((std::numeric_limits<U>::min)(), min) && !IsFlags) {
+ static_assert(!is_valid<E, value<E, min - 1, IsFlags>(0)>(), "magic_enum::enum_range detects enum value smaller than min range size.");
+ }
+ if constexpr (cmp_less(range_size, (std::numeric_limits<U>::max)()) && !IsFlags) {
+ static_assert(!is_valid<E, value<E, min, IsFlags>(range_size + 1)>(), "magic_enum::enum_range detects enum value larger than max range size.");
+ }
+
+ return values<E, IsFlags, reflected_min_v<E, IsFlags>>(std::make_index_sequence<range_size>{});
+ }
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto values_v = values<E, IsFlags>();
+
+ template <typename E, bool IsFlags = false, typename D = std::decay_t<E>>
+ using values_t = decltype((values_v<D, IsFlags>));
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto count_v = values_v<E, IsFlags>.size();
+
+ template <typename E, bool IsFlags = false, typename U = std::underlying_type_t<E>>
+ inline constexpr auto min_v = static_cast<U>(values_v<E, IsFlags>.front());
+
+ template <typename E, bool IsFlags = false, typename U = std::underlying_type_t<E>>
+ inline constexpr auto max_v = static_cast<U>(values_v<E, IsFlags>.back());
+
+ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>
+ constexpr std::size_t range_size() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::range_size requires enum type.");
+ constexpr auto max = IsFlags ? log2(max_v<E, IsFlags>) : max_v<E, IsFlags>;
+ constexpr auto min = IsFlags ? log2(min_v<E, IsFlags>) : min_v<E, IsFlags>;
+ constexpr auto range_size = max - min + U{1};
+ static_assert(range_size > 0, "magic_enum::enum_range requires valid size.");
+ static_assert(range_size < (std::numeric_limits<std::uint16_t>::max)(), "magic_enum::enum_range requires valid size.");
+
+ return static_cast<std::size_t>(range_size);
+ }
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto range_size_v = range_size<E, IsFlags>();
+
+ template <typename E, bool IsFlags = false>
+ using index_t = std::conditional_t<range_size_v<E, IsFlags> < (std::numeric_limits<std::uint8_t>::max)(), std::uint8_t, std::uint16_t>;
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto invalid_index_v = (std::numeric_limits<index_t<E, IsFlags>>::max)();
+
+ template <typename E, bool IsFlags, std::size_t... I>
+ constexpr auto indexes(std::index_sequence<I...>) noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::indexes requires enum type.");
+ constexpr auto min = IsFlags ? log2(min_v<E, IsFlags>) : min_v<E, IsFlags>;
+ [[maybe_unused]] auto i = index_t<E, IsFlags>{0};
+
+ return std::array<decltype(i), sizeof...(I)>{{(is_valid<E, value<E, min, IsFlags>(I)>() ? i++ : invalid_index_v<E, IsFlags>)...}};
+ }
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto indexes_v = indexes<E, IsFlags>(std::make_index_sequence<range_size_v<E, IsFlags>>{});
+
+ template <typename E, bool IsFlags, std::size_t... I>
+ constexpr auto names(std::index_sequence<I...>) noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::names requires enum type.");
+
+ return std::array<string_view, sizeof...(I)>{{enum_name_v<E, values_v<E, IsFlags>[I]>...}};
+ }
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto names_v = names<E, IsFlags>(std::make_index_sequence<count_v<E, IsFlags>>{});
+
+ template <typename E, bool IsFlags = false, typename D = std::decay_t<E>>
+ using names_t = decltype((names_v<D, IsFlags>));
+
+ template <typename E, bool IsFlags, std::size_t... I>
+ constexpr auto entries(std::index_sequence<I...>) noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::entries requires enum type.");
+
+ return std::array<std::pair<E, string_view>, sizeof...(I)>{{{values_v<E, IsFlags>[I], enum_name_v<E, values_v<E, IsFlags>[I]>}...}};
+ }
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr auto entries_v = entries<E, IsFlags>(std::make_index_sequence<count_v<E, IsFlags>>{});
+
+ template <typename E, bool IsFlags = false, typename D = std::decay_t<E>>
+ using entries_t = decltype((entries_v<D, IsFlags>));
+
+ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>
+ constexpr bool is_sparse() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::is_sparse requires enum type.");
+
+ return range_size_v<E, IsFlags> != count_v<E, IsFlags>;
+ }
+
+ template <typename E, bool IsFlags = false>
+ inline constexpr bool is_sparse_v = is_sparse<E, IsFlags>();
+
+ template <typename E, typename U = std::underlying_type_t<E>>
+ constexpr std::size_t undex(U value) noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::undex requires enum type.");
+
+ if (const auto i = static_cast<std::size_t>(value - min_v<E>); value >= min_v<E> && value <= max_v<E>) {
+ if constexpr (is_sparse_v<E>) {
+ if (const auto idx = indexes_v<E>[i]; idx != invalid_index_v<E>) {
+ return idx;
+ }
+ } else {
+ return i;
+ }
+ }
+
+ return invalid_index_v<E>; // Value out of range.
+ }
+
+ template <typename E, typename U = std::underlying_type_t<E>>
+ constexpr std::size_t endex(E value) noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::endex requires enum type.");
+
+ return undex<E>(static_cast<U>(value));
+ }
+
+ template <typename E, typename U = std::underlying_type_t<E>>
+ constexpr U value_ors() noexcept {
+ static_assert(is_enum_v<E>, "magic_enum::detail::endex requires enum type.");
+
+ auto value = U{0};
+ for (std::size_t i = 0; i < count_v<E, true>; ++i) {
+ value |= static_cast<U>(values_v<E, true>[i]);
+ }
+
+ return value;
+ }
+
+ template <bool, typename T, typename R>
+ struct enable_if_enum {};
+
+ template <typename T, typename R>
+ struct enable_if_enum<true, T, R> {
+ using type = R;
+ using D = std::decay_t<T>;
+ static_assert(supported<D>::value, "magic_enum unsupported compiler (https://github.com/Neargye/magic_enum#compiler-compatibility).");
+ };
+
+ template <typename T, typename R = void>
+ using enable_if_enum_t = std::enable_if_t<std::is_enum_v<std::decay_t<T>>, R>;
+
+ template <typename T, typename Enable = std::enable_if_t<std::is_enum_v<std::decay_t<T>>>>
+ using enum_concept = T;
+
+ template <typename T, bool = std::is_enum_v<T>>
+ struct is_scoped_enum : std::false_type {};
+
+ template <typename T>
+ struct is_scoped_enum<T, true> : std::bool_constant<!std::is_convertible_v<T, std::underlying_type_t<T>>> {};
+
+ template <typename T, bool = std::is_enum_v<T>>
+ struct is_unscoped_enum : std::false_type {};
+
+ template <typename T>
+ struct is_unscoped_enum<T, true> : std::bool_constant<std::is_convertible_v<T, std::underlying_type_t<T>>> {};
+
+ template <typename T, bool = std::is_enum_v<std::decay_t<T>>>
+ struct underlying_type {};
+
+ template <typename T>
+ struct underlying_type<T, true> : std::underlying_type<std::decay_t<T>> {};
+
+ } // namespace magic_enum::detail
+
+// Checks is magic_enum supported compiler.
+ inline constexpr bool is_magic_enum_supported = detail::supported<void>::value;
+
+ template <typename T>
+ using Enum = detail::enum_concept<T>;
+
+// Checks whether T is an Unscoped enumeration type.
+// Provides the member constant value which is equal to true, if T is an [Unscoped enumeration](https://en.cppreference.com/w/cpp/language/enum#Unscoped_enumeration) type. Otherwise, value is equal to false.
+ template <typename T>
+ struct is_unscoped_enum : detail::is_unscoped_enum<T> {};
+
+ template <typename T>
+ inline constexpr bool is_unscoped_enum_v = is_unscoped_enum<T>::value;
+
+// Checks whether T is an Scoped enumeration type.
+// Provides the member constant value which is equal to true, if T is an [Scoped enumeration](https://en.cppreference.com/w/cpp/language/enum#Scoped_enumerations) type. Otherwise, value is equal to false.
+ template <typename T>
+ struct is_scoped_enum : detail::is_scoped_enum<T> {};
+
+ template <typename T>
+ inline constexpr bool is_scoped_enum_v = is_scoped_enum<T>::value;
+
+// If T is a complete enumeration type, provides a member typedef type that names the underlying type of T.
+// Otherwise, if T is not an enumeration type, there is no member type. Otherwise (T is an incomplete enumeration type), the program is ill-formed.
+ template <typename T>
+ struct underlying_type : detail::underlying_type<T> {};
+
+ template <typename T>
+ using underlying_type_t = typename underlying_type<T>::type;
+
+// Returns type name of enum.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_type_name() noexcept -> detail::enable_if_enum_t<E, string_view> {
+ using D = std::decay_t<E>;
+ constexpr string_view name = detail::type_name_v<D>;
+ static_assert(name.size() > 0, "Enum type does not have a name.");
+
+ return name;
+ }
+
+// Returns number of enum values.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_count() noexcept -> detail::enable_if_enum_t<E, std::size_t> {
+ using D = std::decay_t<E>;
+
+ return detail::count_v<D>;
+ }
+
+// Returns enum value at specified index.
+// No bounds checking is performed: the behavior is undefined if index >= number of enum values.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_value(std::size_t index) noexcept -> detail::enable_if_enum_t<E, std::decay_t<E>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");
+
+ if constexpr (detail::is_sparse_v<D>) {
+ return assert((index < detail::count_v<D>)), detail::values_v<D>[index];
+ } else {
+ return assert((index < detail::count_v<D>)), detail::value<D, detail::min_v<D>>(index);
+ }
+ }
+
+// Returns std::array with enum values, sorted by enum value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_values() noexcept -> detail::enable_if_enum_t<E, detail::values_t<E>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");
+
+ return detail::values_v<D>;
+ }
+
+// Returns name from static storage enum variable.
+// This version is much lighter on the compile times and is not restricted to the enum_range limitation.
+ template <auto V>
+ [[nodiscard]] constexpr auto enum_name() noexcept -> detail::enable_if_enum_t<decltype(V), string_view> {
+ using D = std::decay_t<decltype(V)>;
+ constexpr string_view name = detail::enum_name_v<D, V>;
+ static_assert(name.size() > 0, "Enum value does not have a name.");
+
+ return name;
+ }
+
+// Returns name from enum value.
+// If enum value does not have name or value out of range, returns empty string.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_name(E value) noexcept -> detail::enable_if_enum_t<E, string_view> {
+ using D = std::decay_t<E>;
+
+ if (const auto i = detail::endex<D>(value); i != detail::invalid_index_v<D>) {
+ return detail::names_v<D>[i];
+ }
+
+ return {}; // Invalid value or out of range.
+ }
+
+// Returns std::array with names, sorted by enum value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_names() noexcept -> detail::enable_if_enum_t<E, detail::names_t<E>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");
+
+ return detail::names_v<D>;
+ }
+
+// Returns std::array with pairs (value, name), sorted by enum value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_entries() noexcept -> detail::enable_if_enum_t<E, detail::entries_t<E>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");
+
+ return detail::entries_v<D>;
+ }
+
+// Obtains enum value from integer value.
+// Returns optional with enum value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_cast(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {
+ using D = std::decay_t<E>;
+
+ if (detail::undex<D>(value) != detail::invalid_index_v<D>) {
+ return static_cast<D>(value);
+ }
+
+ return {}; // Invalid value or out of range.
+ }
+
+// Obtains enum value from name.
+// Returns optional with enum value.
+ template <typename E, typename BinaryPredicate>
+ [[nodiscard]] constexpr auto enum_cast(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {
+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::enum_cast requires bool(char, char) invocable predicate.");
+ using D = std::decay_t<E>;
+
+ for (std::size_t i = 0; i < detail::count_v<D>; ++i) {
+ if (detail::cmp_equal(value, detail::names_v<D>[i], p)) {
+ return enum_value<D>(i);
+ }
+ }
+
+ return {}; // Invalid value or out of range.
+ }
+
+// Obtains enum value from name.
+// Returns optional with enum value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_cast(string_view value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {
+ using D = std::decay_t<E>;
+
+ return enum_cast<D>(value, detail::char_equal_to{});
+ }
+
+// Returns integer value from enum value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_integer(E value) noexcept -> detail::enable_if_enum_t<E, underlying_type_t<E>> {
+ return static_cast<underlying_type_t<E>>(value);
+ }
+
+// Obtains index in enum values from enum value.
+// Returns optional with index.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_index(E value) noexcept -> detail::enable_if_enum_t<E, optional<std::size_t>> {
+ using D = std::decay_t<E>;
+
+ if (const auto i = detail::endex<D>(value); i != detail::invalid_index_v<D>) {
+ return i;
+ }
+
+ return {}; // Invalid value or out of range.
+ }
+
+// Checks whether enum contains enumerator with such enum value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_contains(E value) noexcept -> detail::enable_if_enum_t<E, bool> {
+ using D = std::decay_t<E>;
+
+ return detail::endex<D>(value) != detail::invalid_index_v<D>;
+ }
+
+// Checks whether enum contains enumerator with such integer value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_contains(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, bool> {
+ using D = std::decay_t<E>;
+
+ return detail::undex<D>(value) != detail::invalid_index_v<D>;
+ }
+
+// Checks whether enum contains enumerator with such name.
+ template <typename E, typename BinaryPredicate>
+ [[nodiscard]] constexpr auto enum_contains(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, bool> {
+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::enum_contains requires bool(char, char) invocable predicate.");
+ using D = std::decay_t<E>;
+
+ return enum_cast<D>(value, std::move_if_noexcept(p)).has_value();
+ }
+
+// Checks whether enum contains enumerator with such name.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_contains(string_view value) noexcept -> detail::enable_if_enum_t<E, bool> {
+ using D = std::decay_t<E>;
+
+ return enum_cast<D>(value).has_value();
+ }
+
+ namespace ostream_operators {
+
+ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, E value) {
+ using D = std::decay_t<E>;
+ using U = underlying_type_t<D>;
+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED
+ if (const auto name = magic_enum::enum_name<D>(value); !name.empty()) {
+ for (const auto c : name) {
+ os.put(c);
+ }
+ return os;
+ }
+#endif
+ return (os << static_cast<U>(value));
+ }
+
+ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, optional<E> value) {
+ return value.has_value() ? (os << value.value()) : os;
+ }
+
+ } // namespace magic_enum::ostream_operators
+
+ namespace bitwise_operators {
+
+ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ constexpr E operator~(E rhs) noexcept {
+ return static_cast<E>(~static_cast<underlying_type_t<E>>(rhs));
+ }
+
+ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ constexpr E operator|(E lhs, E rhs) noexcept {
+ return static_cast<E>(static_cast<underlying_type_t<E>>(lhs) | static_cast<underlying_type_t<E>>(rhs));
+ }
+
+ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ constexpr E operator&(E lhs, E rhs) noexcept {
+ return static_cast<E>(static_cast<underlying_type_t<E>>(lhs) & static_cast<underlying_type_t<E>>(rhs));
+ }
+
+ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ constexpr E operator^(E lhs, E rhs) noexcept {
+ return static_cast<E>(static_cast<underlying_type_t<E>>(lhs) ^ static_cast<underlying_type_t<E>>(rhs));
+ }
+
+ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ constexpr E& operator|=(E& lhs, E rhs) noexcept {
+ return lhs = (lhs | rhs);
+ }
+
+ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ constexpr E& operator&=(E& lhs, E rhs) noexcept {
+ return lhs = (lhs & rhs);
+ }
+
+ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ constexpr E& operator^=(E& lhs, E rhs) noexcept {
+ return lhs = (lhs ^ rhs);
+ }
+
+ } // namespace magic_enum::bitwise_operators
+
+ namespace flags {
+
+// Returns type name of enum.
+ using magic_enum::enum_type_name;
+
+// Returns number of enum-flags values.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_count() noexcept -> detail::enable_if_enum_t<E, std::size_t> {
+ using D = std::decay_t<E>;
+
+ return detail::count_v<D, true>;
+ }
+
+// Returns enum-flags value at specified index.
+// No bounds checking is performed: the behavior is undefined if index >= number of enum-flags values.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_value(std::size_t index) noexcept -> detail::enable_if_enum_t<E, std::decay_t<E>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");
+
+ if constexpr (detail::is_sparse_v<D, true>) {
+ return assert((index < detail::count_v<D, true>)), detail::values_v<D, true>[index];
+ } else {
+ constexpr auto min = detail::log2(detail::min_v<D, true>);
+
+ return assert((index < detail::count_v<D, true>)), detail::value<D, min, true>(index);
+ }
+ }
+
+// Returns std::array with enum-flags values, sorted by enum-flags value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_values() noexcept -> detail::enable_if_enum_t<E, detail::values_t<E, true>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");
+
+ return detail::values_v<D, true>;
+ }
+
+// Returns name from enum-flags value.
+// If enum-flags value does not have name or value out of range, returns empty string.
+ template <typename E>
+ [[nodiscard]] auto enum_name(E value) -> detail::enable_if_enum_t<E, string> {
+ using D = std::decay_t<E>;
+ using U = underlying_type_t<D>;
+
+ string name;
+ auto check_value = U{0};
+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {
+ if (const auto v = static_cast<U>(enum_value<D>(i)); (static_cast<U>(value) & v) != 0) {
+ check_value |= v;
+ const auto n = detail::names_v<D, true>[i];
+ if (!name.empty()) {
+ name.append(1, '|');
+ }
+ name.append(n.data(), n.size());
+ }
+ }
+
+ if (check_value != 0 && check_value == static_cast<U>(value)) {
+ return name;
+ }
+
+ return {}; // Invalid value or out of range.
+ }
+
+// Returns std::array with string names, sorted by enum-flags value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_names() noexcept -> detail::enable_if_enum_t<E, detail::names_t<E, true>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");
+
+ return detail::names_v<D, true>;
+ }
+
+// Returns std::array with pairs (value, name), sorted by enum-flags value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_entries() noexcept -> detail::enable_if_enum_t<E, detail::entries_t<E, true>> {
+ using D = std::decay_t<E>;
+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");
+
+ return detail::entries_v<D, true>;
+ }
+
+// Obtains enum-flags value from integer value.
+// Returns optional with enum-flags value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_cast(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {
+ using D = std::decay_t<E>;
+ using U = underlying_type_t<D>;
+
+ if constexpr (detail::is_sparse_v<D, true>) {
+ auto check_value = U{0};
+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {
+ if (const auto v = static_cast<U>(enum_value<D>(i)); (value & v) != 0) {
+ check_value |= v;
+ }
+ }
+
+ if (check_value != 0 && check_value == value) {
+ return static_cast<D>(value);
+ }
+ } else {
+ constexpr auto min = detail::min_v<D, true>;
+ constexpr auto max = detail::value_ors<D>();
+
+ if (value >= min && value <= max) {
+ return static_cast<D>(value);
+ }
+ }
+
+ return {}; // Invalid value or out of range.
+ }
+
+// Obtains enum-flags value from name.
+// Returns optional with enum-flags value.
+ template <typename E, typename BinaryPredicate>
+ [[nodiscard]] constexpr auto enum_cast(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {
+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::flags::enum_cast requires bool(char, char) invocable predicate.");
+ using D = std::decay_t<E>;
+ using U = underlying_type_t<D>;
+
+ auto result = U{0};
+ while (!value.empty()) {
+ const auto d = detail::find(value, '|');
+ const auto s = (d == string_view::npos) ? value : value.substr(0, d);
+ auto f = U{0};
+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {
+ if (detail::cmp_equal(s, detail::names_v<D, true>[i], p)) {
+ f = static_cast<U>(enum_value<D>(i));
+ result |= f;
+ break;
+ }
+ }
+ if (f == U{0}) {
+ return {}; // Invalid value or out of range.
+ }
+ value.remove_prefix((d == string_view::npos) ? value.size() : d + 1);
+ }
+
+ if (result == U{0}) {
+ return {}; // Invalid value or out of range.
+ } else {
+ return static_cast<D>(result);
+ }
+ }
+
+// Obtains enum-flags value from name.
+// Returns optional with enum-flags value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_cast(string_view value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {
+ using D = std::decay_t<E>;
+
+ return enum_cast<D>(value, detail::char_equal_to{});
+ }
+
+// Returns integer value from enum value.
+ using magic_enum::enum_integer;
+
+// Obtains index in enum-flags values from enum-flags value.
+// Returns optional with index.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_index(E value) noexcept -> detail::enable_if_enum_t<E, optional<std::size_t>> {
+ using D = std::decay_t<E>;
+ using U = underlying_type_t<D>;
+
+ if (detail::is_pow2(static_cast<U>(value))) {
+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {
+ if (enum_value<D>(i) == value) {
+ return i;
+ }
+ }
+ }
+
+ return {}; // Invalid value or out of range.
+ }
+
+// Checks whether enum-flags contains enumerator with such enum-flags value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_contains(E value) noexcept -> detail::enable_if_enum_t<E, bool> {
+ using D = std::decay_t<E>;
+ using U = underlying_type_t<D>;
+
+ return enum_cast<D>(static_cast<U>(value)).has_value();
+ }
+
+// Checks whether enum-flags contains enumerator with such integer value.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_contains(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, bool> {
+ using D = std::decay_t<E>;
+
+ return enum_cast<D>(value).has_value();
+ }
+
+// Checks whether enum-flags contains enumerator with such name.
+ template <typename E, typename BinaryPredicate>
+ [[nodiscard]] constexpr auto enum_contains(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, bool> {
+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::flags::enum_contains requires bool(char, char) invocable predicate.");
+ using D = std::decay_t<E>;
+
+ return enum_cast<D>(value, std::move_if_noexcept(p)).has_value();
+ }
+
+// Checks whether enum-flags contains enumerator with such name.
+ template <typename E>
+ [[nodiscard]] constexpr auto enum_contains(string_view value) noexcept -> detail::enable_if_enum_t<E, bool> {
+ using D = std::decay_t<E>;
+
+ return enum_cast<D>(value).has_value();
+ }
+
+ } // namespace magic_enum::flags
+
+ namespace flags::ostream_operators {
+
+ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, E value) {
+ using D = std::decay_t<E>;
+ using U = underlying_type_t<D>;
+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED
+ if (const auto name = magic_enum::flags::enum_name<D>(value); !name.empty()) {
+ for (const auto c : name) {
+ os.put(c);
+ }
+ return os;
+ }
+#endif
+ return (os << static_cast<U>(value));
+ }
+
+ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>
+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, optional<E> value) {
+ return value.has_value() ? (os << value.value()) : os;
+ }
+
+ } // namespace magic_enum::flags::ostream_operators
+
+ namespace flags::bitwise_operators {
+
+ using namespace magic_enum::bitwise_operators;
+
+ } // namespace magic_enum::flags::bitwise_operators
+
+} // namespace magic_enum
+
+#if defined(__clang__)
+# pragma clang diagnostic pop
+#elif defined(__GNUC__)
+# pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+# pragma warning(pop)
+#endif
+
+#endif // NEARGYE_MAGIC_ENUM_HPP
diff --git a/include/string_utils.h b/include/string_utils.h
index 7f641527..adbec688 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -294,4 +294,8 @@ struct StringUtils {
static std::string hash_sha256(const std::string& str);
//static size_t unicode_length(const std::string& bytes);
+
+ static bool begins_with(const std::string& str, const std::string& prefix) {
+ return str.rfind(prefix, 0) == 0;
+ }
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 376f20a5..91b29bed 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -160,10 +160,11 @@ nlohmann::json Collection::get_summary_json() const {
}
Option<nlohmann::json> Collection::add(const std::string & json_str,
- const index_operation_t& operation, const std::string& id) {
+ const index_operation_t& operation, const std::string& id,
+ const DIRTY_VALUES& dirty_values) {
nlohmann::json document;
std::vector<std::string> json_lines = {json_str};
- const nlohmann::json& res = add_many(json_lines, document, operation, id);
+ const nlohmann::json& res = add_many(json_lines, document, operation, id, dirty_values);
if(!res["success"].get<bool>()) {
nlohmann::json res_doc;
@@ -203,7 +204,8 @@ void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::json
}
nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmann::json& document,
- const index_operation_t& operation, const std::string& id) {
+ const index_operation_t& operation, const std::string& id,
+ const DIRTY_VALUES& dirty_values) {
//LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();
std::vector<std::vector<index_record>> iter_batch;
@@ -220,7 +222,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
Option<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, id);
const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get().seq_id : 0;
- index_record record(i, seq_id, document, operation);
+ index_record record(i, seq_id, document, operation, dirty_values);
// NOTE: we overwrite the input json_lines with result to avoid memory pressure
@@ -237,62 +239,9 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
// if `index_all_fields` is enabled, we will have to update schema first before indexing
if(index_all_fields) {
- size_t old_field_count = fields.size();
- for(const auto& kv: document.items()) {
- if (search_schema.count(kv.key()) == 0) {
- std::string field_type;
- bool parseable = field::get_type(kv.value(), field_type);
- if (parseable) {
- // this ensures that we can handle the same field having different types in different docs
- const std::string fname = kv.key();// + "_" + field_type;
- field new_field(fname, field_type, false);
- search_schema.emplace(fname, new_field);
- fields.emplace_back(new_field);
- }
- }
- }
-
- if(fields.size() != old_field_count) {
- // we should persist changes to fields in store
- std::string coll_meta_json;
- StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);
-
- if(status != StoreStatus::FOUND) {
- record.index_failure(500, "Could not fetch collection meta from store.");
- continue;
- }
-
- nlohmann::json collection_meta;
-
- try {
- collection_meta = nlohmann::json::parse(coll_meta_json);
- bool found_default_sorting_field = false;
- nlohmann::json fields_json = nlohmann::json::array();;
-
- Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json,
- found_default_sorting_field);
-
- if(!fields_json_op.ok()) {
- record.index_failure(fields_json_op.code(), fields_json_op.error());
- continue;
- }
-
- collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
- bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());
-
- if(!persisted) {
- record.index_failure(500, "Could not persist collection meta to store.");
- continue;
- }
-
- for(auto index: indices) {
- index->refresh_search_schema(search_schema);
- }
-
- } catch(...) {
- record.index_failure(500, "Unable to parse collection meta.");
- continue;
- }
+ Option<bool> schema_change_op = check_and_update_schema(document);
+ if(!schema_change_op.ok()) {
+ record.index_failure(schema_change_op.code(), schema_change_op.error());
}
}
}
@@ -353,7 +302,7 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch
if(!write_ok) {
// we will attempt to reindex the old doc on a best-effort basis
remove_document(index_record.new_doc, index_record.seq_id, false);
- index_in_memory(index_record.old_doc, index_record.seq_id, false);
+ index_in_memory(index_record.old_doc, index_record.seq_id, false, index_record.dirty_values);
index_record.index_failure(500, "Could not write to on-disk storage.");
} else {
num_indexed++;
@@ -398,13 +347,15 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch
}
}
-Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id, bool is_update) {
+Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t seq_id,
+ bool is_update, const DIRTY_VALUES& dirty_values) {
std::unique_lock lock(mutex);
if(!is_update) {
- // for update, validation should be done prior
+ // for non-update, validation should be done prior
Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,
- search_schema, facet_schema, is_update);
+ search_schema, facet_schema, is_update,
+ index_all_fields, dirty_values);
if(!validation_op.ok()) {
return validation_op;
@@ -433,7 +384,7 @@ size_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> &
CollectionManager::get_instance().get_thread_pool()->enqueue(
[index, index_id, &num_indexed_vec, &iter_batch, this, &m_process, &num_processed, &cv_process]() {
size_t num_indexed = Index::batch_memory_index(index, std::ref(iter_batch[index_id]), default_sorting_field,
- search_schema, facet_schema);
+ search_schema, facet_schema, index_all_fields);
std::unique_lock<std::mutex> lock(m_process);
num_indexed_vec[index_id] = num_indexed;
num_processed++;
@@ -2277,6 +2228,71 @@ spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {
return synonym_definitions;
}
+Option<bool> Collection::check_and_update_schema(nlohmann::json& document) {
+ std::unique_lock lock(mutex);
+
+ std::vector<field> new_fields;
+ for(const auto& kv: document.items()) {
+ // we will not index the special "id" key
+ if (search_schema.count(kv.key()) == 0 && kv.key() != "id") {
+ std::string field_type;
+ bool parseable = field::get_type(kv.value(), field_type);
+ if (parseable) {
+ const std::string& fname = kv.key();
+ field new_field(fname, field_type, false);
+ search_schema.emplace(fname, new_field);
+ fields.emplace_back(new_field);
+ new_fields.emplace_back(new_field);
+
+ if(new_field.is_sortable()) {
+ sort_schema.emplace(new_field.name, new_field);
+ }
+ }
+ }
+ }
+
+ if(!new_fields.empty()) {
+ // we should persist changes to fields in store
+ std::string coll_meta_json;
+ StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);
+
+ if(status != StoreStatus::FOUND) {
+ return Option<bool>(500, "Could not fetch collection meta from store.");
+ }
+
+ nlohmann::json collection_meta;
+
+ try {
+ collection_meta = nlohmann::json::parse(coll_meta_json);
+ bool found_default_sorting_field = false;
+ nlohmann::json fields_json = nlohmann::json::array();;
+
+ Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json,
+ found_default_sorting_field);
+
+ if(!fields_json_op.ok()) {
+ return Option<bool>(fields_json_op.code(), fields_json_op.error());
+ }
+
+ collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
+ bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());
+
+ if(!persisted) {
+ return Option<bool>(500, "Could not persist collection meta to store.");
+ }
+
+ for(auto index: indices) {
+ index->refresh_schemas(new_fields);
+ }
+
+ } catch(...) {
+ return Option<bool>(500, "Unable to parse collection meta.");
+ }
+ }
+
+ return Option<bool>(true);
+}
+
std::vector<Index *> Collection::init_indices() {
std::vector<Index*> index_list;
@@ -2287,7 +2303,7 @@ std::vector<Index *> Collection::init_indices() {
facet_schema.emplace(field.name, field);
}
- if(field.is_single_integer() || field.is_single_float() || field.is_single_bool() || field.is_geopoint()) {
+ if(field.is_sortable()) {
sort_schema.emplace(field.name, field);
}
}
@@ -2299,3 +2315,17 @@ std::vector<Index *> Collection::init_indices() {
return index_list;
}
+
+DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) const {
+ StringUtils::toupper(dirty_values);
+ auto dirty_values_op = magic_enum::enum_cast<DIRTY_VALUES>(dirty_values);
+ DIRTY_VALUES dirty_values_action;
+
+ if(dirty_values_op.has_value()) {
+ dirty_values_action = dirty_values_op.value();
+ } else {
+ dirty_values_action = index_all_fields ? DIRTY_VALUES::COERCE_OR_IGNORE : DIRTY_VALUES::REJECT;
+ }
+
+ return dirty_values_action;
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index b9b288d6..ba6e9447 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -189,6 +189,9 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
auto begin = std::chrono::high_resolution_clock::now();
+ std::string default_dirty_values = "";
+ auto dirty_values = collection->parse_dirty_values_option(default_dirty_values);
+
while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
num_found_docs++;
const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());
@@ -203,7 +206,8 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
}
num_valid_docs++;
- iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE));
+
+ iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));
// Peek and check for last record right here so that we handle batched indexing correctly
// Without doing this, the "last batch" would have to be indexed outside the loop.
diff --git a/src/core_api.cpp b/src/core_api.cpp
index b2070f01..d37da108 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -100,8 +100,8 @@ bool post_create_collection(http_req & req, http_res & res) {
req_json[NUM_MEMORY_SHARDS] = CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;
}
- if(req_json.count("fields") == 0) {
- res.set_400("Parameter `fields` is required.");
+ if(req_json.count("fields") == 0 && req_json.count(INDEX_ALL_FIELDS) == 0) {
+ res.set_400("Parameter `fields` or `index_all_fields` is required.");
return false;
}
@@ -436,6 +436,7 @@ bool post_import_documents(http_req& req, http_res& res) {
//LOG(INFO) << "req.first_chunk=" << req.first_chunk_aggregate << ", last_chunk=" << req.last_chunk_aggregate;
const char *BATCH_SIZE = "batch_size";
const char *ACTION = "action";
+ const char *DIRTY_VALUES = "dirty_values";
if(req.params.count(BATCH_SIZE) == 0) {
req.params[BATCH_SIZE] = "40";
@@ -445,6 +446,10 @@ bool post_import_documents(http_req& req, http_res& res) {
req.params[ACTION] = "create";
}
+ if(req.params.count(DIRTY_VALUES) == 0) {
+ req.params[DIRTY_VALUES] = ""; // set it empty as default will depend on `index_all_fields`
+ }
+
if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {
res.final = true;
res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
@@ -539,7 +544,10 @@ bool post_import_documents(http_req& req, http_res& res) {
if(!single_partial_record_body) {
nlohmann::json document;
- nlohmann::json json_res = collection->add_many(json_lines, document, operation);
+
+ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES]);
+ nlohmann::json json_res = collection->add_many(json_lines, document, operation, "",
+ dirty_values);
//const std::string& import_summary_json = json_res.dump();
//response_stream << import_summary_json << "\n";
@@ -570,6 +578,8 @@ bool post_import_documents(http_req& req, http_res& res) {
bool post_add_document(http_req & req, http_res & res) {
const char *ACTION = "action";
+ const char *DIRTY_VALUES = "dirty_values";
+
if(req.params.count(ACTION) == 0) {
req.params[ACTION] = "create";
}
@@ -579,6 +589,10 @@ bool post_add_document(http_req & req, http_res & res) {
return false;
}
+ if(req.params.count(DIRTY_VALUES) == 0) {
+ req.params[DIRTY_VALUES] = ""; // set it empty as default will depend on `index_all_fields`
+ }
+
CollectionManager & collectionManager = CollectionManager::get_instance();
auto collection = collectionManager.get_collection(req.params["collection"]);
@@ -588,7 +602,9 @@ bool post_add_document(http_req & req, http_res & res) {
}
const index_operation_t operation = get_index_operation(req.params[ACTION]);
- Option<nlohmann::json> inserted_doc_op = collection->add(req.body, operation);
+ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES]);
+
+ Option<nlohmann::json> inserted_doc_op = collection->add(req.body, operation, "", dirty_values);
if(!inserted_doc_op.ok()) {
res.set(inserted_doc_op.code(), inserted_doc_op.error());
diff --git a/src/index.cpp b/src/index.cpp
index 2333fee7..ecacf9df 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -122,7 +122,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
for(const auto& field_pair: search_schema) {
const std::string & field_name = field_pair.first;
- if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {
+ if(document.count(field_name) == 0) {
continue;
}
@@ -258,11 +258,13 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
return Option<>(201);
}
-Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,
+Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
const std::map<std::string, field> & facet_schema,
- bool is_update) {
+ bool is_update,
+ bool index_all_fields,
+ const DIRTY_VALUES& dirty_values) {
bool has_default_sort_field = (document.count(default_sorting_field) != 0);
@@ -284,7 +286,7 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
for(const auto& field_pair: search_schema) {
const std::string& field_name = field_pair.first;
- if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {
+ if((field_pair.second.optional || is_update || index_all_fields) && document.count(field_name) == 0) {
continue;
}
@@ -293,79 +295,83 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
"but is not found in the document.");
}
- if(field_pair.second.type == field_types::STRING) {
- if(!document[field_name].is_string()) {
- return Option<>(400, "Field `" + field_name + "` must be a string.");
+ if(field_pair.second.type == field_types::STRING && !document[field_name].is_string()) {
+ Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, -1);
+ if(!coerce_op.ok()) {
+ return coerce_op;
}
} else if(field_pair.second.type == field_types::INT32) {
if(!document[field_name].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int32.");
+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, -1);
+ if(!coerce_op.ok()) {
+ return coerce_op;
+ }
}
if(document[field_name].get<int64_t>() > INT32_MAX) {
- return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
- }
- } else if(field_pair.second.type == field_types::INT64) {
- if(!document[field_name].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int64.");
- }
- } else if(field_pair.second.type == field_types::FLOAT) {
- if(!document[field_name].is_number()) { // allows integer to be passed to a float field
- return Option<>(400, "Field `" + field_name + "` must be a float.");
- }
- } else if(field_pair.second.type == field_types::BOOL) {
- if(!document[field_name].is_boolean()) {
- return Option<>(400, "Field `" + field_name + "` must be a bool.");
- }
- } else if(field_pair.second.type == field_types::GEOPOINT) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");
- }
- if(document[field_name].size() != 2) {
- return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");
- }
- if(!document[field_name][0].is_number() || !document[field_name][1].is_number()) {
- return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");
- }
- } else if(field_pair.second.type == field_types::STRING_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be a string array.");
- }
- if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
- return Option<>(400, "Field `" + field_name + "` must be a string array.");
- }
- } else if(field_pair.second.type == field_types::INT32_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
+ if(dirty_values == DIRTY_VALUES::IGNORE) {
+ document.erase(field_name);
+ continue;
+ } else {
+ return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
+ }
}
-
- if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
+ } else if(field_pair.second.type == field_types::INT64 && !document[field_name].is_number_integer()) {
+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, false);
+ if(!coerce_op.ok()) {
+ return coerce_op;
}
- } else if(field_pair.second.type == field_types::INT64_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
+ } else if(field_pair.second.type == field_types::FLOAT && !document[field_name].is_number()) {
+ // using `is_number` allows integer to be passed to a float field
+ Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, false);
+ if(!coerce_op.ok()) {
+ return coerce_op;
}
-
- if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
- return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
+ } else if(field_pair.second.type == field_types::BOOL && !document[field_name].is_boolean()) {
+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, false);
+ if(!coerce_op.ok()) {
+ return coerce_op;
}
- } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
+ } else if(field_pair.second.is_array()) {
if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be a float array.");
+ if(dirty_values == DIRTY_VALUES::IGNORE || dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ document.erase(field_name);
+ continue;
+ } else {
+ return Option<>(400, "Field `" + field_name + "` must be an array.");
+ }
}
- if(document[field_name].size() > 0 && !document[field_name][0].is_number()) {
- // allows integer to be passed to a float array field
- return Option<>(400, "Field `" + field_name + "` must be a float array.");
- }
- } else if(field_pair.second.type == field_types::BOOL_ARRAY) {
- if(!document[field_name].is_array()) {
- return Option<>(400, "Field `" + field_name + "` must be a bool array.");
- }
+ for(size_t arr_index=0; arr_index < document[field_name].size(); arr_index++) {
+ const auto& item = document[field_name][arr_index];
- if(document[field_name].size() > 0 && !document[field_name][0].is_boolean()) {
- return Option<>(400, "Field `" + field_name + "` must be a bool array.");
+ if (field_pair.second.type == field_types::STRING_ARRAY && !item.is_string()) {
+ Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, arr_index);
+ if (!coerce_op.ok()) {
+ return coerce_op;
+ }
+ } else if (field_pair.second.type == field_types::INT32_ARRAY && !item.is_number_integer()) {
+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, arr_index);
+ if (!coerce_op.ok()) {
+ return coerce_op;
+ }
+ } else if (field_pair.second.type == field_types::INT64_ARRAY && !item.is_number_integer()) {
+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, true);
+ if (!coerce_op.ok()) {
+ return coerce_op;
+ }
+ } else if (field_pair.second.type == field_types::FLOAT_ARRAY && !item.is_number()) {
+ // we check for `is_number` to allow whole numbers to be passed into float fields
+ Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, true);
+ if (!coerce_op.ok()) {
+ return coerce_op;
+ }
+ } else if (field_pair.second.type == field_types::BOOL_ARRAY && !item.is_boolean()) {
+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, true);
+ if (!coerce_op.ok()) {
+ return coerce_op;
+ }
+ }
}
}
}
@@ -451,7 +457,8 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do
size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,
const std::string & default_sorting_field,
const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema) {
+ const std::map<std::string, field> & facet_schema,
+ bool index_all_fields) {
size_t num_indexed = 0;
@@ -464,7 +471,10 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_
if(index_rec.operation != DELETE) {
Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,
default_sorting_field,
- search_schema, facet_schema, index_rec.is_update);
+ search_schema, facet_schema,
+ index_rec.is_update,
+ index_all_fields,
+ index_rec.dirty_values);
if(!validation_op.ok()) {
index_rec.index_failure(validation_op.code(), validation_op.error());
@@ -1985,7 +1995,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
}
- const int64_t default_score = 0;
+ const int64_t default_score = INT64_MIN;
int64_t scores[3] = {0};
size_t match_score_index = 0;
@@ -2001,7 +2011,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
if (sort_order[0] == -1) {
scores[0] = -scores[0];
}
- scores[1] = 0;
}
if(sort_fields.size() > 1) {
@@ -2015,7 +2024,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
if (sort_order[1] == -1) {
scores[1] = -scores[1];
}
- scores[2] = 0;
}
if(sort_fields.size() > 2) {
@@ -2337,19 +2345,258 @@ const spp::sparse_hash_map<std::string, num_tree_t*>& Index::_get_numerical_inde
return numerical_index;
}
-void Index::refresh_search_schema(const std::unordered_map<std::string, field>& src_search_schema) {
- for(const auto & fname_field: src_search_schema) {
- search_schema.emplace(fname_field.first, fname_field.second);
+void Index::refresh_schemas(const std::vector<field>& new_fields) {
+ std::unique_lock lock(mutex);
+
+ for(const auto & new_field: new_fields) {
+ search_schema.emplace(new_field.name, new_field);
- if(search_index.count(fname_field.first) == 0) {
- if(fname_field.second.is_string()) {
+ if(search_index.count(new_field.name) == 0) {
+ if(new_field.is_string()) {
art_tree *t = new art_tree;
art_tree_init(t);
- search_index.emplace(fname_field.first, t);
+ search_index.emplace(new_field.name, t);
} else {
num_tree_t* num_tree = new num_tree_t;
- numerical_index.emplace(fname_field.first, num_tree);
+ numerical_index.emplace(new_field.name, num_tree);
+ }
+ }
+
+ if(sort_index.count(new_field.name) == 0) {
+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
+ sort_index.emplace(new_field.name, doc_to_score);
+ }
+ }
+}
+
+Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
+ std::string suffix = array_index != -1 ? "an array of" : "a";
+
+ if(dirty_values == DIRTY_VALUES::REJECT) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
+ }
+
+ if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ return Option<uint32_t>(200);
+ }
+
+ // we will try to coerce the value to a string
+
+ if (document[field_name].is_number_integer()) {
+ document[field_name] = std::to_string((int64_t)document[field_name]);
+ }
+
+ else if(document[field_name].is_number_float()) {
+ document[field_name] = std::to_string((float)document[field_name]);
+ }
+
+ else if(document[field_name].is_boolean()) {
+ document[field_name] = document[field_name] == true ? "true" : "false";
+ }
+
+ else {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
}
+ } else {
+ // DIRTY_FIELD_COERCE_REJECT
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
}
+
+ return Option<>(200);
+}
+
+Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
+ std::string suffix = array_index != -1 ? "an array of" : "an";
+
+ if(dirty_values == DIRTY_VALUES::REJECT) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
+ }
+
+ if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ return Option<uint32_t>(200);
+ }
+
+ // try to value coerce into an integer
+
+ if(document[field_name].is_number_float()) {
+ document[field_name] = static_cast<int32_t>(document[field_name].get<float>());
+ }
+
+ else if(document[field_name].is_boolean()) {
+ document[field_name] = document[field_name] == true ? 1 : 0;
+ }
+
+ else if(document[field_name].is_string() && StringUtils::is_int32_t(document[field_name])) {
+ document[field_name] = std::atol(document[field_name].get<std::string>().c_str());
+ }
+
+ else {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ document.erase(field_name);
+ } else {
+ // DIRTY_FIELD_COERCE_REJECT
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
+ }
+ }
+
+ return Option<uint32_t>(200);
+}
+
+Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
+ std::string suffix = array_index != -1 ? "an array of" : "an";
+
+ if(dirty_values == DIRTY_VALUES::REJECT) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
+ }
+
+ if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ return Option<uint32_t>(200);
+ }
+
+ // try to value coerce into an integer
+
+ if(document[field_name].is_number_float()) {
+ document[field_name] = static_cast<int64_t>(document[field_name].get<float>());
+ }
+
+ else if(document[field_name].is_boolean()) {
+ document[field_name] = document[field_name] == true ? 1 : 0;
+ }
+
+ else if(document[field_name].is_string() && StringUtils::is_int64_t(document[field_name])) {
+ document[field_name] = std::atoll(document[field_name].get<std::string>().c_str());
+ }
+
+ else {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ } else {
+ // DIRTY_FIELD_COERCE_REJECT
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
+ }
+ }
+
+ return Option<uint32_t>(200);
+}
+
+Option<uint32_t>
+Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
+ std::string suffix = array_index != -1 ? "a array of" : "a";
+ if(dirty_values == DIRTY_VALUES::REJECT) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
+ }
+
+ if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ return Option<uint32_t>(200);
+ }
+
+ // try to value coerce into a bool
+ if (document[field_name].is_number_integer() &&
+ (document[field_name].get<int64_t>() == 1 || document[field_name].get<int64_t>() == 0)) {
+ document[field_name] = document[field_name].get<int64_t>() == 1;
+ }
+
+ else if(document[field_name].is_string()) {
+ std::string str_val = document[field_name].get<std::string>();
+ StringUtils::tolowercase(str_val);
+ if(str_val == "true") {
+ document[field_name] = true;
+ return Option<uint32_t>(200);
+ } else if(str_val == "false") {
+ document[field_name] = false;
+ return Option<uint32_t>(200);
+ }
+ }
+
+ else {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ } else {
+ // DIRTY_FIELD_COERCE_REJECT
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
+ }
+ }
+
+ return Option<uint32_t>(200);
+}
+
+Option<uint32_t>
+Index::coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,
+ const std::string &field_name, const int array_index) {
+ std::string suffix = array_index != -1 ? "a array of" : "a";
+
+ if(dirty_values == DIRTY_VALUES::REJECT) {
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
+ }
+
+ if(dirty_values == DIRTY_VALUES::IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ return Option<uint32_t>(200);
+ }
+
+ // try to value coerce into a float
+
+ if(document[field_name].is_string() && StringUtils::is_float(document[field_name])) {
+ document[field_name] = std::atof(document[field_name].get<std::string>().c_str());
+ }
+
+ else if(document[field_name].is_boolean()) {
+ document[field_name] = document[field_name] == true ? 1.0 : 0.0;
+ }
+
+ else {
+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {
+ if(array_index == -1) {
+ document.erase(field_name);
+ } else {
+ document[field_name].erase(document[field_name].begin() + array_index);
+ }
+ } else {
+ // DIRTY_FIELD_COERCE_REJECT
+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
+ }
+ }
+
+ return Option<uint32_t>(200);
}
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 7f5a0a41..a5e34765 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -44,7 +44,7 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, true).get();
}
@@ -81,12 +81,62 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {
ASSERT_STREQ("7", results["hits"][2]["document"]["id"].get<std::string>().c_str());
// reject field with a different type than already inferred type
+ // default for `index_all_fields` is `DIRTY_FIELD_COERCE_IGNORE`
+
+ // unable to coerce
auto doc_json = R"({"cast":"William Barnes","points":63,"starring":"Will Ferrell",
"starring_facet":"Will Ferrell","title":"Anchorman 2: The Legend Continues"})";
Option<nlohmann::json> add_op = coll1->add(doc_json);
ASSERT_FALSE(add_op.ok());
- ASSERT_STREQ("Field `cast` must be a string array.", add_op.error().c_str());
+ ASSERT_STREQ("Field `cast` must be an array.", add_op.error().c_str());
+
+ // coerce integer to string
+ doc_json = R"({"cast": ["William Barnes"],"points": 63, "starring":"Will Ferrell",
+ "starring_facet":"Will Ferrell","title": 300})";
+
+ add_op = coll1->add(doc_json);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("300", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("300", results["hits"][0]["document"]["title"].get<std::string>().c_str());
+
+ // with dirty values set to `DIRTY_FIELD_COERCE_IGNORE`
+ // `cast` field should not be indexed into store
+ doc_json = R"({"cast":"William Barnes","points":63,"starring":"Will Ferrell",
+ "starring_facet":"Will Ferrell","title":"With bad cast field."})";
+
+ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_IGNORE);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("With bad cast field", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("With bad cast field.", results["hits"][0]["document"]["title"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["hits"][0]["document"].count("cast"));
+
+ // with dirty values set to `DIRTY_FIELD_IGNORE`
+ // no coercion should happen, `title` field will just be ignored, but record indexed
+ doc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell",
+ "starring_facet":"Will Ferrell","title": 1200 })";
+
+ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::IGNORE);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("1200", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = coll1->search("Jeremy Livingston", {"cast"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(0, results["hits"][0]["document"].count("title"));
+
+ // with dirty values set to `DIRTY_FIELD_REJECT`
+ doc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell",
+ "starring_facet":"Will Ferrell","title": 1200 })";
+
+ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::REJECT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_STREQ("Field `title` must be a string.", add_op.error().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 6b0eb75f..2b49a754 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -278,6 +278,79 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
ASSERT_EQ(4, results["hits"].size());
}
+TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {
+ Collection *coll1;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/optional_fields.jsonl");
+ std::vector<field> fields = {
+ field("max", field_types::INT32, false)
+ };
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "max", 0, true).get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ nlohmann::json document = nlohmann::json::parse(json_line);
+ Option<nlohmann::json> add_op = coll1->add(document.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ infile.close();
+
+ ASSERT_EQ(1, coll1->get_collection_id());
+ ASSERT_EQ(3, coll1->get_sort_fields().size());
+
+ std::unordered_map<std::string, field> schema = collection1->get_schema();
+
+ // create a new collection manager to ensure that it restores the records from the disk backed store
+ CollectionManager & collectionManager2 = CollectionManager::get_instance();
+ collectionManager2.init(store, 1.0, "auth_key");
+ auto load_op = collectionManager2.load();
+
+ if(!load_op.ok()) {
+ LOG(ERROR) << load_op.error();
+ }
+
+ ASSERT_TRUE(load_op.ok());
+
+ auto restored_coll = collectionManager2.get_collection("coll1").get();
+ ASSERT_NE(nullptr, restored_coll);
+
+ std::vector<std::string> facet_fields_expected = {};
+ auto restored_schema = restored_coll->get_schema();
+
+ ASSERT_EQ(1, restored_coll->get_collection_id());
+ ASSERT_EQ(6, restored_coll->get_next_seq_id());
+ ASSERT_EQ(facet_fields_expected, restored_coll->get_facet_fields());
+ ASSERT_EQ(3, restored_coll->get_sort_fields().size());
+ ASSERT_EQ("is_valid", restored_coll->get_sort_fields()[0].name);
+ ASSERT_EQ("average", restored_coll->get_sort_fields()[1].name);
+ ASSERT_EQ("max", restored_coll->get_sort_fields()[2].name);
+
+ // ensures that the "id" field is not added to the schema
+ ASSERT_EQ(6, restored_schema.size());
+
+ ASSERT_EQ("max", restored_coll->get_default_sorting_field());
+
+ ASSERT_EQ(1, restored_schema.count("title"));
+ ASSERT_EQ(1, restored_schema.count("max"));
+ ASSERT_EQ(1, restored_schema.count("description"));
+ ASSERT_EQ(1, restored_schema.count("scores"));
+ ASSERT_EQ(1, restored_schema.count("average"));
+ ASSERT_EQ(1, restored_schema.count("is_valid"));
+
+ for(const auto& kv: restored_schema) {
+ ASSERT_FALSE(kv.second.optional);
+ }
+
+ collectionManager.drop_collection("coll1");
+ collectionManager2.drop_collection("coll1");
+}
+
TEST_F(CollectionManagerTest, DropCollectionCleanly) {
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::string json_line;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index bc9ecbba..b66c7f47 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1339,7 +1339,7 @@ TEST_F(CollectionTest, ImportDocuments) {
"{\"title\": \"Test4\", \"points\": 55, "
"\"cast\": [\"Tom Skerritt\"] }"};
- import_response = coll_mul_fields->add_many(more_records, document);
+ import_response = coll_mul_fields->add_many(more_records, document, CREATE, "", DIRTY_VALUES::REJECT);
ASSERT_FALSE(import_response["success"].get<bool>());
ASSERT_EQ(2, import_response["num_imported"].get<int>());
@@ -1621,7 +1621,19 @@ TEST_F(CollectionTest, IndexingWithBadData) {
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22, \"average\": 78}";
const Option<nlohmann::json> & bad_facet_field_op = sample_collection->add(doc_str);
ASSERT_FALSE(bad_facet_field_op.ok());
- ASSERT_STREQ("Field `tags` must be a string array.", bad_facet_field_op.error().c_str());
+ ASSERT_STREQ("Field `tags` must be an array.", bad_facet_field_op.error().c_str());
+
+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";
+ const Option<nlohmann::json> & bad_array_field_op = sample_collection->add(doc_str);
+ ASSERT_FALSE(bad_array_field_op.ok());
+ ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_op.error().c_str());
+
+ // with coercion should work
+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";
+ const Option<nlohmann::json> &bad_array_field_coercion_op = sample_collection->add(doc_str, CREATE, "",
+ DIRTY_VALUES::COERCE_OR_REJECT);
+ ASSERT_FALSE(bad_array_field_coercion_op.ok());
+ ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_coercion_op.error().c_str());
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";
const Option<nlohmann::json> & empty_facet_field_op = sample_collection->add(doc_str);
@@ -1639,7 +1651,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
bad_default_sorting_field_op3.error().c_str());
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";
- const Option<nlohmann::json> & bad_rank_field_op = sample_collection->add(doc_str);
+ const Option<nlohmann::json> & bad_rank_field_op = sample_collection->add(doc_str, CREATE, "", DIRTY_VALUES::REJECT);
ASSERT_FALSE(bad_rank_field_op.ok());
ASSERT_STREQ("Field `average` must be an int32.", bad_rank_field_op.error().c_str());
@@ -2007,7 +2019,7 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {
auto add_op = coll1->add(doc.dump());
ASSERT_FALSE(add_op.ok());
- ASSERT_STREQ("Field `categories` must be a string array.", add_op.error().c_str());
+ ASSERT_STREQ("Field `categories` must be an array.", add_op.error().c_str());
collectionManager.drop_collection("coll1");
}